2720 files changed, 113434 insertions, 38430 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 94cb5ae..4b6bc90 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,5095 @@
+2025-08-04  Hans-Peter Nilsson  <hp@bitrange.com>
+
+	* defaults.h (MAX_FIXED_MODE_SIZE): Default to 2 * BITS_PER_WORD
+	for larger-than-32-bitters.
+	* doc/tm.texi.in (MAX_FIXED_MODE_SIZE): Adjust accordingly.  Tweak
+	wording.
+	* doc/tm.texi: Regenerate.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	* dump-context.h: Convert "enum optinfo_item_kind" into
+	"enum class kind" within class optinfo_item.
+	* dumpfile.cc: Likewise.  Use "auto" in a few places.
+	Convert "enum optinfo_kind" to "enum class kind" within
+	class optinfo.
+	* opt-problem.cc: Likewise.
+	* optinfo-emit-json.cc: Likewise.
+	* optinfo.cc: Likewise.
+	* optinfo.h: Likewise.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	PR diagnostics/116253
+	* diagnostics/context.cc (context::set_nesting_level): New.
+	* diagnostics/context.h (context::set_nesting_level): New decl.
+	* doc/libgdiagnostics/topics/compatibility.rst
+	(LIBGDIAGNOSTICS_ABI_5): New.
+	* doc/libgdiagnostics/topics/physical-locations.rst
+	(diagnostic_manager_set_debug_physical_locations): New.
+	* libgdiagnostics++.h (manager::set_debug_physical_locations):
+	New.
+	* libgdiagnostics-private.h
+	(private_diagnostic_set_nesting_level): New decl.
+	* libgdiagnostics.cc (diagnostic_manager::diagnostic_manager):
+	Initialize m_debug_physical_locations.
+	(diagnostic_manager::new_location_from_file_and_line): Add debug
+	printing.
+	(diagnostic_manager::new_location_from_file_line_column):
+	Likewise.
+	(diagnostic_manager::new_location_from_range): Likewise.
+	(diagnostic_manager::set_debug_physical_locations): New.
+	(diagnostic_manager::ensure_linemap_for_file_and_line): Avoid
+	redundant calls to linemap_add.
+	(diagnostic_manager::new_location): Add debug printing.
+	(diagnostic_manager::m_debug_physical_locations): New field.
+	(diagnostic::diagnostic): Initialize m_nesting_level.
+	(diagnostic::get_nesting_level): New accessor.
+	(diagnostic::set_nesting_level): New.
+	(diagnostic::m_nesting_level): New field.
+	(diagnostic_manager::emit_va): Set and reset the nesting level
+	of the context from that of the diagnostic.
+	(diagnostic_manager_set_debug_physical_locations): New.
+	(private_diagnostic_set_nesting_level): New.
+	* libgdiagnostics.h
+	(diagnostic_manager_set_debug_physical_locations): New decl.
+	* libgdiagnostics.map (LIBGDIAGNOSTICS_ABI_5): New.
+	* libsarifreplay.cc (sarif_replayer::handle_result_obj): Support
+	the "nestingLevel" property.
+	* libsarifreplay.h (replay_options::m_debug_physical_locations):
+	New field.
+	* sarif-replay.cc: Add -fdebug-physical-locations.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	PR diagnostics/116792
+	* diagnostics/html-sink.cc
+	(html_builder::make_element_for_diagnostic): Don't add the
+	metadata element if it's empty.
+	(html_builder::make_element_for_metadata): Return null rather than
+	an empty element.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/context.h: Move struct counters to its own header
+	and include it.
+	* diagnostics/counters.h: New file, from the above.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/context.h: Split struct source_printing_options out
+	into "diagnostics/source-printing-options.h" and include it.
+	* diagnostics/source-printing-options.h: New file, from the above.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/context.cc: Update for renaming of option_manager to
+	option_id_manager and of context::m_option_mgr to
+	context::m_option_id_mgr.
+	* diagnostics/context.h: Likewise, moving class declaration to a
+	new diagnostics/option-id-manager.h.
+	* diagnostics/lazy-paths.cc: Likewise.
+	* diagnostics/option-id-manager.h: New file, from material in
+	diagnostics/context.h.
+	* lto-wrapper.cc: Update for renaming of option_manager to
+	option_id_manager.
+	* opts-common.cc: Likewise.
+	* opts-diagnostic.h: Likewise.
+	* opts.cc: Likewise.
+	* toplev.cc: Likewise.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/buffering.h: Update comment to refer to output sinks
+	rather than output formats.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	* gimple-warn-recursion.cc (pass_warn_recursion::execute): Add
+	missing auto_diagnostic_group.
+
+2025-08-04  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/121303
+	* avoid-store-forwarding.cc (is_store_forwarding): Add check
+	for `off_val` in `is_store_forwarding`.
+
+2025-08-04  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (vect_analyze_slp): When analyzing a loop
+	and slp instance discovery fails, immediately fail the whole
+	process.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve-builtins.cc
+	(function_expander::expand): Assert that the return value
+	has an appropriate mode.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-protos.h
+	(aarch64_convert_sve_data_to_pred): Remove the mode argument.
+	* config/aarch64/aarch64.cc
+	(aarch64_sve_emit_int_cmp): Allow PRED_MODE to be VNx16BI or
+	the natural predicate mode for the data mode.
+	(aarch64_convert_sve_data_to_pred): Remove the mode argument
+	and instead always create a VNx16BI result.
+	(aarch64_expand_sve_const_pred): Update call accordingly.
+	* config/aarch64/aarch64-sve-builtins-base.cc
+	(svdupq_impl::expand): Likewise, ensuring that the result
+	has mode VNx16BI.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-protos.h
+	(aarch64_emit_sve_pred_vec_duplicate): Declare.
+	* config/aarch64/aarch64.cc
+	(aarch64_emit_sve_pred_vec_duplicate): New function.
+	* config/aarch64/aarch64-sve.md (vec_duplicate<PRED_ALL:mode>): Use it.
+	* config/aarch64/aarch64-sve-builtins-base.cc
+	(svdup_impl::expand): Handle boolean values specially.  Check for
+	constants and fall back on aarch64_emit_sve_pred_vec_duplicate
+	for the variable case, ensuring that the result has mode VNx16BI.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/iterators.md (PNEXT_ONLY): New int iterator.
+	* config/aarch64/aarch64-sve.md
+	(@aarch64_sve_<sve_pred_op><mode>): Restrict SVE_PITER pattern
+	to VNx16BI_ONLY.
+	(@aarch64_sve_<sve_pred_op><mode>): New PNEXT_ONLY pattern for
+	PRED_HSD.
+	(*aarch64_sve_<sve_pred_op><mode>): Likewise.
+	(*aarch64_sve_<sve_pred_op><mode>_cc): Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve2.md (@aarch64_pred_<sve_int_op><mode>):
+	Split SVE2_MATCH pattern into a VNx16QI_ONLY define_ins and a
+	VNx8HI_ONLY define_expand.  Use a VNx16BI destination for the latter.
+	(*aarch64_pred_<sve_int_op><mode>): New SVE2_MATCH pattern for
+	VNx8HI_ONLY.
+	(*aarch64_pred_<sve_int_op><mode>_cc): Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_pred_fac<cmp_op><mode>):
+	Replace with...
+	(@aarch64_pred_fac<cmp_op><mode>_acle): ...this new expander.
+	(*aarch64_pred_fac<cmp_op><mode>_strict_acle): New pattern.
+	* config/aarch64/aarch64-sve-builtins-base.cc
+	(svac_impl::expand): Update accordingly.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_pred_fcm<cmp_op><mode>_acle)
+	(*aarch64_pred_fcm<cmp_op><mode>_acle, @aarch64_pred_fcmuo<mode>_acle)
+	(*aarch64_pred_fcmuo<mode>_acle): New patterns.
+	* config/aarch64/aarch64-sve-builtins-base.cc
+	(svcmp_impl::expand, svcmpuo_impl::expand): Use them.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_pred_cmp<cmp_op><mode>_wide):
+	Split into VNx16QI_ONLY and SVE_FULL_HSI patterns.  Use VNx16BI
+	results for both.
+	(*aarch64_pred_cmp<cmp_op><mode>_wide): New pattern.
+	(*aarch64_pred_cmp<cmp_op><mode>_wide_cc): Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md
+	(*aarch64_pred_cmp<cmp_op><mode>_wide_cc): Turn into a
+	define_insn_and_rewrite and rewrite the governing predicate
+	of the comparison so that it is identical to the PTEST's.
+	(*aarch64_pred_cmp<cmp_op><mode>_wide_ptest): Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_pred_cmp<cmp_op><mode>_wide)
+	(*aarch64_pred_cmp<cmp_op><mode>_wide_cc): Use <VPRED> instead of
+	VNx16BI for the governing predicate.
+	(*aarch64_pred_cmp<cmp_op><mode>_wide_ptest): Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_pred_cmp<cmp_op><mode>_acle)
+	(*aarch64_pred_cmp<cmp_op><mode>_acle, *cmp<cmp_op><mode>_acle_cc)
+	(*cmp<cmp_op><mode>_acle_and): New patterns that yield VNx16BI
+	results for all element types.
+	* config/aarch64/aarch64-sve-builtins-base.cc
+	(svcmp_impl::expand): Use them.
+	(svcmp_wide_impl::expand): Likewise when implementing an svcmp_wide
+	against an in-range constant.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_sve_punpk<perm_hilo>_acle)
+	(*aarch64_sve_punpk<perm_hilo>_acle): New patterns.
+	* config/aarch64/aarch64-sve-builtins-base.cc
+	(svunpk_impl::expand): Use them for boolean svunpk*.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121294
+	* config/aarch64/aarch64.md (UNSPEC_REV_PRED): New unspec.
+	* config/aarch64/aarch64-sve.md (@aarch64_sve_rev<mode>_acle)
+	(*aarch64_sve_rev<mode>_acle): New patterns.
+	* config/aarch64/aarch64-sve-builtins-base.cc
+	(svrev_impl::expand): Use the new patterns for boolean svrev.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121294
+	* config/aarch64/iterators.md (UNSPEC_TRN1_CONV): Delete.
+	(UNSPEC_PERMUTE_PRED): New unspec.
+	* config/aarch64/aarch64-sve.md (@aarch64_sve_trn1_conv<mode>):
+	Replace with...
+	(@aarch64_sve_<perm_insn><mode>_acle)
+	(*aarch64_sve_<perm_insn><mode>_acle): ...these new patterns.
+	* config/aarch64/aarch64.cc (aarch64_expand_sve_const_pred_trn):
+	Update accordingly.
+	* config/aarch64/aarch64-sve-builtins-functions.h
+	(binary_permute::expand): Use the new _acle patterns for
+	predicate operations.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR testsuite/121118
+	* config/aarch64/iterators.md (VNx16BI_ONLY): New mode iterator.
+	* config/aarch64/predicates.md (aarch64_ptrue_all_operand): New
+	predicate.
+	* config/aarch64/aarch64-sve.md
+	(@aarch64_sve_while_<while_optab_cmp><GPI:mode><VNx16BI_ONLY:mode>_acle)
+	(@aarch64_sve_while_<while_optab_cmp><GPI:mode><PRED_HSD:mode>_acle)
+	(*aarch64_sve_while_<while_optab_cmp><GPI:mode><PRED_HSD:mode>_acle)
+	(*while_<while_optab_cmp><GPI:mode><PRED_HSD:mode>_acle_cc): New
+	patterns.
+	* config/aarch64/aarch64-sve-builtins-functions.h
+	(while_comparison::expand): Use the new _acle patterns that
+	always return a VNx16BI.
+	* config/aarch64/aarch64-sve-builtins-sve2.cc
+	(svwhilerw_svwhilewr_impl::expand): Likewise.
+	* config/aarch64/aarch64.cc
+	(aarch64_sve_move_pred_via_while): Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121293
+	* config/aarch64/aarch64-sve-builtins-base.cc (svdupq_lane::expand):
+	Use aarch64_sve_reinterpret instead of subregs.  Explicitly
+	reinterpret the result back to the required mode, rather than
+	leaving the caller to take a subreg.
+
+2025-08-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121362
+	* tree-ssa-sccvn.cc (vn_reference_lookup_3): Generalize
+	aggregate copy handling.
+
+2025-08-04  Filip Kastl  <fkastl@suse.cz>
+
+	* doc/invoke.texi: Add remark about -options being documented
+	under -fdump-tree.  Remove remark about -graph working only for
+	RTL.
+
+2025-08-04  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120941
+	* config/i386/i386-features.cc (x86_cse_kind): Moved before
+	ix86_place_single_vector_set.
+	(redundant_load): Likewise.
+	(ix86_place_single_vector_set): Replace the last argument to the
+	pointer to redundant_load.  For X86_CSE_VEC_DUP, don't place the
+	vector set outside of the loop to avoid extra spills.
+	(remove_redundant_vector_load): Pass load to
+	ix86_place_single_vector_set.
+
+2025-08-03  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr.md (define_insn_and_split) [reload_completed]:
+	For splits that just append a (clobber (reg:CC REG_CC)) to
+	the pattern, use avr_add_ccclobber (curr_insn) instead of
+	repeating the original pattern.
+	* config/avr/avr-dimode.md: Same.
+	* config/avr/avr-fixed.md: Same.
+
+2025-08-03  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr.cc (avr_add_ccclobber): New function.
+	* config/avr/avr-protos.h (avr_add_ccclobber): New proto.
+	(DONE_ADD_CCC): New define.
+
+2025-08-03  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/90242
+	* tree-ssa-sccvn.cc (vn_reference_compute_hash): Use
+	poly_offset_int for offset accumulation.  For hashing
+	truncate to 64 bits and also hash 64 bits.
+	(vn_reference_eq): Likewise.
+
+2025-08-02  Gerald Pfeifer  <gerald@pfeifer.com>
+
+	PR target/69374
+	* doc/install.texi (Specific) <windows>: Drop note on 16-bit
+	Windows support. Streamline note on 32-bit support.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121350
+	* tree-vect-stmts.cc (vectorizable_store): Pass down SLP
+	node when costing scalar stores in vect_body.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121349
+	* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
+	Get full SLP mask, reduce to uniform scalar_mask for further
+	processing if possible.
+	(vect_check_scalar_mask): Remove scalar mask output, remove
+	code conditional on slp_mask.
+	(vectorizable_call): Adjust.
+	(check_scan_store): Get and check SLP mask.
+	(vectorizable_store): Eliminate scalar mask variable.
+	(vectorizable_load): Likewise.
+
+2025-08-01  Gerald Pfeifer  <gerald@pfeifer.com>
+
+	* doc/install.texi (Prerequisites): mdocml.bsd.lv is now
+	mandoc.bsd.lv.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type): Remove,
+	inline into ...
+	(get_load_store_type): ... this.  Remove ncopies parameter.
+	(vectorizable_load): Adjust.
+	(vectorizable_store): Likewise.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type): Remove
+	checks performed at SLP build time.
+	(vect_check_store_rhs): Remove scalar RHS output.
+	(vectorizable_store): Remove uses of scalar RHS.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (VMAT_UNINITIALIZED): New
+	vect_memory_access_type.
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Use it.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121338
+	* tree-ssa-loop-ivopts.cc (avg_loop_niter): Return an
+	unsigned.
+	(adjust_setup_cost): When niters is so large the division
+	result is one or zero avoid it.
+	(create_new_ivs): Adjust.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_simd_clone_data): New.
+	(_slp_tree::simd_clone_info): Remove.
+	(SLP_TREE_SIMD_CLONE_INFO): Likewise.
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Adjust.
+	(_slp_tree::~_slp_tree): Likewise.
+	* tree-vect-stmts.cc (vectorizable_simd_clone_call): Use
+	tyupe specific data to store SLP_TREE_SIMD_CLONE_INFO.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Adjust.
+	(_slp_tree::~_slp_tree): Likewise.
+	* tree-vectorizer.h (vect_data): New base class.
+	(_slp_tree::u): Remove.
+	(_slp_tree::data): Add pointer to vect_data.
+	(_slp_tree::get_data): New helper template.
+
+2025-08-01  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121322
+	* gimple-ssa-store-merging.cc (find_bswap_or_nop): Return NULL if
+	count is 0.
+
+2025-07-31  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr.opt.urls (-mfuse-move2): Add url.
+
+2025-07-31  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr.cc (avr_output_addr_vec) <labl>: Asm out its .type.
+
+2025-07-31  Georg-Johann Lay  <avr@gjlay.de>
+
+	PR rtl-optimization/121340
+	* config/avr/avr.opt (-mfuse-move2): New option.
+	* config/avr/avr-passes.def (avr_pass_2moves): Insert after combine.
+	* config/avr/avr-passes.cc (make_avr_pass_2moves): New function.
+	(pass_data avr_pass_data_2moves): New static variable.
+	(avr_pass_2moves): New rtl_opt_pass.
+	* config/avr/avr-protos.h (make_avr_pass_2moves): New proto.
+	* common/config/avr/avr-common.cc
+	(default_options avr_option_optimization_table) <-mfuse-move2>:
+	Set for -O1 and higher.
+	* doc/invoke.texi (AVR Options) <-mfuse-move2>: Document.
+
+2025-07-31  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120805
+	* tree-vect-loop-manip.cc (vect_gen_vector_loop_niters): Skip setting
+	bounds on epilogues.
+
+2025-07-31  Wilco Dijkstra  <wilco.dijkstra@arm.com>
+
+	* common/config/aarch64/cpuinfo.h: Remove unused features, add FEAT_CSSC
+	and FEAT_MOPS.
+	* config/aarch64/aarch64-option-extensions.def: Remove FMV support
+	for RPRES, use PULL rather than AES, add FMV support for CSSC and MOPS.
+
+2025-07-31  Wilco Dijkstra  <wilco.dijkstra@arm.com>
+
+	* config/aarch64/tuning_models/generic_armv9_a.h
+	(generic_armv9_a_addrcost_table): Use zero cost for himode.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type): Properly
+	compare the scalar type of the gather/scatter offset to
+	the offset vector component type.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* gimple-fold.h (fold_stmt_inplace): Add valueization hook
+	argument, defaulted to no_follow_ssa_edges.
+	* gimple-fold.cc (fold_stmt_inplace): Adjust.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Fix typo.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Handle simultaneous use of regparm and thiscall attributes in
+	case when regparm is set before thiscall.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Fix comments which state that combination of stdcall and fastcall
+	attributes is valid but redundant.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Move 64-bit mode check before regparm handling.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121320
+	* tree-ssa-sccvn.cc (ao_ref_init_from_vn_reference): Convert
+	op->off to poly_offset_int before multiplying by
+	BITS_PER_UNIT.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121323
+	* tree-ssa-alias.cc (ao_ref_init_from_ptr_and_range): Check
+	the pointer offset fits in a HWI when represented in bits.
+
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* config/aarch64/aarch64.cc (build_ifunc_arg_type):
+	Add new fields _hwcap3 and _hwcap4.
+
+2025-07-31  Kishan Parmar  <kishan@linux.ibm.com>
+
+	PR target/118890
+	* config/rs6000/rs6000.cc (can_be_rotated_to_negative_lis): Avoid left
+	shift of negative value and guard shift count.
+	(can_be_built_by_li_and_rldic): Likewise.
+	(rs6000_emit_set_long_const): Likewise.
+	* config/rs6000/rs6000.md (splitter for plus into two 16-bit parts): Fix
+	UB from overflow in addition.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* config/aarch64/aarch64.cc (aarch64_detect_vector_stmt_subtype):
+	Check for node before dereferencing.
+	(aarch64_vector_costs::add_stmt_cost): Likewise.
+
+2025-07-31  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/121028
+	* config/aarch64/aarch64-sme.md (aarch64_smstart_sm): Use the .inst
+	directive if !TARGET_SME.
+	(aarch64_smstop_sm): Likewise.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_stmt_vec_info::memory_access_type): Remove.
+	(STMT_VINFO_MEMORY_ACCESS_TYPE): Likewise.
+	(vect_mem_access_type): Likewise.
+	* tree-vect-stmts.cc (vectorizable_store): Do not set
+	STMT_VINFO_MEMORY_ACCESS_TYPE.  Fix SLP_TREE_MEMORY_ACCESS_TYPE
+	usage.
+	* tree-vect-loop.cc (update_epilogue_loop_vinfo): Remove
+	checking of memory access type.
+	* config/riscv/riscv-vector-costs.cc (costs::compute_local_live_ranges):
+	Use SLP_TREE_MEMORY_ACCESS_TYPE.
+	(costs::need_additional_vector_vars_p): Likewise.
+	(segment_loadstore_group_size): Get SLP node as argument,
+	use SLP_TREE_MEMORY_ACCESS_TYPE.
+	(costs::adjust_stmt_cost): Pass down SLP node.
+	* config/aarch64/aarch64.cc (aarch64_ld234_st234_vectors): Use
+	SLP_TREE_MEMORY_ACCESS_TYPE instead of vect_mem_access_type.
+	(aarch64_detect_vector_stmt_subtype): Likewise.
+	(aarch64_vector_costs::count_ops): Likewise.
+	(aarch64_vector_costs::add_stmt_cost): Likewise.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_transform_loop): Do not verify DRs
+	have not been modified for epilogue loops.
+	(update_epilogue_loop_vinfo): Do not copy modified DRs to
+	the originals.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121264
+	* machmode.h (get_best_mode): Change type of first 2 arguments
+	from int to HOST_WIDE_INT.
+	* stor-layout.cc (get_best_mode): Likewise.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	* gimple-ssa-store-merging.cc (find_bswap_or_nop): Fix comment typos,
+	hanlde -> handle.
+	* config/i386/i386.cc (ix86_gimple_fold_builtin, ix86_rtx_costs):
+	Likewise.
+	* config/i386/i386-features.cc (remove_partial_avx_dependency):
+	Likewise.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (check_scan_store): Remove redundant
+	slp_node check.  Disallow epilogue vectorization.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vector_costs::costing_for_scalar): New
+	accessor.
+	(add_stmt_cost): For scalar costing force vectype to NULL.
+	Verify we do not pass in a SLP node.
+
+2025-07-31  Kito Cheng  <kito.cheng@sifive.com>
+
+	PR target/121312
+	* config/riscv/arch-canonicalize: Add H extension to the
+	canonical order.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR rtl-optimization/121308
+	* simplify-rtx.cc (simplify_context::simplify_subreg): Handle
+	subreg of `not` with word_mode to make it symmetric with the
+	other bitwise operators.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	PR tree-optimization/121295
+	* tree-if-conv.cc (factor_out_operators): Change the phi node
+	to the new result and args.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	Revert:
+	2025-07-28  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* tree-if-conv.cc (is_cond_scalar_reduction): Instead of phi argument,
+	pass bb and res of the phi.
+	(factor_out_operators): Add iterator for the phi. Remove the phi
+	if this is the first time. Return if we had removed the phi.
+	(predicate_scalar_phi): Add the phi iterator argument.
+	Update call to is_cond_scalar_reduction.
+	Update call to factor_out_operators and set the return value to true
+	when factor_out_operators returns true.
+	(predicate_all_scalar_phis): Don't remove the phi if predicate_scalar_phi
+	already removed it.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* auto-profile.cc (string_table::read): Check gcov_is_error.
+	(read_profile): Likewise.
+	* gcov-io.cc (gcov_is_error): Export for gcc linkage.
+	* gcov-io.h (gcov_is_error): Declare.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	* config/i386/i386.cc (ix86_default_vector_cost): Split
+	out from ...
+	(ix86_builtin_vectorization_cost): ... this and use
+	mode instead of vectype as argument.
+	(ix86_vector_costs::add_stmt_cost): Call
+	ix86_default_vector_cost instead of ix86_builtin_vectorization_cost.
+
+2025-07-30  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	PR target/117015
+	* config/s390/s390-protos.h (s390_expand_int_spaceship): New
+	function.
+	(s390_expand_fp_spaceship): New function.
+	* config/s390/s390.cc (s390_expand_int_spaceship): New function.
+	(s390_expand_fp_spaceship): New function.
+	* config/s390/s390.md (spaceship<mode>4): New expander.
+
+2025-07-30  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* cprop.cc (bypass_block): Extract single set.
+	(bypass_conditional_jumps): Ditto.
+
+2025-07-30  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120427
+	* config/i386/i386.md (peephole2): Transform "movq $-1,reg" to
+	"pushq $-1; popq reg" for -Oz if reg is a legacy integer register.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* auto-profile.cc (function_instance::match): Disable warning
+	about bogus locations since dwarf does not represent enough
+	info to output them correctly in all cases.
+	(add_scale): Use nonzero_p instead of orig.force_nonzero () == orig.
+	(afdo_adjust_guessed_profile): Add missing newline in dump
+	file.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* symtab.cc (symbol_table::change_decl_assembler_name): Recompute DECL_RTL
+	in case it is already computed.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* predict.cc (unlikely_executed_edge_p): Ignore EDGE_EH if profile
+	is reliable.
+	(unlikely_executed_stmt_p): special case builtin_trap/unreachable and
+	ignore other heuristics for reliable profiles.
+	(tree_estimate_probability): Disable unlikely bb detection when
+	doing dry run
+
+2025-07-30  Andrew Stubbs  <ams@baylibre.com>
+	     Julian Brown   <julian@codesourcery.com>
+
+	* doc/tm.texi.in (TARGET_VECTORIZE_PREFER_GATHER_SCATTER): Add
+	documentation hook.
+	* doc/tm.texi: Regenerate.
+	* target.def (prefer_gather_scatter): Add target hook under vectorizer.
+	* hooks.cc (hook_bool_mode_int_unsigned_false): New function.
+	* hooks.h (hook_bool_mode_int_unsigned_false): New prototype.
+	* tree-vect-stmts.cc (vect_use_strided_gather_scatters_p): Add
+	parameters group_size and single_element_p, and rework to use
+	targetm.vectorize.prefer_gather_scatter.
+	(get_group_load_store_type): Move some of the condition into
+	vect_use_strided_gather_scatters_p.
+	* config/gcn/gcn.cc (gcn_prefer_gather_scatter): New function.
+	(TARGET_VECTORIZE_PREFER_GATHER_SCATTER): Define hook.
+
+2025-07-30  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn.cc (gcn_option_override): Add note to set default for
+	param_vect_partial_vector_usage to "1".
+	* optc-save-gen.awk: Don't pass through options marked "NoOffload".
+	* params.opt (-param=vect-epilogues-nomask): Add NoOffload.
+	(-param=vect-partial-vector-usage): Likewise.
+	(-param=vect-inner-loop-cost-factor): Likewise.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121130
+	* tree-vect-stmts.cc (vectorizable_call): Bail out for
+	.MASK_CALL.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121020
+	* tree-vect-loop-manip.cc (vect_do_peeling): Update the
+	condition of omitting the skip-vector check.
+	* tree-vectorizer.h (LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING):
+	Add a helper macro.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121190
+	* tree-vect-data-refs.cc (vect_enhance_data_refs_alignment):
+	Increase alignment requirement for speculative loads.
+
+2025-07-30  Alfie Richards  <alfie.richards@arm.com>
+
+	PR target/121300
+	* config/aarch64/aarch64-sve-builtins-sme.def (svamin/svamax): Fix
+	arch gating.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type):
+	Process STMT_VINFO_GATHER_SCATTER before reading
+	memory_access_type.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@cond_<optab><mode>): Extend
+	to support partial FP modes.
+	(*cond_<optab><mode>_2_strict): Extend from SVE_FULL_F to SVE_F,
+	use aarch64_predicate_operand.
+	(*cond_<optab><mode>_4_strict): Extend from SVE_FULL_F_B16B16 to
+	SVE_F_B16B16, use aarch64_predicate_operand.
+	(*cond_<optab><mode>_any_strict):  Likewise.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*cond_<optab><mode>_2_relaxed):
+	Extend from SVE_FULL_F to SVE_F.
+	(*cond_<optab><mode>_4_relaxed): Extend from SVE_FULL_F_B16B16
+	to SVE_F_B16B16.
+	(*cond_<optab><mode>_any_relaxed): Likewise.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (<optab><mode>4): Extend from
+	SVE_FULL_F_B16B16 to SVE_F_B16B16.  Use aarch64_sve_fp_pred instead
+	of aarch64_ptrue_reg.
+	(@aarch64_pred_<optab><mode>): Extend from SVE_FULL_F_B16B16 to
+	SVE_F_B16B16.  Use aarch64_predicate_operand.
+
+2025-07-30  liuhongt  <hongtao.liu@intel.com>
+
+	* config/i386/i386-modes.def: Remove VECTOR_MODES(FLOAT, 256)
+	and VECTOR_MODE (INT, SI, 64).
+	* config/i386/i386.cc (ix86_hard_regno_nregs): Remove related
+	code for V64SF/V64SImode.
+
+2025-07-30  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/121274
+	* config/i386/sse.md (*vec_concatv2di_0): Add a splitter
+	before it.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR rtl-optimization/121302
+	* simplify-rtx.cc (simplify_context::simplify_subreg): Use
+	byte instead of 0 when calling simplify_subreg.
+
+2025-07-29  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-protos.h (aarch64_sve_emit_masked_fp_pred):
+	Declare.
+	* config/aarch64/aarch64-sve.md (and<mode>3):  Change this to...
+	(@and<mode>3): ...this, so that we can use gen_and3.
+	(@cond_<optab><mode>): Extend from SVE_FULL_F_B16B16 to SVE_F_B16B16,
+	use aarch64_predicate_operand.
+	(*cond_<optab><mode>_2_strict): Likewise.
+	(*cond_<optab><mode>_3_strict): Likewise.
+	(*cond_<optab><mode>_any_strict): Likwise.
+	(*cond_<optab><mode>_2_const_strict): Extend from SVE_FULL_F to SVE_F,
+	use aarch64_predicate_operand.
+	(*cond_<optab><mode>_any_const_strict): Likewise.
+	(*cond_sub<mode>_3_const_strict): Likwise.
+	(*cond_sub<mode>_const_strict): Likewise.
+	(*vcond_mask_<mode><vpred>): Use aarch64_predicate_operand, and update
+	the comment here.
+	* config/aarch64/aarch64.cc (aarch64_sve_emit_masked_fp_pred): New
+	function.  Helper to mask the predicate in conditional expanders.
+
+2025-07-29  Dongyan Chen  <chendongyan@isrc.iscas.ac.cn>
+
+	* Makefile.in: Add riscv-mcpu.texi and riscv-mtune.texi to the list
+	of files to be processed by the Texinfo generator.
+	* config/riscv/t-riscv: Add rule for generating riscv-mcpu.texi
+	and riscv-mtune.texi.
+	* doc/invoke.texi: Replace hand‑written extension table with
+	`@include riscv-mcpu.texi` and `@include riscv-mtune.texi` to
+	pull in auto‑generated entries.
+	* config/riscv/gen-riscv-mcpu-texi.cc: New file.
+	* config/riscv/gen-riscv-mtune-texi.cc: New file.
+	* doc/riscv-mcpu.texi: New file.
+	* doc/riscv-mtune.texi: New file.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* simplify-rtx.cc (simplify_context::simplify_subreg): Distribute
+	lowpart subregs through AND/IOR/XOR, if doing so eliminates one
+	of the terms.
+	(test_scalar_int_ext_ops): Add some tests of the above for integers.
+	* config/aarch64/aarch64.cc (aarch64_test_sve_folding): Likewise
+	add tests for predicate modes.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve-builtins.cc
+	(function_expander::get_reg_target): Check whether the target
+	is a valid register_operand.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/120660
+	* avoid-store-forwarding.cc (process_store_forwarding):
+	Fix instruction generation when haveing multiple stores with
+	base offset.
+
+2025-07-29  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* common/config/riscv/riscv-common.cc (riscv_ext_is_subset):
+	Remove use of structured binding to fix compiler warning.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/119795
+	* avoid-store-forwarding.cc
+	(store_forwarding_analyzer::avoid_store_forwarding): Skip
+	transformations for stores that operate on the same address
+	range as deleted ones.
+
+2025-07-29  Pan Li  <pan2.li@intel.com>
+
+	* match.pd: Add mul based unsigned SAT_MUL.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120687
+	* tree-ssa-reassoc.cc (reassociate_bb): Do not disturb
+	the sorted operand order in the early pass.
+	* tree-vect-slp.cc (vect_analyze_slp): Dump when a detected
+	reduction chain fails SLP discovery.
+
+2025-07-29  Alfie Richards  <alfie.richards@arm.com>
+
+	PR middle-end/121261
+	* vec.h: Add null ptr check.
+
+2025-07-29  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121159
+	* calls.cc (can_implement_as_sibling_call_p): Don't reject declared
+	noreturn functions in musttail calls.
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* output.h (MAX_ALIGN_MERGABLE): New define.
+	* tree-switch-conversion.cc (switch_conversion::build_one_array):
+	Use MAX_ALIGN_MERGABLE instead of 256.
+	* varasm.cc (mergeable_string_section): Likewise
+	(mergeable_constant_section): Likewise
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120523
+	* output.h (mergeable_constant_section): New declaration taking
+	unsigned HOST_WIDE_INT for the size.
+	* tree-switch-conversion.cc (switch_conversion::build_one_array):
+	Increase the alignment of CSWTCH for sizes less than 32bytes.
+	* varasm.cc (mergeable_constant_section): Split out twice.
+	One that takes the size in unsigned HOST_WIDE_INT and the
+	other size in a tree.
+	(default_elf_select_section): Pass DECL_SIZE instead of
+	DECL_MODE to mergeable_constant_section.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_load): Un-factor VMAT
+	specific code to their handling blocks.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (gather_scatter_info::offset_dt): Remove.
+	* tree-vect-data-refs.cc (vect_describe_gather_scatter_call):
+	Do not set it.
+	(vect_check_gather_scatter): Likewise.
+	* tree-vect-stmts.cc (vect_truncate_gather_scatter_offset):
+	Likewise.
+	(get_group_load_store_type): Use the vector type of the offset
+	SLP child.  Do not re-check vect_is_simple_use validated by
+	SLP build.
+
+2025-07-28  Georg-Johann Lay  <avr@gjlay.de>
+
+	PR target/121277
+	* config/avr/avr.cc (avr_addr_space_convert): When converting
+	from generic AS to __flashx, don't set bit 23.
+	(avr_convert_to_type): Don't -Waddr-space-convert when NULL
+	is converted to __flashx or to __flash.
+
+2025-07-28  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* tree-if-conv.cc (is_cond_scalar_reduction): Instead of phi argument,
+	pass bb and res of the phi.
+	(factor_out_operators): Add iterator for the phi. Remove the phi
+	if this is the first time. Return if we had removed the phi.
+	(predicate_scalar_phi): Add the phi iterator argument.
+	Update call to is_cond_scalar_reduction.
+	Update call to factor_out_operators and set the return value to true
+	when factor_out_operators returns true.
+	(predicate_all_scalar_phis): Don't remove the phi if predicate_scalar_phi
+	already removed it.
+
+2025-07-28  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/121208
+	* config/i386/i386.cc (ix86_tls_get_addr): Issue an error for
+	-mtls-dialect=gnu with no_caller_saved_registers attribute and
+	suggest -mtls-dialect=gnu2.
+
+2025-07-28  Mikael Pettersson  <mikpelinux@gmail.com>
+
+	PR other/121260
+	* diagnostics/changes.cc: Correct nesting of namespaces
+	and #if CHECKING_P blocks.
+	* diagnostics/context.cc: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/nvptx/nvptx.opt (march-map=): Add sm_100{,f,a},
+	sm_101{,f,a}, sm_103{,a,f}, sm_120{,a,f} and sm_121{,f,a}.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn.md (atomic_load, atomic_store, atomic_exchange):
+	Fix CDNA3 L2 cache write-back before atomic instructions.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
+	Make *gs_info const.
+	(vect_build_one_gather_load_call): Likewise.
+	(vect_build_one_scatter_store_call): Likewise.
+	(vect_get_gather_scatter_ops): Likewise.
+	(vect_get_strided_load_store_ops): Likewise.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn.md  (define_attr "vcmp"): Add with values
+	vcmp/vcmpx/no.
+	(*movbi, cstoredi4.., cstore<mode>4): Set it.
+	* config/gcn/gcn-valu.md (vec_cmp<mode>...): Likewise.
+	* config/gcn/gcn.cc (gcn_cmpx_insn_p): Remove.
+	(gcn_md_reorg): Add two new conditions for MI300.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn-opts.h (enum hsaco_attr_type): Add comment
+	about 'sc0'.
+	* config/gcn/gcn.cc (gcn_md_reorg): Use gen_nops instead of gen_nop.
+	(print_operand_address): Document 'R' and 'V' in the
+	pre-function comment as well.
+	* config/gcn/gcn.md (nops): Add.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121256
+	* tree-vect-loop.cc (vectorizable_recurr): Build a correct
+	initialization vector for SLP_TREE_LANES > 1.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_slp_tree::type): Add.
+	(_slp_tree::u): Likewise.
+	(_stmt_vec_info::type): Remove.
+	(STMT_VINFO_TYPE): Likewise.
+	(SLP_TREE_TYPE): New.
+	* tree-vectorizer.cc (vec_info::new_stmt_vec_info): Do not
+	initialize type.
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Initialize type.
+	(vect_slp_analyze_node_operations): Adjust.
+	(vect_schedule_slp_node): Likewise.
+	* tree-vect-patterns.cc (vect_init_pattern_stmt): Do not
+	copy STMT_VINFO_TYPE.
+	* tree-vect-loop.cc: Set SLP_TREE_TYPE instead of
+	STMT_VINFO_TYPE everywhere.
+	(vect_create_loop_vinfo): Do not set STMT_VINFO_TYPE on
+	loop conditions.
+	* tree-vect-stmts.cc: Set SLP_TREE_TYPE instead of
+	STMT_VINFO_TYPE everywhere.
+	(vect_analyze_stmt): Adjust.
+	(vect_transform_stmt): Likewise.
+	* config/aarch64/aarch64.cc (aarch64_vector_costs::count_ops):
+	Access SLP_TREE_TYPE instead of STMT_VINFO_TYPE.
+	* config/i386/i386.cc (ix86_vector_costs::add_stmt_cost):
+	Remove non-SLP element-wise load/store matching.
+	* config/rs6000/rs6000.cc
+	(rs6000_cost_data::update_target_cost_per_stmt): Pass in
+	the SLP node.  Use that to get at the memory access
+	kind and type.
+	(rs6000_cost_data::add_stmt_cost): Pass down SLP node.
+	* config/riscv/riscv-vector-costs.cc (variable_vectorized_p):
+	Use SLP_TREE_TYPE.
+	(costs::need_additional_vector_vars_p): Likewise.
+	(costs::update_local_live_ranges): Likewise.
+
+2025-07-28  Jennifer Schmitz  <jschmitz@nvidia.com>
+	    Dhruv Chawla  <dhruvc@nvidia.com>
+
+	* config/aarch64/aarch64-cores.def (olympus): Use olympus tuning
+	model.
+	* config/aarch64/aarch64.cc: Include olympus.h.
+	* config/aarch64/tuning_models/olympus.h: New file.
+
+2025-07-28  Lulu Cheng  <chenglulu@loongson.cn>
+
+	* config/loongarch/loongarch.h
+	(CASE_VECTOR_SHORTEN_MODE): Delete.
+
+2025-07-28  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.cc (xtensa_is_insn_L32R_p):
+	Re-rewrite to more accurately capture insns that could be L32R machine
+	instructions wherever possible, and add comments that help understand
+	the intent of the process.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-v.cc (expand_vx_binary_vxrm_vec_vec_dup):
+	Add new case UNSPEC_VAADD.
+	(expand_vx_binary_vxrm_vec_dup_vec): Ditto.
+	* config/riscv/riscv.cc (riscv_rtx_costs): Ditto.
+	* config/riscv/vector-iterators.md: Add new case UNSPEC_VAADD to
+	iterator.
+
+2025-07-27  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR middle-end/120855
+	* cgraphunit.cc (symbol_table::compile): Assert a transparent
+	alias is not an internal identifier.
+	* symtab.cc (symbol_table::change_decl_assembler_name):
+	Likewise.
+	* varasm.cc (assemble_name_resolve): Check for
+	IDENTIFIER_TRANSPARENT_ALIAS instead of just TREE_CHAIN.
+
+2025-07-26  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* diagnostics/context.cc (context::diagnostic_impl)
+	(context::diagnostic_n_impl): In presence of formal parameter
+	'kind', explicitly state 'diagnostics::' scope for 'kind' enum.
+
+2025-07-26  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* diagnostics/option-classifier.cc
+	(update_effective_level_from_pragmas): In presence of local
+	variable 'kind', explicitly state 'diagnostics::' scope for 'kind'
+	enum.
+
+2025-07-26  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* config/riscv/riscv-ext.def: Add allocated group IDs and
+	group bit positions.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/context.cc: Eliminate digraphs::lazy_digraph in
+	favor of lazily_created template.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/digraphs.cc: Likewise, also digraphs::lazy_digraphs.
+	* diagnostics/digraphs.h: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/metadata.h: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/sink.h: Likewise.
+	* diagnostics/text-sink.h: Likewise.
+	* lazily-created.h: New file.
+	* libgdiagnostics.cc: Eliminate digraphs::lazy_digraph in
+	favor of lazily_created template.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h: Use TextOrHtml rather than Sink for template
+	params.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/selftest-context.cc: Use "html_or_text" rather than
+	"sink".
+	* diagnostics/source-printing.cc: Use TextOrHtml rather than Sink
+	for template params.  Use "text_out" and "html_out" rather than
+	"sink".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (diagnostic_set_caret_max_width): Drop forward
+	decl.
+	(diagnostic_same_line): Make
+	diagnostics::context::m_source_printing private.
+	* diagnostics/context.cc (diagnostic_set_caret_max_width): Convert
+	to...
+	(diagnostics::context::set_caret_max_width): ...this.
+	* diagnostics/context.h
+	(diagnostics::context::get_source_printing_options): New
+	accessors.
+	(diagnostics::context::m_source_printing): Make private.
+	* diagnostics/html-sink.cc: Make
+	diagnostics::context::m_source_printing private.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/selftest-context.cc: Likewise.
+	* diagnostics/selftest-context.h
+	(diagnostics::selftest::test_context::colorize_source): New.
+	(diagnostics::selftest::test_context::show_labels): New.
+	(diagnostics::selftest::test_context::show_line_numbers): New.
+	(diagnostics::selftest::test_context::show_ruler): New.
+	(diagnostics::selftest::test_context::show_event_links): New.
+	(diagnostics::selftest::test_context::set_caret_char): New.
+	* diagnostics/source-printing.cc: Make
+	diagnostics::context::m_source_printing private.
+	* diagnostics/text-sink.h: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* opts.cc: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h: Eliminate "diagnostic_info" typedef.
+	* diagnostics/context.cc: Update usage of "diagnostic_info" to
+	explicitly refer to "diagnostics::diagnostic_info".
+	* langhooks.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* rtl-error.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* toplev.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+	* tree-diagnostic.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (enum diagnostics_column_unit): Move to
+	diagnostics/context-options.h.
+	(enum diagnostics_escape_format): Likewise.
+	(enum diagnostics_output_format): Likewise.
+	(enum diagnostic_path_format): Likewise.
+	(enum diagnostics_extra_output_kind): Likewise.
+	(enum diagnostic_text_art_charset): Likewise.
+	* diagnostics/context-options.h: New file, based on the above
+	material.
+	* diagnostics/context.h: Include it.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add diagnostics/file-cache.o.
+	* diagnostics/changes.cc: Update for file_cache and char_span
+	moving from input.h to diagnostics/file-cache.h and into the
+	"diagnostics::" namespace.
+	* diagnostics/context.cc: Likewise.
+	* diagnostics/diagnostics-selftests.cc: Likewise.
+	* diagnostics/diagnostics-selftests.h: Likewise.
+	* diagnostics/file-cache.cc: New file, based on the file_cache
+	and file_cache_slot material in input.cc.
+	* diagnostics/file-cache.h: Likewise for input.h.
+	* diagnostics/selftest-source-printing.h: Update for file_cache
+	and char_span moving from input.h to diagnostics/file-cache.h and
+	into the "diagnostics::" namespace.
+	* diagnostics/source-printing.cc: Likewise.
+	* final.cc: Likewise.
+	* gcc-rich-location.cc: Likewise.
+	* input.cc (default_charset_callback): Move to
+	diagnostics/file-cache.cc.
+	(file_cache::initialize_input_context): Likewise.
+	(class file_cache_slot): Likewise.
+	(file_cache::tune): Likewise.
+	(file_cache::lookup_file): Likewise.
+	(file_cache::forcibly_evict_file): Likewise.
+	(file_cache::missing_trailing_newline_p): Likewise.
+	(file_cache::add_buffered_content): Likewise.
+	(file_cache::evicted_cache_tab_entry): Likewise.
+	(file_cache::add_file): Likewise.
+	(file_cache::file_cache): Likewise.
+	(file_cache::dump): Likewise.
+	(file_cache::dump): Likewise.
+	(file_cache::lookup_or_add_file): Likewise.
+	(find_end_of_line): Likewise.
+	(file_cache::get_source_line): Likewise.
+	(check_line): Likewise.
+	(test_replacement): Likewise.
+	(test_reading_source_line): Likewise.
+	(test_reading_source_buffer): Likewise.
+	* input.h (class char_span): Move to diagnostics/file-cache.h and
+	into the "diagnostics::" namespace.
+	(class file_cache_slot): Likewise.
+	(class file_cache): Likewise.
+	* libgdiagnostics.cc: Update for file_cache and char_span moving
+	from input.h to diagnostics/file-cache.h and into the
+	"diagnostics::" namespace.
+	* selftest.cc: Likewise.
+	* selftest.h: Likewise.
+	* substring-locations.h: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/selftest-paths.cc: Don't include "demangle.h"
+	and "backtrace.h".
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/state-graphs.cc: Don't include "graphviz.h",
+	"xml.h", "xml-printer.h", or "intl.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h: Move include of "unique-argv.h"
+	and various forward decls into...
+	* diagnostics/context.h: ...here.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (num_digits): Move to...
+	(diagnostics::num_digits): ...here.
+	(get_cwe_url): Move to...
+	(diagnostics::get_cwe_url): ...here.
+	(maybe_line_and_column): Move to...
+	(diagnostics::maybe_line_and_column): ...here.
+	* diagnostic.cc: Move to...
+	* diagnostics/context.cc: ...here.
+	* diagnostics/diagnostics-selftests.cc
+	(run_diagnostics_selftests): Add comment about context_cc_tests.
+	* diagnostics/diagnostics-selftests.h: Add context_cc_tests decl.
+	* diagnostics/sarif-sink.cc: Update for file rename.
+	* diagnostics/source-printing.cc: Update for num_digits moving to
+	diagnostics::num_digits.
+	* errors.cc: Update for file rename.
+	* input.cc: Update for num_digits moving to
+	diagnostics::num_digits.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic.o with
+	diagnostics/context.o.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in: Replace diagnostic.def with diagnostics/kinds.def.
+	* config/aarch64/aarch64.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* config/i386/i386-options.cc: Likewise.
+	* config/s390/s390.cc: Likewise.
+	* diagnostic-core.h: Replace typedef diagnostic_t with
+	enum class diagnostics::kind in diagnostics/kinds.h and include
+	it.
+	* diagnostic-global-context.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostics/buffering.cc: Likewise.
+	* diagnostics/buffering.h: Likewise.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/diagnostic-info.h: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostic.def: Move to...
+	* diagnostics/kinds.def: ...here and update for diagnostic_t
+	becoming enum class diagnostics::kind.
+	* diagnostics/kinds.h: New file, based on material in
+	diagnostic-core.h.
+	* diagnostics/lazy-paths.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* diagnostics/option-classifier.cc: Likewise.
+	* diagnostics/option-classifier.h: Likewise.
+	* diagnostics/output-spec.h: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/selftest-context.cc: Likewise.
+	* diagnostics/selftest-context.h: Likewise.
+	* diagnostics/sink.h: Likewise.
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* diagnostics/text-sink.h: Likewise.
+	* gcc.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* lto-wrapper.cc: Likewise.
+	* opts-common.cc: Likewise.
+	* opts-diagnostic.h: Likewise.
+	* opts.cc: Likewise.
+	* rtl-error.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-core.h: Include "diagnostics/option-id.h".
+	(struct diagnostic_option_id): Move there, renaming to
+	diagnostics::option_id.
+	* diagnostic-global-context.cc: Update for renaming of
+	diagnostic_option_id to diagnostics::option_id.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/diagnostic-info.h: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise.
+	* diagnostics/option-classifier.cc: Likewise.
+	* diagnostics/option-classifier.h: Likewise.
+	* diagnostics/option-id.h: New file, taken from material in
+	diagnostic-core.h.
+	* diagnostics/selftest-context.cc: Update for renaming of
+	diagnostic_option_id to diagnostics::option_id.
+	* diagnostics/selftest-context.h: Likewise.
+	* ipa-pure-const.cc: Likewise.
+	* lto-wrapper.cc: Likewise.
+	* opts-common.cc: Likewise.
+	* opts-diagnostic.h: Likewise.
+	* opts.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* substring-locations.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic-spec.o with
+	gcc-diagnostic-spec.o.
+	(GTFILES): Replace diagnostic-spec.h with gcc-diagnostic-spec.h.
+	* diagnostic-spec.cc: Move to...
+	* gcc-diagnostic-spec.cc: ...here.
+	* diagnostic-spec.h: Move to...
+	* gcc-diagnostic-spec.h: ...here.
+	* gengtype.cc (open_base_files): Replace diagnostic-spec.h with
+	gcc-diagnostic-spec.h.
+	* warning-control.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostics/changes.o with
+	diagnostics/edit-context.o.
+	* diagnostic.cc: Update for move of diagnostics/edit-context.h to
+	diagnostics/changes.h.
+	(diagnostics::context::initialize): Update for renaming of
+	m_edit_context_ptr to m_fixits_change_set.
+	(diagnostics::context::finish): Likewise.
+	(diagnostics::context::create_edit_context): Rename to...
+	(diagnostics::context::initialize_fixits_change_set): ...this and
+	update for other renamings.
+	(diagnostics::context::report_diagnostic): Update for renaming of
+	m_edit_context_ptr to m_fixits_change_set.
+	* diagnostic.h (diagnostics::edit_context): Drop forward decl
+	* diagnostics/edit-context.cc: Move to...
+	* diagnostics/changes.cc: ...here.  Update for move of
+	diagnostics/edit-context.h to diagnostics/changes.h.
+	Rename diagnostics::edit_context to
+	diagnostics::changes::change_set.  Rename diagnostics::edited_file
+	to diagnostics::changes::changed_file.  Rename
+	diagnostics::edited_line to diagnostics::changes::changed_line.
+	* diagnostics/edit-context.h: Move to...
+	* diagnostics/changes.h: ...here, renaming as above.
+	* diagnostics/context.h (diagnostics::changes::change_set): New
+	forward decl.
+	(diagnostics::context::create_edit_context): Rename to...
+	(diagnostics::context::initialize_fixits_change_set): ...this.
+	(diagnostics::context::get_edit_context): Rename to...
+	(diagnostics::context::get_fixits_change_set): ...this.
+	(diagnostics::context::m_edit_context_ptr): Rename to...
+	(diagnostics::context::m_fixits_change_set): ..this and update
+	type.
+	* diagnostics/diagnostics-selftests.cc: Update for renaming.
+	* diagnostics/diagnostics-selftests.h: Likewise.
+	* diagnostics/html-sink.cc: Update for above changes.
+	* libgdiagnostics.cc: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Move diagnostics/lazy-paths.o to...
+	(OBJS-libcommon): ...here.  Add
+	diagnostics/diagnostics-selftests.o.
+	* diagnostic.cc: Update for move of
+	selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+	* diagnostics/color.cc: Move selftests from selftest:: to
+	diagnostics::selftest::.
+	* diagnostics/diagnostics-selftests.cc: New file.
+	* diagnostics/diagnostics-selftests.h: New file.
+	* diagnostics/digraphs.cc: Move selftests from selftest:: to
+	diagnostics::selftest::.
+	* diagnostics/edit-context.cc: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise.  Eliminate use of "tree"
+	by porting selftests from simple-diagnostic-path.h to
+	diagnostics/selftest-paths.h.
+	* diagnostics/output-spec.cc: Move selftests from selftest:: to
+	diagnostics::selftest::.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/selftest-context.cc: Move
+	selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+	* diagnostics/selftest-context.h: Likewise.
+	* diagnostics/selftest-logical-locations.cc: Move selftests from
+	selftest:: to diagnostics::logical_locations::selftest::.
+	* diagnostics/selftest-logical-locations.h: Move
+	selftest::logical_location_manager to
+	diagnostics::logical_locations::selftest::logical_location_manager.
+	* diagnostics/selftest-paths.cc: Move
+	selftest::test_diagnostic_path to
+	diagnostics::paths::selftest::test_path.  Move
+	selftest::test_diagnostic_event to
+	diagnostics::paths::selftest::test_event.
+	* diagnostics/selftest-paths.h: Likewise, and move
+	selftest::test_diagnostic_thread to
+	diagnostics::paths::selftest::test_thread.
+	* diagnostics/selftest-source-printing.h: Move
+	selftest::diagnostic_show_locus_fixture to
+	diagnostics::selftest::source_printing_fixture.
+	* diagnostics/source-printing.cc: Move selftests from selftest::
+	to diagnostics::selftest:: and update for renames.
+	* diagnostics/state-graphs.cc: Likewise.
+	* selftest-run-tests.cc: Include
+	"diagnostics/diagnostics-selftests.h".
+	(selftest::run_tests): Replace invocation of the various diagnostics
+	selftests with a call to
+	diagnostics::selftest::run_diagnostics_selftests.
+	* selftest.h: Move decls of the various per-file diagnostics
+	invocation functions to diagnostics/diagnostics-selftests.h,
+	renaming due to diagostics prefix being implied by namespace.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* common.opt: Update for move of diagnostic-url.h to
+	diagnostics/url.h.
+	* diagnostic.cc: Likewise.
+	* diagnostics/color.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* diagnostic-url.h: Move to...
+	* diagnostics/url.h: ...here.
+	* libgdiagnostics.cc: Update for move of diagnostic-url.h to
+	diagnostics/url.h.
+	* pretty-print.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic-color.o with
+	diagnostics/color.o.
+	(PLUGIN_HEADERS): Remove diagnostic-color.h in favor of glob.
+	* common.opt: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+	* diagnostic.cc: Likewise.
+	* diagnostic-color.cc: Move to...
+	* diagnostics/color.cc: ...here.
+	* diagnostic-color.h: Move to...
+	* diagnostics/color.h: ...here.
+	* diagnostics/edit-context.cc: Update for move of
+	diagnostic-color.h to diagnostics/color.h.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* opts.cc: Likewise.
+	* pretty-print-markup.h: Likewise.
+	* pretty-print.cc: Likewise.
+	* selftest-run-tests.cc: Likewise.
+	* selftest.h: Likewise.
+	* text-art/style.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace selftest-diagnostic.o with
+	diagnostics/selftest-context.o.
+	* diagnostic.cc: Update for move of selftest-diagnostic.h to
+	diagnostics/selftest-context.h.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* selftest-diagnostic.cc: Move to...
+	* diagnostics/selftest-context.cc: ...here and update for move of
+	header file.
+	* selftest-diagnostic.h: Move to...
+	* diagnostics/selftest-context.h: ...here.
+	* diagnostics/source-printing.cc: Update for move of
+	selftest-diagnostic.h to diagnostics/selftest-context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add diagnostics/buffering.o.
+	* diagnostic.cc (diagnostics::context::set_diagnostic_buffer): Move to
+	diagnostics/buffering.cc.
+	(diagnostics::context::clear_diagnostic_buffer): Likewise.
+	(diagnostics::context::flush_diagnostic_buffer): Likewise.
+	(diagnostics::buffer::buffer): Likewise.
+	(diagnostics::buffer::~buffer): Likewise.
+	(diagnostics::buffer::dump): Likewise.
+	(diagnostics::buffer::empty_p): Likewise.
+	(diagnostics::buffer::move_to): Likewise.
+	(diagnostics::buffer::ensure_per_sink_buffers): Likewise.
+	* diagnostics/buffering.cc: New file, from material in diagnostic.cc.
+	* diagnostics/buffering.h: Fix comment.
+	* diagnostics/text-sink.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add
+	diagnostics/option-classifier.o.
+	* diagnostic.cc (diagnostics::option_classifier::init): Move to
+	diagnostics/option-classifier.cc.
+	(diagnostics::option_classifier::fini): Likewise.
+	(diagnostics::option_classifier::pch_save): Likewise.
+	(diagnostics::option_classifier::pch_restore): Likewise.
+	(diagnostics::option_classifier::push): Likewise.
+	(diagnostics::option_classifier::pop): Likewise.
+	(diagnostics::option_classifier::classify_diagnostic): Likewise.
+	(diagnostics::option_classifier::update_effective_level_from_pragmas):
+	Likewise.
+	* diagnostics/context.h: Include
+	"diagnostics/option-classifier.h".
+	(class option_classifier): Move to
+	diagnostics/option-classifier.h.
+	* diagnostics/option-classifier.cc: New file, based on material
+	from diagnostic.cc.
+	* diagnostics/option-classifier.h: New file, based on material
+	from diagnostics/context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* coretypes.h: Update for diagnostic_context becoming
+	diagnostics::context.
+	* diagnostic-global-context.cc: Likewise.
+	* diagnostic.cc: Likewise.  Also for diagnostic_option_classifier
+	becoming diagnostics::option_classifier.
+	* diagnostic.h (diagnostic_text_starter_fn): Rename to...
+	(diagnostics::text_starter_fn): ...this, and move to
+	diagnostics/context.h.
+	(get_printer): Rename to...
+	(diagnostics::get_printer): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_option_manager): Rename to...
+	(class diagnostics::option_manager): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_option_classifier): Rename to...
+	(class diagnostics::option_classifier): ...this, and move to
+	diagnostics/context.h.
+	(struct diagnostic_source_printing_options): Rename to...
+	(struct diagnostics::source_printing_options): ...this, and move
+	to diagnostics/context.h.
+	(class diagnostic_column_policy): Rename to...
+	(class diagnostics::column_policy): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_location_print_policy): Rename to...
+	(class diagnostics::location_print_policy): ...this, and move to
+	diagnostics/context.h.
+	(class html_label_writer): Rename to...
+	(class diagnostics::html_label_writer): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_source_print_policy): Rename to...
+	(class diagnostics::source_print_policy): ...this, and move to
+	diagnostics/context.h.
+	(struct diagnostic_counters): Rename to...
+	(struct diagnostics::counters): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_context): Rename to...
+	(class diagnostics::context): ...this, and move to
+	diagnostics/context.h.
+	(diagnostic_text_starter): Rename to...
+	(diagnostics::text_starter): ...this, and move to
+	diagnostics/context.h.
+	(diagnostic_start_span): Rename to...
+	(diagnostics::start_span): ...this, and move to
+	diagnostics/context.h.
+	(diagnostic_text_finalizer): Rename to...
+	(diagnostics::text_finalizer): ...this, and move to
+	diagnostics/context.h.
+	Include "diagnostics/context.h".
+	* diagnostics/buffering.h: Update for diagnostic_context becoming
+	diagnostics::context; similarly for diagnostic_counters.
+	* diagnostics/client-data-hooks.h: Likewise.
+	* diagnostics/context.h: New file, taken from material in
+	diagnostic.h.
+	* diagnostics/html-sink.cc: : Update for diagnostic_context
+	becoming diagnostics::context.
+	* diagnostics/html-sink.h: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise for
+	diagnostic_option_manager.
+	* diagnostics/output-file.h: Likewise for diagnostic_context.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/output-spec.h: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/sarif-sink.h: Likewise.
+	* diagnostics/sink.h: Likewise.
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* diagnostics/text-sink.h: Likewise.
+	* gcc-rich-location.h: Likewise.
+	* gcc.cc: Likewise.
+	* gdbinit.in: Likewise.
+	* langhooks-def.h: Likewise.
+	* langhooks.cc: Likewise.
+	* langhooks.h: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* optc-gen.awk: Likewise.
+	* opth-gen.awk: Likewise.
+	* opts-common.cc: Likewise.
+	* opts-diagnostic.cc: Likewise.
+	* opts-diagnostic.h: Likewise.
+	* opts-global.cc: Likewise.
+	* opts.cc: Likewise.
+	* opts.h: Likewise.
+	* selftest-diagnostic.cc: Likewise.
+	* selftest-diagnostic.h: Likewise.
+	* toplev.cc: Likewise.
+	* tree-diagnostic-client-data-hooks.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+	* tree-diagnostic.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (struct diagnostic_info): Move to
+	diagnostics/diagnostic-info.h as diagnostics::diagnostic_info.
+	Add typedef bringing it back into root namespace, for now.
+	* diagnostics/diagnostic-info.h: New file, based on the above.
+	* langhooks-def.h: Update for diagnostic_info moving into
+	namespace diagnostics.
+	* langhooks.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+	* diagnostic.h: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace diagnostic-macro-unwinding.o with
+	diagnostics/macro-unwinding.o.
+	* diagnostic-macro-unwinding.cc: Move to...
+	* diagnostics/macro-unwinding.cc: ...here, moving content into
+	namespace diagnostics.
+	* diagnostic-macro-unwinding.h: Move to...
+	* diagnostics/macro-unwinding.h: ...here, moving content into
+	namespace diagnostics.
+	* diagnostics/paths-output.cc: Update for move of
+	"diagnostic-macro-unwinding.h" to "diagnostics/macro-unwinding.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace diagnostic-show-locus.o with
+	diagnostics/source-printing.o.
+	* diagnostic.h (class diagnostic_source_effect_info): Replace
+	with...
+	(class diagnotics::source_effect_info): ...this.
+	* diagnostics/paths-output.cc: Likewise.  Update for move of
+	"diagnostic-label-effects.h" to
+	"diagnostics/source-printing-effects.h".
+	* diagnostics/sarif-sink.cc: Update for move of
+	"selftest-diagnostic-show-locus.h" to
+	"diagnostics/selftest-source-printing.h".
+	* selftest-diagnostic-show-locus.h: Move to...
+	* diagnostics/selftest-source-printing.h: ...here.
+	* diagnostic-label-effects.h: Move to...
+	* diagnostics/source-printing-effects.h: ...here, updating
+	for above changes.
+	* diagnostic-show-locus.cc: Move to...
+	* diagnostics/source-printing.cc: ...here.
+	* gcc-rich-location.h: Likewise.
+	* selftest-run-tests.cc: Likewise.
+	* selftest.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace diagnostic-format-html.o with
+	diagnostics/html-sink.o, diagnostic-format-sarif.o with
+	diagnostics/sarif-sink.o, and diagnostic-format-text.o with
+	diagnostics/text-sink.o.
+	* coretypes.h (class diagnostic_text_output_format): Replace
+	with...
+	(class diagnostics::text_sink): ...this.
+	* diagnostic-global-context.cc: Update for move of diagnostics
+	output formats into namespace "diagnostics", as "sinks".
+	* diagnostic-macro-unwinding.cc: Likewise.
+	* diagnostic-macro-unwinding.h: Likewise.
+	* diagnostic-show-locus.cc: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-buffer.h: Rename to...
+	* diagnostics/buffering.h: ...this, updating for above changes.
+	* diagnostics/client-data-hooks.h: Likewise.
+	* diagnostics/digraphs.cc: Likewise.
+	* diagnostic-format-html.cc: Rename to...
+	* diagnostics/html-sink.cc: ...this, updating for above changes.
+	* diagnostic-format-html.h: Rename to...
+	* diagnostics/html-sink.h: ...this, updating for above changes.
+	* diagnostics/lazy-paths.cc: Likewise.
+	* diagnostics/metadata.h: Likewise.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/output-spec.h: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/paths.h: Likewise.
+	* diagnostic-format-sarif.cc: Rename to...
+	* diagnostics/sarif-sink.cc: ...this, updating for above changes.
+	* diagnostic-format-sarif.h: Rename to...
+	* diagnostics/sarif-sink.h: ...this, updating for above changes.
+	* diagnostic-format.h: Rename to...
+	* diagnostics/sink.h: ...this, updating for above changes.
+	* diagnostics/state-graphs-to-dot.cc: Likewise.
+	* diagnostic-format-text.cc: Rename to...
+	* diagnostics/text-sink.cc: ...this, updating for above changes.
+	* diagnostic-format-text.h: Rename to...
+	* diagnostics/text-sink.h: ...this, updating for above changes.
+	* gcc.cc: Likewise.
+	* langhooks-def.h: Likewise.
+	* langhooks.cc: Likewise.
+	* langhooks.h: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* opts-diagnostic.cc: Likewise.
+	* opts.cc: Likewise.
+	* selftest-diagnostic.cc: Likewise.
+	* selftest-run-tests.cc: Likewise.
+	* selftest.h: Likewise.
+	* tree-diagnostic-client-data-hooks.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace lazy-diagnostic-path.o with
+	diagnostics/lazy-paths.o.
+	(OBJS-libcommon): Replace diagnostic-path.o with
+	diagnostics/paths.o, diagnostic-path-output.o with
+	diagnostics/paths-output.o, and selftest-diagnostic-path.o with
+	diagnostics/selftest-paths.o.
+	(EXTRA_BACKEND_OBJS): Replace lazy-diagnostic-path.o with
+	diagnostics/lazy-paths.o.
+	* diagnostic-format-html.cc: Update #include for
+	"diagnostic-path.h" moving to "diagnostics/paths.h",
+	diagnostic_thread_id_t to diagnostics::paths::thread_id_t,
+	diagnostic_event_id_t to diagnostics::paths::event_id_t,
+	diagnostic_path to diagnostics::paths::path, and
+	diagnostic_thread to diagnostics::paths::thread, and
+	diagnostic_event to diagnostics::paths::event.
+	* diagnostic-format-html.h: Likewise.
+	* diagnostic-format-sarif.cc: Likewise.  Update PROPERTY_PREFIX
+	for threadFlowLocations from "gcc/diagnostic_event/" to
+	"gcc/diagnostics/paths/event/".
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic-format-text.h: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-event-id.h: Move to...
+	* diagnostics/event-id.h: ...here, updating header guard.
+	(diagnostics:paths:event_id_t): New typedef.
+	(diagnostic_thread_id_t): Replace with...
+	(diagnostics:paths:thread_id_t): New typedef.
+	* lazy-diagnostic-path.cc: Move to...
+	* diagnostics/lazy-paths.cc: ...here.  Update for above changes,
+	lazy_diagnostic_path becomes diagnostics::paths::lazy_path.
+	(lazy_diagnostic_path_cc_tests): Rename to...
+	(diagnostics_lazy_paths_cc_tests): ...this.
+	* lazy-diagnostic-path.h: Move to...
+	* diagnostics/lazy-paths.h: ...here, updating header guard.
+	Update for above changes.
+	* diagnostic-path-output.cc: Move to...
+	* diagnostics/paths-output.cc: ...here.  Update for above changes.
+	(diagnostic_path_output_cc_tests): Rename to...
+	(diagnostics_paths_output_cc_tests): ...this.
+	* diagnostic-path.cc: Move to...
+	* diagnostics/paths.cc: ...here.  Update for above changes.
+	* diagnostic-path.h: Move to...
+	* diagnostics/paths.h: ...here, updating header guard.
+	Update #include for moving "diagnostic-event-id.h" to
+	"diagnostics/event-id.h".
+	(class diagnostic_event): Convert to...
+	(class diagnostics::paths::event): ...this.
+	(class diagnostic_thread): Convert to...
+	(class diagnostics::paths::thread): ...this.
+	(class diagnostic_path): Convert to...
+	(class diagnostics::paths::path): ...this.
+	* diagnostic-show-locus.cc: Update for above changes.
+	* doc/analyzer.texi: Likewise.
+	* selftest-diagnostic-path.cc: Move to...
+	* diagnostics/selftest-paths.cc: ...here.  Update for
+	above changes, and for "selftest-diagnostic-path.h" moving to
+	"diagnostics/selftest-paths.h".
+	* selftest-diagnostic-path.h: Move to...
+	* diagnostics/selftest-paths.h: ...here, updating header guard.
+	Update for above changes.
+	* libgdiagnostics.cc: Update for above changes.
+	* libsarifreplay.cc: Update property prefix for
+	threadFlowLocations from "gcc/diagnostic_event/" to
+	"gcc/diagnostics/paths/event/".
+	* pretty-print-format-impl.h: Update for above changes.
+	* pretty-print.cc: Likewise.
+	* selftest-run-tests.cc (selftest::run_tests): Update for
+	renaming of lazy_diagnostic_path_cc_tests to
+	diagnostics_lazy_paths_cc_tests, and of
+	diagnostic_path_output_cc_tests to
+	diagnostics_paths_output_cc_tests.
+	* selftest.h (lazy_diagnostic_path_cc_tests): Replace decl with...
+	(diagnostics_lazy_paths_cc_tests): ...this.
+	(diagnostic_path_output_cc_tests): Replace decl with...
+	(diagnostics_paths_output_cc_tests): ...this.
+	* simple-diagnostic-path.cc: Clarify that this relates to "tree"
+	and thus shouldn't be in "diagnostics".  Update for above changes.
+	* simple-diagnostic-path.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic-output-spec.o
+	with diagnostics/output-spec.o.
+	* diagnostic-output-spec.cc: Move to...
+	* diagnostics/output-spec.cc: ...here.  Update #include for
+	move of diagnostic-output-spec.h to diagnostics/output-spec.h.
+	(namespace::diagnostics_output_spec): Replace with...
+	(namespace diagnostics::output_spec): ...this, removing redundant
+	prefixes.
+	(diagnostics_output_spec::gcc_spec_context): Replace with...
+	(diagnostics::output_spec::dc_spec_context): ...this.
+	(diagnostic_output_spec_cc_tests): Rename to...
+	(diagnostics_output_spec_cc_tests): ...this.
+	* diagnostic-output-spec.h: Move to...
+	* diagnostics/output-spec.h: ...here and update inclusion guard.
+	(namespace::diagnostics_output_spec): Replace with...
+	(namespace diagnostics::output_spec): ...this, removing redundant
+	prefixes.
+	(diagnostics_output_spec::gcc_spec_context): Replace with...
+	(diagnostics::output_spec::dc_spec_context): ...this.
+	* libgdiagnostics.cc: Update #include for move of
+	diagnostic-output-spec.h to diagnostics/output-spec.h.
+	* opts-diagnostic.cc: Likewise.  Update for namespace and name
+	changes.
+	* selftest-run-tests.cc (selftest::run_tests): Update for renaming
+	of diagnostic_output_spec_cc_tests to
+	diagnostics_output_spec_cc_tests.
+	* selftest.h (diagnostic_output_spec_cc_tests): Rename decl to...
+	(diagnostics_output_spec_cc_tests): ...this.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Update #include for move of
+	diagnostic-output-file.h to diagnostics/output-file.h.  Update for
+	move of diagnostic_output_file to diagnostics::output_file.
+	* diagnostic-format-html.h: Likewise.
+	* diagnostic-format-sarif.cc: Likewise.
+	* diagnostic-format-sarif.h: Likewise.
+	* diagnostic-output-spec.cc: Likewise.
+	* diagnostic-output-spec.h (along): Likewise.
+	* diagnostic-output-file.h: Move to...
+	* diagnostics/output-file.h: ...here, updating header guard.
+	(class diagnostic_output_file): Move to...
+	(class diagnostics::output_file): ...here.
+	* libgdiagnostics.cc (sarif_sink::sarif_sink): Update for
+	move of diagnostic_output_file to diagnostics::output_file.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Update #include for move of
+	diagnostic-client-data-hooks.h to diagnostics/client-data-hooks.h.
+	* diagnostic-format-sarif.cc: Likewise.  Update for move of
+	diagnostic_client_plugin_info to diagnostics::client_plugin_info.
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-client-data-hooks.h: Move to...
+	* diagnostics/client-data-hooks.h: ...here, updating header guard.
+	(class diagnostic_client_data_hooks): Move to...
+	(class diagnostics::client_data_hooks): ...here.  Remove redundant
+	"diagnostics::" prefixes.
+	(class diagnostic_client_plugin_info): Move to...
+	(class diagnostics::client_plugin_info): ...here.
+	* libgdiagnostics.cc: : Update #include for move of
+	diagnostic-client-data-hooks.h to diagnostics/client-data-hooks.h.
+	Update for move of diagnostic_client_data_hooks to
+	diagnostics::client_data_hooks, and for move of
+	diagnostic_client_version_info to diagnostics::client_version_info.
+	* tree-diagnostic-client-data-hooks.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace edit-context.o with
+	diagnostics/edit-context.o.
+	* diagnostic-format-html.cc: Update #include for move
+	of edit-context.h to diagnostics subdir.
+	* diagnostic.cc: Likewise.  Update for move of edit_context to
+	diagnostics::edit_context.
+	* diagnostic.h: Likewise.
+	* edit-context.cc: Move to...
+	* diagnostics/edit-context.cc: ...here.  Update #include for move
+	of edit-context.h to diagnostics subdir.  Move all code
+	within namespace diagnostics.
+	* edit-context.h: Move to...
+	* diagnostics/edit-context.h: ...here, updating header guard.
+	Move all code within namespace diagnostics.
+	* libgdiagnostics.cc: Update #include for move of edit-context.h
+	to diagnostics subdir.  Update for move of edit_context
+	to diagnostics::edit_context.
+	* toplev.cc: Update #include for move of edit-context.h
+	to diagnostics subdir.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Update for move of diagnostic_diagram
+	to diagnostics::diagram.
+	* diagnostic-format-sarif.cc: Update #include for move of
+	"diagnostic-diagram.h" to "diagnostics/diagram.h". Update for move
+	of diagnostic_diagram to diagnostics::diagram.
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic-format-text.h: Update for move of diagnostic_diagram
+	to diagnostics::diagram.
+	* diagnostic-format.h: Likewise.
+	* diagnostic.cc: Update #include for move of
+	"diagnostic-diagram.h" to "diagnostics/diagram.h". Update for move
+	of diagnostic_diagram to diagnostics::diagram.
+	* diagnostic.h: Update for move of diagnostic_diagram to
+	diagnostics::diagram.
+	* diagnostic-diagram.h: Move to...
+	* diagnostics/diagram.h: ...here.
+	(class diagnostic_diagram): Convert to...
+	(class diagnostics::diagram): ...this.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-core.h: Update for move of diagnostic_metadata to
+	diagnostics::metadata.
+	* diagnostic-format-html.cc: Likewise; also, update #include for
+	move of "diagnostic-metadata.h" to "diagnostics/metadata.h".
+	* diagnostic-format-sarif.cc: Likewise.
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic-global-context.cc: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-metadata.h: Move to...
+	* diagnostics/metadata.h: ...here, updating header guard.
+	(class diagnostic_metadata): Move to...
+	(class diagnostics::metadata): ...here.  Drop redundant
+	"diagnostics::" qualifiers.
+	* libgdiagnostics.cc: : Update #include for move of
+	"diagnostic-metadata.h" to "diagnostics/metadata.h", and update
+	for move of diagnostic_metadata to diagnostics::metadata.
+	* selftest-diagnostic.cc: Update for move of diagnostic_metadata
+	to diagnostics::metadata.
+	* selftest-diagnostic.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace
+	selftest-logical-location.o with
+	diagnostics/selftest-logical-locations.o.
+	* diagnostic-client-data-hooks.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key.
+	* diagnostic-format-html.cc: Add "using namespace diagnostics;",
+	and drop now-redundant "diagnostics::" prefixes.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostic-format-sarif.cc: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".  Add
+	"using namespace diagnostics;", and drop now-redundant
+	"diagnostics::" prefixes.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostic-format-sarif.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* diagnostic-path-output.cc: Add "using namespace diagnostics;".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and
+	of logical_location to diagnostics::logical_locations::key,
+	* diagnostic-path.cc: Add "using namespace diagnostics;".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* diagnostic-path.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key,
+	* diagnostic.cc: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostic.h: Reorganize forward decls.
+	(class logical_location_manager): Replace this forward decl
+	with...
+	(class diagnostics::logical_locations::manager): ...this.
+	(diagnostic_context::get_logical_location_manager): Update
+	return type.
+	* diagnostics/digraphs.h: : Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* logical-location.h: Move to...
+	* diagnostics/logical-locations.h: ...here.
+	(enum class logical_location_kind): Convert to...
+	(enum class diagnostics::logical_locations::kind): ... this.
+	(class logical_location_manager::key): Convert to...
+	(class diagnostics::logical_locations::key): ... this.
+	(class logical_location_manager): Convert to...
+	(class diagnostics::logical_locations::manager): ... this.
+	(logical_location): Drop this typedef in favor of
+	diagnostics::logical_locations::key.
+	* selftest-logical-location.cc: Move to...
+	* diagnostics/selftest-logical-locations.cc: ...here.
+	Update #include for move of "selftest-logical-location.h" to
+	"diagnostics/selftest-logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	(selftest_logical_location_cc_tests): Rename to...
+	(diagnostics_selftest_logical_locations_cc_tests): ...this.
+	* selftest-logical-location.h: Move to...
+	* diagnostics/selftest-logical-locations.h: ...here, updating
+	include guard.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostics/state-graphs-to-dot.cc: Add
+	"using namespace diagnostics;" and drop now-redundant
+	"diagnostics::" prefixes.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager.
+	* diagnostics/state-graphs.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager,
+	of logical_location to diagnostics::logical_locations::key.
+	* libgdiagnostics.cc: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key.
+	* selftest-diagnostic-path.cc: Update for conversion of
+	logical_location to diagnostics::logical_locations::key.
+	* selftest-diagnostic-path.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* selftest-run-tests.cc (selftest::run_tests): Update for
+	renaming of selftest_logical_location_cc_tests to
+	diagnostics_selftest_logical_locations_cc_tests.
+	* selftest.h: Likewise.
+	* simple-diagnostic-path.h: Update for conversion of
+	logical_location to diagnostics::logical_locations::key.
+	* tree-diagnostic-client-data-hooks.cc: Update for conversion of
+	logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key.
+	* tree-logical-location.cc: Update for conversions.
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* tree-logical-location.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location
+	to diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Rename diagnostic-digraphs.o to
+	diagnostics/digraphs.o.  Rename diagnostic-state-graphs.o to
+	diagnostics/state-graphs.o.  Rename diagnostic-state-to-dot.o to
+	diagnostics/state-graphs-to-dot.o.
+	(PLUGIN_HEADERS): Add $(srcdir)/diagnostics/*.h.
+	(install-plugin:): Likewise.
+	* configure: Regenerate.
+	* configure.ac (gccdepdir): Add "diagnostics" to directories.
+	* diagnostic-format-html.cc: Update #includes for move of
+	"diagnostic-digraphs.h" to "diagnostics/digraphs.h" and of
+	move of "diagnostic-state-graphs.h" to
+	"diagnostics/state-graphs.h".
+	* diagnostic-format-sarif.cc: Likewise.
+	* diagnostic-path.cc: Likewise for state-graphs.h
+	* diagnostic-digraphs.cc: Move...
+	* diagnostics/digraphs.cc: ...to here.  Update #include
+	for renaming of digraphs.h.
+	(selftest::diagnostic_digraphs_cc_tests): Rename to...
+	(selftest::diagnostics_digraphs_cc_tests): ...this.
+	* diagnostic-digraphs.h: Move...
+	* diagnostics/digraphs.h: ...to here, updating header guard.
+	* diagnostic-state-to-dot.cc: Move...
+	* diagnostics/state-graphs-to-dot.cc: ...to here.  Update #include
+	of state-graphs.h.
+	* diagnostic-state-graphs.cc: Move...
+	* diagnostics/state-graphs.cc: ...to here.  Update #include
+	of state-graphs.h.
+	(selftest::diagnostic_state_graphs_cc_tests): Rename...
+	(selftest::diagnostics_state_graphs_cc_tests): ...to this.
+	* diagnostic-state-graphs.h: Move...
+	* diagnostics/state-graphs.h: ...to here, updating header guard.
+	* libgdiagnostics.cc: Update #includes for move of
+	"diagnostic-digraphs.h" to "diagnostics/digraphs.h" and of
+	move of "diagnostic-state-graphs.h" to
+	"diagnostics/state-graphs.h".
+	* selftest-run-tests.cc (selftest::run_tests): Update for
+	function renamings that reflect file renamings.
+	* selftest.h (selftest::diagnostic_digraphs_cc_tests): Rename
+	to...
+	(selftest::diagnostics_digraphs_cc_tests): ...this.
+	(selftest::diagnostic_state_graphs_cc_tests): Rename...
+	(selftest::diagnostics_state_graphs_cc_tests): ...to this.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_get_vec_defs): Remove overload
+	with operand vector type.  Remove stmt_vec_info and
+	ncopies argument.
+	* tree-vect-stmts.cc (vect_get_vec_defs): Likewise.
+	(vectorizable_conversion): Adjust by not passing in
+	vector types, stmt_vec_info and ncopies.
+	(vectorizable_bswap): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_scan_store): Likewise.
+	(vectorizable_store): Likewise.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+	* tree-vect-loop.cc (vect_transform_reduction): Likewise.
+	(vect_transform_lc_phi): Likewise.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_is_simple_use): Remove stmt-info
+	with vectype output overload and remove stmt-info argument
+	from SLP based API.
+	* tree-vect-loop.cc (vectorizable_lane_reducing): Remove
+	unused def_stmt_info output argument to vect_is_simple_use.
+	Adjust.
+	* tree-vect-stmts.cc (get_group_load_store_type): Get
+	the gather/scatter offset vector type from the SLP child.
+	(vect_check_scalar_mask): Remove stmt_info argument.  Adjust.
+	(vect_check_store_rhs): Likewise.
+	(vectorizable_call): Likewise.
+	(vectorizable_simd_clone_call): Likewise.
+	(vectorizable_conversion): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_load): Likewise.
+	(vect_is_simple_cond): Remove stmt_info argument.  Adjust.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+	(vectorizable_store): Likewise.
+	(vect_is_simple_use): Remove overload and non-SLP path.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vect_model_simple_cost): Removed code
+	when no SLP node is passed, remove then unused dt and ndts
+	parameters and fix the number of vector stmts calculation.
+	(vectorizable_call): Adjust.
+	(vectorizable_simd_clone_call): Likewise.
+	(vectorizable_conversion): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+
+2025-07-25  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* config/riscv/riscv-vector-costs.cc (get_live_range):
+	Move compute_local_program_points to cost class.
+	(variable_vectorized_p): Add slp node parameter.
+	(need_additional_vector_vars_p): Move from here...
+	(costs::need_additional_vector_vars_p): ... to here and add slp
+	parameter.
+	(compute_estimated_lmul): Move update_local_live_ranges to cost
+	class.
+	(has_unexpected_spills_p): Move from here...
+	(costs::has_unexpected_spills_p): ... to here.
+	(costs::record_lmul_spills): New function.
+	(costs::add_stmt_cost): Add stmt_info, slp mapping.
+	(costs::finish_cost): Analyze loop.
+	* config/riscv/riscv-vector-costs.h: Move declarations to class.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_analyze_loop_2): Get vectype
+	from the SLP node.
+	(vectorize_fold_left_reduction): Likewise.
+	(vect_transform_reduction): Likewise.
+	(vect_transform_cycle_phi): Likewise.
+	(vect_transform_lc_phi): Likewise.
+	* tree-vect-slp.cc (vect_analyze_slp): Likewise.
+	* tree-vect-stmts.cc (vect_truncate_gather_scatter_offset):
+	Pass in vectype and use that.
+	(vect_use_strided_gather_scatters_p): Likewise.
+	(get_group_load_store_type): Adjust.
+	(vect_check_scalar_mask): We always have a SLP node.
+	Use vectype from the SLP node.
+	(vect_check_store_rhs): Likewise.
+	(vect_build_one_gather_load_call): Pass in the vector type
+	and use that.
+	(vect_get_strided_load_store_ops): Likewise.
+	(vectorizable_store): Adjust.
+	(vectorizable_load): Use the vector type from the SLP node
+	and adjust.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* target.h (stmt_vectype): Remove.
+	* tree-vect-stmts.cc (stmt_vectype): Likewise.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* config/riscv/riscv.cc (enum riscv_privilege_levels): Remove USER_MODE.
+	(riscv_handle_type_attribute): Remove "user" interrupts.
+	(riscv_expand_epilogue): Likewise.
+	(riscv_get_interrupt_type): Likewise.
+	* config/riscv/riscv.md (riscv_uret): Remove URET pattern.
+	* doc/extend.texi: Remove documentation of user interrupts.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_stmt_vec_info::vec_stmts): Remove.
+	(STMT_VINFO_VEC_STMTS): Likewise.
+	* tree-vectorizer.cc (vec_info::new_stmt_vec_info): Do not
+	initialize it.
+	(vec_info::free_stmt_vec_info): Nor free it.
+	* tree-vect-data-refs.cc (vect_vfa_access_size): Remove
+	check on STMT_VINFO_VEC_STMTS.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_transform_grouped_load): Remove.
+	(vect_record_grouped_load_vectors): Likewise.
+	* tree-vect-data-refs.cc (vect_permute_load_chain): Likewise.
+	(vect_shift_permute_load_chain): Likewise.
+	(vect_transform_grouped_load): Likewise.
+	(vect_record_grouped_load_vectors): Likewise.
+	* tree-vect-stmts.cc (vectorizable_load): Remove comments
+	about load interleaving.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_permute_store_chain): Remove.
+	* tree-vect-data-refs.cc (vect_permute_store_chain): Likewise.
+	* tree-vect-stmts.cc (vectorizable_store): Remove comment
+	about store interleaving.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_get_vec_defs_for_operand): Remove.
+	* tree-vect-stmts.cc (vect_get_vec_defs_for_operand): Likewise.
+	(vect_get_vec_defs): Remove non-SLP path.
+	(check_load_store_for_partial_vectors): We always have an
+	SLP node.
+	(vect_check_store_rhs): Likewise.
+	(vect_get_gather_scatter_ops): Likewise.
+	(vect_create_vectorized_demotion_stmts): Likewise.
+	(vectorizable_store): Adjust.
+	(vectorizable_load): Likewise.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (VMAT_CONTIGUOUS_PERMUTE): Remove.
+	* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
+	Remove checks on VMAT_CONTIGUOUS_PERMUTE.
+	(vectorizable_load): Likewise.
+	(vectorizable_store): Likewise.  Prune dead code.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_store): Remove trivially
+	dead code.
+
+2025-07-25  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn-opts.h (TARGET_NO_MANUAL_NOPS,
+	TARGET_CDNA3_NOPS): Define.
+	* config/gcn/gcn.md (define_attr "laneselect): Change 'yes' to
+	'read' and 'write'.
+	(define_attr "flatmemaccess"): Add with values store, storex34,
+	load, atomic, atomicwait, cmpswapx2, and no. Replacing ...
+	(define_attr "delayeduse"): Remove.
+	(define_attr "transop"): Add with values yes and no.
+	(various insns): Update 'laneselect', add flatmemaccess and transop,
+	remove delayeduse; fixing an issue for s_load_dwordx4 vs.
+	flat_store_dwordx4 related to delayeduse (now: flatmemaccess).
+	* config/gcn/gcn-valu.md: Update laneselect attribute and add
+	flatmemaccess.
+	* config/gcn/gcn.cc (gcn_cmpx_insn_p): New.
+	(gcn_md_reorg): Update for MI300 to add additional s_nop.
+	Skip s_nop-insertion part for RDNA{2,3}; add "VALU writes EXEC
+	followed by VALU DPP" unconditionally for CDNA2/CDNA3/GCN5.
+
+2025-07-25  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	PR middle-end/121214
+	* genpreds.cc (constraint_max_namelen): Delete.
+	(write_tm_preds_h): Always write insn_constraint_len() and
+	define CONSTRAINT_LEN to it, i.e., remove guard
+	constraint_max_namelen>1.  Remove outer guard
+	constraint_max_namelen>0 and re-indent.
+	(write_insn_preds_c): Remove guard
+	constraint_max_namelen>0 and re-indent.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* config/riscv/riscv.cc (enum riscv_privilege_levels): Add
+	RNMI_MODE.
+	(riscv_handle_type_attribute): Handle 'rnmi' interrupt attribute.
+	(riscv_expand_epilogue): Generate nmret for RNMI handlers.
+	(riscv_get_interrupt_type): Handle 'rnmi' interrupt attribute.
+	* config/riscv/riscv.md (riscv_rnmi): Add nmret INSN.
+	* doc/extend.texi: Add documentation for 'rnmi' interrupt attribute.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vectorizable_induction): Remove
+	gimple **vec_stmt argument.
+	(vectorizable_phi): Likewise.
+	(vectorizable_recurr): Likewise.
+	(vectorizable_early_exit): Likewise.
+	* tree-vect-loop.cc (vectorizable_phi): Likewise and adjust.
+	(vectorizable_recurr): Likewise.
+	(vectorizable_nonlinear_induction): Likewise.
+	(vectorizable_induction): Likewise.
+	* tree-vect-stmts.cc (vectorizable_bswap): Likewise.
+	(vectorizable_call): Likewise.
+	(vectorizable_simd_clone_call): Likewise.
+	(vectorizable_conversion): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_store): Likewise.
+	(vectorizable_load): Likewise.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+	(vectorizable_comparison): Likewise.
+	(vectorizable_early_exit): Likewise.
+	(vect_analyze_stmt): Adjust.
+	(vect_transform_stmt): Likewise.
+	* tree-vect-slp.cc (vect_slp_analyze_operations): Adjust.
+	(vectorize_slp_instance_root_stmt): Likewise.
+
+2025-07-24  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* doc/cpp.texi (#ifdef): Correct typo.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_early_exit): Remove non-SLP
+	path.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_stmt_vec_info::simd_clone_info): Remove.
+	(STMT_VINFO_SIMD_CLONE_INFO): Likewise.
+	* tree-vectorizer.cc (vec_info::free_stmt_vec_info): Do not
+	release it.
+	* tree-vect-stmts.cc (vectorizable_simd_clone_call): Remove
+	non-SLP path.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* internal-fn.cc (internal_fn_len_index): Adjust indices for new
+	alias_ptr param.
+	(internal_fn_else_index): Ditto.
+	(internal_fn_mask_index): Ditto.
+	(internal_fn_stored_value_index): Ditto.
+	(internal_fn_alias_ptr_index): Ditto.
+	(internal_fn_offset_index): Ditto.
+	(internal_fn_scale_index): Ditto.
+	(internal_gather_scatter_fn_supported_p): Ditto.
+	* internal-fn.h (internal_fn_alias_ptr_index): Ditto.
+	* optabs-query.cc (supports_vec_gather_load_p): Ditto.
+	* tree-vect-data-refs.cc (vect_check_gather_scatter): Add alias
+	pointer.
+	* tree-vect-patterns.cc (vect_recog_gather_scatter_pattern): Add
+	alias pointer.
+	* tree-vect-slp.cc (vect_get_operand_map): Adjust for alias
+	pointer.
+	* tree-vect-stmts.cc (vect_truncate_gather_scatter_offset): Add
+	alias pointer and misalignment handling.
+	(get_load_store_type): Move from here...
+	(get_group_load_store_type): ...To here.
+	(vectorizable_store): Add alias pointer.
+	(vectorizable_load): Ditto.
+	* tree-vectorizer.h (struct gather_scatter_info): Ditto.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* config/aarch64/aarch64.cc (aarch64_builtin_support_vector_misalignment):
+	Return true for gather/scatter.
+	* config/arm/arm.cc (arm_builtin_support_vector_misalignment):
+	Ditto.
+	* config/epiphany/epiphany.cc (epiphany_support_vector_misalignment):
+	Ditto.
+	* config/gcn/gcn.cc (gcn_vectorize_support_vector_misalignment):
+	Ditto.
+	* config/loongarch/loongarch.cc (loongarch_builtin_support_vector_misalignment):
+	Ditto.
+	* config/riscv/riscv.cc (riscv_support_vector_misalignment):
+	Add gather/scatter argument.
+	* config/rs6000/rs6000.cc (rs6000_builtin_support_vector_misalignment):
+	Return true for gather/scatter.
+	* config/s390/s390.cc (s390_support_vector_misalignment):
+	Ditto.
+	* doc/tm.texi: Add argument.
+	* target.def: Ditto.
+	* targhooks.cc (default_builtin_support_vector_misalignment):
+	Ditto.
+	* targhooks.h (default_builtin_support_vector_misalignment):
+	Ditto.
+	* tree-vect-data-refs.cc (vect_supportable_dr_alignment):
+	Ditto.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* tree-vect-slp.cc (GATHER_SCATTER_OFFSET): New define.
+	(vect_get_and_check_slp_defs): Use.
+	* tree-vectorizer.h (GATHER_SCATTER_LEGACY_P): New define.
+	(GATHER_SCATTER_IFN_P): Ditto.
+	(GATHER_SCATTER_EMULATED_P): Ditto.
+	* tree-vect-stmts.cc (vectorizable_store): Use.
+	(vectorizable_load): Use.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* internal-fn.cc (expand_scatter_store_optab_fn): Use new
+	function.
+	(expand_gather_load_optab_fn): Ditto.
+	(internal_fn_offset_index): Ditto.
+	(internal_fn_scale_index): Ditto.
+	* internal-fn.h (internal_fn_offset_index): New function.
+	(internal_fn_scale_index): Ditto.
+	* tree-vect-data-refs.cc (vect_describe_gather_scatter_call):
+	Use new function.
+
+2025-07-24  Alfie Richards  <alfie.richards@arm.com>
+
+	* tree-vect-data-refs.cc (vect_create_data_ref_ptr): Remove unnecessary
+	casts to aggr_ptr_type.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_comparison_1): Remove
+	non-SLP path.
+	(vectorizable_comparison): Likewise.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_condition): Remove
+	non-SLP paths.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_scan_store): Remove
+	non-SLP path and unused parameters.
+	(vectorizable_store): Adjust.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_shift): Remove non-SLP paths.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_assignment): Remove
+	non-SLP paths.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_call): Remove non-SLP path.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_bswap): Remove non-SLP path.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_recurr): Remove non-SLP path.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*cond_<optab><mode>_2_relaxed):
+	Extend from SVE_FULL_F_B16B16 to SVE_F_B16B16.
+	(*cond_<optab><mode>_3_relaxed): Likewise.
+	(*cond_<optab><mode>_any_relaxed): Likwise.
+	(*cond_<optab><mode>_any_const_relaxed): Extend from SVE_FULL_F
+	to SVE_F.
+	(*cond_add<mode>_2_const_relaxed): Likewise.
+	(*cond_add<mode>_any_const_relaxed): Likewise.
+	(*cond_sub<mode>_3_const_relaxed): Likewise.
+	(*cond_sub<mode>_const_relaxed): Likewise.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md: (@aarch64_sve_<optab><mode>):
+	Extend from SVE_FULL_F to SVE_F, use aarch64_predicate_operand.
+	(@aarch64_frecpe<mode>): Extend from SVE_FULL_F to SVE_F.
+	(@aarch64_frecps<mode>): Likewise.
+	(div<mode>3): Likewise, use aarch64_sve_fp_pred.
+	* config/aarch64/iterators.md: Add warnings above SVE_FP_UNARY
+	and SVE_FP_BINARY.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (<optab><mode>3): Extend from
+	SVE_FULL_F to SVE_F, use aarch64_sve_fp_pred.
+	(*post_ra_<sve_fp_op><mode>3): Extend from SVE_FULL_F to SVE_F.
+	(@aarch64_pred_<optab><mode>): Extend from SVE_FULL_F to SVE_F,
+	use aarch64_predicate_operand (ADD/SUB/MUL/MAX/MIN).
+	(split for using unpredicated insns): Move SVE_RELAXED_GP into
+	the pattern, rather than testing for it in the condition.
+	* config/aarch64/aarch64-sve2.md (@aarch64_pred_<optab><mode>):
+	Extend from VNx8BF_ONLY to SVE_BF.
+
+2025-07-24  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec-opt.md (*uavg_floor_vx_<mode>): Rename
+	from...
+	(*<sat_op_v_vdup>_vx_<mode>): Rename to...
+	(*<sat_op_vdup_v>_vx_<mode>): Rename to...
+	* config/riscv/riscv-protos.h (enum insn_flags): Add vxrm
+	RNE, ROD type.
+	(enum insn_type): Add RNE_P, ROD_P type.
+	(expand_vx_binary_vxrm_vec_vec_dup): Add new func decl.
+	(expand_vx_binary_vxrm_vec_dup_vec): Ditto.
+	* config/riscv/riscv-v.cc (get_insn_type_by_vxrm_val): Add
+	helper to get insn type by vxrm value.
+	(expand_vx_binary_vxrm_vec_vec_dup): Add new func impl
+	to expand vec + vec_dup pattern.
+	(expand_vx_binary_vxrm_vec_dup_vec): Ditto but for
+	vec_dup + vec pattern.
+	* config/riscv/vector-iterators.md: Add helper iterator
+	for sat vx combine.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*cond_<optab><mode>_2_relaxed):
+	Extend from SVE_FULL_F to SVE_F.
+	(*cond_<optab><mode>_any_relaxed): Likewise.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (<optab><mode>2): Replace use of
+	aarch64_ptrue_reg with aarch64_sve_fp_pred.
+	(@aarch64_pred_<optab><mode>): Extend from SVE_FULL_F to SVE_F,
+	and use aarch64_predicate_operand.
+	* config/aarch64/iterators.md: Split FABS/FNEG out of
+	SVE_COND_FP_UNARY (into new SVE_COND_FP_UNARY_BITWISE).
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md
+	(*cond_<optab>_nontrunc<SVE_PARTIAL_F:mode><SVE_HSDI:mode>_relaxed):
+	New FCVT/SEL combiner pattern.
+	(*cond_<optab>_trunc<VNx2DF_ONLY:mode><VNx2SI_ONLY:mode>_relaxed):
+	New FCVTZ{S,U}/SEL combiner pattern.
+	(*cond_<optab>_nonextend<SVE_HSDI:mode><SVE_PARTIAL_F:mode>_relaxed):
+	New {S,U}CVTF/SEL combiner pattern.
+	(*cond_<optab>_trunc<SVE_SDF:mode><SVE_PARTIAL_HSF:mode>):
+	New FCVT/SEL combiner pattern.
+	(*cond_<optab>_nontrunc<SVE_PARTIAL_HSF:mode><SVE_SDF:mode>_relaxed):
+	New FCVTZ{S,U}/SEL combiner pattern.
+	* config/aarch64/iterators.md: New mode iterator for VNx2SI.
+
+2025-07-23  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/121073
+	* config/riscv/autovec-opt.md: Use new helpers.
+	* config/riscv/autovec.md: Ditto.
+	* config/riscv/predicates.md (strided_broadcast_mask_operand):
+	New predicate.
+	(strided_broadcast_operand): Ditto.
+	(any_broadcast_operand): Ditto.
+	* config/riscv/riscv-protos.h (expand_broadcast): Declare.
+	(expand_set_first): Ditto.
+	(expand_set_first_tu): Ditto.
+	(strided_broadcast_p): Ditto.
+	* config/riscv/riscv-string.cc (expand_vec_setmem): Use new
+	helpers.
+	* config/riscv/riscv-v.cc (expand_broadcast): New functionk.
+	(expand_set_first): Ditto.
+	(expand_set_first_tu): Ditto.
+	(expand_const_vec_duplicate): Use new helpers.
+	(expand_const_vector_duplicate_repeating): Ditto.
+	(expand_const_vector_duplicate_default): Ditto.
+	(sew64_scalar_helper): Ditto.
+	(expand_vector_init_merge_repeating_sequence): Ditto.
+	(expand_reduction): Ditto.
+	(strided_broadcast_p): New function.
+	(whole_reg_to_reg_move_p): Use new helpers.
+	* config/riscv/riscv-vector-builtins-bases.cc: Use either
+	broadcast or strided broadcast.
+	* config/riscv/riscv-vector-builtins.cc (function_expander::use_ternop_insn):
+	Ditto.
+	(function_expander::use_widen_ternop_insn): Ditto.
+	(function_expander::use_scalar_broadcast_insn): Ditto.
+	* config/riscv/riscv-vector-builtins.h: Declare scalar
+	broadcast.
+	* config/riscv/vector.md (*pred_broadcast<mode>): Split into
+	regular and strided broadcast.
+	(*pred_broadcast<mode>_zvfh): Split.
+	(pred_broadcast<mode>_zvfh): Ditto.
+	(*pred_broadcast<mode>_zvfhmin): Ditto.
+	(@pred_strided_broadcast<mode>): Ditto.
+	(*pred_strided_broadcast<mode>): Ditto.
+	(*pred_strided_broadcast<mode>_zvfhmin): Ditto.
+
+2025-07-23  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR target/120119
+	* config/aarch64/cortex-a57-fma-steering.cc (func_fma_steering::analyze):
+	Skip if renaming fails.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121220
+	* tree-ssa-sink.cc (statement_sink_location): For stores
+	handle sinking to paths ending in a store.  Skip loads
+	that do not use the store.
+
+2025-07-23  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/119085
+	* tree-sra.cc (sort_and_splice_var_accesses): Prevent total
+	scalarization if two incompatible aggregates access the same place.
+
+2025-07-23  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* config.gcc <sparc*-*-solaris2*> (with_cpu): Default to ultrasparc3.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	* config/aarch64/aarch64.cc (aarch64_vector_costs::add_stmt_cost):
+	Check vectype is non-NULL before accessing it.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/121216
+	* expr.cc (constant_byte_string): Check the string type
+	size fits an uhwi before converting to uhwi.
+
+2025-07-22  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/109267
+	* tree-cfgcleanup.cc (execute_cleanup_cfg_post_optimizing): If the first
+	non debug statement in the first (and only) basic block is a call
+	to __builtin_unreachable change it to a call to __builtin_trap or an
+	infinite loop.
+
+2025-07-22  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.cc (xtensa_is_insn_L32R_p):
+	Rewrite to capture insns that could be L32R machine instructions
+	wherever possible.
+	(xtensa_rtx_costs): Fix to consider that moves larger than a
+	natural word can take multiple L32R machine instructions.
+	(constantpool_address_p): Cosmetics.
+	* config/xtensa/xtensa.md (movdi_internal, movdf_internal):
+	Add missing insn attributes.
+
+2025-07-22  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.md
+	(movsi_internal, movhi_internal, movsf_internal):
+	Change the value of the "type" attribute from "move" to "load"
+	when the source operand constraint is "Y".
+
+2025-07-22  Karl Meakin  <karl.meakin@arm.com>
+
+	* tree-vect-data-refs.cc (vect_check_gather_scatter): Swap
+	`base` and `off` in more scenarios. Also assert at the end of
+	the function that `base` and `off` are loop-invariant and not
+	loop-invariant respectively.
+
+2025-07-22  Jeff Law  <jlaw@ventanamicro.com>
+
+	* config/riscv/generic-vector-ooo.md: Restrict insn reservations to
+	generic_ooo and generic tuning models.
+
+2025-07-22  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121202
+	* tree-vect-slp.cc (vect_schedule_slp_node): Do not take
+	an out-of-region stmt as "last".
+
+2025-07-22  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* genpreds.cc (write_insn_constraint_len): Replace rawmemchr by
+	a loop.
+
+2025-07-21  Jeff Law  <jlaw@ventanamicro.com>
+
+	* config/riscv/mips-p8700.md: Add support for "ghost" insn types.
+	* config/riscv/xiangshan.md: Add support for "sf_vc" and "sf_vc_se"
+	insn types.
+
+2025-07-21  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/110949
+	PR tree-optimization/95906
+	* match.pd (cmp - 1): New pattern.
+
+2025-07-21  Andreas Schwab  <schwab@suse.de>
+
+	PR target/121121
+	* config/riscv/riscv.cc (riscv_allocate_and_probe_stack_space):
+	Use temp2 instead of temp1 for the CFA note.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/vector.md: Allow VLS DImode for sat_op vx pattern.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec-opt.md (*uavg_floor_vx_<mode>): Add
+	pattern for vaaddu.vx combine.
+	* config/riscv/riscv.cc (get_vector_binary_rtx_cost): Add UNSPEC
+	handling for UNSPEC_VAADDU.
+	(riscv_rtx_costs): Ditto.
+
+2025-07-21  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (aarch64_simd_vec_set_zero<mode>):
+	Enable only when optimizing for size.
+
+2025-07-21  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64.cc (aarch64_rtx_costs): Add extra_cost values
+	only when speed is true for CONST_VECTOR, VEC_DUPLICATE, VEC_SELECT
+	cases.
+	* config/aarch64/aarch64-cost-tables.h (qdf24xx_extra_costs,
+	thunderx_extra_costs, thunderx2t99_extra_costs,
+	thunderx3t110_extra_costs, tsv110_extra_costs, a64fx_extra_costs,
+	ampere1_extra_costs, ampere1a_extra_costs, ampere1b_extra_costs):
+	Reduce cost of movi, dup, extract fields by COSTS_N_INSNS (1).
+	* config/arm/aarch-cost-tables.h (generic_extra_costs,
+	cortexa53_extra_costs, cortexa57_extra_costs, cortexa76_extra_costs,
+	exynosm1_extra_costs, xgene1_extra_costs): Likewise.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121194
+	* tree-vect-loop.cc (vectorizable_lc_phi): Verify
+	vector types are compatible.
+
+2025-07-21  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (gather_load<mode><vndi>): New.
+	(scatter_store<mode><vndi>): New.
+	(mask_gather_load<mode><vndi>): New.
+	(mask_scatter_store<mode><vndi>): New.
+	* config/gcn/gcn.cc (gcn_expand_scaled_offsets): Support DImode.
+
+2025-07-21  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn.cc (GEN_VNM_NOEXEC): Use USE_QHF.
+	(GEN_VNM): Likewise, and call for new ashl and mul variants.
+
+2025-07-21  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (add<mode>3_dup): New.
+	(add<mode>3_dup_exec): New.
+	(<su>mul<mode>3_highpart_dup<exec>): New.
+	(mul<mode>3_dup): Move the vec_duplicate to operand 1.
+	(mul<mode>3_dup_exec): New.
+	(vec_series<mode>): Adjust call to gen_mul<mode>3_dup.
+	* config/gcn/gcn.cc (gcn_expand_vector_init): Likewise.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* genoutput.cc (main): Emit function
+	verify_reg_names_in_constraints() for run-time validation.
+	(mdep_constraint_len): Deal with hard register constraints.
+	* output.h (verify_reg_names_in_constraints): New function
+	declaration.
+	* toplev.cc (backend_init): If checking is enabled, call into
+	verify_reg_names_in_constraints().
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* cfgexpand.cc (n_occurrences): Move this ...
+	(check_operand_nalternatives): and this ...
+	(expand_asm_stmt): and the call to gimplify.cc.
+	* config/s390/s390.cc (s390_md_asm_adjust): Pass null pointer to
+	parse_{input,output}_constraint().
+	* gimple-walk.cc (walk_gimple_asm): Pass null pointer to
+	parse_{input,output}_constraint().
+	(walk_stmt_load_store_addr_ops): Ditto.
+	* gimplify-me.cc (gimple_regimplify_operands): Ditto.
+	* gimplify.cc (num_occurrences): Moved from cfgexpand.cc.
+	(num_alternatives): Ditto.
+	(gimplify_asm_expr): Deal with hard register constraints.
+	* stmt.cc (eliminable_regno_p): New helper.
+	(hardreg_ok_p): Perform a similar check as done in
+	make_decl_rtl().
+	(parse_output_constraint): Add parameter for gimplify_reg_info
+	and validate hard register constrained operands.
+	(parse_input_constraint): Ditto.
+	* stmt.h (class gimplify_reg_info): Forward declaration.
+	(parse_output_constraint): Add parameter.
+	(parse_input_constraint): Ditto.
+	* tree-ssa-operands.cc
+	(operands_scanner::get_asm_stmt_operands): Pass null pointer
+	to parse_{input,output}_constraint().
+	* tree-ssa-structalias.cc (find_func_aliases): Pass null pointer
+	to parse_{input,output}_constraint().
+	* varasm.cc (assemble_asm): Pass null pointer to
+	parse_{input,output}_constraint().
+	* gimplify_reg_info.h: New file.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/cris/cris.cc (cris_md_asm_adjust): Deal with hard
+	register constraint.
+	* config/i386/i386.cc (map_egpr_constraints): Ditto.
+	* config/s390/s390.cc (f_constraint_p): Ditto.
+	* doc/extend.texi: Document hard register constraints.
+	* doc/md.texi: Ditto.
+	* function.cc (match_asm_constraints_2): Have a unique pseudo
+	for each operand with a hard register constraint.
+	(pass_match_asm_constraints::execute): Calling into new helper
+	match_asm_constraints_2().
+	* genoutput.cc (mdep_constraint_len): Return the length of a
+	hard register constraint.
+	* genpreds.cc (write_insn_constraint_len): Support hard register
+	constraints for insn_constraint_len().
+	* ira.cc (valid_replacement_for_asm_input_p_1): New helper.
+	(valid_replacement_for_asm_input_p): New helper.
+	(decrease_live_ranges_number): Similar to
+	match_asm_constraints_2() ensure that each operand has a unique
+	pseudo if constrained by a hard register.
+	* lra-constraints.cc (process_alt_operands): Install hard
+	register filter according to constraint.
+	* recog.cc (asm_operand_ok): Accept register type for hard
+	register constrained asm operands.
+	(constrain_operands): Validate hard register constraints.
+	* stmt.cc (decode_hard_reg_constraint): Parse a hard register
+	constraint into the corresponding register number or bail out.
+	(parse_output_constraint): Parse hard register constraint and
+	set *ALLOWS_REG.
+	(parse_input_constraint): Ditto.
+	* stmt.h (decode_hard_reg_constraint): Declaration of new
+	function.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_analyze_data_refs): Remove min_vf
+	output.
+	* tree-vect-data-refs.cc (vect_analyze_data_refs): Likewise.
+	* tree-vect-loop.cc (vect_analyze_loop_2): Remove early
+	out based on bogus min_vf.
+	* tree-vect-slp.cc (vect_slp_analyze_bb_1): Adjust.
+
+2025-07-20  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec.md: Add (const_int 1) as the op2 of
+	ashiftrt.
+
+2025-07-19  Dimitar Dimitrov  <dimitar@dinux.eu>
+
+	PR target/121124
+	* config/pru/pru-pragma.cc (pru_pragma_ctable_entry): Handle the
+	ctable base address as signed 32-bit value, and sign-extend to
+	HOST_WIDE_INT.
+	* config/pru/pru-protos.h (struct pru_ctable_entry): Store the
+	ctable base address as signed.
+	(pru_get_ctable_exact_base_index): Pass base address as signed.
+	(pru_get_ctable_base_index): Ditto.
+	(pru_get_ctable_base_offset): Ditto.
+	* config/pru/pru.cc (pru_get_ctable_exact_base_index): Ditto.
+	(pru_get_ctable_base_index): Ditto.
+	(pru_get_ctable_base_offset): Ditto.
+	(pru_print_operand_address): Ditto.
+
+2025-07-19  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* config/riscv/autovec-opt.md (*vfwnmacc_vf_<mode>): New pattern.
+	(*vfwnmsac_vf_<mode>): New pattern.
+	* config/riscv/riscv.cc (get_vector_binary_rtx_cost): Add support for a
+	vec_duplicate in a neg.
+
+2025-07-19  Artemiy Volkov  <artemiyv@acm.org>
+
+	* config/riscv/riscv.cc (riscv_macro_fusion_pair_p): Protect
+	from a NULL PREV_SET or CURR_SET.
+
+2025-07-19  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr-passes.cc (avr_optimize_casesi): Fuse
+	get_insns() with end_sequence().
+
+2025-07-18  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec.md (avg<mode>3_ceil): Add new pattern
+	of avg3_ceil for RVV DImode
+
+2025-07-18  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/117423
+	* tree-sra.cc (analyze_access_subtree): Fix computation of grp_covered
+	flag.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121126
+	* tree-vect-stmts.cc (vect_analyze_stmt): Analyze the
+	live lane extract for LC PHIs that are vect_internal_def.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_live_operation_1):
+	Remove stmt_info and ncopies parameters.  Remove !slp_node
+	paths.
+	(vectorizable_live_operation): Remove !slp_node paths.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120924
+	* params.opt (uninit-max-chain-len): Up from 8 to 12.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121048
+	* tree-vect-loop.cc (vect_determine_vectype_for_stmt_1):
+	Remove rejecting vector(1) vector types.
+	(vect_set_stmts_vectype): Likewise.
+	* tree-vect-slp.cc (vect_make_slp_decision): Only
+	count instances with non-vector(1) root towards whether
+	we have any interesting instances to vectorize.
+
+2025-07-18  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121131
+	* gimple-fold.cc (fold_nonarray_ctor_reference): Use
+	TREE_INT_CST_LOW (TYPE_SIZE ()) instead of
+	GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE ()) for BLKmode BITINT_TYPEs.
+	Don't compute encoding_size at all for little endian targets.
+
+2025-07-17  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/121095
+	* gcse.cc (execute_hardreg_pre): Skip if the hardreg which is never live.
+
+2025-07-17  Filip Kastl  <fkastl@suse.cz>
+
+	* tree-ssa-structalias.cc (lookup_vi_for_tree): Fix GNU style.
+	(process_constraint): Fix GNU style.
+	(get_constraint_for_component_ref): Fix GNU style.
+	(get_constraint_for_1): Fix GNU style.
+	(get_function_part_constraint): Fix GNU style.
+	(handle_lhs_call): Fix GNU style.
+	(find_func_aliases_for_builtin_call): Fix GNU style.
+	(find_func_aliases): Fix GNU style.
+	(find_func_clobbers): Fix GNU style.
+	(struct shared_bitmap_hasher): Fix GNU style.
+	(shared_bitmap_hasher::hash): Fix GNU style.
+	(pt_solution_includes_global): Fix GNU style.
+	(init_base_vars): Fix GNU style.
+	(visit_loadstore): Fix GNU style.
+	(compute_dependence_clique): Fix GNU style.
+	(struct pt_solution): Fix GNU style.
+	(ipa_pta_execute): Fix GNU style.
+
+2025-07-17  Filip Kastl  <fkastl@suse.cz>
+
+	* pta-andersen.cc (struct constraint_graph): Fix GNU style.
+	(constraint_equal): Fix GNU style.
+	(set_union_with_increment): Fix GNU style.
+	(insert_into_complex): Fix GNU style.
+	(merge_node_constraints): Fix GNU style.
+	(unify_nodes): Fix GNU style.
+	(do_ds_constraint): Fix GNU style.
+	(scc_info::scc_info): Fix GNU style.
+	(find_indirect_cycles): Fix GNU style.
+	(equiv_class_lookup_or_add): Fix GNU style.
+	(label_visit): Fix GNU style.
+	(dump_pred_graph): Fix GNU style.
+	(perform_var_substitution): Fix GNU style.
+	(eliminate_indirect_cycles): Fix GNU style.
+	(solve_graph): Fix GNU style.
+	(solve_constraints): Fix GNU style.
+	* tree-ssa-structalias.cc (first_vi_for_offset): Fix GNU style.
+	(debug_constraint): Fix GNU style.
+	* tree-ssa-structalias.h (struct constraint_expr): Fix GNU
+	style.
+	(struct variable_info): Fix GNU style.
+
+2025-07-17  H.J. Lu  <hjl.tools@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_ternlog): Don't change
+	mode for XOR.
+
+2025-07-17  Filip Kastl  <fkastl@suse.cz>
+
+	* Makefile.in: Add pta-andersen.o.
+	* tree-ssa-structalias.cc (create_variable_info_for): Just move
+	around.
+	(unify_nodes): Move to pta-andersen.cc.
+	(struct constraint): Move to tree-ssa-structalias.h.
+	(EXECUTE_IF_IN_NONNULL_BITMAP): Move to pta-andersen.cc.
+	(struct variable_info): Move to tree-ssa-structalias.h.
+	(struct constraint_stats): Move to tree-ssa-structalias.h.
+	(first_vi_for_offset): External linkage, move to namespace
+	pointer_analysis.
+	(first_or_preceding_vi_for_offset): External linkage, move to namespace
+	pointer_analysis.
+	(dump_constraint): External linkage, move to namespace
+	pointer_analysis.
+	(debug_constraint): External linkage, move to namespace
+	pointer_analysis.
+	(dump_constraints): External linkage, move to namespace
+	pointer_analysis.
+	(debug_constraints): External linkage, move to namespace
+	pointer_analysis.
+	(lookup_vi_for_tree): Move around inside tree-ssa-structalias.cc.
+	(type_can_have_subvars): Move around inside tree-ssa-structalias.cc.
+	(make_param_constraints): Move around inside tree-ssa-structalias.cc.
+	(dump_solution_for_var): External linkage, move to namespace
+	pointer_analysis.  find (...) -> var_rep[...].
+	(get_varinfo): Move to tree-ssa-structalias.h.
+	(debug_solution_for_var): External linkage, move to namespace
+	pointer_analysis.
+	(vi_next): Move to tree-ssa-structalias.h.
+	(dump_sa_stats): External linkage, move to namespace pointer_analysis.
+	(new_var_info): Just move around.
+	(dump_sa_points_to_info): External linkage, move to namespace
+	pointer_analysis.
+	(debug_sa_points_to_info): External linkage, move to namespace
+	pointer_analysis.
+	(get_call_vi): Just move around.
+	(dump_varinfo): External linkage, move to namespace pointer_analysis.
+	(lookup_call_use_vi): Just move around.
+	(lookup_call_clobber_vi): Just move around.
+	(get_call_use_vi): Just move around.
+	(get_call_clobber_vi): Just move around.
+	(enum constraint_expr_type): Move to tree-ssa-structalias.h.
+	(struct constraint_expr): Move to tree-ssa-structalias.h.
+	(UNKNOWN_OFFSET): Move to tree-ssa-structalias.h.
+	(get_constraint_for_1): Just move around.
+	(get_constraint_for): Just move around.
+	(get_constraint_for_rhs): Just move around.
+	(do_deref): Just move around.
+	(constraint_pool): Just move around.
+	(struct constraint_graph): Move to pta-andersen.h.
+	(FIRST_REF_NODE): Move to pta-andersen.cc.
+	(LAST_REF_NODE): Move to pta-andersen.cc.
+	(find): Move to pta-andersen.cc.
+	(unite): Move to pta-andersen.cc.
+	(new_constraint): Just move around.
+	(debug_constraint_graph): External linkage, move to namespace
+	pointer_analysis.
+	(debug_varinfo): External linkage, move to namespace pointer_analysis.
+	(debug_varmap): External linkage, move to namespace pointer_analysis.
+	(dump_constraint_graph): External linkage, move to namespace
+	pointer_analysis.
+	(constraint_expr_equal): Move to pta-andersen.cc.
+	(constraint_expr_less): Move to pta-andersen.cc.
+	(constraint_less): Move to pta-andersen.cc.
+	(constraint_equal): Move to pta-andersen.cc.
+	(constraint_vec_find): Move to pta-andersen.cc.
+	(constraint_set_union): Move to pta-andersen.cc.
+	(solution_set_expand): Move to pta-andersen.cc.
+	(set_union_with_increment): Move to pta-andersen.cc.
+	(insert_into_complex): Move to pta-andersen.cc.
+	(merge_node_constraints): Move to pta-andersen.cc.
+	(clear_edges_for_node): Move to pta-andersen.cc.
+	(merge_graph_nodes): Move to pta-andersen.cc.
+	(add_implicit_graph_edge): Move to pta-andersen.cc.
+	(add_pred_graph_edge): Move to pta-andersen.cc.
+	(add_graph_edge): Move to pta-andersen.cc.
+	(init_graph): Move to pta-andersen.cc.  Initialize
+	predbitmap_obstack here.
+	(build_pred_graph): Move to pta-andersen.cc.
+	(build_succ_graph): Move to pta-andersen.cc.
+	(class scc_info): Move to pta-andersen.cc.
+	(scc_visit): Move to pta-andersen.cc.
+	(solve_add_graph_edge): Move to pta-andersen.cc.
+	(do_sd_constraint): Move to pta-andersen.cc.
+	(do_ds_constraint): Move to pta-andersen.cc.
+	(do_complex_constraint): Move to pta-andersen.cc.
+	(scc_info::scc_info): Move to pta-andersen.cc.
+	(scc_info::~scc_info): Move to pta-andersen.cc.
+	(find_indirect_cycles): Move to pta-andersen.cc.
+	(topo_visit): Move to pta-andersen.cc.
+	(compute_topo_order): Move to pta-andersen.cc.
+	(struct equiv_class_hasher): Move to pta-andersen.cc.
+	(equiv_class_hasher::hash): Move to pta-andersen.cc.
+	(equiv_class_hasher::equal): Move to pta-andersen.cc.
+	(equiv_class_lookup_or_add): Move to pta-andersen.cc.
+	(condense_visit): Move to pta-andersen.cc.
+	(label_visit): Move to pta-andersen.cc.
+	(dump_pred_graph): External linkage, move to namespace
+	pointer_analysis.
+	(dump_varmap): External linkage, move to namespace pointer_analysis.
+	(perform_var_substitution): Move to pta-andersen.cc.
+	(free_var_substitution_info): Move to pta-andersen.cc.
+	(find_equivalent_node): Move to pta-andersen.cc.
+	(unite_pointer_equivalences): Move to pta-andersen.cc.
+	(move_complex_constraints): Move to pta-andersen.cc.
+	(rewrite_constraints): Move to pta-andersen.cc.
+	(eliminate_indirect_cycles): Move to pta-andersen.cc.
+	(solve_graph): Move to pta-andersen.cc.
+	(set_uids_in_ptset): find (...) -> var_rep[...].
+	(find_what_var_points_to): find (...) -> var_rep[...].
+	(init_alias_vars): Don't initialize predbitmap_obstack here.
+	(remove_preds_and_fake_succs): Move to pta-andersen.cc.
+	(solve_constraints): Move to pta-andersen.cc.  Call
+	delete_graph() at the end.
+	(delete_points_to_sets): Don't delete graph here.  Delete var_rep here.
+	(visit_loadstore): find (...) -> var_rep[...].
+	(compute_dependence_clique): find (...) -> var_rep[...].
+	(ipa_pta_execute): find (...) -> var_rep[...].
+	* pta-andersen.cc: New file.
+	* pta-andersen.h: New file.
+	* tree-ssa-structalias.h: New file.
+
+2025-07-17  Richard Sandiford  <richard.sandiford@arm.com>
+	    Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* doc/sourcebuild.texi (aarch64_sme_hw): Document.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/s390.md (signbit_tdc): Rename expander.
+	(signbit<mode>2): New expander.
+	(signbit<mode>2_z10): New expander.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/s390.cc (s390_register_move_cost): Add costing for
+	vlvg/vlgv.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/vector.md (bhfgq): Add scalar modes.
+	(*movdi<mode>_zero_extend_A): New insn.
+	(*movsi<mode>_zero_extend_A): New insn.
+	(*movdi<mode>_zero_extend_B): New insn.
+	(*movsi<mode>_zero_extend_B): New insn.
+
+2025-07-17  Xi Ruoyao  <xry111@xry111.site>
+
+	PR target/121064
+	* config/loongarch/lsx.md (lsx_vshuf_<lsxfmt_f>): Add '@' to
+	generate a mode-aware helper.  Use <VIMODE> as the mode of the
+	operand 1 (selector).
+	* config/loongarch/lasx.md (lasx_xvshuf_<lasxfmt_f>): Likewise.
+	* config/loongarch/loongarch.cc
+	(loongarch_try_expand_lsx_vshuf_const): Create a new pseudo for
+	the selector.  Use the mode-aware helper to simplify the code.
+	(loongarch_expand_vec_perm_const): Likewise.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (vect_build_slp_tree_1): Reject
+	single-lane vector types.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121035
+	* tree-ssa-pre.cc (find_or_generate_expression): Handle
+	values without expression.
+
+2025-07-16  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-state-to-dot.cc (state_diagram::m_show_tags): Drop
+	unused field.
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* gimplify.cc (gimplify_omp_affinity): Use OMP_ITERATOR_DECL_P.
+	(compute_omp_iterator_count): New.
+	(build_omp_iterator_loop): New.
+	(gimplify_omp_depend): Use OMP_ITERATOR_DECL_P,
+	compute_omp_iterator_count and build_omp_iterator_loop.
+	* tree-inline.cc (copy_tree_body_r): Use OMP_ITERATOR_DECL_P.
+	* tree-pretty-print.cc (dump_omp_clause): Likewise.
+	* tree.h (OMP_ITERATOR_DECL_P): New macro.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121062
+	* config/i386/i386.cc (ix86_convert_const_vector_to_integer):
+	Handle E_V1SImode and E_V1DImode.
+	* config/i386/mmx.md (V_16_32_64): Add V1SI, V2BF and V1DI.
+	(mmxinsnmode): Add V1DI and V1SI.
+	Add V_16_32_64 splitter for constant vector loads from constant
+	vector pool.
+	(V_16_32_64:*mov<mode>_imm): Moved after V_16_32_64 splitter.
+	Replace lowpart_subreg with adjust_address.
+
+2025-07-16  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	PR testsuite/121078
+	* config/i386/i386-options.cc (ix86_option_override_internal):
+	Warn -pg without -mfentry only on glibc targets.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_move):
+	Use MEM_P predicate instead of open coding it.
+	(ix86_erase_embedded_rounding):
+	Use NONJUMP_INSN_P predicate instead of open coding it.
+	* config/i386/i386-features.cc (convertible_comparison_p):
+	Use REG_P predicate instead of open coding it.
+	* config/i386/i386.cc (ix86_rtx_costs):
+	Use SUBREG_P predicate instead of open coding it.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386.cc (symbolic_reference_mentioned_p):
+	Use LABEL_REF_P predicate instead of open coding it.
+	(ix86_legitimate_constant_p): Ditto.
+	(legitimate_pic_address_disp_p): Ditto.
+	(ix86_legitimate_address_p): Ditto.
+	(legitimize_pic_address): Ditto.
+	(ix86_print_operand): Ditto.
+	(ix86_print_operand_address_as): Ditto.
+	(ix86_rip_relative_addr_p): Ditto.
+	* config/i386/i386.h (SYMBOLIC_CONST): Ditto.
+	* config/i386/i386.md (*anddi_1 to *andsi_1_zext splitter): Ditto.
+	* config/i386/predicates.md (symbolic_operand): Ditto.
+	(local_symbolic_operand): Ditto.
+	(vsib_address_operand): Ditto.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_move):
+	Use SYMBOL_REF_P predicate instead of open coding it.
+	(ix86_split_long_move): Ditto.
+	(construct_plt_address): Ditto.
+	(ix86_expand_call): Ditto.
+	(ix86_notrack_prefixed_insn_p): Ditto.
+	* config/i386/i386-features.cc
+	(rest_of_insert_endbr_and_patchable_area): Ditto.
+	* config/i386/i386.cc (symbolic_reference_mentioned_p): Ditto.
+	(ix86_force_load_from_GOT_p): Ditto.
+	(ix86_legitimate_constant_p): Ditto.
+	(legitimate_pic_operand_p): Ditto.
+	(legitimate_pic_address_disp_p): Ditto.
+	(ix86_legitimate_address_p): Ditto.
+	(legitimize_pic_address): Ditto.
+	(ix86_legitimize_address): Ditto.
+	(ix86_delegitimize_tls_address): Ditto.
+	(ix86_print_operand): Ditto.
+	(ix86_print_operand_address_as): Ditto.
+	(ix86_rip_relative_addr_p): Ditto.
+	(symbolic_base_address_p): Ditto.
+	* config/i386/i386.h (SYMBOLIC_CONST): Ditto.
+	* config/i386/i386.md (*anddi_1 to *andsi_1_zext splitter): Ditto.
+	* config/i386/predicates.md (symbolic_operand): Ditto.
+	(local_symbolic_operand): Ditto.
+	(local_func_symbolic_operand): Ditto.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_vector_logical_operator):
+	Use CONST_VECTOR_P instead of open coding it.
+	(ix86_expand_int_sse_cmp): Ditto.
+	(ix86_extract_perm_from_pool_constant): Ditto.
+	(ix86_split_to_parts): Ditto.
+	(const_vector_equal_evenodd_p): Ditto.
+	* config/i386/i386.cc (ix86_print_operand): Ditto.
+	* config/i386/predicates.md (zero_extended_scalar_load_operand): Ditto.
+	(float_vector_all_ones_operand): Ditto.
+	* config/i386/sse.md (avx512vl_vextractf128<mode>): Ditto.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121049
+	* internal-fn.h (widening_evenodd_fn_p): Declare.
+	* internal-fn.cc (widening_evenodd_fn_p): New function.
+	* tree-vect-stmts.cc (vectorizable_conversion): When using
+	an even/odd widening function disable loop masking.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/119920
+	PR tree-optimization/112324
+	PR tree-optimization/110015
+	* tree-if-conv.cc (find_different_opnum): New function.
+	(factor_out_operators): New function.
+	(predicate_scalar_phi): Call factor_out_operators when
+	there is only 2 elements of a phi.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* tree-if-conv.cc (fold_build_cond_expr): Return early if lhs and rhs
+	are the same.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* tree-if-conv.cc (combine_blocks): Remove predicated
+	dynamic array.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121116
+	* tree-vect-loop.cc (vectorizable_induction): Use the
+	step vector element type for further processing.
+
+2025-07-16  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (add<mode>3_vcc_dup<exec_vcc>): Change
+	operand 2 to allow gcn_alu_operand.  Swap the operands in the VCC
+	update RTL.
+	(add<mode>3_vcc_zext_dup): Likewise.
+	(add<mode>3_vcc_zext_dup_exec): Likewise.
+	(add<mode>3_vcc_zext_dup2): Likewise.
+	(add<mode>3_vcc_zext_dup2_exec): Likewise.
+
+2025-07-16  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/117850
+	* config/aarch64/aarch64-builtins.cc (LO_HI_PAIRINGS): New, group the
+	lo/hi pairs from aarch64-builtin-pairs.def.
+	(aarch64_get_highpart_builtin): New function.
+	(aarch64_v128_highpart_ref): New function, helper to look for vector
+	highparts.
+	(aarch64_build_vector_cst): New function, helper to build duplicated
+	VECTOR_CSTs.
+	(aarch64_fold_lo_call_to_hi): New function.
+	(aarch64_general_gimple_fold_builtin): Add cases for the lo builtins
+	in aarch64-builtin-pairs.def.
+	* config/aarch64/aarch64-builtin-pairs.def: New file, declare the
+	parirs of lowpart-operating and highpart-operating builtins.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* tree.cc (get_clone_versions): New function.
+	(get_clone_attr_versions): New function.
+	(get_version): New function.
+	* tree.h (get_clone_versions): New function.
+	(get_clone_attr_versions): New function.
+	(get_target_version): New function.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* attribs.cc (make_attribute): Change arguments.
+	* attribs.h (make_attribute): Change arguments.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* pretty-print.cc (format_phase_2): Add support for string_slice.
+	* vec.cc (string_slice::tokenize): New static method.
+	(string_slice::strcmp): New static method.
+	(string_slice::strip): New method.
+	(test_string_slice_initializers): New test.
+	(test_string_slice_tokenize): Ditto.
+	(test_string_slice_strcmp): Ditto.
+	(test_string_slice_equality): Ditto.
+	(test_string_slice_inequality): Ditto.
+	(test_string_slice_invalid): Ditto.
+	(test_string_slice_strip): Ditto.
+	(vec_cc_tests): Add new tests.
+	* vec.h (class string_slice): New class.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR middle-end/121065
+	* cfgexpand.cc (expand_debug_expr): Allow fixed-point modes for
+	RDIV_EXPR.
+	* optabs-tree.cc (optab_for_tree_code): Ditto.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120297
+	* config/riscv/riscv-vsetvl.def: Do not forget ratio demand of
+	previous vsetvl.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-sve2.md (*aarch64_sve2_bsl2n_eon<mode>):
+	New pattern.
+	(*aarch64_sve2_eon_bsl2n_unpred<mode>): Likewise.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-sve2.md (*aarch64_sve2_unpred_nor<mode>):
+	New define_insn.
+	(*aarch64_sve2_nand_unpred<mode>): Likewise.
+
+2025-07-16  Jeremy Rifkin  <jeremy@rifkin.dev>
+
+	PR c/82134
+	* gimplify.cc (gimplify_modify_expr): Add suppress_warning
+	* tree-cfg.cc (do_warn_unused_result): Check warning_suppressed_p
+
+2025-07-16  Haochen Jiang  <haochen.jiang@intel.com>
+
+	* common/config/i386/i386-common.cc
+	(OPTION_MASK_ISA2_AMX_AVX512_SET): Do not set AVX10.2.
+	(OPTION_MASK_ISA2_AVX10_2_UNSET): Remove AMX-AVX512 unset.
+	(OPTION_MASK_ISA2_AVX512F_UNSET): Unset AMX-AVX512.
+	(ix86_handle_option): Imply AVX512F for AMX-AVX512.
+
+2025-07-16  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec.md (avg<mode>3_floor): Add new
+	pattern of avg3_floor for rvv DImode.
+
+2025-07-15  David Malcolm  <dmalcolm@redhat.com>
+
+	* spellcheck.cc: Define INCLUDE_ALGORITHM.
+	(CASE_COST, BASE_COST): Convert to...
+	(case_cost, base_cost): ...these, in an anonymous namespace.
+	(get_edit_distance): Update for above.  Use std::min rather than
+	MIN.
+	(get_edit_distance_cutoff): Likewise.  Use std::max rather than
+	MAX.
+	(selftest::test_edit_distances): Update for BASE_COST renaming.
+	(selftest::get_old_cutoff): Likewise.  Use std::max.
+	(selftest::assert_not_suggested_for): Use nullptr.
+	(selftest::test_find_closest_string): Likewise.
+	* spellcheck.h: Replace TYPE with StringLikeType in templates,
+	and use CamelCase.
+
+2025-07-15  David Malcolm  <dmalcolm@redhat.com>
+
+	PR sarif-replay/120792
+	* auto-obstack.h: New file, based on material taken from
+	pretty-print.cc.
+	* diagnostic-digraphs.h
+	(diagnostics::digraphs::digraph::set_description): New.
+	(diagnostics::digraphs::node::set_label): New.
+	* doc/libgdiagnostics/topics/compatibility.rst: Add
+	LIBGDIAGNOSTICS_ABI_4.
+	* doc/libgdiagnostics/topics/diagnostics.rst
+	(diagnostic_finish_via_msg_buf): Document new entrypoint.
+	* doc/libgdiagnostics/topics/execution-paths.rst
+	(diagnostic_execution_path_add_event_via_msg_buf): Document new
+	entrypoint.
+	* doc/libgdiagnostics/topics/index.rst: Add message-buffers.rst.
+	* doc/libgdiagnostics/topics/message-buffers.rst: New file.
+	* doc/libgdiagnostics/topics/message-formatting.rst: Add note
+	about message buffers.
+	* doc/libgdiagnostics/topics/physical-locations.rst
+	(diagnostic_add_location_with_label_via_msg_buf): Add.
+	* doc/libgdiagnostics/tutorial/07-execution-paths.rst: Link to
+	next section.
+	* doc/libgdiagnostics/tutorial/08-message-buffers.rst: New file.
+	* doc/libgdiagnostics/tutorial/index.rst: Add
+	08-message-buffers.rst.
+	* libgdiagnostics++.h (libgdiagnostics::message_buffer): New
+	class.
+	(libgdiagnostics::execution_path::add_event_via_msg_buf): New.
+	(libgdiagnostics::diagnostic::add_location_with_label): New.
+	(libgdiagnostics::diagnostic::finish_via_msg_buf): New.
+	(libgdiagnostics::graph::set_description): New overload.
+	(libgdiagnostics::graph::add_edge): New overload.
+	(libgdiagnostics::node::set_label): New overload.
+	* libgdiagnostics-private.h
+	(private_diagnostic_execution_path_add_event_2): Drop decl.
+	(private_diagnostic_execution_path_add_event_3): New decl.
+	* libgdiagnostics.cc:  Include "pretty-print-format-impl.h",
+	"pretty-print-markup.h", and "auto-obstack.h".
+	(class copying_token_printer): New.
+	(struct diagnostic_message_buffer): New.
+	(class pp_element_message_buffer): New.
+	(libgdiagnostics_path_event::libgdiagnostics_path_event): Replace
+	params "gmsgid" and "args" with "msg_buf".
+	(libgdiagnostics_path_event::print_desc): Reimplement using
+	pp_element_message_buffer to replay m_msg_buf into "pp".
+	(libgdiagnostics_path_event::m_desc_uncolored): Drop field.
+	(libgdiagnostics_path_event::m_desc_colored): Drop field.
+	(libgdiagnostics_path_event::msg_buf): New field.
+	(diagnostic_execution_path::add_event_va): Reimplement.
+	(diagnostic_execution_path::add_event_via_msg_buf): New.
+	(diagnostic::add_location_with_label): New overload, using
+	msg_buf.
+	(diagnostic_manager::emit): Reimplement with...
+	(diagnostic_manager::emit_va): ...this.
+	(diagnostic_manager::emit_msg_buf): New.
+	(FAIL_IF_NULL): Rename "p" to "ptr_arg".
+	(diagnostic_finish_va): Update to use diagnostic_manager::emit_va.
+	(diagnostic_graph::add_node_with_id): Rename "id" to "node_id".
+	(diagnostic_graph_add_node): Likewise.
+	(diagnostic_graph_add_edge): Rename "id" to "edge_id".
+	(diagnostic_graph_get_node_by_id): Rename "id" to "node_id".
+	(diagnostic_graph_get_edge_by_id): Rename "id" to "edge_id".
+	(private_diagnostic_execution_path_add_event_2): Delete.
+	(diagnostic_message_buffer_new): New public entrypoint.
+	(diagnostic_message_buffer_release): Likewise.
+	(diagnostic_message_buffer_append_str): Likewise.
+	(diagnostic_message_buffer_append_text): Likewise.
+	(diagnostic_message_buffer_append_byte): Likewise.
+	(diagnostic_message_buffer_append_printf): Likewise.
+	(diagnostic_message_buffer_append_event_id): Likewise.
+	(diagnostic_message_buffer_begin_url): Likewise.
+	(diagnostic_message_buffer_end_url): Likewise.
+	(diagnostic_message_buffer_begin_quote): Likewise.
+	(diagnostic_message_buffer_end_quote): Likewise.
+	(diagnostic_message_buffer_begin_color): Likewise.
+	(diagnostic_message_buffer_end_color): Likewise.
+	(diagnostic_message_buffer_dump): Likewise.
+	(diagnostic_finish_via_msg_buf): Likewise.
+	(diagnostic_add_location_with_label_via_msg_buf): Likewise.
+	(diagnostic_execution_path_add_event_via_msg_buf): Likewise.
+	(diagnostic_graph_set_description_via_msg_buf): Likewise.
+	(diagnostic_graph_add_edge_via_msg_buf): Likewise.
+	(diagnostic_node_set_label_via_msg_buf): Likewise.
+	(private_diagnostic_execution_path_add_event_3): New private
+	entrypoint.
+	* libgdiagnostics.h (LIBGDIAGNOSTICS_PARAM_FORMAT_STRING): New macro.
+	(LIBGDIAGNOSTICS_PARAM_PRINTF_FORMAT_STRING): New macro.
+	(diagnostic_message_buffer): New typedef.
+	(LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer): New define.
+	(diagnostic_message_buffer_new): New decl.
+	(diagnostic_message_buffer_release): New decl.
+	(diagnostic_message_buffer_append_str): New decl.
+	(diagnostic_message_buffer_append_text): New decl.
+	(diagnostic_message_buffer_append_byte): New decl.
+	(diagnostic_message_buffer_append_printf): New decl.
+	(diagnostic_message_buffer_append_event_id): New decl.
+	(diagnostic_message_buffer_begin_url): New decl.
+	(diagnostic_message_buffer_end_url): New decl.
+	(diagnostic_message_buffer_begin_quote): New decl.
+	(diagnostic_message_buffer_end_quote): New decl.
+	(diagnostic_message_buffer_begin_color): New decl.
+	(diagnostic_message_buffer_end_color): New decl.
+	(diagnostic_message_buffer_dump): New decl.
+	(diagnostic_finish_via_msg_buf): New decl.
+	(diagnostic_add_location_with_label_via_msg_buf): New decl.
+	(diagnostic_execution_path_add_event_via_msg_buf): New decl.
+	(diagnostic_graph_set_description_via_msg_buf): New decl.
+	(diagnostic_graph_add_edge_via_msg_buf): New decl.
+	(diagnostic_node_set_label_via_msg_buf): New decl.
+	* libgdiagnostics.map (LIBGDIAGNOSTICS_ABI_3): Drop
+	private_diagnostic_execution_path_add_event_2.
+	(LIBGDIAGNOSTICS_ABI_4): New.
+	* libsarifreplay.cc (class annotation): Use
+	libgdiagnostics::message_buffer rather than label_text.
+	(add_any_annotations): Likewise.
+	(sarif_replayer::handle_result_obj): Likewise.
+	(make_plain_text_within_result_message): Likewise.
+	(handle_thread_flow_location_object): Likewise.
+	(handle_location_object): Likewise.
+	(sarif_replayer::handle_graph_object): Likewise.
+	(sarif_replayer::handle_node_object): Likewise.
+	(sarif_replayer::handle_edge_object): Likewise.
+	* pretty-print-format-impl.h (pp_token_list::push_back_byte): New
+	decl.
+	* pretty-print-markup.h (pp_markup::context::begin_url): New decl.
+	(pp_markup::context::end_url): New decl.
+	(pp_markup::context::add_event_id): New decl.
+	* pretty-print.cc: Include "auto-obstack.h".
+	(pp_token_list::push_back_byte): New.
+	(struct auto_obstack): Move to auto-obstack.h.
+	(default_token_printer): Make non-static.
+	(pp_markup::context::begin_url): New.
+	(pp_markup::context::end_url): New.
+	(pp_markup::context::add_event_id): New.
+
+2025-07-15  Umesh Kalappa  <ukalappa.mips@gmail.com>
+
+	* config/riscv/riscv-cores.def (RISCV_CORE): Updated the supported march.
+	* config/riscv/riscv-ext-mips.def (DEFINE_RISCV_EXT):
+	New file added for mips conditional mov extension.
+	* config/riscv/riscv-ext.def: Likewise.
+	* config/riscv/t-riscv: Generates riscv-ext.opt
+	* config/riscv/riscv-ext.opt: Generated file.
+	* config/riscv/riscv.cc (riscv_expand_conditional_move): Updated for mips cmov
+	and outlined some code that handle arch cond move.
+	* config/riscv/riscv.md (mov<mode>cc): updated expand for MIPS CCMOV.
+	* config/riscv/mips-insn.md: New file for mips-p8700 ccmov insn.
+	* doc/riscv-ext.texi: Updated for mips cmov.
+
+2025-07-15  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	* avoid-store-forwarding.cc (generate_bit_insert_sequence):
+	Remove adjustment of bitfield insertion's starting position
+	when BYTES_BIG_ENDIAN != BITS_BIG_ENDIAN.
+	(process_store_forwarding): Update offset check in base reg
+	initialization to take into account the target's endianness.
+
+2025-07-15  Soumya AR  <soumyaa@nvidia.com>
+
+	* config/aarch64/aarch64-tuning-flags.def (AARCH64_EXTRA_TUNING_OPTION):
+	Add AVOID_LDAPUR tuning flag.
+	* config/aarch64/aarch64.cc (aarch64_adjust_generic_arch_tuning):
+	Set AVOID_LDAPUR for architectures before armv8.8-a.
+	(aarch64_override_options_internal): Apply generic tuning adjustments
+	to generic_armv8_a_tunings and generic_armv9_a_tunings.
+	* config/aarch64/aarch64.h (TARGET_ENABLE_LDAPUR): New macro to
+	control LDAPUR usage based on RCPC2 and tuning flags.
+	* config/aarch64/aarch64.md: Add enable_ldapur attribute.
+	* config/aarch64/atomics.md (aarch64_atomic_load<mode>_rcpc): Modify
+	to emit LDAPUR for cores with RCPC2.
+	(*aarch64_atomic_load<ALLX:mode>_rcpc_zext): Likewise.
+	(*aarch64_atomic_load<ALLX:mode>_rcpc_sext): Update constraint to Ust.
+	* config/aarch64/tuning_models/cortexx925.h: Add AVOID_LDAPUR flag.
+	* config/aarch64/tuning_models/neoversev2.h: Likewise.
+	* config/aarch64/tuning_models/neoversev3.h: Likewise.
+	* config/aarch64/tuning_models/neoversev3ae.h: Likewise.
+
+2025-07-15  Richard Biener  <rguenther@suse.de>
+	    Richard Sandiford   <richard.sandiford@arm.com>
+
+	PR tree-optimization/121059
+	* tree-vect-stmts.cc (vectorizable_operation): Query
+	scalar_cond_masked_set with the correct number of masks.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* common.opt (Wunused-but-set-parameter=, Wunused-but-set-variable=):
+	New options.
+	(Wunused-but-set-parameter, Wunused-but-set-variable): Turn into
+	aliases.
+	* common.opt.urls: Regenerate.
+	* diagnostic-spec.cc (nowarn_spec_t::nowarn_spec_t): Use
+	OPT_Wunused_but_set_variable_ instead of OPT_Wunused_but_set_variable
+	and OPT_Wunused_but_set_parameter_ instead of
+	OPT_Wunused_but_set_parameter.
+	* gimple-ssa-store-merging.cc (find_bswap_or_nop_1): Remove unused
+	but set variable tmp.
+	* ipa-strub.cc (pass_ipa_strub::execute): Cast named_args to
+	(void) if ATTR_FNSPEC_DECONST_WATERMARK is not defined.
+	* doc/invoke.texi (Wunused-but-set-parameter=,
+	Wunused-but-set-variable=): Document new options.
+	(Wunused-but-set-parameter, Wunused-but-set-variable): Adjust
+	documentation now that they are just aliases.
+
+2025-07-15  Alfie Richards  <alfie.richards@arm.com>
+
+	* config/aarch64/aarch64-sme.md (@aarch64_sme_<faminmax_uns_op><mode>):
+	Change gating and comment.
+
+2025-07-15  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	Revert:
+	2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (*eor3qdi4): New
+	define_insn_and_split.
+
+2025-07-15  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*fcm<cmp_op><mode>_and_combine):
+	Extend from SVE_FULL_F to SVE_F.
+	(*fcmuo<mode>_and_combine): Likewise.
+	(*fcm<cmp_op><mode>_bic_combine): Likewise.
+	(*fcm<cmp_op><mode>_nor_combine): Likewise.
+	(*fcmuo<mode>_bic_combine): Likewise.
+	(*fcmuo<mode>_nor_combine): Likewise.  Move the comment here to
+	above fcmuo<mode>_bic_combine, since it applies to both patterns.
+
+2025-07-15  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* tree.cc: Include memmodel.h.
+
+2025-07-14  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (vec_cmpu<mode>di_exec): Call gen_vec_cmp*,
+	not gen_vec_cmpu*.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	Revert:
+	2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* tree-vect-stmts.cc (vectorizable_operation): Record a
+	loop mask for mask AND operations.
+
+2025-07-14  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/vector.md (reduc_plus_scal_<mode>): Implement.
+	(reduc_plus_scal_v2df): Implement.
+	(reduc_plus_scal_v4sf): Implement.
+	(REDUC_FMINMAX): New int iterator.
+	(reduc_fminmax_name): New int attribute.
+	(reduc_minmax): New code iterator.
+	(reduc_minmax_name): New code attribute.
+	(reduc_<reduc_fminmax_name>_scal_v2df): Implement.
+	(reduc_<reduc_fminmax_name>_scal_v4sf): Implement.
+	(reduc_<reduc_minmax_name>_scal_v2df): Implement.
+	(reduc_<reduc_minmax_name>_scal_v4sf): Implement.
+	(REDUCBIN): New code iterator.
+	(reduc_bin_insn): New code attribute.
+	(reduc_<reduc_bin_insn>_scal_v2di): Implement.
+	(reduc_<reduc_bin_insn>_scal_v4si): Implement.
+	(reduc_<reduc_bin_insn>_scal_v8hi): Implement.
+	(reduc_<reduc_bin_insn>_scal_v16qi): Implement.
+
+2025-07-14  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.cc (s390_option_override_internal): Remove override.
+
+2025-07-14  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (add<mode>3<exec_clobber>): Rename ...
+	(add<mode>3<exec>): ... to this, remove the clobber, and change the
+	instruction from v_add_co_u32 to v_add_u32.
+	(add<mode>3_dup<exec_clobber>): Rename ...
+	(add<mode>3_dup<exec>): ... to this, and likewise.
+	(sub<mode>3<exec_clobber>): Rename ...
+	(sub<mode>3<exec>): ... to this, and likewise
+	* config/gcn/gcn.md (addsi3): Remove the DI clobber, and change the
+	instruction from v_add_co_u32 to v_add_u32.
+	(addsi3_scc): Likewise.
+	(subsi3): Likewise, but for v_sub_co_u32.
+	(muldi3): Likewise.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* tree-vect-stmts.cc (vectorizable_operation): Record a
+	loop mask for mask AND operations.
+
+2025-07-14  Pan Li  <pan2.li@intel.com>
+
+	* match.pd: Make sure widen mul has twice bitsize
+	of the inputs in SAT_MUL pattern.
+
+2025-07-14  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121015
+	* config/i386/i386-features.cc (ix86_broadcast_inner): Check all
+	0s/1s vectors with standard_sse_constant_p.
+
+2025-07-14  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	* config.in: Regenerated.
+	* configure: Likewise.
+	* configure.ac: Add --enable-x86-64-mfentry.
+	* config/i386/i386-options.cc (ix86_option_override_internal):
+	Enable __fentry__ in 64-bit mode if ENABLE_X86_64_MFENTRY is set
+	to 1.  Warn -pg without -mfentry with shrink wrapping enabled.
+	* doc/install.texi: Document --enable-x86-64-mfentry.
+
+2025-07-14  François-Xavier Coudert  <fxcoudert@gcc.gnu.org>
+
+	PR target/120645
+	* config/darwin-driver.cc: Account for latest macOS numbering
+	scheme.
+
+2025-07-14  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* config/riscv/autovec-opt.md (*vfwmacc_vf_<mode>): New pattern to
+	handle both vfwmacc and vfwmsac.
+	(*extend_vf_<mode>): New pattern that serves as an intermediate combine
+	step.
+	* config/riscv/vector-iterators.md (vsubel): New mode attribute. This is
+	just the lower-case version of VSUBEL.
+	* config/riscv/vector.md (@pred_widen_mul_<optab><mode>_scalar): Reorder
+	and swap operands to match the RTL emitted by expand, i.e. first
+	float_extend then vec_duplicate.
+
+2025-07-14  Alfie Richards  <alfie.richards@arm.com>
+
+	* config/aarch64/aarch64-sme.md (@aarch64_sme_<faminmax_uns_op><mode>):
+	New patterns.
+	* config/aarch64/aarch64-sve-builtins-sme.def (svamin): New intrinsics.
+	(svamax): New intrinsics.
+	* config/aarch64/aarch64-sve-builtins-sve2.cc (class faminmaximpl): New
+	class.
+	(svamin): New function.
+	(svamax): New function.
+
+2025-07-14  Haochen Jiang  <haochen.jiang@intel.com>
+
+	* config/i386/i386.h (PTA_PANTHERLAKE): Revmoe KL and WIDEKL.
+	(PTA_CLEARWATERFOREST): Ditto.
+	* doc/invoke.texi: Revise documentation.
+
+2025-07-13  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120866
+	* tree.cc: Add include to tm_p.h.
+
+2025-07-13  Benjamin Wu  <bwu25@cs.washington.edu>
+
+	* gimple.h (GTMA_DOES_GO_IRREVOCABLE): Fix typo.
+
+2025-07-12  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (function_instance::~function_instance):
+	Move down in source.
+	(string_table::get_cgraph_node): New member function with
+	logic broken out from ...
+	(function_instance::get_cgraph_node): ... here.
+	(match_with_target): Fix formating.
+	(function_instance::match): Fix formating; do not use iterators
+	after modifying map; remove incorrect set of warned flag.
+	(autofdo_source_profile::offline_external_functions): Keep
+	seen set up to date.
+	(function_instance::read_function_instance): Fix formating.
+
+2025-07-12  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/mmx.md (mov<V_32:mode>):
+	Use nonimm_or_0_operand predicate for operand 1.
+	(*mov<V_32:mode>_internal): Ditto.
+	(movv2qi): Ditto.
+	(*movv2qi_internal): Ditto.  Use ix86_hardreg_mov_ok
+	in insn condition.
+
+2025-07-12  Xi Ruoyao  <xry111@xry111.site>
+
+	PR rtl-optimization/120983
+	* lra-constraints.cc (process_alt_operands): Allow reloading
+	user hard registers unless the insn is an asm.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Include "diagnostic-format-sarif.h",
+	Replace include of "diagnostic-state.h" with includes of
+	"diagnostic-digraphs.h" and "diagnostic-state-graphs.h".
+	(html_generation_options::html_generation_options): Update for
+	field renaming.
+	(html_builder::m_body_element): New field.
+	(html_builder::html_builder): Initialize m_body_element.
+	(html_builder::maybe_make_state_diagram): Port from XML
+	implementation to state graph implementation.
+	(html_builder::make_element_for_diagnostic): Add any
+	per-diagnostic graphs.
+	(html_builder::add_graph): New.
+	(html_builder::emit_global_graph): New.
+	(html_output_format::report_global_digraph): New.
+	* diagnostic-format-html.h
+	(html_generation_options::m_show_state_diagram_xml): Replace
+	with...
+	(html_generation_options::m_show_state_diagrams_sarif): ...this.
+	(html_generation_options::m_show_state_diagram_dot_src): Rename
+	to...
+	(html_generation_options::m_show_state_diagrams_dot_src): ...this.
+	* diagnostic-format-sarif.cc: Include "diagnostic-digraphs.h" and
+	"diagnostic-state-graphs.h".
+	(sarif_builder::m_run_graphs): New field.
+	(sarif_result::on_nested_diagnostic): Update call to
+	make_location_object to pass arg by pointer.
+	(sarif_builder::sarif_builder): Initialize m_run_graphs.
+	(sarif_builder::report_global_digraph): New.
+	(sarif_builder::make_result_object): Add any graphs to
+	the result object.
+	(sarif_builder::make_locations_arr): Update call to
+	make_location_object to pass arg by pointer.
+	(sarif_builder::make_location_object): Pass param "loc_mgr" by
+	pointer rather than by reference so that it can be null, and
+	handle this case.
+	(copy_any_property_bag): New.
+	(make_sarif_graph): New.
+	(make_sarif_node): New.
+	(make_sarif_edge): New.
+	(sarif_property_bag::set_graph): New.
+	(populate_thread_flow_location_object): Port from XML
+	implementation to state graph implementation.
+	(make_run_object): Store any graphs.
+	(sarif_output_format::report_global_digraph): New.
+	(sarif_generation_options::sarif_generation_options): Rename
+	m_xml_state to m_state_graph.
+	(selftest::test_make_location_object): Update for change to
+	make_location_object.
+	* diagnostic-format-sarif.h:
+	(sarif_generation_options::m_xml_state): Replace with...
+	(sarif_generation_options::m_state_graph): ...this.
+	(class sarif_location_manager): Add forward decl.
+	(diagnostics::digraphs::digraph): New forward decl.
+	(diagnostics::digraphs::node): New forward decl.
+	(diagnostics::digraphs::edge): New forward decl.
+	(sarif_property_bag::set_graph): New decl.
+	(class sarif_graph): New.
+	(class sarif_node): New.
+	(class sarif_edge): New.
+	(make_sarif_graph): New decl.
+	(make_sarif_node): New decl.
+	(make_sarif_edge): New decl.
+	* diagnostic-format-text.h
+	(diagnostic_text_output_format::report_global_digraph): New.
+	* diagnostic-format.h
+	(diagnostic_output_format::report_global_digraph): New vfunc.
+	* diagnostic-digraphs.cc: New file.
+	* diagnostic-digraphs.h: New file.
+	* diagnostic-metadata.h (diagnostics::digraphs::lazy_digraphs):
+	New forward decl.
+	(diagnostic_metadata::diagnostic_metadata): Initialize
+	m_lazy_digraphs.
+	(diagnostic_metadata::set_lazy_digraphs): New.
+	(diagnostic_metadata::get_lazy_digraphs): New.
+	(diagnostic_metadata::m_lazy_digraphs): New field.
+	* diagnostic-output-spec.cc (sarif_scheme_handler::make_sink):
+	Update for XML to state graph changes.
+	(sarif_scheme_handler::make_sarif_gen_opts): Likewise.
+	(html_scheme_handler::make_sink): Rename "show-state-diagram-xml"
+	to "show-state-diagrams-sarif" and use pluralization consistently.
+	* diagnostic-path.cc: Replace include of "xml.h" with
+	"diagnostic-state-graphs.h".
+	(diagnostic_event::maybe_make_xml_state): Replace with...
+	(diagnostic_event::maybe_make_diagnostic_state_graph): ...this.
+	* diagnostic-path.h (diagnostics::digraphs::digraph): New forward
+	decl.
+	(diagnostic_event::maybe_make_xml_state): Replace with...
+	(diagnostic_event::maybe_make_diagnostic_state_graph): ...this.
+	* diagnostic-state-graphs.cc: New file.
+	* diagnostic-state-graphs.h: New file.
+	* diagnostic-state-to-dot.cc: Port implementation from XML to
+	state graphs.
+	* diagnostic-state.h: Deleted file.
+	* diagnostic.cc (diagnostic_context::report_global_digraph): New.
+	* diagnostic.h (diagnostics::digraphs::lazy_digraph): New forward
+	decl.
+	(diagnostic_context::report_global_digraph): New decl.
+	* doc/analyzer.texi (Debugging the Analyzer): Update to reflect
+	change from XML to state graphs.
+	* doc/invoke.texi ("sarif" diagnostics sink): Replace "xml-state"
+	with "state-graphs".
+	("experimental-html" diagnostics sink): Replace
+	"show-state-diagrams-xml" with "show-state-diagrams-sarif"
+	* doc/libgdiagnostics/topics/compatibility.rst
+	(LIBGDIAGNOSTICS_ABI_3): New.
+	* doc/libgdiagnostics/topics/graphs.rst: New file.
+	* doc/libgdiagnostics/topics/index.rst: Add graphs.rst.
+	* graphviz.h (node_id::operator=): New.
+	* json.h (json::value::dyn_cast_string): New.
+	(json::object::get_num_keys): New accessor.
+	(json::object::get_key): New accessor.
+	(json::string::dyn_cast_string): New.
+	* libgdiagnostics++.h (class libgdiagnostics::graph): New.
+	(class libgdiagnostics::node): New.
+	(class libgdiagnostics::edge): New.
+	(class libgdiagnostics::diagnostic::take_graph): New.
+	(class libgdiagnostics::manager::take_global_graph): New.
+	(class libgdiagnostics::graph::set_description): New.
+	(class libgdiagnostics::graph::get_node_by_id): New.
+	(class libgdiagnostics::graph::get_edge_by_id): New.
+	(class libgdiagnostics::graph::add_edge): New.
+	(class libgdiagnostics::node::set_label): New.
+	(class libgdiagnostics::node::set_location): New.
+	(class libgdiagnostics::node::set_logical_location): New.
+	* libgdiagnostics-private.h: New file.
+	* libgdiagnostics.cc: Define INCLUDE_STRING.  Include
+	"diagnostic-digraphs.h", "diagnostic-state-graphs.h", and
+	"libgdiagnostics-private.h".
+	(struct diagnostic_graph): New.
+	(struct diagnostic_node): New.
+	(struct diagnostic_edge): New.
+	(libgdiagnostics_path_event::libgdiagnostics_path_event): Add
+	state_graph param.
+	(libgdiagnostics_path_event::maybe_make_diagnostic_state_graph):
+	New.
+	(libgdiagnostics_path_event::m_state_graph): New field.
+	(diagnostic_execution_path::add_event_va): Add state_graph param.
+	(class prebuilt_digraphs): New.
+	(diagnostic::diagnostic): Use m_graphs in m_metadata.
+	(diagnostic::take_graph): New.
+	(diagnostic::get_graphs): New accessor.
+	(diagnostic::m_graphs): New field.
+	(diagnostic_manager::take_global_graph): New.
+	(diagnostic_execution_path_add_event): Update for new param to
+	add_event_va.
+	(diagnostic_execution_path_add_event_va): Likewise.
+	(diagnostic_graph::add_node_with_id): New public entrypoint.
+	(diagnostic_graph::add_edge_with_label): New public entrypoint.
+	(diagnostic_manager_new_graph): New public entrypoint.
+	(diagnostic_manager_take_global_graph): New public entrypoint.
+	(diagnostic_take_graph): New public entrypoint.
+	(diagnostic_graph_release): New public entrypoint.
+	(diagnostic_graph_set_description): New public entrypoint.
+	(diagnostic_graph_add_node): New public entrypoint.
+	(diagnostic_graph_add_edge): New public entrypoint.
+	(diagnostic_graph_get_node_by_id): New public entrypoint.
+	(diagnostic_graph_get_edge_by_id): New public entrypoint.
+	(diagnostic_node_set_location): New public entrypoint.
+	(diagnostic_node_set_label): New public entrypoint.
+	(diagnostic_node_set_logical_location): New public entrypoint.
+	(private_diagnostic_execution_path_add_event_2): New private
+	entrypoint.
+	(private_diagnostic_graph_set_property_bag): New private
+	entrypoint.
+	(private_diagnostic_node_set_property_bag): New private
+	entrypoint.
+	(private_diagnostic_edge_set_property_bag): New private
+	entrypoint.
+	* libgdiagnostics.h (diagnostic_graph): New typedef.
+	(diagnostic_node): New typedef.
+	(diagnostic_edge): New typedef.
+	(diagnostic_manager_new_graph): New decl.
+	(diagnostic_manager_take_global_graph): New decl.
+	(diagnostic_take_graph): New decl.
+	(diagnostic_graph_release): New decl.
+	(diagnostic_graph_set_description): New decl.
+	(diagnostic_graph_add_node): New decl.
+	(diagnostic_graph_add_edge): New decl.
+	(diagnostic_graph_get_node_by_id): New decl.
+	(diagnostic_graph_get_edge_by_id): New decl.
+	(diagnostic_node_set_label): New decl.
+	(diagnostic_node_set_location): New decl.
+	(diagnostic_node_set_logical_location): New decl.
+	* libgdiagnostics.map (LIBGDIAGNOSTICS_ABI_3): New.
+	* libsarifreplay.cc: Include "libgdiagnostics-private.h".
+	(id_map): New "using".
+	(sarif_replayer::report_invalid_sarif): Update for change to
+	report_problem params.
+	(sarif_replayer::report_unhandled_sarif): Likewise.
+	(sarif_replayer::report_note): New.
+	(sarif_replayer::report_problem): Pass param "ref" by
+	pointer rather than reference and handle it being null.
+	(sarif_replayer::maybe_get_property_bag): New.
+	(sarif_replayer::maybe_get_property_bag_value): New.
+	(sarif_replayer::handle_run_obj): Handle run-level "graphs" as per
+	§3.14.20.
+	(sarif_replayer::handle_result_obj): Handle result-level "graphs"
+	as per §3.27.19.
+	(handle_thread_flow_location_object): Optionally handle graphs
+	stored in property "gcc/diagnostic_event/state_graph" as state
+	graphs.
+	(sarif_replayer::handle_graph_object): New.
+	(sarif_replayer::handle_node_object): New.
+	(sarif_replayer::handle_edge_object): New.
+	(sarif_replayer::get_graph_node_by_id_property): New.
+	* selftest-run-tests.cc (selftest::run_tests): Call
+	selftest::diagnostic_graph_cc_tests and
+	selftest::diagnostic_state_graph_cc_tests.
+	* selftest.h (selftest::diagnostic_graph_cc_tests): New decl.
+	(selftest::diagnostic_state_graph_cc_tests): New decl.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add diagnostic-digraphs.o and
+	diagnostic-state-graphs.o.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* json.cc (json::object::clone): New.
+	(json::object::clone_as_object): New.
+	(json::array::clone): New.
+	(json::float_number::clone): New.
+	(json::integer_number::clone): New.
+	(json::string::clone): New.
+	(json::literal::clone): New.
+	(selftest::test_cloning): New test.
+	(selftest::json_cc_tests): Call it.
+	* json.h (json::value::clone): New vfunc.
+	(json::object::clone): New decl.
+	(json::object::clone_as_object): New decl.
+	(json::array::clone): New decl.
+	(json::float_number::clone): New decl.
+	(json::integer_number::clone): New decl.
+	(json::string::clone): New decl.
+	(json::literal::clone): New decl.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* json.cc (string::string): When constructing from pointer and
+	length, ensure the new buffer is null-terminated.
+	(selftest::test_strcmp): New.
+	(selftest::json_cc_tests): Likewise.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* doc/libgdiagnostics/topics/compatibility.rst
+	(_LIBGDIAGNOSTICS_ABI_2): Add missing anchor.
+	* doc/libgdiagnostics/topics/diagnostic-manager.rst
+	(diagnostic_manager_add_sink_from_spec): Add links to GCC's
+	documentation of "-fdiagnostics-add-output=".  Fix parameter
+	markup.
+	(diagnostic_manager_set_analysis_target): Fix parameter markup.
+	Add link to SARIF spec.
+	* doc/libgdiagnostics/topics/logical-locations.rst: Markup fix.
+	* doc/libgdiagnostics/tutorial/02-physical-locations.rst: Clarify
+	wording of what "the source file" means, and that a range can't
+	have multiple files.
+
+2025-07-11  Vladimir N. Makarov  <vmakarov@redhat.com>
+
+	* lra-constraints.cc (process_address_1): When changing base reg
+	on a reg of the base class, fall back to reload of whole inner address.
+	(process_address): Constrain the iteration number.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* doc/invoke.texi (Wc++26-compat): Document.
+
+2025-07-11  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121027
+	* config/aarch64/aarch64.cc (aarch64_evpc_sve_tbl): Punt on 2-input
+	operations that can be handled by vec_perm.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (*eor3qdi4): New
+	define_insn_and_split.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (*bcaxqdi4): New
+	define_insn_and_split.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (eor3q<mode>4): Use VDQ_I mode
+	iterator.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (bcaxq<mode>4): Use VDQ_I mode
+	iterator.
+
+2025-07-11  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121034
+	* tree-vect-loop.cc (vectorizable_reduction): Cleanup
+	reduction chain following code.
+
+2025-07-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* opts.cc (finish_options): Enable debug_nonbind_markers_p for
+	auto-profile.
+	* tree-cfg.cc (struct locus_discrim_map): Remove.
+	(struct locus_discrim_hasher): Remove.
+	(locus_discrim_hasher::hash): Remove.
+	(locus_discrim_hasher::equal): Remove.
+	(first_non_label_nondebug_stmt): Remove.
+	(build_gimple_cfg): Do not allocate discriminator tables.
+	(next_discriminator_for_locus): Remove.
+	(same_line_p): Remove.
+	(struct discrim_entry): New structure.
+	(assign_discriminator): Rewrite.
+	(assign_discriminators): Rewrite.
+
+2025-07-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/114790
+	* cgraph.cc (cgraph_update_edges_for_call_stmt_node): Resolve devirtualization
+	if call statement was optimized out or turned to direct call.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+	    Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/121023
+	* ipa-fnsummary.cc (compute_fn_summary): Disallow signature changes
+	on cfun->has_musttail functions.
+
+2025-07-11  Hu, Lin1  <lin1.hu@intel.com>
+
+	PR target/91384
+	* config/i386/i386.md: Add new peeophole2 for optimize *negsi_1
+	followed by *cmpsi_ccno_1 with APX_F.
+
+2025-07-11  Richard Biener  <rguenther@suse.de>
+
+	* config/i386/i386.cc (ix86_vector_costs::add_stmt_cost): Use
+	the LHS of a scalar stmt to determine mode and whether it is FP.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.cc (aarch64_vector_costs::add_stmt_cost):
+	Guard VF-based costing with !m_costing_for_scalar.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	* internal-fn.cc (expand_ACCESS_WITH_SIZE): Update comments.
+	* internal-fn.def (ACCESS_WITH_SIZE): Update comments.
+	* tree-object-size.cc (access_with_size_object_size): Update comments.
+	Adjust the arguments per the new design.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	PR middle-end/121000
+	* internal-fn.cc (expand_ACCESS_WITH_SIZE): Update comments.
+	* internal-fn.def (ACCESS_WITH_SIZE): Update comments.
+	* tree-object-size.cc (access_with_size_object_size): Update comments.
+	Get the element_size from the 6th argument directly.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve2.md (aarch64_gather_ld1q): Replace with...
+	(@aarch64_gather_ld1q<mode>): ...this, parameterizing based on mode.
+	* config/aarch64/aarch64-sve-builtins-sve2.cc
+	(svld1q_gather_impl::expand): Update accordingly.
+	(svst1q_scatter_impl::expand): Use aarch64_sve_reinterpret
+	instead of force_lowpart_subreg.
+
+2025-07-10  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc: Include output.h.
+	(function_instance::set_call_location): Also sanity check
+	that location is known.
+	(raw_symbol_name): Two new static functions.
+	(dump_inline_stack): Use it.
+	(string_table::get_index_by_decl): Likewise.
+	(function_instance::get_cgraph_node): Likewise.
+	(function_instance::get_function_instance_by_decl): Fix typo
+	in warning; use raw names; fix lineno decoding.
+	(match_with_target): Add containing funciton parameter;
+	correctly output function and call location in warning.
+	(function_instance::lookup_count): Fix warning locations.
+	(function_instance::match): Fix warning locations; avoid
+	crash with mismatched callee; do not warn about broken callsites
+	twice.
+	(autofdo_source_profile::offline_external_functions): Use
+	raw_assembler_name.
+	(walk_block): Use raw_assembler_name.
+
+2025-07-10  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/121014
+	* cfgexpand.cc (expand_debug_expr): Assert FLOAT_MODE_P.
+	* optabs-tree.cc (optab_for_tree_code): Assert FLOAT_TYPE_P.
+	* tree-vect-loop.cc (vect_get_loop_len): Use EXACT_DIV_EXPR.
+
+2025-07-10  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/118734
+	* config/riscv/constraints.md (Wdm): Use tunable for Wdm
+	constraint.
+	* config/riscv/riscv-protos.h (emit_avltype_insn): Declare.
+	(can_be_broadcasted_p): Rename to...
+	(can_be_broadcast_p): ...this.
+	* config/riscv/predicates.md: Use renamed function.
+	(strided_load_broadcast_p): Declare.
+	* config/riscv/riscv-selftests.cc (run_broadcast_selftests):
+	Only run broadcast selftest if strided broadcasts are OK.
+	* config/riscv/riscv-v.cc (emit_avltype_insn): New function.
+	(sew64_scalar_helper): Only emit a pred_broadcast if the new
+	tunable says so.
+	(can_be_broadcasted_p): Rename to...
+	(can_be_broadcast_p): ...this and use new tunable.
+	* config/riscv/riscv.cc (struct riscv_tune_param): Add strided
+	broad tunable.
+	(strided_load_broadcast_p): Implement.
+	* config/riscv/vector.md: Use strided_load_broadcast_p () and
+	work around 64-bit broadcast on rv32 targets.
+
+2025-07-10  Co-authored-by: Jeff Law  <jlaw@ventanamicro.com>
+
+	* config/riscv/riscv.cc (riscv_fusion_pairs): Add new cases.
+	(riscv_set_is_add): New function.
+	(riscv_set_is_addi, riscv_set_is_adduw, riscv_set_is_shNadd): Likewise.
+	(riscv_set_is_shNadduw): Likewise.
+	(riscv_macro_fusion_pair_p): Add new fusion cases.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (vect_analyze_slp): Fail for non-canonical
+	gconds.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (vect_build_slp_instance): Do not use
+	SLP_TREE_VECTYPE to determine the conversion back to the
+	reduction IV.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_reduction): Avoid
+	vect_is_simple_use and record a vector type if we come
+	up with one.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_load_store_type): Do not use
+	vect_is_simple_use to fill gather/scatter offset operand
+	vectype and dt.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_model_reduction_cost): Get SLP
+	node instead of stmt_info and use that when recording costs.
+
+2025-07-10  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	PR target/120999
+	* config/aarch64/aarch64-sve2.md (*aarch64_sve2_nor<mode>):
+	Adjust movprfx alternative.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* doc/sourcebuild.texi (aarch64_sve2_hw, aarch64_sve2p1_hw): Document.
+	* config/aarch64/aarch64.cc (aarch64_evpc_hvla): Extend to
+	BYTES_BIG_ENDIAN.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_analyze_stmt): Remove stmt-info
+	and need_to_vectorize arguments.
+	* tree-vect-slp.cc (vect_slp_analyze_node_operations_1):
+	Adjust.
+	* tree-vect-stmts.cc (can_vectorize_live_stmts): Remove
+	stmt_info argument and remove non-SLP path.
+	(vect_analyze_stmt): Remove stmt_info and need_to_vectorize
+	argument and prune paths no longer reachable.
+	(vect_transform_stmt): Adjust.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* config/i386/x86-tune.def: Change "Tunning the" to "tuning" in
+	comment and use semicolon instead of dot in comment.
+	* loop-unroll.cc (decide_unroll_stupid): Comment spelling fix,
+	tunning -> tuning.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* tree-vect-loop.cc (scale_profile_for_vect_loop): Comment
+	spelling fix: bellow -> below.
+	* ipa-polymorphic-call.cc (record_known_type): Likewise.
+	* config/i386/x86-tune.def: Likewise.
+	* config/riscv/vector.md (*vsetvldi_no_side_effects_si_extend):
+	Likewise.
+	* tree-scalar-evolution.cc (iv_can_overflow_p): Likewise.
+	* ipa-devirt.cc (add_type_duplicate): Likewise.
+	* tree-ssa-loop-niter.cc (maybe_lower_iteration_bound): Likewise.
+	* gimple-ssa-sccopy.cc: Likewise.
+	* cgraphunit.cc: Likewise.
+	* graphite.h (struct poly_dr): Likewise.
+	* ipa-reference.cc (ignore_edge_p): Likewise.
+	* tree-ssa-alias.cc (ao_compare::compare_ao_refs): Likewise.
+	* profile-count.h (profile_probability::probably_reliable_p):
+	Likewise.
+	* ipa-inline-transform.cc (inline_call): Likewise.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_dissolve_slp_only_groups): Remove.
+	(vect_analyze_loop_2): Do not call it.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_active_double_reduction_p): Remove.
+	(vect_analyze_loop_operations): Remove.
+	(vect_analyze_loop_2): Do not call it.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_determine_vf_for_stmt_1): Rename
+	to ...
+	(vect_determine_vectype_for_stmt_1): ... this and only set
+	STMT_VINFO_VECTYPE.  Fail for single-element vector types.
+	(vect_determine_vf_for_stmt): Rename to ...
+	(vect_determine_vectype_for_stmt): ... this and only set
+	STMT_VINFO_VECTYPE. Fail for single-element vector types.
+	(vect_determine_vectorization_factor): Rename to ...
+	(vect_set_stmts_vectype): ... this and only set STMT_VINFO_VECTYPE.
+	(vect_update_vf_for_slp): Remove.
+	(vect_analyze_loop_operations): Remove walk over stmts.
+	(vect_analyze_loop_2): Call vect_set_stmts_vectype instead of
+	vect_determine_vectorization_factor.  Set vectorization factor
+	from LOOP_VINFO_SLP_UNROLLING_FACTOR.  Fail if vect_detect_hybrid_slp
+	detects hybrid stmts or when vect_make_slp_decision finds
+	nothing to SLP.
+	* tree-vect-slp.cc (vect_detect_hybrid_slp): Move check
+	whether we have any hybrid stmts here from vect_update_vf_for_slp
+	* tree-vect-stmts.cc (vect_analyze_stmt): Remove loop over
+	stmts.
+	* tree-vectorizer.h (vect_detect_hybrid_slp): Update.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.cc (aarch64_simd_valid_imm): Account
+	for FLOAT_WORDS_BIG_ENDIAN when building a floating-point value.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (afdo_adjust_guessed_profile): Add forgotten
+	if (dump_file) guard.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.cc (aarch64_sve_index_series_p): New
+	function, split out from...
+	(aarch64_simd_valid_imm): ...here.  Account for the different
+	SVE and Advanced SIMD element orders on big-endian targets.
+	Check each vector in a structure mode.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* read-rtl-function.cc (function_reader::read_rtx_operand_r): Use
+	hard_regno_nregs to work out REG_NREGS for hard registers.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-v.cc (expand_vx_binary_vec_vec_dup): Add
+	new case SS_MINUS.
+	* config/riscv/riscv.cc (riscv_rtx_costs): Ditto.
+	* config/riscv/vector-iterators.md: Add new op ss_minus.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* ext-dce.cc (ext_dce_process_uses): Apply is_constant directly
+	to the subreg_lsb.
+
+2025-07-09  Jan Dubiec  <jdx@o2.pl>
+
+	PR target/109286
+	* config.gcc: Include elfos.h before h8300/h8300.h.
+	* config/h8300/h8300.h (INIT_SECTION_ASM_OP): Override
+	default version from elfos.h.
+	(FINI_SECTION_ASM_OP): Ditto.
+	(ASM_DECLARE_FUNCTION_NAME): Ditto.
+	(ASM_GENERATE_INTERNAL_LABEL): Macro removed because it was
+	being overridden in elfos.h anyway.
+	(ASM_OUTPUT_SKIP): Ditto.
+
+2025-07-09  Icen Zeyada  <Icen.Zeyada2@arm.com>
+
+	PR tree-optimization/119196
+	* match.pd: Allow scalar optimizations with bitwise AND/OR/XOR to apply to vectors.
+
+2025-07-09  Icen Zeyada  <Icen.Zeyada2@arm.com>
+
+	PR tree-optimization/119196
+	* match.pd: Merge multiple vec_cond_expr in a single one for
+	bit_and, bit_ior and bit_xor.
+
+2025-07-09  Jeff Law  <jlaw@ventanamicro.com>
+
+	PR target/120642
+	* config/riscv/riscv-avlprop.cc (pass_avlprop::execute): Do not do
+	constant AVL propagation for xtheadvector.
+
+2025-07-09  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_reduction): Get the
+	output vector type from slp_for_stmt_info.
+	* tree-vect-stmts.cc (vect_analyze_stmt): Bail out earlier
+	for PURE_SLP_STMT when doing loop stmt analysis.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (struct scale): New structure.
+	(add_scale): Also record weights.
+	(afdo_adjust_guessed_profile): Compute robust average
+	of scales and cap by max count in function.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* tree-inline.cc (initialize_cfun): Use num and den for scaling.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (get_original_name): Fix loop walking the
+	suffixes.
+
+2025-07-09  Christophe Lyon  <christophe.lyon@linaro.org>
+
+	* config/arm/arm_neon.h: Remove useless push/pop pragmas.
+
+2025-07-09  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120922
+	* tree-vect-loop-manip.cc (vect_gen_vector_loop_niters): Support range
+	for partial vectors.
+
+2025-07-09  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120922
+	* tree-vect-loop-manip.cc (vect_gen_vector_loop_niters): Don't set range
+	for partial vectors.
+
+2025-07-08  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.cc (xtensa_b4const_or_zero):
+	Remove.
+	(xtensa_b4const): Add a case where the value is 0, and rename
+	to xtensa_b4const_or_zero.
+	(xtensa_rtx_costs): Fix to also consider the result of
+	xtensa_b4constu().
+
+2025-07-08  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/s390.md (stack_protect_get_tpsi): New insn.
+	(stack_protect_get_tpdi): New insn.
+	(stack_protect_set): Use new insn.
+	(stack_protect_test): Use new insn.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120461
+	* config/riscv/riscv-v.cc (emit_vlmax_insn_lra): Do not emit
+	vsetivli for XTHeadVector.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/113829
+	* config/riscv/riscv-vector-builtins.cc (registered_function::overloaded_hash):
+	Skip non-type arguments.
+
+2025-07-08  Andreas Schwab  <schwab@suse.de>
+
+	PR target/120995
+	* config/riscv/sync.md (zacas_atomic_cas_value_strong<mode>):
+	Allow op3 to be zero.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* config/i386/x86-tune.def (X86_TUNE_AVX512_MASKED_EPILOGUES):
+	New tunable, default on for m_ZNVER4 and m_ZNVER5.
+	* config/i386/i386.cc (ix86_vector_costs::finish_cost): With
+	X86_TUNE_AVX512_MASKED_EPILOGUES and when the main loop
+	had a vectorization factor > 2 use a masked epilogue when
+	possible and when not obviously problematic.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vector_costs::suggested_epilogue_mode):
+	Add masked output parameter and return m_masked_epilogue.
+	(vector_costs::m_masked_epilogue): New tristate flag.
+	(vector_costs::vector_costs): Initialize m_masked_epilogue.
+	* tree-vect-loop.cc (vect_analyze_loop_1): Pass in masked
+	flag to optionally initialize can_use_partial_vectors_p.
+	(vect_analyze_loop): For epilogues also get whether to use
+	a masked epilogue for this loop from the target and use
+	that for the first epilogue mode we try.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120358
+	* tree-ssa-structalias.cc (get_constraint_for_1): Adjust
+	pruning of sub-variables according to the imprecise
+	known start offset.
+
+2025-07-08  Alexandre Oliva  <oliva@adacore.com>
+
+	* config/vxworks-dummy.h (TARGET_VXWORKS_VAROFF): New.
+	(TARGET_VXWORKS_GOTTPIC): New.
+	* config/vxworks.h (TARGET_VXWORKS_VAROFF): Override.
+	(TARGET_VXWORKS_GOTTPIC): Likewise.
+	* config/i386/i386.cc (output_set_got): Disable VxWorks6 GOT
+	sequence on VxWorks7.
+	(legitimize_pic_address): Accept relative addressing of
+	labels on VxWorks7.
+	(ix86_delegitimize_address_1): Likewise.
+	(ix86_output_addr_diff_elt): Likewise.
+	* config/i386/i386.md (tablejump): Likewise.
+	(set_got, set_got_labelled): Set no-red-zone flag on VxWorks7.
+	* config/i386/predicates.md (gotoff_operand): Test
+	TARGET_VXWORKS_VAROFF.
+
+2025-07-08  Alexandre Oliva  <oliva@adacore.com>
+
+	* config.gcc (vxworks-dummy.h): Add to aarch64-*-* as well.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* doc/extend.texi: Extend counted_by attribute to pointer fields in
+	structures. Add one more requirement to pointers with counted_by
+	attribute.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
+
+	* tree-object-size.cc (access_with_size_object_size): Update comments
+	for pointers with .ACCESS_WITH_SIZE.
+	(collect_object_sizes_for): Propagate size info through GIMPLE_ASSIGN
+	for pointers with .ACCESS_WITH_SIZE.
+
+2025-07-07  Martin Jambor  <mjambor@suse.cz>
+
+	* value-range.h (class irange): Mark member function verify_range
+	with override.
+	(class prange): Mark member function verify_range with final override.
+	(class frange): Mark member function verify_range with override.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120888
+	* config/xtensa/xtensa.cc (xtensa_promote_function_mode): New.
+	(TARGET_PROMOTE_FUNCTION_MODE): Use.
+	(TARGET_PROMOTE_PROTOTYPES): Removed.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.md: Update UNSPECs
+	* config/s390/vector.md (fmax<mode>3): New expander.
+	(fmin<mode>3): New expander.
+	* config/s390/vx-builtins.md (*fmin<mode>): New insn.
+	(vfmin<mode>): Redefined to use new insn.
+	(*fmax<mode>): New insn.
+	(vfmax<mode>): Redefined to use new insn.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* doc/invoke.texi: Add -Wno-abbreviated-auto-in-template-arg.
+
+2025-07-07  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64.md (popcountti2): Add TARGET_SVE path.
+
+2025-07-07  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120817
+	* tree-ssa-dse.cc (initialize_ao_ref_for_dse): Use
+	ao_ref_init_from_ptr_and_range with unknown size for
+	.MASK_STORE and .MASK_LEN_STORE.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-protos.h (riscv_expand_usmul): Add new func
+	decl.
+	* config/riscv/riscv.cc (riscv_expand_xmode_usmul): Add new func
+	to expand Xmode SAT_MUL.
+	(riscv_expand_non_xmode_usmul): Ditto but for non-Xmode.
+	(riscv_expand_usmul): Add new func to implment SAT_MUL.
+	* config/riscv/riscv.md (usmul<mode>3): Add new pattern to match
+	standard name usmul.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* match.pd: Add new match pattern for unsigned SAT_MUL.
+	* tree-ssa-math-opts.cc (gimple_unsigned_integer_sat_mul):
+	new decl for pattern match func.
+	(match_unsigned_saturation_mul): Add new func to match unsigned
+	SAT_MUL.
+	(math_opts_dom_walker::after_dom_children): Try to match
+	unsigned SAT_MUL on NOP.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* internal-fn.cc (commutative_binary_fn_p): Add new case
+	for SAT_MUL.
+	* internal-fn.def (SAT_MUL): Add new IFN_SAT_MUL.
+	* optabs.def (OPTAB_NL): Remove fixed point limitation.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.md: Removed unused unspecs.
+	* config/s390/vector.md (avg<mode>3_ceil): New expander.
+	(uavg<mode>3_ceil): New expander.
+	(smul<mode>3_highpart): New expander.
+	(umul<mode>3_highpart): New expander.
+	* config/s390/vx-builtins.md (vec_umulh<mode>): Remove unspec.
+	(vec_smulh<mode>): Remove unspec.
+
+2025-07-07  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (vec_cmp<mode><vpred>): Extend
+	to handle partial FP modes.
+	(@aarch64_pred_fcm<cmp_op><mode>): Likewise.
+	(@aarch64_pred_fcmuo<mode>): Likewise.
+	(*one_cmpl<mode>3): Rename to...
+	(@aarch64_pred_one_cmpl<mode>_z): ... this.
+	* config/aarch64/aarch64.cc (aarch64_emit_sve_fp_cond): Allow the
+	target and governing predicates to have different modes.
+	(aarch64_emit_sve_or_fp_conds): Likewise.
+	(aarch64_emit_sve_invert_fp_cond): Likewise.
+	(aarch64_expand_sve_vec_cmp_float): Likewise.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR tree-optimization/118891
+	* tree-vect-stmts.cc (supportable_widening_operation): Swap the
+	hi and lo internal functions on big-endian targets.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* ext-dce.cc (ext_dce_process_uses): Apply is_constant directly
+	to the subreg_lsb.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_sve_set_neonq_<mode>):
+	Use %Z instead of lowpart_subreg.  Tweak formatting.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/118891
+	* config/aarch64/aarch64.cc (aarch64_expand_vector_init): Fix the
+	ZIP1 operand order for big-endian targets.
+
+2025-07-07  Jan Hubicka  <hubicka@ucw.cz>
+
+	* tree-ssa-live.cc (dump_scope_block): Print discriminators
+	of inlined functions.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120670
+	PR target/120683
+	* config/i386/i386-expand.cc (expand_set_or_cpymem_via_loop):
+	Don't generate the loop if the loop count is 1.
+	(expand_cpymem_epilogue): Use move_by_pieces.
+	(setmem_epilogue_gen_val): New.
+	(expand_setmem_epilogue): Use store_by_pieces.
+	(expand_small_cpymem_or_setmem): Choose cpymem mode from MOVE_MAX.
+	For memset with vector and the size is smaller than the vector
+	size, first try the narrower vector, otherwise, use the scalar
+	value.
+	(promote_duplicated_reg): Duplicate the scalar value for vector.
+	(ix86_expand_set_or_cpymem): Always expand vector-version of
+	memset for vector_loop.  Use misaligned prologue if alignment
+	isn't needed and destination isn't aligned.  Always initialize
+	vec_promoted_val from the promoted scalar value for vector_loop.
+
+2025-07-07  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120709
+	* builtins.cc (expand_builtin_crc_table_based): Error out
+	instead of asserting the 3rd argument is an integer constant.
+	* internal-fn.cc (expand_crc_optab_fn): Likewise.
+	* doc/extend.texi (crc): Document requirement of the poly argument
+	being a constant.
+
+2025-07-06  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr-mcus.def: -mmcu= takes lower case MCU names.
+	* doc/avr-mmcu.texi: Rebuild.
+
+2025-07-06  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr-mcus.def (avr32da28S, avr32da32S, avr32da48S)
+	(avr64da28S, avr64da32S, avr64da48S avr64da64S)
+	(avr128da28S, avr128da32S, avr128da48S, avr128da64S): Add devices.
+	* doc/avr-mmcu.texi: Rebuild.
+
+2025-07-06  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/120951
+	* tree-call-cdce.cc (use_internal_fn): For non-call exceptions
+	with EQ_EXPR can throw for floating point types, then create
+	the EQ_EXPR seperately.
+
+2025-07-06  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120921
+	* tree-cfg.cc (verify_gimple_assign_single): Reject constant and address expression LHS.
+	For non-empty vector constructors, make sure the LHS is an is_gimple_reg.
+
+2025-07-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc
+	(autofdo_source_profile::read): Scale cutoff.
+	(read_autofdo_file): Initialize cutoff
+	* coverage.cc (read_counts_file): Initialize cutoff to 1.
+	* gcov-io.h (struct gcov_summary): Add cutoff field.
+	* ipa-inline.cc (inline_small_functions): mac_count can be non-zero
+	also with auto_profile.
+	* lto-cgraph.cc (output_profile_summary): Write cutoff
+	and sum_max.
+	(input_profile_summary): Read cutoff and sum max.
+	(merge_profile_summaries): Initialize and scale global cutoffs
+	and sum max.
+	* profile-count.cc: Include profile.h
+	(profile_count::force_nonzero): move here from ...; use cutoff.
+	* profile-count.h: (profile_count::force_nonzero): ... here.
+
+2025-07-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	* profile-count.cc (profile_count::operator*): fix overflow check.
+
+2025-07-05  Alexandre Oliva  <oliva@adacore.com>
+
+	* config/rs6000/vxworks.h (SUBTARGET_DRIVER_SELF_SPECS):
+	Redefine to select word size matching TARGET_VXWORKS64.
+	(TARGET_VXWORKS64): Redefine in terms of TARGET_64BIT.
+
+2025-07-04  Vineet Gupta  <vineetg@rivosinc.com>
+
+	PR target/118241
+	* config/riscv/riscv.md (prefetch): Add alternative "r".
+
+2025-07-04  Vineet Gupta  <vineetg@rivosinc.com>
+
+	* config/riscv/predicates.md (prefetch_operand): mack 5 bits.
+
+2025-07-04  Raphael Moreira Zinsly  <rzinsly@ventanamicro.com>
+
+	* config/sh/predicates.md
+	(treg_set_expr_not_const01): call sh_recog_treg_set_expr_not_01
+	* config/sh/sh-protos.h
+	(sh_recog_treg_set_expr_not_01): New function
+	* config/sh/sh.cc (sh_recog_treg_set_expr_not_01): Likewise
+
+2025-07-04  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c/118948
+	* fold-const.cc (tree_expr_nonnegative_warnv_p): Use
+	error_operand_p instead of checking for error_mark_node directly.
+
+2025-07-04  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-v.cc (expand_vx_binary_vec_dup_vec): Add
+	new case SS_PLUS.
+	(expand_vx_binary_vec_vec_dup): Ditto.
+	* config/riscv/riscv.cc (riscv_rtx_costs): Ditto.
+	* config/riscv/vector-iterators.md: Add new op ss_plus.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120944
+	* tree-ssa-sccvn.cc (vn_reference_lookup_3): Gate optimizations
+	invalid when volatile is involved.
+
+2025-07-04  Jan Hubicka  <hubicka@ucw.cz>
+
+	* common.opt: Add period.
+	* common.opt.urls: Regenerate.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120927
+	* tree-vect-data-refs.cc (vect_compute_data_ref_alignment):
+	Do not force a DRs base alignment when analyzing an
+	epilog loop.  Check whether the step preserves alignment
+	for all VFs possibly involved sofar.
+
+2025-07-04  Xi Ruoyao  <xry111@xry111.site>
+
+	* config/loongarch/loongarch.md (crc_combine): Avoid nested
+	subreg.
+
+2025-07-04  Shreya Munnangi  <smunnangi1@ventanamicro.com>
+
+	* config/riscv/riscv.cc (riscv_macro_fusion_pair_p): Add basic
+	instrumentation to all cases where fusion is detected.  Fix
+	minor formatting goofs found in the process.
+
+2025-07-04  panciyan  <panciyan@eswincomputing.com>
+
+	* match.pd: Add signed scalar SAT_ADD IMM form2 matching.
+
+2025-07-03  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.cc (expand_perm_with_merge): Add size change cases.
+	(expand_perm_with_pack): New function.
+	(vectorize_vec_perm_const_1): Wire up new function.
+
+2025-07-03  Co-authored-by: Daniel Barboza  <dbarboza@ventanamicro.com>
+	    Co-authored-by: Shreya Munnangi  <smunnangi1@ventanamicro.com>
+
+	PR target/118886
+	* config/riscv/riscv.cc (riscv_macro_fusion_pair_p): Check
+	for fusion being disabled earlier.  If PREV is already fused,
+	then it can't be fused again.  Be more selective about fusing
+	when the destination registers do not match.  Don't fuse into
+	loads that aren't scalar integer modes.  Revamp store pair
+	commit support.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (aarch64_cbz<optab><mode>1): Move
+	above rules for CBB<cond>/CBH<cond>/CB<cond>.
+	(*aarch64_tbz<optab><mode>1): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64-protos.h (aarch64_cb_rhs): New function.
+	* config/aarch64/aarch64.cc (aarch64_cb_rhs): Likewise.
+	* config/aarch64/aarch64.md (cbranch<mode>4): Rename to ...
+	(cbranch<GPI:mode>4): ...here, and emit CMPBR if possible.
+	(cbranch<SHORT:mode>4): New expand rule.
+	(aarch64_cb<INT_CMP:code><GPI:mode>): New insn rule.
+	(aarch64_cb<INT_CMP:code><SHORT:mode>): Likewise.
+	* config/aarch64/constraints.md (Uc0): New constraint.
+	(Uc1): Likewise.
+	(Uc2): Likewise.
+	* config/aarch64/iterators.md (cmpbr_suffix): New mode attr.
+	(INT_CMP): New code iterator.
+	(cmpbr_imm_constraint): New code attr.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64-option-extensions.def (cmpbr): New
+	option.
+	* config/aarch64/aarch64.h (TARGET_CMPBR): New macro.
+	* doc/invoke.texi (cmpbr): New option.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (far_branch): Replace 0/1 with
+	no/yes.
+	(aarch64_bcond): Handle rename.
+	(aarch64_cbz<optab><mode>1): Likewise.
+	(*aarch64_tbz<optab><mode>1): Likewise.
+	(@aarch64_tbz<optab><ALLI:mode><GPI:mode>): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (BRANCH_LEN_P_1MiB): New constant.
+	(BRANCH_LEN_N_1MiB): Likewise.
+	(BRANCH_LEN_P_32KiB): Likewise.
+	(BRANCH_LEN_N_32KiB): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (condjump): Rename to ...
+	(aarch64_bcond): ...here.
+	(*compare_condjump<GPI:mode>): Rename to ...
+	(*aarch64_bcond_wide_imm<GPI:mode>): ...here.
+	(aarch64_cb<optab><mode>): Rename to ...
+	(aarch64_cbz<optab><mode>1): ...here.
+	(*cb<optab><mode>1): Rename to ...
+	(*aarch64_tbz<optab><mode>1): ...here.
+	(@aarch64_tb<optab><ALLI:mode><GPI:mode>): Rename to ...
+	(@aarch64_tbz<optab><ALLI:mode><GPI:mode>): ...here.
+	(restore_stack_nonlocal): Handle rename.
+	(stack_protect_combined_test): Likewise.
+	* config/aarch64/aarch64-simd.md (cbranch<mode>4): Likewise.
+	* config/aarch64/aarch64-sme.md (aarch64_restore_za): Likewise.
+	* config/aarch64/aarch64.cc (aarch64_gen_test_and_branch): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (cbranch<mode>4): Reformat.
+	(cbranchcc4): Likewise.
+	(condjump): Likewise.
+	(*compare_condjump<GPI:mode>): Likewise.
+	(aarch64_cb<optab><mode>1): Likewise.
+	(*cb<optab><mode>1): Likewise.
+	(tbranch_<code><mode>3): Likewise.
+	(@aarch64_tb<optab><ALLI:mode><GPI:mode>): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (condjump): Move.
+	(*compare_condjump<GPI:mode>): Likewise.
+	(aarch64_cb<optab><mode>1): Likewise.
+	(*cb<optab><mode>1): Likewise.
+	(tbranch_<code><mode>3): Likewise.
+	(@aarch64_tb<optab><ALLI:mode><GPI:mode>): Likewise.
+
+2025-07-03  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	PR tree-optimization/120780
+	* tree-object-size.cc (inner_at_offset,
+	get_wholesize_for_memref): New functions.
+	(addr_object_size): Call get_wholesize_for_memref.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120936
+	* config/i386/i386.cc (x86_print_call_or_nop): Add a label
+	argument and use it to print label.
+	(x86_function_profiler): Emit label only when __mcount_loc
+	section is used.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (get_combined_location): Handle negative
+	offsets; output better diagnostics.
+	(get_relative_location_for_locus): Reutrn -1 for unknown location.
+	(function_instance::get_cgraph_node): New member function.
+	(match_with_target): New function.
+	(dump_stmt): New function.
+	(function_instance::lookup_count): New function.
+	(mark_expr_locations): New function.
+	(function_instance::match): New function.
+	(autofdo_source_profile::offline_external_functions): Do
+	not repeat renaming; manage two worklists and do matching.
+	(autofdo_source_profile::offline_unrealized_inlines): Simplify.
+	(afdo_set_bb_count): do not look for lost discriminators.
+	(auto_profile): Do not ICE when profile reading failed.
+	* common.opt (Wauto-profile): New warning flag
+	* doc/invoke.texi (-Wauto-profile): Document.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-fnsummary.cc (analyze_function_body): For loop
+	heuristics use header count instead of preheader count.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-cp.cc (update_profiling_info): Watch for division by zero.
+
+2025-07-03  Alex Coplan  <alex.coplan@arm.com>
+
+	* config/aarch64/aarch64-sve.md
+	(vec_load_lanes<mode><vsingle>): Expand else operand in
+	subvector mode, as per optab documentation.
+	(vec_mask_load_lanes<mode><vsingle>): Add missing mode for
+	operand 3.
+	* config/aarch64/predicates.md (aarch64_maskload_else_operand):
+	Remove const_int.
+
+2025-07-03  Alex Coplan  <alex.coplan@arm.com>
+
+	* doc/md.texi (Standard Names): Clarify mode of else operand for
+	vec_mask_load_lanesmn optab.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-cp.cc (cs_interesting_for_ipcp_p): Handle
+	correctly GLOBAL0 afdo counts.
+	(ipcp_cloning_candidate_p): Do not rule out nodes
+	!node->optimize_for_size_p ().
+	(good_cloning_opportunity_p): Handle afdo counts
+	as non-zero.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-cp.cc (hint_time_bonus): Return sreal and avoid
+	conversions to integer.
+	(good_cloning_opportunity_p): Avoid sreal to integer
+	conversions
+	(perform_estimation_of_a_value): Update.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (afdo_hot_bb_threshod): New global
+	variable.
+	(maybe_hot_afdo_count_p): New function.
+	(autofdo_source_profile::read): Do not set up dump file;
+	set afdo_hot_bb_threshod.
+	(afdo_annotate_cfg): Handle partial training.
+	(afdo_callsite_hot_enough_for_early_inline):
+	Use maybe_hot_afdo_count_p.
+	(auto_profile_offline::execute): Read autofdo file.
+	* auto-profile.h (maybe_hot_afdo_count_p): Declare.
+	(afdo_hot_bb_threshold): Declare.
+	* coverage.cc (read_counts_file): Also set gcov_profile_info.
+	(coverage_init): Do not read autofdo file.
+	* opts.cc (enable_fdo_optimizations): Add autofdo parameter;
+	do not set flag_branch_probabilities and flag_profile_values
+	with it.
+	(common_handle_option): Update.
+	* passes.cc (finish_optimization_passes): Do not end branch
+	prob here.
+	(pass_manager::dump_profile_report): Also mark change after
+	autofdo pass.
+	* profile.cc: Include auto-profile.h
+	(gcov_profile_info): New global variable.
+	(struct afdo_fdo_record): New struture.
+	(compute_branch_probabilities): Record afdo profile.
+	(end_branch_prob): Dump afdo/fdo profile comparsion.
+	* profile.h (gcov_profile_info): Declarre.
+	* tree-profile.cc (tree_profiling): Call end_branch_prob
+	(pass_ipa_tree_profile::gate): Also enable with autoFDO
+
+2025-07-03  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/118669
+	* tree-vect-stmts.cc (vectorizable_load): Emit loads
+	with proper (element) alignment.
+	(vectorizable_store): Likewise.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120908
+	* config/i386/i386.cc (legitimize_tls_address): Pass RDI to
+	gen_tls_local_dynamic_64.
+	* config/i386/i386.md (*tls_global_dynamic_64_largepic): Add
+	RDI clobber and use it to generate LEA.
+	(*tls_local_dynamic_64_<mode>): Likewise.
+	(*tls_local_dynamic_base_64_largepic): Likewise.
+	(@tls_local_dynamic_64_<mode>): Add a clobber.
+
 2025-07-02  H.J. Lu  <hjl.tools@gmail.com>
 
 	PR target/120908
diff --git a/gcc/DATESTAMP b/gcc/DATESTAMP
index 6952979..3724f15 100644
--- a/gcc/DATESTAMP
+++ b/gcc/DATESTAMP
@@ -1 +1 @@
-20250703
+20250805
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index d5ceccc..d7d5cbe 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1069,7 +1069,7 @@ TREE_SSA_H = tree-ssa.h tree-ssa-operands.h \
 PRETTY_PRINT_H = pretty-print.h $(INPUT_H) $(OBSTACK_H) wide-int-print.h
 TREE_PRETTY_PRINT_H = tree-pretty-print.h $(PRETTY_PRINT_H)
 GIMPLE_PRETTY_PRINT_H = gimple-pretty-print.h $(TREE_PRETTY_PRINT_H)
-DIAGNOSTIC_CORE_H = diagnostic-core.h $(INPUT_H) bversion.h diagnostic.def
+DIAGNOSTIC_CORE_H = diagnostic-core.h $(INPUT_H) bversion.h diagnostics/kinds.def
 DIAGNOSTIC_H = diagnostic.h $(DIAGNOSTIC_CORE_H) $(PRETTY_PRINT_H)
 C_PRETTY_PRINT_H = c-family/c-pretty-print.h $(PRETTY_PRINT_H) \
 	$(C_COMMON_H) $(TREE_H)
@@ -1595,7 +1595,6 @@ OBJS = \
 	jump.o \
 	langhooks.o \
 	late-combine.o \
-	lazy-diagnostic-path.o \
 	lcm.o \
 	lists.o \
 	loop-doloop.o \
@@ -1796,6 +1795,7 @@ OBJS = \
 	tree-ssa-sink.o \
 	tree-ssa-strlen.o \
 	tree-ssa-structalias.o \
+	pta-andersen.o \
 	tree-ssa-tail-merge.o \
 	tree-ssa-ter.o \
 	tree-ssa-threadbackward.o \
@@ -1851,28 +1851,39 @@ OBJS = \
 
 # Objects in libcommon.a, potentially used by all host binaries and with
 # no target dependencies.
-OBJS-libcommon = diagnostic-spec.o diagnostic.o diagnostic-color.o \
-	diagnostic-format-html.o \
-	diagnostic-format-sarif.o \
-	diagnostic-format-text.o \
+OBJS-libcommon = \
 	diagnostic-global-context.o \
-	diagnostic-macro-unwinding.o \
-	diagnostic-output-spec.o \
-	diagnostic-path.o \
-	diagnostic-path-output.o \
-	diagnostic-show-locus.o \
-	diagnostic-state-to-dot.o \
-	edit-context.o \
+	diagnostics/buffering.o \
+	diagnostics/changes.o \
+	diagnostics/color.o \
+	diagnostics/context.o \
+	diagnostics/digraphs.o \
+	diagnostics/file-cache.o \
+	diagnostics/output-spec.o \
+	diagnostics/html-sink.o \
+	diagnostics/sarif-sink.o \
+	diagnostics/text-sink.o \
+	diagnostics/lazy-paths.o \
+	diagnostics/macro-unwinding.o \
+	diagnostics/option-classifier.o \
+	diagnostics/paths.o \
+	diagnostics/paths-output.o \
+	diagnostics/source-printing.o \
+	diagnostics/state-graphs.o \
+	diagnostics/state-graphs-to-dot.o \
+	diagnostics/selftest-context.o \
+	diagnostics/selftest-logical-locations.o \
+	diagnostics/selftest-paths.o \
+	diagnostics/diagnostics-selftests.o \
+	gcc-diagnostic-spec.o \
 	graphviz.o pex.o \
 	pretty-print.o intl.o \
 	json.o json-parsing.o \
 	xml.o \
 	sbitmap.o \
 	vec.o input.o hash-table.o ggc-none.o memory-block.o \
-	selftest.o selftest-diagnostic.o sort.o \
-	selftest-diagnostic-path.o \
+	selftest.o sort.o \
 	selftest-json.o \
-	selftest-logical-location.o \
 	text-art/box-drawing.o \
 	text-art/canvas.o \
 	text-art/ruler.o \
@@ -1919,7 +1930,7 @@ endif
 
 # libbackend.a objs that might not be in some cases linked into the compiler,
 # yet they are supposed to be part of the plugin ABI.  See PR116143.
-EXTRA_BACKEND_OBJS = simple-diagnostic-path.o lazy-diagnostic-path.o
+EXTRA_BACKEND_OBJS = simple-diagnostic-path.o diagnostics/lazy-paths.o
 
 BACKEND = $(EXTRA_BACKEND_OBJS) libbackend.a main.o libcommon-target.a \
 	libcommon.a $(CPPLIB) $(LIBDECNUMBER)
@@ -3036,7 +3047,7 @@ GTFILES = $(CPPLIB_H) $(srcdir)/input.h $(srcdir)/coretypes.h \
   $(srcdir)/ipa-modref-tree.h \
   $(srcdir)/ipa-locality-cloning.cc \
   $(srcdir)/signop.h \
-  $(srcdir)/diagnostic-spec.h $(srcdir)/diagnostic-spec.cc \
+  $(srcdir)/gcc-diagnostic-spec.h $(srcdir)/gcc-diagnostic-spec.cc \
   $(srcdir)/dwarf2out.h \
   $(srcdir)/dwarf2asm.cc \
   $(srcdir)/dwarf2cfi.cc \
@@ -3709,7 +3720,7 @@ TEXI_GCC_FILES = gcc.texi gcc-common.texi gcc-vers.texi frontends.texi	\
 	 contribute.texi compat.texi funding.texi gnu.texi gpl_v3.texi	\
 	 fdl.texi contrib.texi cppenv.texi cppopts.texi avr-mmcu.texi	\
 	 implement-c.texi implement-cxx.texi gcov-tool.texi gcov-dump.texi \
-	 lto-dump.texi riscv-ext.texi
+	 lto-dump.texi riscv-ext.texi riscv-mcpu.texi riscv-mtune.texi
 
 # we explicitly use $(srcdir)/doc/tm.texi here to avoid confusion with
 # the generated tm.texi; the latter might have a more recent timestamp,
@@ -4069,7 +4080,7 @@ PLUGIN_HEADERS = $(TREE_H) $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
   fold-const.h fold-const-call.h tree-cfg.h tree-into-ssa.h tree-ssanames.h \
   print-tree.h varasm.h context.h tree-phinodes.h stor-layout.h \
   ssa-iterators.h $(RESOURCE_H) tree-cfgcleanup.h attribs.h calls.h \
-  cfgexpand.h diagnostic-color.h gcc-symtab.h gimple-builder.h gimple-low.h \
+  cfgexpand.h gcc-symtab.h gimple-builder.h gimple-low.h \
   gimple-walk.h gimplify-me.h pass_manager.h print-rtl.h stmt.h \
   tree-dfa.h tree-hasher.h tree-nested.h tree-object-size.h tree-outof-ssa.h \
   tree-parloops.h tree-ssa-address.h tree-ssa-coalesce.h tree-ssa-dom.h \
@@ -4079,7 +4090,7 @@ PLUGIN_HEADERS = $(TREE_H) $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
   hash-set.h dominance.h cfg.h cfgrtl.h cfganal.h cfgbuild.h cfgcleanup.h \
   lcm.h cfgloopmanip.h file-prefix-map.h builtins.def $(INSN_ATTR_H) \
   pass-instances.def params.list $(srcdir)/../include/gomp-constants.h \
-  $(EXPR_H) $(srcdir)/analyzer/*.h
+  $(EXPR_H) $(srcdir)/analyzer/*.h $(srcdir)/diagnostics/*.h
 
 # generate the 'build fragment' b-header-vars
 s-header-vars: Makefile
@@ -4117,6 +4128,7 @@ install-plugin: installdirs lang.install-plugin s-header-vars install-gengtype
 	  fi; \
 	  case $$path in \
 	  "$(srcdir)"/analyzer/* \
+	  | "$(srcdir)"/diagnostics/* \
 	  | "$(srcdir)"/config/* | "$(srcdir)"/common/config/* \
 	  | "$(srcdir)"/c-family/* | "$(srcdir)"/*.def ) \
 	    base=`echo "$$path" | sed -e "s|$$srcdirstrip/||"`;; \
diff --git a/gcc/ada/ChangeLog b/gcc/ada/ChangeLog
index 8aaa006..063d6a7 100644
--- a/gcc/ada/ChangeLog
+++ b/gcc/ada/ChangeLog
@@ -1,3 +1,1135 @@
+2025-08-04  Viljar Indus  <indus@adacore.com>
+
+	* contracts.adb: Use Is_Ignored_In_Codegen instead of just
+	using Is_Ignored.
+	* exp_ch6.adb: Likewise.
+	* exp_prag.adb: Likewise.
+	* exp_util.adb: Likewise.
+	* frontend.adb: Avoid removal of ignored nodes in GNATProve_Mode.
+	* gnat1drv.adb: Avoid forcing Assertions_Enabled in GNATProve_Mode.
+	* lib-writ.adb (Write_With_File_Names): Avoid early exit
+	with ignored entities in GNATProve_Mode.
+	* lib-xref.adb: Likewise.
+	* opt.adb: Remove check for Assertions_Enabled.
+	* sem_attr.adb: Use Is_Ignored_In_Codegen instead of Is_Ignored.
+	* sem_ch13.adb: Likewise. Additionally always add predicates in
+	GNATProve_Mode.
+	* sem_prag.adb: Likewise. Additionally remove modifications
+	to applied policies in GNATProve_Mode.
+	* sem_util.adb (Is_Ignored_In_Codegen): New function that overrides
+	Is_Ignored in GNATProve_Mode and Codepeer_Mode.
+	(Is_Ignored_Ghost_Pragma_In_Codegen): Likewise for
+	Is_Ignored_Ghost_Pragma.
+	(Is_Ignored_Ghost_Entity_In_Codegen): Likewise for
+	Is_Ignored_Ghost_Entity.
+	(Policy_In_List): Remove overriding of policies in GNATProve_Mode.
+	* sem_util.ads: Add specs for new functions.
+	* (Predicates_Enabled): Always generate predicates in
+	GNATProve_Mode.
+
+2025-08-04  Bob Duff  <duff@adacore.com>
+
+	* treepr.adb (Print_Node_Ref): Protect against
+	Entity (N) being empty before calling
+	Compile_Time_Known_Value.
+
+2025-08-04  Viljar Indus  <indus@adacore.com>
+
+	* sem_prag.adb (Validate_Compile_Time_Warning_Errors):
+	Check if the original compile time pragma was replaced and
+	validate the original node instead.
+
+2025-08-04  Viljar Indus  <indus@adacore.com>
+
+	* sem_prag.adb (Validate_Compile_Time_Warning_Or_Error):
+	simplify the implementation.
+
+2025-08-04  Steve Baird  <baird@adacore.com>
+
+	* exp_ch6.adb (Apply_Access_Discrims_Accessibility_Check): If the
+	accessibility level being checked is known statically, then
+	statically check it against the level of the function being
+	returned from.
+
+2025-08-04  Viljar Indus  <indus@adacore.com>
+
+	* atree.adb: update references to Ghost_Mode.
+	* exp_ch3.adb: use a structure type to store all of the existing
+	ghost mode related state variables.
+	* exp_disp.adb: Likewise.
+	* exp_spark.adb: Likewise.
+	* exp_util.adb: Likewise.
+	* expander.adb: Likewise.
+	* freeze.adb: Likewise and replace references to existing ghost
+	mode variables.
+	* ghost.adb (Install_Ghost_Region): install the changes of
+	the region in to the new Ghost_Config structure.
+	(Restore_Ghost_Region): Use the new Ghost_Config instead.
+	In general replace all references to the existing ghost mode
+	variables with the new structure equivalent.
+	* ghost.ads (Restore_Ghost_Region): update the spec.
+	* opt.ads (Ghost_Config_Type): A new type that has two of the
+	previous ghost code related global variables as memembers -
+	Ghost_Mode and Ignored_Ghost_Region.
+	(Ghost_Config) New variable to store the previous Ghost_Mode and
+	Ignored_Ghost_Region info.
+	* rtsfind.adb: Replace references to existing ghost mode variables.
+	* sem.adb: Likewise.
+	* sem_ch12.adb: Likewise.
+	* sem_ch13.adb: Likewise.
+	* sem_ch3.adb: Likewise.
+	* sem_ch5.adb: Likewise.
+	* sem_ch6.adb: Likewise.
+	* sem_ch7.adb: Likewise.
+	* sem_prag.adb: Likewise.
+	* sem_util.adb: Likewise.
+
+2025-08-04  Steve Baird  <baird@adacore.com>
+
+	* freeze.adb (Freeze_Profile): Do not emit a warning stating that
+	a formal parameter's size is 8 if the parameter's size is not 8.
+
+2025-08-04  Viljar Indus  <indus@adacore.com>
+
+	* table.adb (Max): Move variable to the body and initialize
+	it with the same value as in the Init function.
+	* table.ads (Max): Likewise.
+
+2025-08-04  Bob Duff  <duff@adacore.com>
+
+	* par.adb: Move and rewrite some comments.
+	(Util): Shared code and comments for dealing with
+	defining_identifier_lists.
+	* par-util.adb (Append): Shared code for appending
+	one identifier onto Defining_Identifiers.
+	(P_Def_Ids): Shared code for parsing a defining_identifier_list.
+	Unfortunately, this is not used in all cases, because some of
+	them mix in sophisticated error recovery, which we do not
+	modify here.
+	* par-ch12.adb (P_Formal_Object_Declarations):
+	Use Defining_Identifiers and related code.
+	* par-ch3.adb (P_Identifier_Declarations): Likewise.
+	(P_Known_Discriminant_Part_Opt): Likewise.
+	(P_Component_Items): Likewise.
+	* par-ch6.adb (P_Formal_Part): Likewise.
+
+2025-07-31  Eric Botcazou  <ebotcazou@gcc.gnu.org>
+
+	Revert:
+	2025-07-31  Nicolas Boulenguez  <nicolas@debian.org>
+
+	PR ada/114065
+	* Makefile.rtl (GNATRTL_NONTASKING_OBJS): Add g-c_time$(objext) and
+	s-c_time$(objext).
+	(Aarch64/Android): Do not use s-osinte__android.adb.
+	(SPARC/Solaris): Do not use s-osprim__solaris.adb.
+	(x86/Solaris): Likewise.
+	(LynxOS178): Do not use s-parame__posix2008.ads.
+	(RTEMS): Likewise.
+	(x32/Linux): Likewise, as well as s-linux__x32.ads.  Replace
+	s-osprim__x32.adb with s-osprim__posix.adb.
+	(LIBGNAT_OBJS): Remove cal.o.
+	* cal.c: Delete.
+	* doc/gnat_rm/the_gnat_library.rst (GNAT.C_Time): New entry.
+	(GNAT.Calendar): Do not mention the obsolete conversion functions.
+	* impunit.adb (Non_Imp_File_Names_95): Add g-c_time.
+	* libgnarl/a-exetim__posix.adb: Add with clause for System.C_Time
+	(Clock): Use type and functions from System.C_Time.
+	* libgnarl/s-linux.ads: Remove with clause for System.Parameters.
+	Remove declarations of C time types.
+	* libgnarl/s-linux__alpha.ads: Likewise.
+	* libgnarl/s-linux__android-aarch64.ads: Likewise.
+	* libgnarl/s-linux__android-arm.ads: Likewise.
+	* libgnarl/s-linux__hppa.ads: Likewise.
+	* libgnarl/s-linux__loongarch.ads: Likewise.
+	* libgnarl/s-linux__mips.ads: Likewise.
+	* libgnarl/s-linux__riscv.ads: Likewise.
+	* libgnarl/s-linux__sparc.ads: Likewise.
+	* libgnarl/s-osinte__aix.ads: Likewise.
+	* libgnarl/s-osinte__android.ads: Likewise.
+	* libgnarl/s-osinte__cheribsd.ads: Likewise.
+	* libgnarl/s-osinte__darwin.ads: Likewise.
+	* libgnarl/s-osinte__dragonfly.ads: Likewise.
+	* libgnarl/s-osinte__freebsd.ads: Likewise.
+	* libgnarl/s-osinte__gnu.ads: Likewise.
+	* libgnarl/s-osinte__hpux.ads: Likewise.
+	* libgnarl/s-osinte__kfreebsd-gnu.ads: Likewise.
+	* libgnarl/s-osinte__linux.ads: Likewise.
+	* libgnarl/s-osinte__lynxos178e.ads: Likewise.
+	* libgnarl/s-osinte__qnx.ads: Likewise.
+	* libgnarl/s-osinte__rtems.ads: Likewise.
+	* libgnarl/s-osinte__solaris.ads: Likewise.
+	* libgnarl/s-osinte__vxworks.ads: Likewise.
+	* libgnarl/s-qnx.ads: Likewise.
+	* libgnarl/s-linux__x32.ads: Delete.
+	* libgnarl/s-osinte__darwin.adb (To_Duration): Remove.
+	(To_Timespec): Likewise.
+	* libgnarl/s-osinte__aix.adb: Likewise.
+	* libgnarl/s-osinte__dragonfly.adb: Likewise.
+	* libgnarl/s-osinte__freebsd.adb: Likewise.
+	* libgnarl/s-osinte__gnu.adb: Likewise.
+	* libgnarl/s-osinte__lynxos178.adb: Likewise.
+	* libgnarl/s-osinte__posix.adb: Likewise.
+	* libgnarl/s-osinte__qnx.adb: Likewise.
+	* libgnarl/s-osinte__rtems.adb: Likewise.
+	* libgnarl/s-osinte__solaris.adb: Likewise.
+	* libgnarl/s-osinte__vxworks.adb: Likewise.
+	* libgnarl/s-osinte__x32.adb: Likewise.
+	* libgnarl/s-taprop__solaris.adb: Add with clause for System.C_Time.
+	(Monotonic_Clock): Use type and functions from System.C_Time.
+	(RT_Resolution): Likewise.
+	(Timed_Sleep): Likewise.
+	(Timed_Delay): Likewise.
+	* libgnarl/s-taprop__vxworks.adb: Likewise.
+	* libgnarl/s-tpopmo.adb: Likewise.
+	* libgnarl/s-osinte__android.adb: Delete.
+	* libgnat/g-c_time.ads: New file.
+	* libgnat/g-calend.adb: Delegate to System.C_Time.
+	* libgnat/g-calend.ads: Likewise.
+	* libgnat/g-socket.adb: Likewise.
+	* libgnat/g-socthi.adb: Likewise.
+	* libgnat/g-socthi__vxworks.adb: Likewise.
+	* libgnat/g-sothco.ads: Likewise.
+	* libgnat/g-spogwa.adb: Likewise.
+	* libgnat/s-c_time.adb: New file.
+	* libgnat/s-c_time.ads: Likewise.
+	* libgnat/s-optide.adb: Import nanosleep here.
+	* libgnat/s-os_lib.ads (time_t): Remove.
+	(To_Ada): Adjust.
+	(To_C): Likewise.
+	* libgnat/s-os_lib.adb: Likewise.
+	* libgnat/s-osprim__darwin.adb: Delegate to System.C_Time.
+	* libgnat/s-osprim__posix.adb: Likewise.
+	* libgnat/s-osprim__posix2008.adb: Likewise.
+	* libgnat/s-osprim__rtems.adb: Likewise.
+	* libgnat/s-osprim__unix.adb: Likewise.
+	* libgnat/s-osprim__solaris.adb: Delete.
+	* libgnat/s-osprim__x32.adb: Likewise.
+	* libgnat/s-parame.ads (time_t_bits): Remove.
+	* libgnat/s-parame__hpux.ads: Likewise.
+	* libgnat/s-parame__vxworks.ads: Likewise.
+	* libgnat/s-parame__posix2008.ads: Delete.
+	* s-oscons-tmplt.c (SIZEOF_tv_nsec): New constant.
+
+2025-07-31  Eric Botcazou  <ebotcazou@gcc.gnu.org>
+
+	PR ada/120440
+	* gcc-interface/Makefile.in (GNATLINK_OBJS): Add s-excmac.o.
+	(GNATMAKE_OBJS): Likewise.
+
+2025-07-31  Nicolas Boulenguez  <nicolas@debian.org>
+
+	PR ada/114065
+	* Makefile.rtl (GNATRTL_NONTASKING_OBJS): Add g-c_time$(objext) and
+	s-c_time$(objext).
+	(Aarch64/Android): Do not use s-osinte__android.adb.
+	(SPARC/Solaris): Do not use s-osprim__solaris.adb.
+	(x86/Solaris): Likewise.
+	(LynxOS178): Do not use s-parame__posix2008.ads.
+	(RTEMS): Likewise.
+	(x32/Linux): Likewise, as well as s-linux__x32.ads.  Replace
+	s-osprim__x32.adb with s-osprim__posix.adb.
+	(LIBGNAT_OBJS): Remove cal.o.
+	* cal.c: Delete.
+	* doc/gnat_rm/the_gnat_library.rst (GNAT.C_Time): New entry.
+	(GNAT.Calendar): Do not mention the obsolete conversion functions.
+	* impunit.adb (Non_Imp_File_Names_95): Add g-c_time.
+	* libgnarl/a-exetim__posix.adb: Add with clause for System.C_Time
+	(Clock): Use type and functions from System.C_Time.
+	* libgnarl/s-linux.ads: Remove with clause for System.Parameters.
+	Remove declarations of C time types.
+	* libgnarl/s-linux__alpha.ads: Likewise.
+	* libgnarl/s-linux__android-aarch64.ads: Likewise.
+	* libgnarl/s-linux__android-arm.ads: Likewise.
+	* libgnarl/s-linux__hppa.ads: Likewise.
+	* libgnarl/s-linux__loongarch.ads: Likewise.
+	* libgnarl/s-linux__mips.ads: Likewise.
+	* libgnarl/s-linux__riscv.ads: Likewise.
+	* libgnarl/s-linux__sparc.ads: Likewise.
+	* libgnarl/s-osinte__aix.ads: Likewise.
+	* libgnarl/s-osinte__android.ads: Likewise.
+	* libgnarl/s-osinte__cheribsd.ads: Likewise.
+	* libgnarl/s-osinte__darwin.ads: Likewise.
+	* libgnarl/s-osinte__dragonfly.ads: Likewise.
+	* libgnarl/s-osinte__freebsd.ads: Likewise.
+	* libgnarl/s-osinte__gnu.ads: Likewise.
+	* libgnarl/s-osinte__hpux.ads: Likewise.
+	* libgnarl/s-osinte__kfreebsd-gnu.ads: Likewise.
+	* libgnarl/s-osinte__linux.ads: Likewise.
+	* libgnarl/s-osinte__lynxos178e.ads: Likewise.
+	* libgnarl/s-osinte__qnx.ads: Likewise.
+	* libgnarl/s-osinte__rtems.ads: Likewise.
+	* libgnarl/s-osinte__solaris.ads: Likewise.
+	* libgnarl/s-osinte__vxworks.ads: Likewise.
+	* libgnarl/s-qnx.ads: Likewise.
+	* libgnarl/s-linux__x32.ads: Delete.
+	* libgnarl/s-osinte__darwin.adb (To_Duration): Remove.
+	(To_Timespec): Likewise.
+	* libgnarl/s-osinte__aix.adb: Likewise.
+	* libgnarl/s-osinte__dragonfly.adb: Likewise.
+	* libgnarl/s-osinte__freebsd.adb: Likewise.
+	* libgnarl/s-osinte__gnu.adb: Likewise.
+	* libgnarl/s-osinte__lynxos178.adb: Likewise.
+	* libgnarl/s-osinte__posix.adb: Likewise.
+	* libgnarl/s-osinte__qnx.adb: Likewise.
+	* libgnarl/s-osinte__rtems.adb: Likewise.
+	* libgnarl/s-osinte__solaris.adb: Likewise.
+	* libgnarl/s-osinte__vxworks.adb: Likewise.
+	* libgnarl/s-osinte__x32.adb: Likewise.
+	* libgnarl/s-taprop__solaris.adb: Add with clause for System.C_Time.
+	(Monotonic_Clock): Use type and functions from System.C_Time.
+	(RT_Resolution): Likewise.
+	(Timed_Sleep): Likewise.
+	(Timed_Delay): Likewise.
+	* libgnarl/s-taprop__vxworks.adb: Likewise.
+	* libgnarl/s-tpopmo.adb: Likewise.
+	* libgnarl/s-osinte__android.adb: Delete.
+	* libgnat/g-c_time.ads: New file.
+	* libgnat/g-calend.adb: Delegate to System.C_Time.
+	* libgnat/g-calend.ads: Likewise.
+	* libgnat/g-socket.adb: Likewise.
+	* libgnat/g-socthi.adb: Likewise.
+	* libgnat/g-socthi__vxworks.adb: Likewise.
+	* libgnat/g-sothco.ads: Likewise.
+	* libgnat/g-spogwa.adb: Likewise.
+	* libgnat/s-c_time.adb: New file.
+	* libgnat/s-c_time.ads: Likewise.
+	* libgnat/s-optide.adb: Import nanosleep here.
+	* libgnat/s-os_lib.ads (time_t): Remove.
+	(To_Ada): Adjust.
+	(To_C): Likewise.
+	* libgnat/s-os_lib.adb: Likewise.
+	* libgnat/s-osprim__darwin.adb: Delegate to System.C_Time.
+	* libgnat/s-osprim__posix.adb: Likewise.
+	* libgnat/s-osprim__posix2008.adb: Likewise.
+	* libgnat/s-osprim__rtems.adb: Likewise.
+	* libgnat/s-osprim__unix.adb: Likewise.
+	* libgnat/s-osprim__solaris.adb: Delete.
+	* libgnat/s-osprim__x32.adb: Likewise.
+	* libgnat/s-parame.ads (time_t_bits): Remove.
+	* libgnat/s-parame__hpux.ads: Likewise.
+	* libgnat/s-parame__vxworks.ads: Likewise.
+	* libgnat/s-parame__posix2008.ads: Delete.
+	* s-oscons-tmplt.c (SIZEOF_tv_nsec): New constant.
+
+2025-07-28  Marc Poulhiès  <poulhies@adacore.com>
+
+	* gcc-interface/trans.cc (gnat_to_gnu): Fix typo in comment.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc-interface/misc.cc: Make
+	diagnostics::context::m_source_printing private.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc-interface/trans.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc-interface/misc.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+
+2025-07-25  Marc Poulhiès  <poulhies@adacore.com>
+
+	* gcc-interface/utils.cc (update_pointer_to): Renamed ptr/old_ptr, ref/old_ref.
+
+2025-07-25  Alexandre Oliva  <oliva@adacore.com>
+
+	* s-oscons-tmplt.c (CLOCK_RT_Ada) [__vxworks]: Define to
+	CLOCK_REALTIME on VxWorks6.
+	* gsocket.h [__vxworks]: Include strings.h if available.
+	* sysdep.c [__vxworks]: Likewise.
+
+2025-07-25  Steve Baird  <baird@adacore.com>
+
+	* exp_ch6.adb (Apply_Access_Discrims_Accessibility_Check): Do
+	nothing and simply return if either Ada_Version <= Ada_95 or if
+	the function being returned from lacks the extra formal parameter
+	needed to perform the check (typically because the result is
+	tagged).
+
+2025-07-25  Bob Duff  <duff@adacore.com>
+
+	* sem_ch12.adb (Check_Formal_Package_Instance):
+	Do nothing in case of E_Subprogram_Body.
+
+2025-07-25  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch3.adb (Process_Discriminants): Update comments
+
+2025-07-25  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch3.adb (Build_Derived_Record_Type): Set flag appropriately.
+
+2025-07-25  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* libgnat/a-calend.adb (To_Struct_Timespec_64): Deal with negative
+	Duration values and truncate the nanoseconds too.
+	* libgnat/g-calend.adb (timeval_to_duration): Unsuppress overflow
+	checks.
+	(duration_to_timeval): Likewise.  Deal with negative Duration values
+	as well as integral Duration values.
+	* libgnat/g-socket.adb (To_Timeval): Simplify the implementation.
+
+2025-07-24  Steve Baird  <baird@adacore.com>
+
+	* exp_aggr.adb (Convert_To_Assignments): Add calls to Ensure_Defined
+	before generating assignments to components that could be
+	associated with a not-yet-defined itype.
+
+2025-07-24  Steve Baird  <baird@adacore.com>
+
+	* accessibility.adb (Function_Call_Or_Allocator_Level): Handle the
+	case where a function that has an Extra_Accessibility_Of_Result
+	parameter returns as its result a call to another such function.
+	In that case, the extra parameter should be passed along.
+	(Check_Return_Construct_Accessibility): Replace a warning about an
+	inevitable failure of a dynamic check with a legality-rule-violation
+	error message; adjust the text of the message accordingly.
+	* exp_ch6.ads (Apply_Access_Discrims_Accessibility_Check): New
+	procedure, following example of the existing
+	Apply_CW_Accessibility procedure.
+	* exp_ch6.adb (Apply_Access_Discrims_Accessibility_Check): body
+	for new procedure.
+	(Expand_Simple_Function_Return): Add call to new
+	Apply_Access_Discrims_Accessibility_Check procedure.
+	* exp_ch3.adb (Make_Allocator_For_Return): Add call to new
+	Apply_Access_Discrims_Accessibility_Check procedure.
+
+2025-07-24  Tonu Naks  <naks@adacore.com>
+
+	* doc/gnat_rm/standard_and_implementation_defined_restrictions.rst:
+	clarify parameter description.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* sem_ch8.adb (End_Use_Type): Add a test for there not being an earlier
+	use_type_clause for the same type as an additional criterion for turning
+	off In_Use and Current_Use_Clause.
+
+2025-07-22  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_attr.adb (Eval_Attribute): Only fold array attributes when prefix
+	is static or at least safe to evaluate
+
+2025-07-22  Ronan Desplanques  <desplanques@adacore.com>
+
+	* einfo.ads (Is_Controlled_Active): Fix pasto in comment.
+	* sem_util.ads (Propagate_Controlled_Flags): Update comment for
+	Destructor aspect.
+
+2025-07-22  Ronan Desplanques  <desplanques@adacore.com>
+
+	* doc/gnat_rm/gnat_language_extensions.rst: Document new extension.
+	* snames.ads-tmpl: Add name for new aspect.
+	* gen_il-fields.ads (Has_Destructor, Is_Destructor): Add new fields.
+	* gen_il-gen-gen_entities.adb (E_Procedure, Type_Kind): Add new fields.
+	* einfo.ads (Has_Destructor, Is_Destructor): Document new fields.
+	* aspects.ads: Add new aspect.
+	* sem_ch13.adb (Analyze_Aspect_Specifications,
+	Check_Aspect_At_Freeze_Point, Check_Aspect_At_End_Of_Declarations):
+	Add semantic analysis for new aspect.
+	(Resolve_Finalization_Procedure): New function.
+	(Resolve_Finalizable_Argument): Use new function above.
+	* sem_util.adb (Propagate_Controlled_Flags): Extend for new field.
+	* freeze.adb (Freeze_Entity): Add legality check for new aspect.
+	* exp_ch3.adb (Expand_Freeze_Record_Type, Predefined_Primitive_Bodies):
+	Use new field.
+	* exp_ch7.adb (Build_Finalize_Statements): Add expansion for
+	destructors.
+	(Make_Final_Call, Build_Record_Deep_Procs): Adapt to new Has_Destructor
+	field.
+	(Build_Adjust_Statements): Tweak to handle cases of empty lists.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-22  Denis Mazzucato  <mazzucato@adacore.com>
+
+	* sem_ch6.adb (Might_Need_BIP_Task_Actuals): Before retrieving the original corresponding
+	operation we retrieve first the root of the aliased chain.
+
+2025-07-22  Ronan Desplanques  <desplanques@adacore.com>
+
+	* gen_il-fields.ads (Is_Implicit_Full_View): New field.
+	* gen_il-gen-gen_entities.adb (Type_Kind): Use new field.
+	* einfo.ads (Is_Implicit_Full_View): Document new field.
+	* exp_ch7.adb (Make_Adjust_Call, Make_Init_Call, Make_Final_Call): Use
+	new field.
+	* exp_util.adb (Finalize_Address): Likewise.
+	* sem_ch3.adb (Copy_And_Build): Set new field.
+
+2025-07-22  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_util.ads (May_Generate_Large_Temp): Delete.
+	* exp_util.adb (May_Generate_Large_Temp): Likewise.
+	(Safe_Unchecked_Type_Conversion): Do not take stack checking into
+	account to compute the result.
+
+2025-07-22  Javier Miranda  <miranda@adacore.com>
+
+	* sinfo.ads (Is_Expanded_Dispatching_Call): New flag.
+	(Tag_Propagated): New flag.
+	* exp_ch6.adb (Expand_Call_Helper): Propagate the tag when
+	the dispatching call is placed in conditionl expressions or
+	case-expressions.
+	* sem_ch5.adb (Analyze_Assignment): For assignment of tag-
+	indeterminate expression, do not propagate the tag if
+	previously done.
+	* sem_disp.adb (Is_Tag_Indeterminate): Add missing support
+	for conditional expression and case expression.
+	* exp_disp.ads (Is_Expanded_Dispatching_Call): Removed. Function
+	replaced by a new flag in the nodes.
+	* exp_disp.adb (Expand_Dispatching_Call): Set a flag in the
+	call node to remember that the call has been expanded.
+	(Is_Expanded_Dispatching_Call): Function removed.
+	* gen_il-fields.ads (Tag_Propagated): New flag.
+	(Is_Expanded_Dispatching_Call): New flag.
+	* gen_il-gen-gen_nodes.adb (Tag_Propagated): New flag.
+	(Is_Expanded_Dispatching_Call): New flag.
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* libgnat/a-cbhama.ads (Empty): Add missing default to Capacity formal.
+	* libgnat/a-cbhama.adb (Empty): Add missing default to Capacity formal.
+	* exp_aggr.adb (Build_Size_Expr): Test for presence of Capacity
+	discriminant as additional criterion for generating the call to
+	the Length function. Update comments.
+
+2025-07-22  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_util.adb (Safe_Unchecked_Type_Conversion): Always return True
+	if the expression is the prefix of an N_Selected_Component.
+
+2025-07-22  Denis Mazzucato  <mazzucato@adacore.com>
+
+	* sem_ch6.adb (Might_Need_BIP_Task_Actuals): Check whether No_Task_Parts is enabled in any
+	of the derived types, or interfaces, from the user-defined primitive return type.
+	* sem_ch13.adb (Analyze_Aspect_Specifications): Add No_Task_Parts and No_Controlled_Parts to
+	the representation chain to be visible in the full view of private types.
+	* aspects.ads (Nonoverridable_Aspect_Id): As per GNAT RM, No_Task_Parts is nonoverridable.
+	* sem_util.adb (Check_Inherited_Nonoverridable_Aspects): Likewise.
+	* sem_util.ads: Fix typo and style.
+	* sem_disp.adb: Missing comment.
+
+2025-07-22  Javier Miranda  <miranda@adacore.com>
+
+	* einfo.ads (Extra_Formals): Complete documentation.
+	(Has_First_Controlling_Parameter_Aspect): Place it in alphabetical order.
+	(Has_Frozen_Extra_Formals): New attribute.
+	* gen_il-fields.ads (Has_Frozen_Extra_Formals): New entity field.
+	* gen_il-gen-gen_entities.adb (Has_Frozen_Extra_Formals): Adding new
+	entity flag to subprograms, subprogram types, and and entries.
+	* gen_il-internals.adb (Image): Adding Has_Frozen_Extra_Formals.
+	* exp_ch3.adb (Build_Array_Init_Proc): Freeze its extra formals.
+	(Build_Init_Procedure): Freeze its extra formals.
+	(Expand_Freeze_Record_Type): For tagged types with foreign convention
+	create the extra formals of primitives with convention Ada.
+	* exp_ch6.ads (Create_Extra_Actuals): New subprogram.
+	* exp_ch6.adb (Check_BIP_Actuals): Adding assertions.
+	(Create_Extra_Actuals): New subprogram that factorizes code from
+	Expand_Call_Helper.
+	(Expand_Call_Helper): Adding support to defer the addition of extra
+	actuals. Move the code that adds the extra actuals to a new subprogram.
+	(Is_Unchecked_Union_Equality): Renamed as Is_Unchecked_Union_Predefined_
+	Equality_Call.
+	* exp_ch7.adb (Create_Finalizer): Freeze its extra formals.
+	(Wrap_Transient_Expression): Link the temporary with its relocated
+	expression to facilitate locating the expression in the expanded code.
+	* exp_ch9.ads (Expand_N_Entry_Declaration): Adding one formal.
+	* exp_ch9.adb (Expand_N_Entry_Declaration): Defer the expansion of
+	the entry if the extra formals are not available; analyze the built
+	declarations for the record type that holds all the parameters if
+	the expansion of the entry declaration was deferred.
+	* exp_disp.adb (Expand_Dispatching_Call): Handle deferred extra formals.
+	(Set_CPP_Constructors): Freeze its extra formals.
+	* freeze.adb (Freeze_Entity): Create the extra actuals of acccess to
+	subprograms whose designated type is a subprogram type.
+	(Freeze_Subprogram): Adjust assertion to support deferred extra formals,
+	and freeze extra formals of non-dispatching subprograms with foreign
+	convention. Added assertion to check matching of formals in thunks.
+	* sem_aux.adb (Get_Called_Entity): Adding documentation.
+	* sem_ch3.adb (Analyze_Full_Type_Declaration): Create the extra formals
+	of deferred subprograms, subprogram types and entries; create also the
+	extra actuals of deferred calls.
+	* sem_ch6.ads (Freeze_Extra_Formals): New subprogram.
+	(Deferred_Extra_Formals_Support): New package.
+	* sem_ch6.adb (Analyze_Subprogram_Body_Helper): Create the extra formals
+	of subprograms without separate spec.
+	(Add_Extra_Formal): Add documentation.
+	(Has_Extra_Formals): Removed.
+	(Parent_Subprogram): Adding documentation.
+	(Create_Extra_Formals): Defer adding extra formals if the underlying_type
+	of some formal type or return type is not available.
+	(Extra_Formals_Match_OK): Add missing check on the extra formals of
+	unchecked unions.
+	(Freeze_Extra_Formals): New subprogram.
+	(Deferred_Extra_Formals_Support): New package.
+	* sem_ch9.adb (Analyze_Entry_Declaration): Freeze its extra formals.
+	* sem_ch13.adb (New_Put_Image_Subprogram): ditto.
+	* sem_util.ads (Is_Unchecked_Union_Equality): New subprogram.
+	* sem_util.adb (Is_Unchecked_Union_Equality): ditto.
+
+2025-07-22  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_util.adb (Get_Actual_Subtype): Do the same for GCC and GNATprove
+	backends.
+
+2025-07-22  Martin Clochard  <clochard@adacore.com>
+
+	* exp_spark.adb (Expand_SPARK): Add expansion of continue statements.
+	(Expand_SPARK_N_Continue_Statement): Expand continue statements resolved
+	as procedure calls into said procedure calls.
+
+2025-07-22  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_res.adb (Resolve_Call): Look at the base type of actual parameter
+	when checking call to Set_Handler.
+
+2025-07-22  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sem_util.adb (Get_Actual_Subtype): Only create a new subtype when
+	the expander is active.  Remove a useless test of type inequality,
+	as well as a useless call to Set_Has_Delayed_Freeze on the subtype.
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* exp_aggr.adb (Build_Size_Expr): Change test of "not Present (...)"
+	to "No (...)".
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* exp_aggr.adb (Build_Size_Expr): Determine the length of a container
+	aggregate association in the case where it's an iteration over an
+	object of a container type coming from an instantiation of a predefined
+	container generic. Minor updates to existing comments.
+
+2025-07-22  Ghjuvan Lacambre  <lacambre@adacore.com>
+
+	* exp_util.adb (Finalize_Address): Prevent infinite loop
+
+2025-07-22  Steve Baird  <baird@adacore.com>
+
+	* sem_aux.ads: Declare new function Unique_Component_Name.
+	* sem_aux.adb: Implement new function Unique_Component_Name.
+
+2025-07-22  Viljar Indus  <indus@adacore.com>
+
+	* sem_ch13.adb (Check_Aspect_At_End_Of_Declarations):
+	Ensure the Expression_Copy always has a parent before
+	calling any analyze.
+
+2025-07-22  Steve Baird  <baird@adacore.com>
+
+	* exp_aggr.adb (Build_Record_Aggr_Code.Gen_Assign): In the case of
+	an aggregate component where the component type is mutably tagged
+	and the component value is provided by a qualified aggregate (and
+	qualified with a specific type), avoid incorrectly rejecting the
+	inner aggregate for violating the rule that the type of an
+	aggregate shall not be class-wide.
+	* exp_attr.adb: For a predefined streaming operation (i.e., Read,
+	Write, Input, or Output) of a class-wide type, the external name
+	of the tag of the value is normally written out by Output and read
+	in by Input. In the case of a mutably tagged type, this is instead
+	done in Write and Read.
+	* exp_ch4.adb (Expand_Composite_Equality): In the case of an
+	equality comparison for a type having a mutably tagged component,
+	we want the component comparison to compare two values of the
+	mutably tagged type, not two values of the corresponding
+	array-of-bytes-ish representation type. Even if there are no
+	user-defined equality functions anywhere in sight, comparing the
+	array values still doesn't work because undefined bits may end up
+	participating in the comparison (resulting in an incorrect result
+	of False).
+	* exp_put_image.adb: In the case of a class-wide type, the
+	predefined Image attribute includes the name of the specific type
+	(and a "'" character, to follow qualified expression syntax) to
+	indicate the tag of the value. With the introduction of mutably
+	tagged types, this case can now arise in the case of a component
+	(of either an enclosing array or an enclosing record), not just
+	for a top-level object. So we factor the code to do this into a
+	new procedure, Put_Specific_Type_Name_Qualifier, so that it can be
+	called from more than one place. This reorganization also involves
+	replacing the procedure Put_String_Exp with a new procedure,
+	Put_String_Exp_To_Buffer, declared in a less nested scope. For
+	mutably tagged components (at the source level) the component type
+	(at the GNAT tree level) is an array of bytes (actually a two
+	field record containing an array of bytes, but that's a detail).
+	Appropriate conversions need to be generated so that we don't end
+	up generating an image for an array of bytes; this is done at the
+	same places where Put_Specific_Type_Name_Qualifier is called
+	(for components) by calling Make_Mutably_Tagged_Conversion.
+	* exp_strm.adb (Make_Field_Attribute): Add
+	Make_Mutably_Tagged_Conversion call where we construct a
+	Selected_Component node and the corresponding component type is
+	the internal representation type for a mutably tagged type.
+	(Stream_Base_Type): Return the mutably
+	tagged type if given the corresponding internal representation type.
+	* sem_ch3.adb (Array_Type_Declaration): In the case where the
+	source-level component type of an array type is mutably tagged,
+	set the Component_Type field of the base type of the declared
+	array type (as opposed to that of the first subtype of the array
+	type) to the corresponding internal representation type.
+	* sem_ch4.adb (Analyze_Selected_Component): In the case of a
+	selected component name which references a component whose type is
+	the internal representation type of a mutably tagged type,
+	generate a conversion to the mutably tagged type.
+
+2025-07-21  Eric Botcazou  <ebotcazou@gcc.gnu.org>
+
+	PR ada/121184
+	* styleg.adb (Check_Comment): Use consistent warning message.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc-interface/trans.cc (gnat_to_gnu): Pass null pointer to
+	parse_{input,output}_constraint().
+
+2025-07-18  Steve Baird  <baird@adacore.com>
+
+	* sem_ch12.adb (Validate_Derived_Type_Instance): Cope with the case
+	where the ancestor type for a formal derived type is declared in
+	an earlier formal package but Get_Instance_Of does not return the
+	corresponding type from the corresponding actual package.
+
+2025-07-18  Bob Duff  <duff@adacore.com>
+
+	* tbuild.adb (Unchecked_Convert_To): Back out
+	change.
+
+2025-07-18  Marc Poulhiès  <poulhies@adacore.com>
+	    Eric Botcazou  <botcazou@adacore.com>
+
+	* exp_ch6.adb (Convert): Do not call Expand_Inlined_Call for
+	unsupported cases.
+	* inline.adb (Expand_Inlined_Call): Add assert to catch unsupported
+	case.
+
+2025-07-18  Gary Dismukes  <dismukes@adacore.com>
+
+	* einfo.ads: Document new field Overridden_Inherited_Operation and
+	list it as a field for the entity kinds that it applies to.
+	* gen_il-fields.ads (type Opt_Field_Enum): Add new literal
+	Overridden_Inherited_Operation to the type.
+	* gen_il-gen-gen_entities.adb: Add Overridden_Inherited_Operation as
+	a field of entities of kinds E_Enumeration_Literal and Subprogram_Kind.
+	* sem_ch4.adb (Is_Callable_Private_Overriding): Change name (was
+	Is_Private_Overriding). Replace Is_Hidden test on Overridden_Operation
+	with test of Is_Hidden on the new field Overridden_Inherited_Operation.
+	* sem_ch6.adb (New_Overloaded_Entity): Set the new field
+	Overridden_Inherited_Operation on an operation derived from
+	an interface to refer to the inherited operation of a private
+	extension that's overridden by the derived operation. Also set
+	that field in the more common cases of an explicit subprogram
+	that overrides, to refer to the inherited subprogram that is
+	overridden. (Contrary to its name, the Overridden_Operation
+	field of the overriding subprogram, which is also set in these
+	places, refers to the *parent* subprogram from which the inherited
+	subprogram is derived.) Also, remove a redundant Present (Alias (S))
+	test in an if_statement and the dead "else" part of that statement.
+
+2025-07-18  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_util.adb (Build_Elaboration_Entity): Set ghost mode to none
+	before creating the elaboration entity; restore the ghost mode
+	afterwards.
+
+2025-07-18  Javier Miranda  <miranda@adacore.com>
+
+	* exp_aggr.adb (Gen_Assign): Code cleanup.
+	(Initialize_Component): Do not adjust the tag when the type of
+	the aggregate components is a mutably tagged type.
+
+2025-07-14  Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR ada/121056
+	* sem_ch4.adb (Try_Object_Operation.Try_Primitive_Operation): Add
+	test on Is_Record_Type before accessing Underlying_Record_View.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* par-load.adb: Comment spelling fix: bellow -> below.
+	* libgnarl/s-taskin.ads: Likewise.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/Make-lang.in (ACATSDIR): Change to acats-4.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/utils.cc (make_packable_type): Clear the TYPE_PACKED
+	flag in the case where the alignment is bumped.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/trans.cc (Subprogram_Body_to_gnu): Do not generate
+	a block-copy out for a null initialization procedure when the _Init
+	parameter is not passed in.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_subprog_type): Only apply the
+	transformation to integer types.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_subprog_type): Add guards.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_subprog_type): In the case of a
+	subprogram using the Copy-In/Copy-Out mechanism, deal specially with
+	the case of 2 parameters of differing sizes.
+	* gcc-interface/trans.cc (Subprogram_Body_to_gnu): In the case of a
+	subprogram using the Copy-In/Copy-Out mechanism, make sure the types
+	are consistent on the two sides for all the parameters.
+
+2025-07-04  Steve Baird  <baird@adacore.com>
+
+	* sem_res.adb (Resolve_Type_Conversion): Replace code for
+	detecting a similar case with a more comprehensive test.
+
+2025-07-04  Bob Duff  <duff@adacore.com>
+
+	* doc/gnat_rm/implementation_defined_pragmas.rst
+	(Short_Circuit_And_Or): Add more documentation.
+	* sem_ch8.adb (Analyze_Subprogram_Renaming):
+	Disallow renamings.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-04  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Make_Final_Call): Tweak search of Finalize primitive.
+	* exp_util.adb (Finalize_Address): Likewise.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* freeze.adb (Check_Compile_Time_Size): Try harder to see whether
+	the bounds of array types are known at compile time.
+
+2025-07-04  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_aux.ads (First_Discriminant): Remove space before period.
+
+2025-07-04  Steve Baird  <baird@adacore.com>
+
+	* sem_ch13.adb (Analyze_Record_Representation_Clause): In deciding
+	whether to generate a warning about a missing component clause, in
+	addition to calling Is_Unchecked_Union also call a new local
+	function, Unchecked_Union_Pragma_Pending, which checks for the
+	case of a not-yet-analyzed Unchecked_Union pragma occurring later
+	in the declaration list.
+
+2025-07-04  Steve Baird  <baird@adacore.com>
+
+	* mutably_tagged.adb (Make_CW_Size_Compile_Check): Include the
+	value of the Size'Class limit in the message generated via a
+	Compile_Time_Error pragma.
+
+2025-07-04  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch13.adb (Check_Aspect_At_Freeze_Point): Remove obsolete bits.
+
+2025-07-04  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch13.adb (Analyze_Aspect_Specifications): Fix error emission.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/Makefile.in (gnatlib-sjlj): Delete.
+	(gnatlib-zcx): Do not modify Frontend_Exceptions constant.
+	* libgnat/system-linux-loongarch.ads (Frontend_Exceptions): Delete.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (type_contains_only_integral_data): Do not
+	return false only because the type contains pointer data.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_entity): Use default messages
+	for errors reported for Object_Size clauses.
+	(validate_size): Give an error for stand-alone objects of composite
+	types if the specified size is not a multiple of the alignment.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/trans.cc (addressable_p): Add COMPG third parameter.
+	<COMPONENT_REF>: Do not return true out of alignment considerations
+	for non-strict-alignment targets if COMPG is set.
+	(Call_to_gnu): Pass true as COMPG in the call to the addressable_p
+	predicate if the called subprogram is an initialization procedure.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/trans.cc (gnat_to_gnu) <N_Allocator>: Allocate the
+	bounds alongside the data if the Is_Constr_Array_Subt_With_Bounds
+	flag is set on the designated type.
+	<N_Free_Statement>: Take into account the allocated bounds if the
+	Is_Constr_Array_Subt_With_Bounds flag is set on the designated type.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_component_type): Validate the
+	Component_Size like the size of a type only if the component type
+	is actually packed.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_elab.adb (Check_Overriding_Primitive): Find early call region
+	of the subprogram body declaration, not of the subprogram body stub.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* gen_il-gen-gen_nodes.adb (N_Unchecked_Type_Conversion):
+	Remove useless Nmake_Assert.
+	* tbuild.adb (Unchecked_Convert_To):
+	Narrow the bitfield-related conditions.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_util.adb (Insert_Actions): Fix check.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch6.adb (Expand_Ctrl_Function_Call): Precisify comment.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_ch6.adb (Expand_Ctrl_Function_Call): Do not bail out for the
+	declarations of return objects.
+
+2025-07-03  Daniel King  <dmking@adacore.com>
+
+	* Makefile.rtl (LIBGNAT_TARGET_PAIRS): New unit s-tsgsba__cheri.adb for morello-freebsd.
+	* libgnarl/s-tassta.adb (Get_Stack_Base): New function.
+	* libgnarl/s-tsgsba__cheri.adb: New file for CHERI targets.
+	* libgnarl/s-tsgsba.adb: New default file for non-CHERI targets.
+	* libgnat/s-stausa.adb (Fill_Stack, Compute_Result): Port to CHERI.
+	* libgnat/s-stausa.ads (Initialize_Analyzer, Stack_Analyzer): Port to CHERI.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Use Original_Node.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Use type from
+	explicit subtype indication, when possible.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Adjust error message
+	to match the RM wording.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Use the nominal
+	subtype of a return object; literally implement the RM rule about
+	elementary types; check for static subtype compatibility both when
+	the subtype is given as a subtype mark and a subtype indication.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* repinfo.adb (First_Comp_Or_Discr.Is_Placed_Before): Return True
+	only if the components are in the same component list.
+
+2025-07-03  Denis Mazzucato  <mazzucato@adacore.com>
+
+	* sem_disp.adb (Check_Dispatching_call): Fix uninitialized Subp_Entity.
+	* sem_util.adb (Update_Controlling_Argument): No need to replace controlling argument
+	in case of functions.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* errid.ads: Adjust header to renaming and fix copyright line.
+	* errid.adb: Adjust header to renaming and add blank line.
+	* erroutc-pretty_emitter.ads: Adjust header to renaming.
+	* erroutc-pretty_emitter.adb: Likewise.
+	* erroutc-sarif_emitter.ads: Likewise.
+	* erroutc-sarif_emitter.adb: Likewise.
+	* errsw.ads: Adjust header to renaming and add blank line.
+	* errsw.adb: Likewise.
+	* json_utils.ads: Likewise.
+	* json_utils.adb: Adjust header to renaming.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* errid.ads (Diagnostic_Entries): Now a constant.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* errid.ads (Diagnostic_Entries): Remove nested aggregate.
+	* errsw.adb (Switches): Likewise.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Make_Deep_Record_Body): Fix case of absent Initialize
+	primitive.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch3.adb (Count_Default_Sized_Task_Stacks): Refine subtypes of
+	parameters; same for callsites.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_imgv.adb (Expand_Value_Attribute): Do not call Set_Etype on N
+	after rewriting it by means of OK_Convert_To.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_aggr.adb (Generate_Finalization_Actions): Stop assuming that
+	initialize primitive exists.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Build_Record_Deep_Procs): Fix typo in comment.
+
+2025-07-03  Gary Dismukes  <dismukes@adacore.com>
+
+	* sem_ch12.adb (Install_Spec): Remove "not Is_Generic_Instance (Par)"
+	in test for setting Instance_Parent_Unit. Revise comment to no longer
+	say "noninstance", plus remove "???".
+	(Remove_Parent): Restructure if_statement to allow for both "elsif"
+	parts to be executed (by changing them to be separate if_statements
+	within an "else" part).
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch3.adb (Predefined_Primitive_Bodies): Fix comment.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_tss.adb (TSS): Refactor IF condition to make code smaller.
+	* lib.adb (Increment_Serial_Number, Synchronize_Serial_Number):
+	Use type of renamed object when creating renaming.
+	* lib.ads (Unit_Record): Refine subtype of dependency number.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* libgnat/s-valuef.adb: Document the prerequisites more precisely.
+	* libgnat/a-tifiio.adb (OK_Get_32): Adjust to the prerequisites.
+	(OK_Get_64): Likewise.
+	* libgnat/a-tifiio__128.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	(OK_Get_128): Likewise.
+	* libgnat/a-wtfiio.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	* libgnat/a-wtfiio__128.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	(OK_Get_128): Likewise.
+	* libgnat/a-ztfiio.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	* libgnat/a-ztfiio__128.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	(OK_Get_128): Likewise.
+	* exp_imgv.adb (Expand_Value_Attribute): Adjust the conditions under
+	which the RE_Value_Fixed{32,64,128} routines are called for ordinary
+	fixed-point types.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch3.adb (Make_Predefined_Primitive_Specs): Fix comment.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Insert_Actions_In_Scope_Around): Fix condition.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* checks.adb: Remove unnecessary "return;" statements.
+	* eval_fat.adb: Likewise.
+	* exp_aggr.adb: Likewise.
+	* exp_attr.adb: Likewise.
+	* exp_ch3.adb: Likewise.
+	* exp_ch4.adb: Likewise.
+	* exp_ch5.adb: Likewise.
+	* exp_ch6.adb: Likewise.
+	* exp_unst.adb: Likewise.
+	* krunch.adb: Likewise.
+	* layout.adb: Likewise.
+	* libgnat/s-excdeb.adb: Likewise.
+	* libgnat/s-trasym__dwarf.adb: Likewise.
+	* par-endh.adb: Likewise.
+	* par-tchk.adb: Likewise.
+	* sem.adb: Likewise.
+	* sem_attr.adb: Likewise.
+	* sem_ch6.adb: Likewise.
+	* sem_elim.adb: Likewise.
+	* sem_eval.adb: Likewise.
+	* sfn_scan.adb: Likewise.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* doc/gnat_rm/implementation_defined_characteristics.rst:
+	Change Ignore to Disable.
+	* sem_ch13.ads (Analyze_Aspect_Specifications):
+	Minor: Remove incorrect comment; there is no need to check
+	Has_Aspects (N) at the call site.
+	* gnat_rm.texi: Regenerate.
+	* gnat_ugn.texi: Regenerate.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* types.ads (Empty_Or_Error): Remove.
+	* atree.adb: Remove reference to Empty_Or_Error.
+	* par-endh.adb: Likewise.
+	* sem_ch12.adb: Likewise.
+	* sem_ch3.adb: Likewise.
+	* sem_util.adb: Likewise.
+	* treepr.adb: Likewise.
+
+2025-07-03  Viljar Indus  <indus@adacore.com>
+
+	* sem_ch10.adb(Analyze_With_Clause): Call Semantics instead
+	of Analyze to bring Current_Sem_Unit up to date.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* lib-xref-spark_specific.adb
+	(Enclosing_Subprogram_Or_Library_Package): Traverse subunits and body
+	stubs.
+
+2025-07-03  Tonu Naks  <naks@adacore.com>
+
+	* libgnat/i-cstrin.ads (Value): add documentation
+
+2025-07-03  Aleksandra Pasek  <pasek@adacore.com>
+
+	* libgnat/a-strsup.adb (Super_Delete): Fix index check.
+	* libgnat/a-stwisu.adb (Super_Delete): Likewise.
+	* libgnat/a-stzsup.adb (Super_Delete): Likewise.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_ch4.adb (Handle_Changed_Representation): Alphabetize local
+	variables.  Set the No_Finalize_Actions flag on the assignment.
+
+2025-07-03  Joffrey Huguet  <huguet@adacore.com>
+
+	* aspects.ads: Define an identifier for Potentially_Invalid.
+	* doc/gnat_rm/implementation_defined_aspects.rst: Add section for Potentially_Invalid.
+	* sem_attr.adb (Analyze_Attribute_Old_Result): Attribute Old is allowed to occur in a
+	Potentially_Invalid aspect.
+	* sem_ch13.adb (Analyze_Aspect_Specifications): Handle Potentially_Invalid.
+	* sem_util.adb (Has_Potentially_Invalid): Returns True iff an entity is subject to the
+	Potentially_Invalid aspect.
+	* sem_util.ads (Has_Potentially_Invalid): Idem.
+	* snames.ads-tmpl: New name for Potentially_Invalid.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch10.adb (Analyze_Compilation_Unit): Ignored ghost unit need no
+	elaboration checks.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* libgnat/s-valued.adb (Integer_to_Decimal): Use truncation for the
+	scaled divide operation performed for bases other than 10.
+
 2025-07-01  Eric Botcazou  <ebotcazou@adacore.com>
 
 	PR ada/120705
diff --git a/gcc/ada/accessibility.adb b/gcc/ada/accessibility.adb
index 0b8d3f7..c780054 100644
--- a/gcc/ada/accessibility.adb
+++ b/gcc/ada/accessibility.adb
@@ -327,8 +327,23 @@ package body Accessibility is
                if In_Return_Value (N)
                  or else In_Return_Context
                then
-                  return Make_Level_Literal
-                           (Subprogram_Access_Level (Current_Subprogram));
+                  if Present (Extra_Accessibility_Of_Result
+                                (Current_Subprogram))
+                  then
+                     --  If a function is passed an extra "level of the
+                     --  master of the call" parameter and that function
+                     --  returns a call to another such function (or
+                     --  possibly to the same function, in the case of a
+                     --  recursive call), then that parameter should be
+                     --  "passed along".
+
+                     return New_Occurrence_Of
+                              (Extra_Accessibility_Of_Result
+                                (Current_Subprogram), Loc);
+                  else
+                     return Make_Level_Literal
+                              (Subprogram_Access_Level (Current_Subprogram));
+                  end if;
                end if;
             end if;
 
@@ -1683,16 +1698,14 @@ package body Accessibility is
                 Condition => Check_Cond,
                 Reason    => PE_Accessibility_Check_Failed));
 
-            --  If constant folding has happened on the condition for the
-            --  generated error, then warn about it being unconditional when
-            --  we know an error will be raised.
+            --  ??? Is this how we want to detect RM 6.5(5.9) violations?
 
             if Nkind (Check_Cond) = N_Identifier
               and then Entity (Check_Cond) = Standard_True
             then
                Error_Msg_N
-                 ("access discriminant in return object could be a dangling"
-                  & " reference??", Return_Stmt);
+                 ("level of type of access discriminant value of return object"
+                    & " is statically too deep", Return_Stmt);
             end if;
          end if;
 
diff --git a/gcc/ada/aspects.ads b/gcc/ada/aspects.ads
index d8861bf..737f151 100644
--- a/gcc/ada/aspects.ads
+++ b/gcc/ada/aspects.ads
@@ -92,6 +92,7 @@ package Aspects is
       Aspect_Default_Value,
       Aspect_Depends,                       -- GNAT
       Aspect_Designated_Storage_Model,      -- GNAT
+      Aspect_Destructor,                    -- GNAT
       Aspect_Dimension,                     -- GNAT
       Aspect_Dimension_System,              -- GNAT
       Aspect_Dispatching_Domain,
@@ -259,6 +260,7 @@ package Aspects is
                                  | Aspect_Iterator_Element
                                  | Aspect_Max_Entry_Queue_Length
                                  | Aspect_No_Controlled_Parts
+                                 | Aspect_No_Task_Parts
                                  | Aspect_Real_Literal
                                  | Aspect_String_Literal
                                  | Aspect_Variable_Indexing;
@@ -293,6 +295,7 @@ package Aspects is
       Aspect_CUDA_Global                => True,
       Aspect_Depends                    => True,
       Aspect_Designated_Storage_Model   => True,
+      Aspect_Destructor                 => True,
       Aspect_Dimension                  => True,
       Aspect_Dimension_System           => True,
       Aspect_Disable_Controlled         => True,
@@ -447,6 +450,7 @@ package Aspects is
       Aspect_Default_Value              => Expression,
       Aspect_Depends                    => Expression,
       Aspect_Designated_Storage_Model   => Name,
+      Aspect_Destructor                 => Name,
       Aspect_Dimension                  => Expression,
       Aspect_Dimension_System           => Expression,
       Aspect_Dispatching_Domain         => Expression,
@@ -551,6 +555,7 @@ package Aspects is
       Aspect_Default_Value                => True,
       Aspect_Depends                      => False,
       Aspect_Designated_Storage_Model     => True,
+      Aspect_Destructor                   => False,
       Aspect_Dimension                    => False,
       Aspect_Dimension_System             => False,
       Aspect_Dispatching_Domain           => False,
@@ -726,6 +731,7 @@ package Aspects is
       Aspect_Default_Value                => Name_Default_Value,
       Aspect_Depends                      => Name_Depends,
       Aspect_Designated_Storage_Model     => Name_Designated_Storage_Model,
+      Aspect_Destructor                   => Name_Destructor,
       Aspect_Dimension                    => Name_Dimension,
       Aspect_Dimension_System             => Name_Dimension_System,
       Aspect_Disable_Controlled           => Name_Disable_Controlled,
@@ -994,6 +1000,7 @@ package Aspects is
       Aspect_Default_Value                => Always_Delay,
       Aspect_Default_Component_Value      => Always_Delay,
       Aspect_Designated_Storage_Model     => Always_Delay,
+      Aspect_Destructor                   => Always_Delay,
       Aspect_Discard_Names                => Always_Delay,
       Aspect_Dispatching_Domain           => Always_Delay,
       Aspect_Dynamic_Predicate            => Always_Delay,
diff --git a/gcc/ada/atree.adb b/gcc/ada/atree.adb
index 20ca189..0ff3d6e 100644
--- a/gcc/ada/atree.adb
+++ b/gcc/ada/atree.adb
@@ -1802,12 +1802,12 @@ package body Atree is
 
       --  The Ghost node is created within a Ghost region
 
-      if Ghost_Mode = Check then
+      if Ghost_Config.Ghost_Mode = Check then
          if Nkind (N) in N_Entity then
             Set_Is_Checked_Ghost_Entity (N);
          end if;
 
-      elsif Ghost_Mode = Ignore then
+      elsif Ghost_Config.Ghost_Mode = Ignore then
          if Nkind (N) in N_Entity then
             Set_Is_Ignored_Ghost_Entity (N);
          end if;
diff --git a/gcc/ada/contracts.adb b/gcc/ada/contracts.adb
index 70e9487..7e4e4a2 100644
--- a/gcc/ada/contracts.adb
+++ b/gcc/ada/contracts.adb
@@ -2714,10 +2714,11 @@ package body Contracts is
 
       procedure Append_Enabled_Item (Item : Node_Id; List : in out List_Id) is
       begin
-         --  Do not chain ignored or disabled pragmas
+         --  Do not chain ignored or disabled pragmas. Note that disabled
+         --  pragmas are also considered ignored.
 
          if Nkind (Item) = N_Pragma
-           and then (Is_Ignored (Item) or else Is_Disabled (Item))
+           and then Is_Ignored_In_Codegen (Item)
          then
             null;
 
diff --git a/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst b/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst
index f313179..ff111dd 100644
--- a/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst
+++ b/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst
@@ -1793,3 +1793,71 @@ statement in the sequence of statements of the specified loop_statement.
 
 Note that ``continue`` is a keyword but it is not a reserved word. This is a
 configuration that does not exist in standard Ada.
+
+Destructors
+-----------
+
+The ``Destructor`` aspect can be applied to any record type, tagged or not.
+It must denote a primitive of the type that is a procedure with one parameter
+of the type and of mode ``in out``:
+
+.. code-block:: ada
+
+   type T is record
+      ...
+   end record with Destructor => Foo;
+
+   procedure Foo (X : in out T);
+
+This is equivalent to the following code that uses ``Finalizable``:
+
+.. code-block:: ada
+
+   type T is record
+      ...
+   end record with Finalizable => (Finalize => Foo);
+
+   procedure Foo (X : in out T);
+
+Unlike ``Finalizable``, however, ``Destructor`` can be specified on a derived
+type. And when it is, the effect of the aspect combines with the destructors of
+the parent type. Take, for example:
+
+.. code-block:: ada
+
+   type T1 is record
+      ...
+   end record with Destructor => Foo;
+
+   procedure Foo (X : in out T1);
+
+   type T2 is new T1 with Destructor => Bar;
+
+   procedure Bar (X : in out T2);
+
+Here, when an object of type ``T2`` is finalized, a call to ``Bar``
+will be performed and it will be followed by a call to ``Foo``.
+
+The ``Destructor`` aspect comes with a legality rule: if a primitive procedure
+of a type is denoted by a ``Destructor`` aspect specification, it is illegal to
+override this procedure in a derived type. For example, the following is illegal:
+
+.. code-block:: ada
+
+   type T1 is record
+      ...
+   end record with Destructor => Foo;
+
+   procedure Foo (X : in out T1);
+
+   type T2 is new T1;
+
+   overriding
+   procedure Foo (X : in out T2); -- Error here
+
+It is possible to specify ``Destructor`` on the completion of a private type,
+but there is one more restriction in that case: the denoted primitive must
+be private to the enclosing package. This is necessary due to the previously
+mentioned legality rule, to prevent breaking the privacy of the type when
+imposing that rule on outside types that derive from the private view of the
+type.
diff --git a/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst b/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
index 02013f1..3986298 100644
--- a/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
+++ b/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
@@ -5914,13 +5914,33 @@ Syntax:
   pragma Short_Circuit_And_Or;
 
 
-This configuration pragma causes any occurrence of the AND operator applied to
-operands of type Standard.Boolean to be short-circuited (i.e. the AND operator
-is treated as if it were AND THEN). Or is similarly treated as OR ELSE. This
-may be useful in the context of certification protocols requiring the use of
-short-circuited logical operators. If this configuration pragma occurs locally
-within the file being compiled, it applies only to the file being compiled.
+This configuration pragma causes the predefined AND and OR operators of
+type Standard.Boolean to have short-circuit semantics. That is, they
+behave like AND THEN and OR ELSE; the right-hand side is not evaluated
+if the left-hand side determines the result. This may be useful in the
+context of certification protocols requiring the use of short-circuited
+logical operators.
+
 There is no requirement that all units in a partition use this option.
+However, mixing of short-circuit and non-short-circuit semantics can be
+confusing. Therefore, the recommended use is to put the pragma in a
+configuration file that applies to the whole program. Alternatively, if
+you have a legacy library that should not use this pragma, you can put
+it in a separate library project that does not use the pragma.
+In any case, fine-grained mixing of the different semantics is not
+recommended. If pragma ``Short_Circuit_And_Or`` is specified, then it
+is illegal to rename the predefined Boolean AND and OR, or to pass
+them to generic formal functions; this corresponds to the fact that
+AND THEN and OR ELSE cannot be renamed nor passed as generic formal
+functions.
+
+Note that this pragma has no effect on other logical operators --
+predefined operators of modular types, array-of-boolean types and types
+derived from Standard.Boolean, nor user-defined operators.
+
+See also the pragma ``Unevaluated_Use_Of_Old`` and the restriction
+``No_Direct_Boolean_Operators``, which may be useful in conjunction
+with ``Short_Circuit_And_Or``.
 
 Pragma Short_Descriptors
 ========================
diff --git a/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst b/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst
index 0e9162a..1f0aa03 100644
--- a/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst
+++ b/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst
@@ -333,9 +333,9 @@ must define with the following profile:
                  "__gnat_last_chance_handler");
 
 
-The parameter is a C null-terminated string representing a message to be
-associated with the exception (typically the source location of the raise
-statement generated by the compiler). The Line parameter when nonzero
+The ``Source_Location`` parameter is a C null-terminated string representing a
+message to be associated with the exception (typically the source location of
+the raise statement generated by the compiler). The Line parameter when nonzero
 represents the line number in the source program where the raise occurs.
 
 No_Exception_Propagation
diff --git a/gcc/ada/einfo.ads b/gcc/ada/einfo.ads
index ba79fe4..916d9c6 100644
--- a/gcc/ada/einfo.ads
+++ b/gcc/ada/einfo.ads
@@ -1279,9 +1279,10 @@ package Einfo is
 --       that represents an activation record pointer is an extra formal.
 
 --    Extra_Formals
---       Applies to subprograms, subprogram types, entries, and entry
---       families. Returns first extra formal of the subprogram or entry.
---       Returns Empty if there are no extra formals.
+--       Applies to subprograms, subprogram types, entries, and entry families.
+--       Returns the first extra formal of the subprogram or entry. An entity
+--       has no extra formals when this attribute is Empty, and its attribute
+--       Extra_Formals_Known is True.
 
 --    Finalization_Collection [root type only]
 --       Defined in access-to-controlled or access-to-class-wide types. The
@@ -1594,6 +1595,11 @@ package Einfo is
 --       set, signalling that Freeze.Inherit_Delayed_Rep_Aspects must be called
 --       at the freeze point of the derived type.
 
+--    Has_Destructor
+--       Defined in all type and subtype entities. Set only for record type
+--       entities for which at least one ancestor has the Destructor aspect
+--       specified.
+
 --    Has_DIC (synthesized)
 --       Defined in all type entities. Set for a private type and its full view
 --       when the type is subject to pragma Default_Initial_Condition (DIC), or
@@ -1640,11 +1646,6 @@ package Einfo is
 --       that this does not imply a representation with holes, since the rep
 --       clause may merely confirm the default 0..N representation.
 
---    Has_First_Controlling_Parameter_Aspect
---       Defined in tagged types, concurrent types and concurrent record types.
---       Set to indicate that the type has a First_Controlling_Parameter of
---       True (whether by an aspect_specification, a pragma, or inheritance).
-
 --    Has_Exit
 --       Defined in loop entities. Set if the loop contains an exit statement.
 
@@ -1654,6 +1655,12 @@ package Einfo is
 --       flag prevents double expansion of a contract when a construct is
 --       rewritten into something else and subsequently reanalyzed/expanded.
 
+--    Has_First_Controlling_Parameter_Aspect
+--       Defined in tagged types, concurrent types, and concurrent record
+--       types. Set to indicate that the type has a First_Controlling_Parameter
+--       of True (whether by an aspect_specification, a pragma, or
+--       inheritance).
+
 --    Has_Foreign_Convention (synthesized)
 --       Applies to all entities. Determines if the Convention for the entity
 --       is a foreign convention, i.e. non-native: other than Convention_Ada,
@@ -1668,6 +1675,12 @@ package Einfo is
 --       the instance will conflict with the linear elaboration of front-end
 --       inlining.
 
+--    Extra_Formals_Known
+--       Defined in subprograms, subprogram types, entries, and entry families.
+--       Set when the extra formals have been determined. An entity has no
+--       extra formals when this attribute is True, and its attribute
+--       Extra_Formals is Empty.
+
 --    Has_Fully_Qualified_Name
 --       Defined in all entities. Set if the name in the Chars field has been
 --       replaced by the fully qualified name, as used for debug output. See
@@ -2515,11 +2528,12 @@ package Einfo is
 
 --    Is_Controlled_Active [base type only]
 --       Defined in all type entities. Indicates that the type is controlled,
---       i.e. has been declared with the Finalizable aspect or has inherited
---       the Finalizable aspect from an ancestor. Can only be set for record
---       types, tagged or untagged. System.Finalization_Root.Root_Controlled
---       is an example of the former case while Ada.Finalization.Controlled
---       and Ada.Finalization.Limited_Controlled are examples of the latter.
+--       i.e. has been declared with the Finalizable or the Destructor aspect
+--       or has inherited the aspect from an ancestor. Can only be set for
+--       record types, tagged or untagged.
+--       System.Finalization_Root.Root_Controlled is an example of the former
+--       case while Ada.Finalization.Controlled and
+--       Ada.Finalization.Limited_Controlled are examples of the latter.
 
 --    Is_Controlled (synth) [base type only]
 --       Defined in all type entities. Set if Is_Controlled_Active is set for
@@ -2545,6 +2559,10 @@ package Einfo is
 --       Defined in all entities. True if the entity is type System.Address,
 --       or (recursively) a subtype or derived type of System.Address.
 
+--    Is_Destructor
+--       Defined in procedure entities. True if the procedure is denoted by the
+--       Destructor aspect on some type.
+
 --    Is_DIC_Procedure
 --       Defined in functions and procedures. Set for a generated procedure
 --       which verifies the assumption of pragma Default_Initial_Condition at
@@ -2779,6 +2797,10 @@ package Einfo is
 --       identifiers in standard library packages, and to implement the
 --       restriction No_Implementation_Identifiers.
 
+--    Is_Implicit_Full_View
+--       Defined in types. Set on types that the compiler generates to act as
+--       full views of types that are derivations of private types.
+
 --    Is_Imported
 --       Defined in all entities. Set if the entity is imported. For now we
 --       only allow the import of exceptions, functions, procedures, packages,
@@ -3940,9 +3962,21 @@ package Einfo is
 --       Defined in constants and variables. Set if there is an address clause
 --       that causes the entity to overlay a constant object.
 
+--    Overridden_Inherited_Operation
+--       Defined in subprograms and enumeration literals. When set on a
+--       subprogram S, indicates an inherited subprogram that S overrides.
+--       In the case of a privately declared explicit subprogram E that
+--       overrides a private inherited subprogram, and the inherited
+--       subprogram itself overrides another inherited subprogram declared
+--       for a private extension, the field on E will reference the subprogram
+--       inherited by the private extension. This field is used for properly
+--       handling visibility for such privately declared subprograms. This
+--       field is always Empty for enumeration literal entities.
+
 --    Overridden_Operation
 --       Defined in subprograms. For overriding operations, points to the
---       user-defined parent subprogram that is being overridden.
+--       user-defined parent subprogram from which the inherited subprogram
+--       that is being overridden is derived.
 
 --    Package_Instantiation
 --       Defined in packages and generic packages. When defined, this field
@@ -5381,11 +5415,12 @@ package Einfo is
    --    Scope_Depth_Value
    --    Protection_Object                    (protected kind)
    --    Contract_Wrapper
-   --    Extra_Formals
    --    Contract
    --    SPARK_Pragma                         (protected kind)
    --    Default_Expressions_Processed
    --    Entry_Accepted
+   --    Extra_Formals
+   --    Extra_Formals_Known
    --    Has_Yield_Aspect
    --    Has_Expanded_Contract
    --    Ignore_SPARK_Mode_Pragmas
@@ -5413,6 +5448,7 @@ package Einfo is
    --    Enumeration_Pos
    --    Enumeration_Rep
    --    Alias
+   --    Overridden_Inherited_Operation
    --    Enumeration_Rep_Expr
    --    Interface_Name $$$
    --    Renamed_Object $$$
@@ -5502,9 +5538,11 @@ package Einfo is
    --    Subps_Index                          (non-generic case only)
    --    Interface_Alias
    --    LSP_Subprogram                       (non-generic case only)
+   --    Overridden_Inherited_Operation
    --    Overridden_Operation
    --    Wrapped_Entity                       (non-generic case only)
    --    Extra_Formals
+   --    Extra_Formals_Known                  (non-generic case only)
    --    Anonymous_Collections                (non-generic case only)
    --    Corresponding_Equality               (implicit /= only)
    --    Thunk_Entity                         (thunk case only)
@@ -5705,9 +5743,12 @@ package Einfo is
    --    Extra_Accessibility_Of_Result
    --    Last_Entity
    --    Subps_Index
+   --    Overridden_Inherited_Operation
    --    Overridden_Operation
    --    Linker_Section_Pragma
    --    Contract
+   --    Extra_Formals
+   --    Extra_Formals_Known
    --    Import_Pragma
    --    LSP_Subprogram
    --    SPARK_Pragma
@@ -5858,9 +5899,11 @@ package Einfo is
    --    Subps_Index                          (non-generic case only)
    --    Interface_Alias
    --    LSP_Subprogram                       (non-generic case only)
+   --    Overridden_Inherited_Operation
    --    Overridden_Operation                 (never for init proc)
    --    Wrapped_Entity                       (non-generic case only)
    --    Extra_Formals
+   --    Extra_Formals_Known                  (non-generic case only)
    --    Anonymous_Collections                (non-generic case only)
    --    Static_Initialization                (init_proc only)
    --    Thunk_Entity                         (thunk case only)
@@ -5899,6 +5942,7 @@ package Einfo is
    --    Is_Class_Wide_Wrapper
    --    Is_Constructor
    --    Is_CUDA_Kernel
+   --    Is_Destructor                        (non-generic case only)
    --    Is_DIC_Procedure                     (non-generic case only)
    --    Is_Elaboration_Checks_OK_Id
    --    Is_Elaboration_Warnings_OK_Id
@@ -6088,6 +6132,7 @@ package Einfo is
    --    Last_Entity
    --    Scope_Depth_Value
    --    Extra_Formals
+   --    Extra_Formals_Known
    --    Anonymous_Collections
    --    Contract
    --    SPARK_Pragma
@@ -6101,6 +6146,7 @@ package Einfo is
    --    Extra_Accessibility_Of_Result
    --    Directly_Designated_Type
    --    Extra_Formals
+   --    Extra_Formals_Known
    --    Access_Subprogram_Wrapper
    --    First_Formal                         (synth)
    --    First_Formal_With_Extras             (synth)
diff --git a/gcc/ada/exp_aggr.adb b/gcc/ada/exp_aggr.adb
index 9ff69ec..cd98369 100644
--- a/gcc/ada/exp_aggr.adb
+++ b/gcc/ada/exp_aggr.adb
@@ -1422,8 +1422,11 @@ package body Exp_Aggr is
                elsif Is_Mutably_Tagged_Type (Comp_Typ)
                  and then Nkind (Expr) = N_Qualified_Expression
                then
-                  Analyze_And_Resolve (Expr_Q, Etype (Expr));
+                  --  Avoid class-wide expected type for aggregate
+                  --  (which would be rejected as illegal)
+                  --  if the aggregate is explicitly qualified.
 
+                  Analyze_And_Resolve (Expr_Q, Etype (Expr));
                else
                   Analyze_And_Resolve (Expr_Q, Comp_Typ);
                end if;
@@ -1457,54 +1460,12 @@ package body Exp_Aggr is
          end if;
 
          if Present (Expr) then
-
-            --  For mutably tagged abstract class-wide types, we rely on the
-            --  type of the initializing expression to initialize the tag of
-            --  each array component.
-
-            --  Generate:
-            --     expr_type!(Indexed_Comp) := expr;
-            --     expr_type!(Indexed_Comp)._tag := expr_type'Tag;
-
-            if Is_Mutably_Tagged_Type (Comp_Typ)
-              and then Is_Abstract_Type (Root_Type (Comp_Typ))
-            then
-               declare
-                  Expr_Type : Entity_Id;
-
-               begin
-                  if Nkind (Expr) in N_Has_Etype
-                    and then Present (Etype (Expr))
-                  then
-                     Expr_Type := Etype (Expr);
-
-                  elsif Nkind (Expr) = N_Qualified_Expression then
-                     Analyze (Subtype_Mark (Expr));
-                     Expr_Type := Etype (Subtype_Mark (Expr));
-
-                  --  Unsupported case
-
-                  else
-                     pragma Assert (False);
-                     raise Program_Error;
-                  end if;
-
-                  Initialize_Component
-                    (N          => N,
-                     Comp       => Unchecked_Convert_To (Expr_Type,
-                                     Indexed_Comp),
-                     Comp_Typ   => Expr_Type,
-                     Init_Expr  => Expr,
-                     Stmts      => Stmts);
-               end;
-            else
-               Initialize_Component
-                 (N          => N,
-                  Comp       => Indexed_Comp,
-                  Comp_Typ   => Comp_Typ,
-                  Init_Expr  => Expr,
-                  Stmts      => Stmts);
-            end if;
+            Initialize_Component
+              (N          => N,
+               Comp       => Indexed_Comp,
+               Comp_Typ   => Comp_Typ,
+               Init_Expr  => Expr,
+               Stmts      => Stmts);
 
          --  Ada 2005 (AI-287): In case of default initialized component, call
          --  the initialization subprogram associated with the component type.
@@ -1519,10 +1480,10 @@ package body Exp_Aggr is
 
          else
             --  For mutably tagged class-wide types, default initialization is
-            --  performed by the init procedure of their root type.
+            --  performed by the init procedure of their specific type.
 
             if Is_Mutably_Tagged_Type (Comp_Typ) then
-               Comp_Typ := Root_Type (Comp_Typ);
+               Comp_Typ := Find_Specific_Type (Comp_Typ);
             end if;
 
             if Present (Base_Init_Proc (Comp_Typ)) then
@@ -4388,6 +4349,7 @@ package body Exp_Aggr is
         and then Is_Limited_Type (Typ)
       then
          Target_Expr := New_Copy_Tree (Name (Parent_Node));
+         Ensure_Defined (Typ, Parent_Node);
          Insert_Actions (Parent_Node,
            Build_Record_Aggr_Code (N, Typ, Target_Expr));
          Rewrite (Parent_Node, Make_Null_Statement (Loc));
@@ -4413,6 +4375,7 @@ package body Exp_Aggr is
 
             if Nkind (N) in N_Aggregate | N_Extension_Aggregate then
                Target_Expr := New_Copy_Tree (Lhs);
+               Ensure_Defined (Typ, Parent_Node);
                Insert_Actions (Parent_Node,
                  Build_Record_Aggr_Code (N, Typ, Target_Expr));
                Rewrite (Parent_Node, Make_Null_Statement (Loc));
@@ -6771,6 +6734,7 @@ package body Exp_Aggr is
       function Build_Size_Expr (Comp : Node_Id) return Node_Id is
          Lo, Hi       : Node_Id;
          It           : Node_Id;
+         It_Subt      : Entity_Id;
          Siz_Exp      : Node_Id := Empty;
          Choice       : Node_Id;
          Temp_Siz_Exp : Node_Id;
@@ -6845,20 +6809,22 @@ package body Exp_Aggr is
          elsif Nkind (Comp) = N_Iterated_Component_Association then
             if Present (Iterator_Specification (Comp)) then
 
-               --  If the static size of the iterable object is known,
+               --  If the size of the iterable object can be determined,
                --  attempt to return it.
 
                It := Name (Iterator_Specification (Comp));
                Preanalyze (It);
 
-               --  Handle the simplest cases for now where It denotes an array
-               --  object.
+               It_Subt := Etype (It);
+
+               --  Handle the simplest cases for now, where It denotes an array
+               --  object or a container object.
 
                if Nkind (It) in N_Identifier
-                 and then Ekind (Etype (It)) = E_Array_Subtype
+                 and then Ekind (It_Subt) = E_Array_Subtype
                then
                   declare
-                     Idx_N : Node_Id := First_Index (Etype (It));
+                     Idx_N : Node_Id := First_Index (It_Subt);
                      Siz_Exp : Node_Id := Empty;
                   begin
                      while Present (Idx_N) loop
@@ -6892,6 +6858,96 @@ package body Exp_Aggr is
 
                      return Siz_Exp;
                   end;
+
+               --  Case of iterating over a container object. Note that this
+               --  must be a simple object, and not something like a function
+               --  call (which might have side effects, and we wouldn't want
+               --  it to be evaluated more than once). We take advantage of
+               --  RM22 4.3.5(40/5), which allows implementation-defined
+               --  behavior for the parameter passed to the Empty function,
+               --  and here use the container Length function when available.
+               --  Class-wide objects are also excluded, since those would
+               --  lead to dispatching, which could call a user-defined
+               --  overriding of Length that might have arbitrary effects.
+
+               elsif Is_Entity_Name (It)
+                 and then Is_Object (Entity (It))
+                 and then Ekind (It_Subt) in Record_Kind
+                 and then not Is_Class_Wide_Type (It_Subt)
+               then
+                  declare
+                     Aggr_Base    : constant Entity_Id := Base_Type (Typ);
+                     It_Base      : constant Entity_Id := Base_Type (It_Subt);
+                     Empty_Formal : constant Entity_Id :=
+                                      First_Formal (Entity (Empty_Subp));
+                     Length_Subp  : Entity_Id;
+                     Param_List   : List_Id;
+
+                  begin
+                     --  We only determine a nondefault capacity in the case
+                     --  of containers of predefined container types, which
+                     --  generally have a Length function. User-defined
+                     --  containers don't necessarily have such a function,
+                     --  or it may be named differently, or it may have
+                     --  the wrong semantics. The base subtypes are tested,
+                     --  since their Sloc will refer to the original container
+                     --  generics in the predefined library, even though the
+                     --  types are declared in a package instantiation in some
+                     --  other unit. Also, this is only done when Empty_Subp
+                     --  has a formal parameter (generally named Capacity),
+                     --  and not in the case of a parameterless Empty function.
+                     --  Finally, we test for the container aggregate's type
+                     --  having a first discriminant with the name Capacity,
+                     --  since determining capacity via Length is only sensible
+                     --  for container types with that discriminant (bounded
+                     --  containers).
+
+                     if Present (Empty_Formal)
+                       and then In_Predefined_Unit (It_Base)
+                       and then In_Predefined_Unit (Aggr_Base)
+                       and then Has_Discriminants (Aggr_Base)
+                       and then
+                         Get_Name_String
+                           (Chars (First_Discriminant (Aggr_Base)))
+                           = "capacity"
+                     then
+                        --  Look for the container type's Length function in
+                        --  the package where it's defined.
+
+                        Push_Scope (Scope (It_Base));
+
+                        Length_Subp := Current_Entity_In_Scope (Name_Length);
+
+                        Pop_Scope;
+
+                        --  If we found a Length function that has a single
+                        --  parameter of the iterator object's container type,
+                        --  then expand a call to that, passing the object,
+                        --  and return that call, which will be used as the
+                        --  "size" of the current element association of the
+                        --  bounded container aggregate.
+
+                        if Present (Length_Subp)
+                          and then Ekind (Length_Subp) = E_Function
+                          and then
+                            Present (First_Entity (Length_Subp))
+                          and then
+                            No (Next_Entity (First_Entity (Length_Subp)))
+                          and then
+                            Base_Type
+                              (Etype (First_Entity (Length_Subp))) = It_Base
+                        then
+                           Param_List :=
+                             New_List (New_Occurrence_Of (Entity (It), Loc));
+
+                           return
+                             Make_Function_Call (Loc,
+                               Name =>
+                                 New_Occurrence_Of (Length_Subp, Loc),
+                               Parameter_Associations => Param_List);
+                        end if;
+                     end if;
+                  end;
                end if;
 
                return Empty;
@@ -8864,7 +8920,15 @@ package body Exp_Aggr is
       else
          Set_No_Ctrl_Actions (Init_Stmt);
 
-         if Tagged_Type_Expansion and then Is_Tagged_Type (Comp_Typ) then
+         if Tagged_Type_Expansion
+           and then Is_Tagged_Type (Comp_Typ)
+
+         --  Cannot adjust the tag when the expected type of the component is
+         --  a mutably tagged (and therefore class-wide) type; each component
+         --  of the aggregate has the tag of its initializing expression.
+
+           and then not Is_Mutably_Tagged_Type (Comp_Typ)
+         then
             declare
                Typ : Entity_Id := Underlying_Type (Comp_Typ);
 
diff --git a/gcc/ada/exp_attr.adb b/gcc/ada/exp_attr.adb
index 4f9f16c..810248d 100644
--- a/gcc/ada/exp_attr.adb
+++ b/gcc/ada/exp_attr.adb
@@ -1915,6 +1915,15 @@ package body Exp_Attr is
       --  call to the appropriate TSS procedure. Pname is the entity for the
       --  procedure to call.
 
+      procedure Read_Controlling_Tag
+        (P_Type : Entity_Id; Cntrl : out Node_Id);
+      --  Read the external tag from the stream and use it to construct the
+      --  controlling operand for a dispatching call.
+
+      procedure Write_Controlling_Tag (P_Type : Entity_Id);
+      --  Write the external tag of the given attribute prefix type to
+      --  the stream. Also perform the accompanying accessibility check.
+
       -------------------------------------
       -- Build_And_Insert_Type_Attr_Subp --
       -------------------------------------
@@ -2175,6 +2184,153 @@ package body Exp_Attr is
          Analyze (N);
       end Rewrite_Attribute_Proc_Call;
 
+      --------------------------
+      -- Read_Controlling_Tag --
+      --------------------------
+
+      procedure Read_Controlling_Tag
+        (P_Type : Entity_Id; Cntrl : out Node_Id)
+      is
+         Strm    : constant Node_Id := First (Exprs);
+         Expr    : Node_Id; -- call to Descendant_Tag
+         Get_Tag : Node_Id; -- expression to read the 'Tag
+
+      begin
+         --  Read the internal tag (RM 13.13.2(34)) and use it to
+         --  initialize a dummy tag value. We used to unconditionally
+         --  generate:
+         --
+         --     Descendant_Tag (String'Input (Strm), P_Type);
+         --
+         --  which turns into a call to String_Input_Blk_IO. However,
+         --  if the input is malformed, that could try to read an
+         --  enormous String, causing chaos. So instead we call
+         --  String_Input_Tag, which does the same thing as
+         --  String_Input_Blk_IO, except that if the String is
+         --  absurdly long, it raises an exception.
+         --
+         --  However, if the No_Stream_Optimizations restriction
+         --  is active, we disable this unnecessary attempt at
+         --  robustness; we really need to read the string
+         --  character-by-character.
+         --
+         --  This value is used only to provide a controlling
+         --  argument for the eventual _Input call. Descendant_Tag is
+         --  called rather than Internal_Tag to ensure that we have a
+         --  tag for a type that is descended from the prefix type and
+         --  declared at the same accessibility level (the exception
+         --  Tag_Error will be raised otherwise). The level check is
+         --  required for Ada 2005 because tagged types can be
+         --  extended in nested scopes (AI-344).
+
+         --  Note: we used to generate an explicit declaration of a
+         --  constant Ada.Tags.Tag object, and use an occurrence of
+         --  this constant in Cntrl, but this caused a secondary stack
+         --  leak.
+
+         if Restriction_Active (No_Stream_Optimizations) then
+            Get_Tag :=
+              Make_Attribute_Reference (Loc,
+                Prefix         =>
+                  New_Occurrence_Of (Standard_String, Loc),
+                Attribute_Name => Name_Input,
+                Expressions    => New_List (
+                  Relocate_Node (Duplicate_Subexpr (Strm))));
+         else
+            Get_Tag :=
+              Make_Function_Call (Loc,
+                Name                   =>
+                  New_Occurrence_Of
+                    (RTE (RE_String_Input_Tag), Loc),
+                Parameter_Associations => New_List (
+                  Relocate_Node (Duplicate_Subexpr (Strm))));
+         end if;
+
+         Expr :=
+           Make_Function_Call (Loc,
+             Name                   =>
+               New_Occurrence_Of (RTE (RE_Descendant_Tag), Loc),
+             Parameter_Associations => New_List (
+               Get_Tag,
+               Make_Attribute_Reference (Loc,
+                 Prefix         => New_Occurrence_Of (P_Type, Loc),
+                 Attribute_Name => Name_Tag)));
+
+         Set_Etype (Expr, RTE (RE_Tag));
+
+         --  Construct a controlling operand for a dispatching call.
+
+         Cntrl := Unchecked_Convert_To (P_Type, Expr);
+         Set_Etype (Cntrl, P_Type);
+         Set_Parent (Cntrl, N);
+      end Read_Controlling_Tag;
+
+      ----------------------------
+      --  Write_Controlling_Tag --
+      ----------------------------
+
+      procedure Write_Controlling_Tag (P_Type : Entity_Id) is
+         Strm : constant Node_Id := First (Exprs);
+         Item : constant Node_Id := Next (Strm);
+      begin
+         --  Ada 2005 (AI-344): Check that the accessibility level
+         --  of the type of the output object is not deeper than
+         --  that of the attribute's prefix type.
+
+         --  if Get_Access_Level (Item'Tag)
+         --       /= Get_Access_Level (P_Type'Tag)
+         --  then
+         --     raise Tag_Error;
+         --  end if;
+
+         --  String'Output (Strm, External_Tag (Item'Tag));
+
+         --  We cannot figure out a practical way to implement this
+         --  accessibility check on virtual machines, so we omit it.
+
+         if Ada_Version >= Ada_2005
+           and then Tagged_Type_Expansion
+         then
+            Insert_Action (N,
+              Make_Implicit_If_Statement (N,
+                Condition =>
+                  Make_Op_Ne (Loc,
+                    Left_Opnd  =>
+                      Build_Get_Access_Level (Loc,
+                        Make_Attribute_Reference (Loc,
+                          Prefix         =>
+                            Relocate_Node (
+                              Duplicate_Subexpr (Item,
+                                Name_Req => True)),
+                          Attribute_Name => Name_Tag)),
+
+                    Right_Opnd =>
+                      Make_Integer_Literal (Loc,
+                        Type_Access_Level (P_Type))),
+
+                Then_Statements =>
+                  New_List (Make_Raise_Statement (Loc,
+                              New_Occurrence_Of (
+                                RTE (RE_Tag_Error), Loc)))));
+         end if;
+
+         Insert_Action (N,
+           Make_Attribute_Reference (Loc,
+             Prefix => New_Occurrence_Of (Standard_String, Loc),
+             Attribute_Name => Name_Output,
+             Expressions => New_List (
+               Relocate_Node (Duplicate_Subexpr (Strm)),
+               Make_Function_Call (Loc,
+                 Name =>
+                   New_Occurrence_Of (RTE (RE_External_Tag), Loc),
+                 Parameter_Associations => New_List (
+                  Make_Attribute_Reference (Loc,
+                    Prefix =>
+                      Relocate_Node
+                        (Duplicate_Subexpr (Item, Name_Req => True)),
+                    Attribute_Name => Name_Tag))))));
+      end Write_Controlling_Tag;
+
       Typ  : constant Entity_Id    := Etype (N);
       Btyp : constant Entity_Id    := Base_Type (Typ);
       Ptyp : constant Entity_Id    := Etype (Pref);
@@ -4487,94 +4643,57 @@ package body Exp_Attr is
 
             elsif Is_Class_Wide_Type (P_Type) then
 
-               --  No need to do anything else compiling under restriction
-               --  No_Dispatching_Calls. During the semantic analysis we
-               --  already notified such violation.
+               if Is_Mutably_Tagged_Type (P_Type) then
 
-               if Restriction_Active (No_Dispatching_Calls) then
-                  return;
-               end if;
+                  --  In mutably tagged case, rewrite
+                  --    T'Class'Input (Strm)
+                  --  as (roughly)
+                  --    declare
+                  --       Result : T'Class;
+                  --       T'Class'Read (Strm, Result);
+                  --    begin
+                  --      Result;
+                  --    end;
 
-               declare
-                  Rtyp : constant Entity_Id := Root_Type (P_Type);
+                  declare
+                     Result_Temp : constant Entity_Id :=
+                       Make_Temporary (Loc, 'I');
 
-                  Expr    : Node_Id; -- call to Descendant_Tag
-                  Get_Tag : Node_Id; -- expression to read the 'Tag
+                     --  Gets default initialization
+                     Result_Temp_Decl : constant Node_Id :=
+                       Make_Object_Declaration (Loc,
+                         Defining_Identifier => Result_Temp,
+                         Object_Definition =>
+                           New_Occurrence_Of (P_Type, Loc));
 
-               begin
-                  --  Read the internal tag (RM 13.13.2(34)) and use it to
-                  --  initialize a dummy tag value. We used to unconditionally
-                  --  generate:
-                  --
-                  --     Descendant_Tag (String'Input (Strm), P_Type);
-                  --
-                  --  which turns into a call to String_Input_Blk_IO. However,
-                  --  if the input is malformed, that could try to read an
-                  --  enormous String, causing chaos. So instead we call
-                  --  String_Input_Tag, which does the same thing as
-                  --  String_Input_Blk_IO, except that if the String is
-                  --  absurdly long, it raises an exception.
-                  --
-                  --  However, if the No_Stream_Optimizations restriction
-                  --  is active, we disable this unnecessary attempt at
-                  --  robustness; we really need to read the string
-                  --  character-by-character.
-                  --
-                  --  This value is used only to provide a controlling
-                  --  argument for the eventual _Input call. Descendant_Tag is
-                  --  called rather than Internal_Tag to ensure that we have a
-                  --  tag for a type that is descended from the prefix type and
-                  --  declared at the same accessibility level (the exception
-                  --  Tag_Error will be raised otherwise). The level check is
-                  --  required for Ada 2005 because tagged types can be
-                  --  extended in nested scopes (AI-344).
-
-                  --  Note: we used to generate an explicit declaration of a
-                  --  constant Ada.Tags.Tag object, and use an occurrence of
-                  --  this constant in Cntrl, but this caused a secondary stack
-                  --  leak.
-
-                  if Restriction_Active (No_Stream_Optimizations) then
-                     Get_Tag :=
-                       Make_Attribute_Reference (Loc,
-                         Prefix         =>
-                           New_Occurrence_Of (Standard_String, Loc),
-                         Attribute_Name => Name_Input,
-                         Expressions    => New_List (
-                           Relocate_Node (Duplicate_Subexpr (Strm))));
-                  else
-                     Get_Tag :=
-                       Make_Function_Call (Loc,
-                         Name                   =>
-                           New_Occurrence_Of
-                             (RTE (RE_String_Input_Tag), Loc),
-                         Parameter_Associations => New_List (
-                           Relocate_Node (Duplicate_Subexpr (Strm))));
-                  end if;
+                     function Result_Temp_Name return Node_Id is
+                       (New_Occurrence_Of (Result_Temp, Loc));
 
-                  Expr :=
-                    Make_Function_Call (Loc,
-                      Name                   =>
-                        New_Occurrence_Of (RTE (RE_Descendant_Tag), Loc),
-                      Parameter_Associations => New_List (
-                        Get_Tag,
-                        Make_Attribute_Reference (Loc,
-                          Prefix         => New_Occurrence_Of (P_Type, Loc),
-                          Attribute_Name => Name_Tag)));
+                     Actions : constant List_Id := New_List (
+                       Result_Temp_Decl,
+                       Make_Attribute_Reference (Loc,
+                         Prefix => New_Occurrence_Of (P_Type, Loc),
+                         Attribute_Name => Name_Read,
+                         Expressions => New_List (
+                           Relocate_Node (Strm), Result_Temp_Name)));
+                  begin
+                     Rewrite (N, Make_Expression_With_Actions (Loc,
+                                   Actions, Result_Temp_Name));
+                     Analyze_And_Resolve (N, P_Type);
+                     return;
+                  end;
+               end if;
 
-                  Set_Etype (Expr, RTE (RE_Tag));
+               --  No need to do anything else compiling under restriction
+               --  No_Dispatching_Calls. During the semantic analysis we
+               --  already notified such violation.
 
-                  --  Now we need to get the entity for the call, and construct
-                  --  a function call node, where we preset a reference to Dnn
-                  --  as the controlling argument (doing an unchecked convert
-                  --  to the class-wide tagged type to make it look like a real
-                  --  tagged object).
+               if Restriction_Active (No_Dispatching_Calls) then
+                  return;
+               end if;
 
-                  Fname := Find_Prim_Op (Rtyp, TSS_Stream_Input);
-                  Cntrl := Unchecked_Convert_To (P_Type, Expr);
-                  Set_Etype (Cntrl, P_Type);
-                  Set_Parent (Cntrl, N);
-               end;
+               Read_Controlling_Tag (P_Type, Cntrl);
+               Fname := Find_Prim_Op (Root_Type (P_Type), TSS_Stream_Input);
 
             --  For tagged types, use the primitive Input function
 
@@ -5957,6 +6076,14 @@ package body Exp_Attr is
                      Attr_Ref => N);
                end;
 
+            --  In the mutably tagged case, T'Class'Output calls T'Class'Write;
+            --  T'Write will take care of writing out the external tag.
+
+            elsif Is_Mutably_Tagged_Type (P_Type) then
+               Set_Attribute_Name (N, Name_Write);
+               Analyze (N);
+               return;
+
             --  Class-wide case, first output external tag, then dispatch
             --  to the appropriate primitive Output function (RM 13.13.2(31)).
 
@@ -5970,68 +6097,7 @@ package body Exp_Attr is
                   return;
                end if;
 
-               Tag_Write : declare
-                  Strm : constant Node_Id := First (Exprs);
-                  Item : constant Node_Id := Next (Strm);
-
-               begin
-                  --  Ada 2005 (AI-344): Check that the accessibility level
-                  --  of the type of the output object is not deeper than
-                  --  that of the attribute's prefix type.
-
-                  --  if Get_Access_Level (Item'Tag)
-                  --       /= Get_Access_Level (P_Type'Tag)
-                  --  then
-                  --     raise Tag_Error;
-                  --  end if;
-
-                  --  String'Output (Strm, External_Tag (Item'Tag));
-
-                  --  We cannot figure out a practical way to implement this
-                  --  accessibility check on virtual machines, so we omit it.
-
-                  if Ada_Version >= Ada_2005
-                    and then Tagged_Type_Expansion
-                  then
-                     Insert_Action (N,
-                       Make_Implicit_If_Statement (N,
-                         Condition =>
-                           Make_Op_Ne (Loc,
-                             Left_Opnd  =>
-                               Build_Get_Access_Level (Loc,
-                                 Make_Attribute_Reference (Loc,
-                                   Prefix         =>
-                                     Relocate_Node (
-                                       Duplicate_Subexpr (Item,
-                                         Name_Req => True)),
-                                   Attribute_Name => Name_Tag)),
-
-                             Right_Opnd =>
-                               Make_Integer_Literal (Loc,
-                                 Type_Access_Level (P_Type))),
-
-                         Then_Statements =>
-                           New_List (Make_Raise_Statement (Loc,
-                                       New_Occurrence_Of (
-                                         RTE (RE_Tag_Error), Loc)))));
-                  end if;
-
-                  Insert_Action (N,
-                    Make_Attribute_Reference (Loc,
-                      Prefix => New_Occurrence_Of (Standard_String, Loc),
-                      Attribute_Name => Name_Output,
-                      Expressions => New_List (
-                        Relocate_Node (Duplicate_Subexpr (Strm)),
-                        Make_Function_Call (Loc,
-                          Name =>
-                            New_Occurrence_Of (RTE (RE_External_Tag), Loc),
-                          Parameter_Associations => New_List (
-                           Make_Attribute_Reference (Loc,
-                             Prefix =>
-                               Relocate_Node
-                                 (Duplicate_Subexpr (Item, Name_Req => True)),
-                             Attribute_Name => Name_Tag))))));
-               end Tag_Write;
+               Write_Controlling_Tag (P_Type);
 
                Pname := Find_Prim_Op (U_Type, TSS_Stream_Output);
 
@@ -6793,6 +6859,7 @@ package body Exp_Attr is
          P_Type  : constant Entity_Id := Entity (Pref);
          B_Type  : constant Entity_Id := Base_Type (P_Type);
          U_Type  : constant Entity_Id := Underlying_Type (P_Type);
+         Cntrl   : Node_Id := Empty; -- nonempty only if P_Type mutably tagged
          Pname   : Entity_Id;
          Decl    : Node_Id;
          Prag    : Node_Id;
@@ -6941,6 +7008,11 @@ package body Exp_Attr is
             --  this will dispatch in the class-wide case which is what we want
 
             elsif Is_Tagged_Type (U_Type) then
+
+               if Is_Mutably_Tagged_Type (U_Type) then
+                  Read_Controlling_Tag (P_Type, Cntrl);
+               end if;
+
                Pname := Find_Prim_Op (U_Type, TSS_Stream_Read);
 
             --  All other record type cases, including protected records. The
@@ -7001,6 +7073,46 @@ package body Exp_Attr is
 
          Rewrite_Attribute_Proc_Call (Pname);
 
+         if Present (Cntrl) then
+            pragma Assert (Is_Mutably_Tagged_Type (U_Type));
+            pragma Assert (Nkind (N) = N_Procedure_Call_Statement);
+
+            --  Assign the Tag value that was read from the stream
+            --  to the tag of the out-mode actual parameter so that
+            --  we dispatch correctly. This isn't quite right.
+            --  We should assign a complete object (not just
+            --  the tag), but that would require a dispatching call to
+            --  perform default initialization of the source object and
+            --  dispatching default init calls are currently not supported.
+
+            declare
+               function Select_Tag (Prefix : Node_Id) return Node_Id is
+                 (Make_Selected_Component (Loc,
+                    Prefix => Prefix,
+                    Selector_Name =>
+                      New_Occurrence_Of (First_Tag_Component
+                                           (Etype (Prefix)), Loc)));
+
+               Controlling_Actual : constant Node_Id :=
+                 Next (First (Parameter_Associations (N)));
+
+               pragma Assert (Is_Controlling_Actual (Controlling_Actual));
+
+               Assign_Tag : Node_Id;
+            begin
+               Remove_Side_Effects (Controlling_Actual, Name_Req => True);
+
+               Assign_Tag :=
+                 Make_Assignment_Statement (Loc,
+                   Name =>
+                     Select_Tag (New_Copy_Tree (Controlling_Actual)),
+                   Expression => Select_Tag (Cntrl));
+
+               Insert_Before (Before => N, Node => Assign_Tag);
+               Analyze (Assign_Tag);
+            end;
+         end if;
+
          if not Is_Tagged_Type (P_Type) then
             Cached_Attribute_Ops.Read_Map.Set (U_Type, Pname);
          end if;
@@ -8611,6 +8723,14 @@ package body Exp_Attr is
             --  this will dispatch in the class-wide case which is what we want
 
             elsif Is_Tagged_Type (U_Type) then
+
+               --  If T'Class is mutably tagged, then the external tag
+               --  is written out by T'Class'Write, not by T'Class'Output.
+
+               if Is_Mutably_Tagged_Type (U_Type) then
+                  Write_Controlling_Tag (P_Type);
+               end if;
+
                Pname := Find_Prim_Op (U_Type, TSS_Stream_Write);
 
             --  All other record type cases, including protected records.
diff --git a/gcc/ada/exp_ch3.adb b/gcc/ada/exp_ch3.adb
index 5a47a5a..00b3aae 100644
--- a/gcc/ada/exp_ch3.adb
+++ b/gcc/ada/exp_ch3.adb
@@ -942,10 +942,11 @@ package body Exp_Ch3 is
                Make_Handled_Sequence_Of_Statements (Loc,
                  Statements => Body_Stmts)));
 
-         Mutate_Ekind       (Proc_Id, E_Procedure);
-         Set_Is_Public      (Proc_Id, Is_Public (A_Type));
-         Set_Is_Internal    (Proc_Id);
-         Set_Has_Completion (Proc_Id);
+         Mutate_Ekind         (Proc_Id, E_Procedure);
+         Set_Is_Public        (Proc_Id, Is_Public (A_Type));
+         Set_Is_Internal      (Proc_Id);
+         Set_Has_Completion   (Proc_Id);
+         Freeze_Extra_Formals (Proc_Id);
 
          if not Debug_Generated_Code then
             Set_Debug_Info_Off (Proc_Id);
@@ -3204,6 +3205,7 @@ package body Exp_Ch3 is
          end if;
 
          Set_Parameter_Specifications (Proc_Spec_Node, Parameters);
+         Freeze_Extra_Formals (Proc_Id);
          Set_Specification (Body_Node, Proc_Spec_Node);
          Set_Declarations (Body_Node, Decls);
 
@@ -6496,7 +6498,7 @@ package body Exp_Ch3 is
          end;
       end if;
 
-      if Has_Controlled_Component (Typ) then
+      if Has_Controlled_Component (Typ) or else Has_Destructor (Typ) then
          Build_Controlling_Procs (Typ);
       end if;
 
@@ -6570,17 +6572,16 @@ package body Exp_Ch3 is
       --  procedure, because a self-referential type might call one of these
       --  primitives in the body of the init_proc itself.
       --
-      --  This is not needed:
-      --    1) If expansion is disabled, because extra formals are only added
-      --       when we are generating code.
+      --  This is not needed when expansion is disabled, because extra formals
+      --  are only added when we are generating code.
       --
-      --    2) For types with foreign convention since primitives with foreign
-      --       convention don't have extra formals and AI95-117 requires that
-      --       all primitives of a tagged type inherit the convention.
+      --  Notice that for tagged types with foreign convention this is also
+      --  required because (although primitives with foreign convention don't
+      --  have extra formals), a tagged type with foreign convention may have
+      --  primitives with convention Ada.
 
       if Expander_Active
         and then Is_Tagged_Type (Typ)
-        and then not Has_Foreign_Convention (Typ)
       then
          declare
             Elmt : Elmt_Id;
@@ -7500,6 +7501,12 @@ package body Exp_Ch3 is
                Apply_CW_Accessibility_Check (Expr, Func_Id);
             end if;
 
+            if Has_Anonymous_Access_Discriminant (Etype (Expr)) then
+               --  Check that access discrims do not designate entities
+               --  that the function result could outlive.
+               Apply_Access_Discrims_Accessibility_Check (Expr, Func_Id);
+            end if;
+
             Alloc_Expr := New_Copy_Tree (Expr);
 
             if Etype (Alloc_Expr) /= Alloc_Typ then
@@ -9058,6 +9065,10 @@ package body Exp_Ch3 is
                if Is_Class_Wide_Type (Etype (Func_Id)) then
                   Apply_CW_Accessibility_Check (Expr_Q, Func_Id);
                end if;
+
+               --  ??? Usually calls to Apply_CW_Accessibility_Check and to
+               --  Apply_Access_Discrims_Accessibility_Check come in pairs.
+               --  Do we need a (conditional) call here to A_A_D_A_C ?
             end;
          end if;
 
@@ -9590,8 +9601,7 @@ package body Exp_Ch3 is
 
       Def_Id : constant Entity_Id := Entity (N);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Result : Boolean := False;
@@ -9945,13 +9955,13 @@ package body Exp_Ch3 is
          end if;
       end if;
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
 
       return Result;
 
    exception
       when RE_Not_Available =>
-         Restore_Ghost_Region (Saved_GM, Saved_IGR);
+         Restore_Ghost_Region (Saved_Ghost_Config);
 
          return False;
    end Freeze_Type;
@@ -12845,25 +12855,27 @@ package body Exp_Ch3 is
             Append_To (Res, Decl);
          end if;
 
-         Fin_Call := Empty;
-         Decl     := Predef_Deep_Spec (Loc, Tag_Typ, TSS_Deep_Finalize, True);
+         if not Has_Destructor (Tag_Typ) then
+            Fin_Call := Empty;
+            Decl := Predef_Deep_Spec (Loc, Tag_Typ, TSS_Deep_Finalize, True);
 
-         if Is_Controlled (Tag_Typ) then
-            Fin_Call :=
-              Make_Final_Call
-                (Obj_Ref => Make_Identifier (Loc, Name_V),
-                 Typ     => Tag_Typ);
-         end if;
+            if Is_Controlled (Tag_Typ) then
+               Fin_Call :=
+                 Make_Final_Call
+                   (Obj_Ref => Make_Identifier (Loc, Name_V), Typ => Tag_Typ);
+            end if;
 
-         if No (Fin_Call) then
-            Fin_Call := Make_Null_Statement (Loc);
-         end if;
+            if No (Fin_Call) then
+               Fin_Call := Make_Null_Statement (Loc);
+            end if;
 
-         Set_Handled_Statement_Sequence (Decl,
-           Make_Handled_Sequence_Of_Statements (Loc,
-             Statements => New_List (Fin_Call)));
+            Set_Handled_Statement_Sequence
+              (Decl,
+               Make_Handled_Sequence_Of_Statements
+                 (Loc, Statements => New_List (Fin_Call)));
 
-         Append_To (Res, Decl);
+            Append_To (Res, Decl);
+         end if;
       end if;
 
       return Res;
diff --git a/gcc/ada/exp_ch4.adb b/gcc/ada/exp_ch4.adb
index 76386fc..43c94f3 100644
--- a/gcc/ada/exp_ch4.adb
+++ b/gcc/ada/exp_ch4.adb
@@ -2354,6 +2354,7 @@ package body Exp_Ch4 is
       Rhs        : Node_Id) return Node_Id
    is
       Loc       : constant Source_Ptr := Sloc (Nod);
+      CW_Comp   : Boolean := False;
       Full_Type : Entity_Id;
       Eq_Op     : Entity_Id;
 
@@ -2383,10 +2384,17 @@ package body Exp_Ch4 is
          Full_Type := Underlying_Type (Full_Type);
       end if;
 
+      if Is_Class_Wide_Equivalent_Type (Full_Type) then
+         CW_Comp := True;
+         Full_Type :=
+           Get_Corresponding_Mutably_Tagged_Type_If_Present (Full_Type);
+         pragma Assert (Is_Tagged_Type (Full_Type));
+      end if;
+
       --  Case of tagged record types
 
       if Is_Tagged_Type (Full_Type) then
-         Eq_Op := Find_Primitive_Eq (Comp_Type);
+         Eq_Op := Find_Primitive_Eq (if CW_Comp then Full_Type else Comp_Type);
          pragma Assert (Present (Eq_Op));
 
          return
diff --git a/gcc/ada/exp_ch6.adb b/gcc/ada/exp_ch6.adb
index c24c8c6..e877469 100644
--- a/gcc/ada/exp_ch6.adb
+++ b/gcc/ada/exp_ch6.adb
@@ -734,6 +734,268 @@ package body Exp_Ch6 is
       Add_Extra_Actual_To_Call (Function_Call, Chain_Formal, Chain_Actual);
    end Add_Task_Actuals_To_Build_In_Place_Call;
 
+   ----------------------------------------------
+   -- Apply_Access_Discrims_Accesibility_Check --
+   ----------------------------------------------
+
+   procedure Apply_Access_Discrims_Accessibility_Check
+     (Exp : Node_Id; Func : Entity_Id)
+   is
+      Loc : constant Source_Ptr := Sloc (Exp);
+
+      --  Some of the code here in this procedure may need to be factored
+      --  out at some point because it seems like some of the same
+      --  functionality would be needed for accessibility checking of a
+      --  return statement when the function result type is an anonymous
+      --  access type (as opposed to a type that has an anonymous access
+      --  discriminant).
+      --
+      --  Another case that is not addressed today is the case where
+      --  we need to check an access discriminant subcomponent of the
+      --  function result other than a discriminant of the function result.
+      --  This can only happen if the function result type has an unconstrained
+      --  subcomponent subtype that has an access discriminant (which implies
+      --  that the function result type must be limited).
+      --
+      --  A further corner case of that corner case arises if the limited
+      --  function result type is class-wide and it is not known statically
+      --  that this access-discriminant-bearing subcomponent exists. The
+      --  easiest way to address this properly would probably involve adding
+      --  a new compiler-generated dispatching procedure; a dispatching call
+      --  could then be used to perform the check in a context where we know
+      --  statically the specific type of the function result. Finding a
+      --  less important unimplemented case would be challenging.
+
+      function Constraint_Bearing_Subtype_If_Any
+        (Exp : Node_Id) return Node_Id;
+      --  If we can locate a constrained subtype whose constraint applies
+      --  to Exp, then return that. Otherwise, return Etype (Exp).
+
+      function Discr_Expression
+        (Typ : Entity_Id; Discr_Index : Positive) return Node_Id;
+      --  Typ is a constrained discriminated subtype.
+      --  Return the constraint expression for the indexed discriminant.
+
+      function Has_Level_Tied_To_Explicitly_Aliased_Param
+        (Constraint_Exp : Node_Id) return Boolean;
+      --  Constraint_Exp is the value given for an access discriminant
+      --  in a discriminant constraint for Exp. Return True iff the
+      --  accessibility of the type of that discriminant of Exp is the level
+      --  of an explicitly aliased parameter of Func. If true, this indicates
+      --  that no check should be performed for this discriminant.
+
+      ---------------------------------------
+      -- Constraint_Bearing_Subtype_If_Any --
+      ---------------------------------------
+
+      function Constraint_Bearing_Subtype_If_Any
+        (Exp : Node_Id) return Entity_Id
+      is
+         Result : Entity_Id := Etype (Exp);
+      begin
+         if Is_Constrained (Result) then
+            return Result;
+         end if;
+
+         --  Look through expansion-generated levels of indirection
+         --  to find a constrained subtype. Yuck. This comes up in
+         --  some cases when the unexpanded source returns an aggregate.
+
+         if Nkind (Exp) = N_Explicit_Dereference
+           and then Nkind (Prefix (Exp)) = N_Identifier
+           and then Ekind (Entity (Prefix (Exp))) = E_Constant
+         then
+            declare
+               Acc_Const       : Entity_Id := Entity (Prefix (Exp));
+               Acc_Const_Value : Node_Id := Empty;
+            begin
+               --  look through constants initialized to constants
+               loop
+                  exit when Nkind (Parent (Acc_Const)) /= N_Object_Declaration;
+
+                  Acc_Const_Value := Expression (Parent (Acc_Const));
+
+                  if Nkind (Acc_Const_Value) = N_Identifier
+                    and then Ekind (Entity (Acc_Const_Value)) = E_Constant
+                  then
+                     Acc_Const := Entity (Acc_Const_Value);
+                  else
+                     exit;
+                  end if;
+               end loop;
+
+               if Nkind (Acc_Const_Value) = N_Allocator
+                 and then Nkind (Expression (Acc_Const_Value))
+                             = N_Qualified_Expression
+               then
+                  Result :=
+                    Etype (Expression (Acc_Const_Value));
+               end if;
+            end;
+         end if;
+
+         if Is_Constrained (Result) then
+            return Result;
+         end if;
+
+         --  no constrained subtype found
+         return Etype (Exp);
+      end Constraint_Bearing_Subtype_If_Any;
+
+      ----------------------
+      -- Discr_Expression --
+      ----------------------
+
+      function Discr_Expression
+        (Typ : Entity_Id; Discr_Index : Positive) return Node_Id
+      is
+         Constraint_Elmt : Elmt_Id :=
+           First_Elmt (Discriminant_Constraint (Typ));
+      begin
+         for Skip in 1 .. Discr_Index - 1 loop
+            Next_Elmt (Constraint_Elmt);
+         end loop;
+         return Node (Constraint_Elmt);
+      end Discr_Expression;
+
+      -------------------------------------------------
+      --  Has_Level_Tied_To_Explicitly_Aliased_Param --
+      -------------------------------------------------
+
+      function Has_Level_Tied_To_Explicitly_Aliased_Param
+        (Constraint_Exp : Node_Id) return Boolean
+      is
+         Discr_Exp   : Node_Id := Constraint_Exp;
+         Attr_Prefix : Node_Id;
+      begin
+         --  look through constants
+         while Nkind (Discr_Exp) = N_Identifier
+           and then Ekind (Entity (Discr_Exp)) = E_Constant
+           and then Nkind (Parent (Entity (Discr_Exp))) = N_Object_Declaration
+         loop
+            Discr_Exp := Expression (Parent (Entity (Discr_Exp)));
+         end loop;
+
+         if Nkind (Discr_Exp) = N_Attribute_Reference
+           and then Get_Attribute_Id
+                      (Attribute_Name (Discr_Exp)) = Attribute_Access
+         then
+            Attr_Prefix := Ultimate_Prefix (Prefix (Discr_Exp));
+            if Is_Entity_Name (Attr_Prefix)
+              and then Is_Explicitly_Aliased (Entity (Attr_Prefix))
+              and then Scope (Entity (Attr_Prefix)) = Func
+            then
+               return True;
+            end if;
+         end if;
+
+         return False;
+      end Has_Level_Tied_To_Explicitly_Aliased_Param;
+
+      Discr       : Entity_Id := First_Discriminant (Etype (Exp));
+      Discr_Index : Positive  := 1;
+      Discr_Exp   : Node_Id;
+
+      Constrained_Subtype : constant Entity_Id :=
+        Constraint_Bearing_Subtype_If_Any (Exp);
+   begin
+      --  ??? Do not generate a check if version is Ada 95 (or earlier).
+      --  It is unclear whether this is really correct, or is just a stopgap
+      --  measure. Investigation is needed to decide how post-Ada-95 binding
+      --  interpretation changes in RM 3.10.2 should interact with Ada 95's
+      --  return-by-reference model for functions with limited result types
+      --  (which was abandoned in Ada 2005).
+
+      if Ada_Version <= Ada_95 then
+         return;
+      end if;
+
+      --  If we are returning a function call then that function will
+      --  perform the needed check.
+
+      if Nkind (Unqualify (Exp)) = N_Function_Call then
+         return;
+      end if;
+
+     --  ??? Cope with the consequences of the Disable_Tagged_Cases flag
+     --  in accessibility.adb (which can cause the extra formal parameter
+     --  needed for the check(s) generated here to be missing in the case
+     --  of a tagged result type); this is a workaround and can
+     --  prevent generation of a required check (or even a required
+     --  legality check - see "statically too deep" check below).
+
+      if No (Extra_Accessibility_Of_Result (Func)) then
+         return;
+      end if;
+
+      Remove_Side_Effects (Exp);
+
+      while Present (Discr) loop
+         if Is_Anonymous_Access_Type (Etype (Discr)) then
+            if Is_Constrained (Constrained_Subtype) then
+               Discr_Exp :=
+                 New_Copy_Tree
+                   (Discr_Expression (Constrained_Subtype, Discr_Index));
+            else
+               Discr_Exp :=
+                 Make_Selected_Component (Loc,
+                   Prefix => New_Copy_Tree (Exp),
+                   Selector_Name => New_Occurrence_Of (Discr, Loc));
+            end if;
+
+            if not Has_Level_Tied_To_Explicitly_Aliased_Param (Discr_Exp) then
+               declare
+                  --  We could do this min operation earlier, as is done
+                  --  for other implicit level parameters. Motivation for
+                  --  doing this min operation (earlier or not) is as for
+                  --  Generate_Minimum_Accessibility (see sem_ch6.adb):
+                  --  if a level value is too big, then the caller and the
+                  --  callee disagree about what it means.
+
+                  Level_Of_Master_Of_Call : constant Node_Id :=
+                    Make_Attribute_Reference (Loc,
+                      Prefix => New_Occurrence_Of (Standard_Natural, Loc),
+                      Attribute_Name => Name_Min,
+                      Expressions => New_List (
+                        Make_Integer_Literal (Loc, Scope_Depth (Func)),
+                        New_Occurrence_Of
+                          (Extra_Accessibility_Of_Result (Func), Loc)));
+
+                  Discrim_Level : Node_Id;
+               begin
+                  Analyze (Level_Of_Master_Of_Call);
+                  Analyze (Discr_Exp);
+
+                  Discrim_Level :=
+                    Accessibility_Level (Discr_Exp, Level => Dynamic_Level);
+                  Analyze (Discrim_Level);
+
+                  if Nkind (Discrim_Level) = N_Integer_Literal
+                    and then Intval (Discrim_Level) > Scope_Depth (Func)
+                  then
+                     Error_Msg_N
+                        ("level of type of access discriminant value of "
+                         & "return expression is statically too deep",
+                         Enclosing_Declaration_Or_Statement (Exp));
+                  end if;
+
+                  Insert_Action (Exp,
+                    Make_Raise_Program_Error (Loc,
+                      Condition =>
+                        Make_Op_Gt (Loc,
+                          Left_Opnd  => Discrim_Level,
+                          Right_Opnd => Level_Of_Master_Of_Call),
+                      Reason    => PE_Accessibility_Check_Failed),
+                    Suppress => Access_Check);
+               end;
+            end if;
+         end if;
+
+         Next_Discriminant (Discr);
+         Discr_Index := Discr_Index + 1;
+      end loop;
+   end Apply_Access_Discrims_Accessibility_Check;
+
    ----------------------------------
    -- Apply_CW_Accessibility_Check --
    ----------------------------------
@@ -1155,13 +1417,18 @@ package body Exp_Ch6 is
      (Subp_Call : Node_Id;
       Subp_Id   : Entity_Id) return Boolean
    is
-      Formal : Entity_Id;
+      use Deferred_Extra_Formals_Support;
+
       Actual : Node_Id;
+      Formal : Entity_Id;
 
    begin
       pragma Assert (Nkind (Subp_Call) in N_Entry_Call_Statement
                                         | N_Function_Call
                                         | N_Procedure_Call_Statement);
+      pragma Assert (Extra_Formals_Known (Subp_Id)
+        or else not Expander_Active
+        or else Is_Unsupported_Extra_Actuals_Call (Subp_Call, Subp_Id));
 
       --  In CodePeer_Mode, the tree for `'Elab_Spec` procedures will be
       --  malformed because GNAT does not perform the usual expansion that
@@ -2866,15 +3133,17 @@ package body Exp_Ch6 is
    -----------------
 
    procedure Expand_Call (N : Node_Id) is
-      function Is_Unchecked_Union_Equality (N : Node_Id) return Boolean;
+      function Is_Unchecked_Union_Predefined_Equality_Call
+        (N : Node_Id) return Boolean;
       --  Return True if N is a call to the predefined equality operator of an
       --  unchecked union type, or a renaming thereof.
 
-      ---------------------------------
-      -- Is_Unchecked_Union_Equality --
-      ---------------------------------
+      -------------------------------------------------
+      -- Is_Unchecked_Union_Predefined_Equality_Call --
+      -------------------------------------------------
 
-      function Is_Unchecked_Union_Equality (N : Node_Id) return Boolean is
+      function Is_Unchecked_Union_Predefined_Equality_Call
+        (N : Node_Id) return Boolean is
       begin
          if Is_Entity_Name (Name (N))
            and then Ekind (Entity (Name (N))) = E_Function
@@ -2899,7 +3168,7 @@ package body Exp_Ch6 is
          else
             return False;
          end if;
-      end Is_Unchecked_Union_Equality;
+      end Is_Unchecked_Union_Predefined_Equality_Call;
 
       --  If this is an indirect call through an Access_To_Subprogram
       --  with contract specifications, it is rewritten as a call to
@@ -2996,7 +3265,7 @@ package body Exp_Ch6 is
       --  Case of a call to the predefined equality operator of an unchecked
       --  union type, which requires specific processing.
 
-      elsif Is_Unchecked_Union_Equality (N) then
+      elsif Is_Unchecked_Union_Predefined_Equality_Call (N) then
          declare
             Eq : constant Entity_Id := Entity (Name (N));
 
@@ -3020,29 +3289,12 @@ package body Exp_Ch6 is
       end if;
    end Expand_Call;
 
-   ------------------------
-   -- Expand_Call_Helper --
-   ------------------------
-
-   --  This procedure handles expansion of function calls and procedure call
-   --  statements (i.e. it serves as the body for Expand_N_Function_Call and
-   --  Expand_N_Procedure_Call_Statement). Processing for calls includes:
-
-   --    Replace call to Raise_Exception by Raise_Exception_Always if possible
-   --    Provide values of actuals for all formals in Extra_Formals list
-   --    Replace "call" to enumeration literal function by literal itself
-   --    Rewrite call to predefined operator as operator
-   --    Replace actuals to in-out parameters that are numeric conversions,
-   --     with explicit assignment to temporaries before and after the call.
-
-   --   Note that the list of actuals has been filled with default expressions
-   --   during semantic analysis of the call. Only the extra actuals required
-   --   for the 'Constrained attribute and for accessibility checks are added
-   --   at this point.
+   --------------------------
+   -- Create_Extra_Actuals --
+   --------------------------
 
-   procedure Expand_Call_Helper (N : Node_Id; Post_Call : out List_Id) is
-      Loc           : constant Source_Ptr := Sloc (N);
-      Call_Node     : Node_Id := N;
+   procedure Create_Extra_Actuals (Call_Node : Node_Id) is
+      Loc           : constant Source_Ptr := Sloc (Call_Node);
       Extra_Actuals : List_Id := No_List;
       Prev          : Node_Id := Empty;
 
@@ -3072,88 +3324,6 @@ package body Exp_Ch6 is
       --  expression for the value of the actual, EF is the entity for the
       --  extra formal.
 
-      procedure Add_View_Conversion_Invariants
-        (Formal : Entity_Id;
-         Actual : Node_Id);
-      --  Adds invariant checks for every intermediate type between the range
-      --  of a view converted argument to its ancestor (from parent to child).
-
-      function Can_Fold_Predicate_Call (P : Entity_Id) return Boolean;
-      --  Try to constant-fold a predicate check, which often enough is a
-      --  simple arithmetic expression that can be computed statically if
-      --  its argument is static. This cleans up the output of CCG, even
-      --  though useless predicate checks will be generally removed by
-      --  back-end optimizations.
-
-      procedure Check_Subprogram_Variant;
-      --  Emit a call to the internally generated procedure with checks for
-      --  aspect Subprogram_Variant, if present and enabled.
-
-      function Inherited_From_Formal (S : Entity_Id) return Entity_Id;
-      --  Within an instance, a type derived from an untagged formal derived
-      --  type inherits from the original parent, not from the actual. The
-      --  current derivation mechanism has the derived type inherit from the
-      --  actual, which is only correct outside of the instance. If the
-      --  subprogram is inherited, we test for this particular case through a
-      --  convoluted tree traversal before setting the proper subprogram to be
-      --  called.
-
-      function In_Unfrozen_Instance (E : Entity_Id) return Boolean;
-      --  Return true if E comes from an instance that is not yet frozen
-
-      function Is_Class_Wide_Interface_Type (E : Entity_Id) return Boolean;
-      --  Return True when E is a class-wide interface type or an access to
-      --  a class-wide interface type.
-
-      function Is_Direct_Deep_Call (Subp : Entity_Id) return Boolean;
-      --  Determine if Subp denotes a non-dispatching call to a Deep routine
-
-      function New_Value (From : Node_Id) return Node_Id;
-      --  From is the original Expression. New_Value is equivalent to a call
-      --  to Duplicate_Subexpr with an explicit dereference when From is an
-      --  access parameter.
-
-      --------------------------
-      -- Add_Actual_Parameter --
-      --------------------------
-
-      procedure Add_Actual_Parameter (Insert_Param : Node_Id) is
-         Actual_Expr : constant Node_Id :=
-                         Explicit_Actual_Parameter (Insert_Param);
-
-      begin
-         --  Case of insertion is first named actual
-
-         if No (Prev) or else
-            Nkind (Parent (Prev)) /= N_Parameter_Association
-         then
-            Set_Next_Named_Actual
-              (Insert_Param, First_Named_Actual (Call_Node));
-            Set_First_Named_Actual (Call_Node, Actual_Expr);
-
-            if No (Prev) then
-               if No (Parameter_Associations (Call_Node)) then
-                  Set_Parameter_Associations (Call_Node, New_List);
-               end if;
-
-               Append (Insert_Param, Parameter_Associations (Call_Node));
-
-            else
-               Insert_After (Prev, Insert_Param);
-            end if;
-
-         --  Case of insertion is not first named actual
-
-         else
-            Set_Next_Named_Actual
-              (Insert_Param, Next_Named_Actual (Parent (Prev)));
-            Set_Next_Named_Actual (Parent (Prev), Actual_Expr);
-            Append (Insert_Param, Parameter_Associations (Call_Node));
-         end if;
-
-         Prev := Actual_Expr;
-      end Add_Actual_Parameter;
-
       --------------------------------------
       -- Add_Cond_Expression_Extra_Actual --
       --------------------------------------
@@ -3368,14 +3538,14 @@ package body Exp_Ch6 is
             if Etype (Formal) = Standard_Natural then
                Actual := Make_Integer_Literal (Loc, Uint_0);
                Analyze_And_Resolve (Actual, Standard_Natural);
-               Add_Extra_Actual_To_Call (N, Formal, Actual);
+               Add_Extra_Actual_To_Call (Call_Node, Formal, Actual);
 
             --  BIPtaskmaster
 
             elsif Etype (Formal) = Standard_Integer then
                Actual := Make_Integer_Literal (Loc, Uint_0);
                Analyze_And_Resolve (Actual, Standard_Integer);
-               Add_Extra_Actual_To_Call (N, Formal, Actual);
+               Add_Extra_Actual_To_Call (Call_Node, Formal, Actual);
 
             --  BIPstoragepool, BIPcollection, BIPactivationchain,
             --  and BIPaccess.
@@ -3383,7 +3553,7 @@ package body Exp_Ch6 is
             elsif Is_Access_Type (Etype (Formal)) then
                Actual := Make_Null (Loc);
                Analyze_And_Resolve (Actual, Etype (Formal));
-               Add_Extra_Actual_To_Call (N, Formal, Actual);
+               Add_Extra_Actual_To_Call (Call_Node, Formal, Actual);
 
             else
                pragma Assert (False);
@@ -3402,6 +3572,47 @@ package body Exp_Ch6 is
          pragma Assert (Check_BIP_Actuals (Call_Node, Function_Id));
       end Add_Dummy_Build_In_Place_Actuals;
 
+      --------------------------
+      -- Add_Actual_Parameter --
+      --------------------------
+
+      procedure Add_Actual_Parameter (Insert_Param : Node_Id) is
+         Actual_Expr : constant Node_Id :=
+                         Explicit_Actual_Parameter (Insert_Param);
+
+      begin
+         --  Case of insertion is first named actual
+
+         if No (Prev)
+           or else Nkind (Parent (Prev)) /= N_Parameter_Association
+         then
+            Set_Next_Named_Actual
+              (Insert_Param, First_Named_Actual (Call_Node));
+            Set_First_Named_Actual (Call_Node, Actual_Expr);
+
+            if No (Prev) then
+               if No (Parameter_Associations (Call_Node)) then
+                  Set_Parameter_Associations (Call_Node, New_List);
+               end if;
+
+               Append (Insert_Param, Parameter_Associations (Call_Node));
+
+            else
+               Insert_After (Prev, Insert_Param);
+            end if;
+
+         --  Case of insertion is not first named actual
+
+         else
+            Set_Next_Named_Actual
+              (Insert_Param, Next_Named_Actual (Parent (Prev)));
+            Set_Next_Named_Actual (Parent (Prev), Actual_Expr);
+            Append (Insert_Param, Parameter_Associations (Call_Node));
+         end if;
+
+         Prev := Actual_Expr;
+      end Add_Actual_Parameter;
+
       ----------------------
       -- Add_Extra_Actual --
       ----------------------
@@ -3427,6 +3638,421 @@ package body Exp_Ch6 is
          end if;
       end Add_Extra_Actual;
 
+      --  Local variables
+
+      use Deferred_Extra_Formals_Support;
+
+      Actual        : Node_Id;
+      Formal        : Entity_Id;
+      Param_Count   : Positive;
+      Subp          : constant Entity_Id := Get_Called_Entity (Call_Node);
+
+   --  Start of processing for Create_Extra_Actuals
+
+   begin
+      --  Special case: Thunks must not compute the extra actuals; they must
+      --  just propagate their extra actuals to the target primitive.
+
+      if Is_Thunk (Current_Scope)
+        and then Thunk_Entity (Current_Scope) = Subp
+      then
+         declare
+            Target_Formal : Entity_Id;
+            Thunk_Formal  : Entity_Id;
+
+         begin
+            pragma Assert (Extra_Formals_Known (Subp)
+              and then Extra_Formals_Match_OK (Current_Scope, Subp));
+
+            Target_Formal := Extra_Formals (Subp);
+            Thunk_Formal  := Extra_Formals (Current_Scope);
+            while Present (Target_Formal) loop
+               Add_Extra_Actual
+                  (Expr => New_Occurrence_Of (Thunk_Formal, Loc),
+                   EF   => Thunk_Formal);
+
+               Target_Formal := Extra_Formal (Target_Formal);
+               Thunk_Formal  := Extra_Formal (Thunk_Formal);
+            end loop;
+
+            while Is_Non_Empty_List (Extra_Actuals) loop
+               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
+            end loop;
+
+            return;
+         end;
+      end if;
+
+      pragma Assert (Extra_Formals_Known (Subp)
+        or else Is_Unsupported_Extra_Formals_Entity (Subp));
+
+      --  First step, compute extra actuals, corresponding to any Extra_Formals
+      --  present. Note that we do not access Extra_Formals directly; instead
+      --  we generate and collect the corresponding actuals in Extra_Actuals.
+
+      Formal := First_Formal (Subp);
+      Actual := First_Actual (Call_Node);
+      Param_Count := 1;
+      while Present (Formal) loop
+         --  Prepare to examine current entry
+
+         Prev := Actual;
+
+         --  Create possible extra actual for constrained case. Usually, the
+         --  extra actual is of the form actual'constrained, but since this
+         --  attribute is only available for unconstrained records, TRUE is
+         --  expanded if the type of the formal happens to be constrained (for
+         --  instance when this procedure is inherited from an unconstrained
+         --  record to a constrained one) or if the actual has no discriminant
+         --  (its type is constrained). An exception to this is the case of a
+         --  private type without discriminants. In this case we pass FALSE
+         --  because the object has underlying discriminants with defaults.
+
+         if Present (Extra_Constrained (Formal)) then
+            if Is_Mutably_Tagged_Type (Etype (Actual))
+              or else (Is_Private_Type (Etype (Prev))
+                        and then not Has_Discriminants
+                                       (Base_Type (Etype (Prev))))
+            then
+               Add_Extra_Actual
+                 (Expr => New_Occurrence_Of (Standard_False, Loc),
+                  EF   => Extra_Constrained (Formal));
+
+            elsif Is_Constrained (Etype (Formal))
+              or else not Has_Discriminants (Etype (Prev))
+            then
+               Add_Extra_Actual
+                 (Expr => New_Occurrence_Of (Standard_True, Loc),
+                  EF   => Extra_Constrained (Formal));
+
+            --  Do not produce extra actuals for Unchecked_Union parameters.
+            --  Jump directly to the end of the loop.
+
+            elsif Is_Unchecked_Union (Base_Type (Etype (Actual))) then
+               goto Skip_Extra_Actual_Generation;
+
+            else
+               --  If the actual is a type conversion, then the constrained
+               --  test applies to the actual, not the target type.
+
+               declare
+                  Act_Prev : Node_Id;
+
+               begin
+                  --  Test for unchecked conversions as well, which can occur
+                  --  as out parameter actuals on calls to stream procedures.
+
+                  Act_Prev := Prev;
+                  while Nkind (Act_Prev) in N_Type_Conversion
+                                          | N_Unchecked_Type_Conversion
+                  loop
+                     Act_Prev := Expression (Act_Prev);
+                  end loop;
+
+                  --  If the expression is a conversion of a dereference, this
+                  --  is internally generated code that manipulates addresses,
+                  --  e.g. when building interface tables. No check should
+                  --  occur in this case, and the discriminated object is not
+                  --  directly at hand.
+
+                  if not Comes_From_Source (Actual)
+                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
+                    and then Nkind (Act_Prev) = N_Explicit_Dereference
+                  then
+                     Add_Extra_Actual
+                       (Expr => New_Occurrence_Of (Standard_False, Loc),
+                        EF   => Extra_Constrained (Formal));
+
+                  else
+                     Add_Extra_Actual
+                       (Expr =>
+                          Make_Attribute_Reference (Sloc (Prev),
+                            Prefix         =>
+                              Duplicate_Subexpr_No_Checks
+                                (Act_Prev, Name_Req => True),
+                            Attribute_Name => Name_Constrained),
+                        EF   => Extra_Constrained (Formal));
+                  end if;
+               end;
+            end if;
+         end if;
+
+         --  Create possible extra actual for accessibility level
+
+         if Present (Extra_Accessibility (Formal)) then
+
+            --  Ada 2005 (AI-251): Thunks must propagate the extra actuals of
+            --  accessibility levels.
+
+            if Is_Thunk (Current_Scope) then
+               declare
+                  Parm_Ent : Entity_Id;
+
+               begin
+                  if Is_Controlling_Actual (Actual) then
+
+                     --  Find the corresponding actual of the thunk
+
+                     Parm_Ent := First_Entity (Current_Scope);
+                     for J in 2 .. Param_Count loop
+                        Next_Entity (Parm_Ent);
+                     end loop;
+
+                  --  Handle unchecked conversion of access types generated
+                  --  in thunks (cf. Expand_Interface_Thunk).
+
+                  elsif Is_Access_Type (Etype (Actual))
+                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
+                  then
+                     Parm_Ent := Entity (Expression (Actual));
+
+                  else pragma Assert (Is_Entity_Name (Actual));
+                     Parm_Ent := Entity (Actual);
+                  end if;
+
+                  Add_Extra_Actual
+                    (Expr => Accessibility_Level
+                               (Expr            => Parm_Ent,
+                                Level           => Dynamic_Level,
+                                Allow_Alt_Model => False),
+                     EF   => Extra_Accessibility (Formal));
+               end;
+
+            --  Conditional expressions
+
+            elsif Nkind (Prev) = N_Expression_With_Actions
+              and then Nkind (Original_Node (Prev)) in
+                         N_If_Expression | N_Case_Expression
+            then
+               Add_Cond_Expression_Extra_Actual (Formal);
+
+            --  Internal constant generated to remove side effects (normally
+            --  from the expansion of dispatching calls).
+
+            --  First verify the actual is internal
+
+            elsif not Comes_From_Source (Prev)
+              and then not Is_Rewrite_Substitution (Prev)
+
+              --  Next check that the actual is a constant
+
+              and then Nkind (Prev) = N_Identifier
+              and then Ekind (Entity (Prev)) = E_Constant
+              and then Nkind (Parent (Entity (Prev))) = N_Object_Declaration
+            then
+               --  Generate the accessibility level based on the expression in
+               --  the constant's declaration.
+
+               declare
+                  Ent : Entity_Id := Entity (Prev);
+
+               begin
+                  --  Handle deferred constants
+
+                  if Present (Full_View (Ent)) then
+                     Ent := Full_View (Ent);
+                  end if;
+
+                  Add_Extra_Actual
+                    (Expr => Accessibility_Level
+                               (Expr            => Expression (Parent (Ent)),
+                                Level           => Dynamic_Level,
+                                Allow_Alt_Model => False),
+                     EF   => Extra_Accessibility (Formal));
+               end;
+
+            --  Normal case
+
+            else
+               Add_Extra_Actual
+                 (Expr => Accessibility_Level
+                            (Expr            => Prev,
+                             Level           => Dynamic_Level,
+                             Allow_Alt_Model => False),
+                  EF   => Extra_Accessibility (Formal));
+            end if;
+         end if;
+
+         --  This label is required when skipping extra actual generation for
+         --  Unchecked_Union parameters.
+
+         <<Skip_Extra_Actual_Generation>>
+
+         Param_Count := Param_Count + 1;
+         Next_Actual (Actual);
+         Next_Formal (Formal);
+      end loop;
+
+      --  If we are calling an Ada 2012 function which needs to have the
+      --  "accessibility level determined by the point of call" (AI05-0234)
+      --  passed in to it, then pass it in.
+
+      if Ekind (Subp) in E_Function | E_Operator | E_Subprogram_Type
+        and then
+          Present (Extra_Accessibility_Of_Result (Ultimate_Alias (Subp)))
+      then
+         declare
+            Extra_Form : Node_Id := Empty;
+            Level      : Node_Id := Empty;
+
+         begin
+            --  Detect cases where the function call has been internally
+            --  generated by examining the original node and return library
+            --  level - taking care to avoid ignoring function calls expanded
+            --  in prefix notation.
+
+            if Nkind (Original_Node (Call_Node)) not in N_Function_Call
+                                                      | N_Selected_Component
+                                                      | N_Indexed_Component
+            then
+               Level := Make_Integer_Literal
+                          (Loc, Scope_Depth (Standard_Standard));
+
+            --  Otherwise get the level normally based on the call node
+
+            else
+               Level := Accessibility_Level
+                          (Expr            => Call_Node,
+                           Level           => Dynamic_Level,
+                           Allow_Alt_Model => False);
+            end if;
+
+            --  It may be possible that we are re-expanding an already
+            --  expanded call when are are dealing with dispatching ???
+
+            if No (Parameter_Associations (Call_Node))
+              or else Nkind (Last (Parameter_Associations (Call_Node)))
+                        /= N_Parameter_Association
+              or else not Is_Accessibility_Actual
+                              (Last (Parameter_Associations (Call_Node)))
+            then
+               Extra_Form := Extra_Accessibility_Of_Result
+                               (Ultimate_Alias (Subp));
+
+               Add_Extra_Actual
+                 (Expr => Level,
+                  EF   => Extra_Form);
+            end if;
+         end;
+      end if;
+
+      --  Second step: In the previous loop we gathered the extra actuals (the
+      --  ones that correspond to Extra_Formals), so now they can be appended.
+
+      if Is_Non_Empty_List (Extra_Actuals) then
+         declare
+            Num_Extra_Actuals : constant Nat := List_Length (Extra_Actuals);
+
+         begin
+            while Is_Non_Empty_List (Extra_Actuals) loop
+               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
+            end loop;
+
+            --  Add dummy extra BIP actuals if we are calling a function that
+            --  inherited the BIP extra actuals but does not require them.
+
+            if Nkind (Call_Node) = N_Function_Call
+              and then Is_Function_Call_With_BIP_Formals (Call_Node)
+              and then not Is_Build_In_Place_Function_Call (Call_Node)
+            then
+               Add_Dummy_Build_In_Place_Actuals (Subp,
+                 Num_Added_Extra_Actuals => Num_Extra_Actuals);
+            end if;
+         end;
+
+      --  Add dummy extra BIP actuals if we are calling a function that
+      --  inherited the BIP extra actuals but does not require them.
+
+      elsif Nkind (Call_Node) = N_Function_Call
+        and then Is_Function_Call_With_BIP_Formals (Call_Node)
+        and then not Is_Build_In_Place_Function_Call (Call_Node)
+      then
+         Add_Dummy_Build_In_Place_Actuals (Subp);
+      end if;
+
+      --  For non build-in-place calls formals and actuals must match;
+      --  for build-in-place function calls, the pending bip actuals are
+      --  added by the following subprograms as part of the bottom-up
+      --  expansion of the call (and this check will be performed there):
+      --    Make_Build_In_Place_Call_In_Allocator
+      --    Make_Build_In_Place_Call_In_Anonymous_Context
+      --    Make_Build_In_Place_Call_In_Assignment
+      --    Make_Build_In_Place_Call_In_Object_Declaration
+      --    Make_Build_In_Place_Iface_Call_In_Allocator
+      --    Make_Build_In_Place_Iface_Call_In_Anonymous_Context
+      --    Make_Build_In_Place_Iface_Call_In_Object_Declaration
+
+      pragma Assert (Is_Build_In_Place_Function_Call (Call_Node)
+        or else (Check_Number_Of_Actuals (Call_Node, Subp)
+                   and then Check_BIP_Actuals (Call_Node, Subp)));
+   end Create_Extra_Actuals;
+
+   ------------------------
+   -- Expand_Call_Helper --
+   ------------------------
+
+   --  This procedure handles expansion of function calls and procedure call
+   --  statements (i.e. it serves as the body for Expand_N_Function_Call and
+   --  Expand_N_Procedure_Call_Statement). Processing for calls includes:
+
+   --    Replace call to Raise_Exception by Raise_Exception_Always if possible
+   --    Provide values of actuals for all formals in Extra_Formals list
+   --    Replace "call" to enumeration literal function by literal itself
+   --    Rewrite call to predefined operator as operator
+   --    Replace actuals to in-out parameters that are numeric conversions,
+   --     with explicit assignment to temporaries before and after the call.
+
+   --   Note that the list of actuals has been filled with default expressions
+   --   during semantic analysis of the call. Only the extra actuals required
+   --   for the 'Constrained attribute and for accessibility checks are added
+   --   at this point.
+
+   procedure Expand_Call_Helper (N : Node_Id; Post_Call : out List_Id) is
+      Loc       : constant Source_Ptr := Sloc (N);
+      Call_Node : Node_Id := N;
+      Prev      : Node_Id := Empty;
+
+      procedure Add_View_Conversion_Invariants
+        (Formal : Entity_Id;
+         Actual : Node_Id);
+      --  Adds invariant checks for every intermediate type between the range
+      --  of a view converted argument to its ancestor (from parent to child).
+
+      function Can_Fold_Predicate_Call (P : Entity_Id) return Boolean;
+      --  Try to constant-fold a predicate check, which often enough is a
+      --  simple arithmetic expression that can be computed statically if
+      --  its argument is static. This cleans up the output of CCG, even
+      --  though useless predicate checks will be generally removed by
+      --  back-end optimizations.
+
+      procedure Check_Subprogram_Variant;
+      --  Emit a call to the internally generated procedure with checks for
+      --  aspect Subprogram_Variant, if present and enabled.
+
+      function Inherited_From_Formal (S : Entity_Id) return Entity_Id;
+      --  Within an instance, a type derived from an untagged formal derived
+      --  type inherits from the original parent, not from the actual. The
+      --  current derivation mechanism has the derived type inherit from the
+      --  actual, which is only correct outside of the instance. If the
+      --  subprogram is inherited, we test for this particular case through a
+      --  convoluted tree traversal before setting the proper subprogram to be
+      --  called.
+
+      function In_Unfrozen_Instance (E : Entity_Id) return Boolean;
+      --  Return true if E comes from an instance that is not yet frozen
+
+      function Is_Class_Wide_Interface_Type (E : Entity_Id) return Boolean;
+      --  Return True when E is a class-wide interface type or an access to
+      --  a class-wide interface type.
+
+      function Is_Direct_Deep_Call (Subp : Entity_Id) return Boolean;
+      --  Determine if Subp denotes a non-dispatching call to a Deep routine
+
+      function New_Value (From : Node_Id) return Node_Id;
+      --  From is the original Expression. New_Value is equivalent to a call
+      --  to Duplicate_Subexpr with an explicit dereference when From is an
+      --  access parameter.
+
       ------------------------------------
       -- Add_View_Conversion_Invariants --
       ------------------------------------
@@ -3943,6 +4569,9 @@ package body Exp_Ch6 is
       Subp          : Entity_Id;
 
       CW_Interface_Formals_Present : Boolean := False;
+      Defer_Extra_Actuals          : Boolean := False;
+
+      use Deferred_Extra_Formals_Support;
 
    --  Start of processing for Expand_Call_Helper
 
@@ -4029,12 +4658,6 @@ package body Exp_Ch6 is
          end if;
       end if;
 
-      --  Ensure that the called subprogram has all its formals
-
-      if not Is_Frozen (Subp) then
-         Create_Extra_Formals (Subp);
-      end if;
-
       --  Ada 2005 (AI-345): We have a procedure call as a triggering
       --  alternative in an asynchronous select or as an entry call in
       --  a conditional or timed select. Check whether the procedure call
@@ -4080,6 +4703,50 @@ package body Exp_Ch6 is
          end;
       end if;
 
+      --  Ensure that the called subprogram has all its formals; extra formals
+      --  of init procs were added when they were built.
+
+      if not Extra_Formals_Known (Subp) then
+         Create_Extra_Formals (Subp);
+
+         --  If the previous call to Create_Extra_Formals could not add the
+         --  extra formals, then we must defer adding the extra actuals of
+         --  this call until we know the underlying type of all the formals
+         --  and return type of the called subprogram or entry. Deferral of
+         --  extra actuals occurs in two cases:
+         --  1) In the body of internally built dynamic call helpers of
+         --     class-wide preconditions.
+         --  2) In the body of expanded expression functions.
+
+         if not Extra_Formals_Known (Subp) then
+            declare
+               Scop_Id : Entity_Id := Current_Scope;
+
+            begin
+               --  Locate the enclosing subprogram or entry since it is
+               --  required to register this deferred call.
+
+               Scop_Id := Current_Scope;
+               while Present (Scop_Id)
+                 and then Scop_Id /= Standard_Standard
+                 and then not Is_Subprogram_Or_Entry (Scop_Id)
+               loop
+                  Scop_Id := Scope (Scop_Id);
+               end loop;
+
+               pragma Assert (Is_Subprogram_Or_Entry (Scop_Id));
+               pragma Assert (Is_Deferred_Extra_Formals_Entity (Subp));
+               Register_Deferred_Extra_Formals_Call (Call_Node, Scop_Id);
+
+               Defer_Extra_Actuals := True;
+            end;
+         end if;
+      end if;
+
+      pragma Assert (Extra_Formals_Known (Subp)
+        or else Is_Deferred_Extra_Formals_Entity (Subp)
+        or else Is_Unsupported_Extra_Formals_Entity (Subp));
+
       --  If this is a call to a predicate function, try to constant fold it
 
       if Nkind (Call_Node) = N_Function_Call
@@ -4091,56 +4758,39 @@ package body Exp_Ch6 is
       end if;
 
       --  First step, compute extra actuals, corresponding to any Extra_Formals
-      --  present. Note that we do not access Extra_Formals directly, instead
+      --  present. Note that we do not access Extra_Formals directly; instead
       --  we simply note the presence of the extra formals as we process the
       --  regular formals collecting corresponding actuals in Extra_Actuals.
 
-      --  We also generate any required range checks for actuals for in formals
-      --  as we go through the loop, since this is a convenient place to do it.
-      --  (Though it seems that this would be better done in Expand_Actuals???)
+      --  We also generate any required range checks for actuals for in-mode
+      --  formals as we go through the loop, since this is a convenient place
+      --  to do it. (Though it seems that this would be better done in
+      --  Expand_Actuals???)
 
       --  Special case: Thunks must not compute the extra actuals; they must
-      --  just propagate to the target primitive their extra actuals.
+      --  just propagate their extra actuals to the target primitive (this
+      --  propagation is performed by Create_Extra_Actuals).
 
       if Is_Thunk (Current_Scope)
         and then Thunk_Entity (Current_Scope) = Subp
+        and then Extra_Formals_Known (Subp)
         and then Present (Extra_Formals (Subp))
       then
-         pragma Assert (Extra_Formals_Match_OK (Current_Scope, Subp));
+         Create_Extra_Actuals (N);
 
-         declare
-            Target_Formal : Entity_Id;
-            Thunk_Formal  : Entity_Id;
-
-         begin
-            Target_Formal := Extra_Formals (Subp);
-            Thunk_Formal  := Extra_Formals (Current_Scope);
-            while Present (Target_Formal) loop
-               Add_Extra_Actual
-                 (Expr => New_Occurrence_Of (Thunk_Formal, Loc),
-                  EF   => Thunk_Formal);
-
-               Target_Formal := Extra_Formal (Target_Formal);
-               Thunk_Formal  := Extra_Formal (Thunk_Formal);
-            end loop;
-
-            while Is_Non_Empty_List (Extra_Actuals) loop
-               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
-            end loop;
+         --  Mark the call as an expanded build-in-place call; required
+         --  to avoid adding the extra formals twice.
 
-            --  Mark the call as processed build-in-place call; required
-            --  to avoid adding the extra formals twice.
+         if Nkind (Call_Node) = N_Function_Call then
+            Set_Is_Expanded_Build_In_Place_Call (Call_Node);
+         end if;
 
-            if Nkind (Call_Node) = N_Function_Call then
-               Set_Is_Expanded_Build_In_Place_Call (Call_Node);
-            end if;
+         Expand_Actuals (Call_Node, Subp, Post_Call);
 
-            Expand_Actuals (Call_Node, Subp, Post_Call);
-            pragma Assert (Is_Empty_List (Post_Call));
-            pragma Assert (Check_Number_Of_Actuals (Call_Node, Subp));
-            pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
-            return;
-         end;
+         pragma Assert (Is_Empty_List (Post_Call));
+         pragma Assert (Check_Number_Of_Actuals (Call_Node, Subp));
+         pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
+         return;
       end if;
 
       Formal := First_Formal (Subp);
@@ -4158,180 +4808,6 @@ package body Exp_Ch6 is
            CW_Interface_Formals_Present
              or else Is_Class_Wide_Interface_Type (Etype (Formal));
 
-         --  Create possible extra actual for constrained case. Usually, the
-         --  extra actual is of the form actual'constrained, but since this
-         --  attribute is only available for unconstrained records, TRUE is
-         --  expanded if the type of the formal happens to be constrained (for
-         --  instance when this procedure is inherited from an unconstrained
-         --  record to a constrained one) or if the actual has no discriminant
-         --  (its type is constrained). An exception to this is the case of a
-         --  private type without discriminants. In this case we pass FALSE
-         --  because the object has underlying discriminants with defaults.
-
-         if Present (Extra_Constrained (Formal)) then
-            if Is_Mutably_Tagged_Type (Etype (Actual))
-              or else (Is_Private_Type (Etype (Prev))
-                        and then not Has_Discriminants
-                                       (Base_Type (Etype (Prev))))
-            then
-               Add_Extra_Actual
-                 (Expr => New_Occurrence_Of (Standard_False, Loc),
-                  EF   => Extra_Constrained (Formal));
-
-            elsif Is_Constrained (Etype (Formal))
-              or else not Has_Discriminants (Etype (Prev))
-            then
-               Add_Extra_Actual
-                 (Expr => New_Occurrence_Of (Standard_True, Loc),
-                  EF   => Extra_Constrained (Formal));
-
-            --  Do not produce extra actuals for Unchecked_Union parameters.
-            --  Jump directly to the end of the loop.
-
-            elsif Is_Unchecked_Union (Base_Type (Etype (Actual))) then
-               goto Skip_Extra_Actual_Generation;
-
-            else
-               --  If the actual is a type conversion, then the constrained
-               --  test applies to the actual, not the target type.
-
-               declare
-                  Act_Prev : Node_Id;
-
-               begin
-                  --  Test for unchecked conversions as well, which can occur
-                  --  as out parameter actuals on calls to stream procedures.
-
-                  Act_Prev := Prev;
-                  while Nkind (Act_Prev) in N_Type_Conversion
-                                          | N_Unchecked_Type_Conversion
-                  loop
-                     Act_Prev := Expression (Act_Prev);
-                  end loop;
-
-                  --  If the expression is a conversion of a dereference, this
-                  --  is internally generated code that manipulates addresses,
-                  --  e.g. when building interface tables. No check should
-                  --  occur in this case, and the discriminated object is not
-                  --  directly at hand.
-
-                  if not Comes_From_Source (Actual)
-                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
-                    and then Nkind (Act_Prev) = N_Explicit_Dereference
-                  then
-                     Add_Extra_Actual
-                       (Expr => New_Occurrence_Of (Standard_False, Loc),
-                        EF   => Extra_Constrained (Formal));
-
-                  else
-                     Add_Extra_Actual
-                       (Expr =>
-                          Make_Attribute_Reference (Sloc (Prev),
-                            Prefix         =>
-                              Duplicate_Subexpr_No_Checks
-                                (Act_Prev, Name_Req => True),
-                            Attribute_Name => Name_Constrained),
-                        EF   => Extra_Constrained (Formal));
-                  end if;
-               end;
-            end if;
-         end if;
-
-         --  Create possible extra actual for accessibility level
-
-         if Present (Extra_Accessibility (Formal)) then
-            --  Ada 2005 (AI-251): Thunks must propagate the extra actuals of
-            --  accessibility levels.
-
-            if Is_Thunk (Current_Scope) then
-               declare
-                  Parm_Ent : Entity_Id;
-
-               begin
-                  if Is_Controlling_Actual (Actual) then
-
-                     --  Find the corresponding actual of the thunk
-
-                     Parm_Ent := First_Entity (Current_Scope);
-                     for J in 2 .. Param_Count loop
-                        Next_Entity (Parm_Ent);
-                     end loop;
-
-                  --  Handle unchecked conversion of access types generated
-                  --  in thunks (cf. Expand_Interface_Thunk).
-
-                  elsif Is_Access_Type (Etype (Actual))
-                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
-                  then
-                     Parm_Ent := Entity (Expression (Actual));
-
-                  else pragma Assert (Is_Entity_Name (Actual));
-                     Parm_Ent := Entity (Actual);
-                  end if;
-
-                  Add_Extra_Actual
-                    (Expr => Accessibility_Level
-                               (Expr            => Parm_Ent,
-                                Level           => Dynamic_Level,
-                                Allow_Alt_Model => False),
-                     EF   => Extra_Accessibility (Formal));
-               end;
-
-            --  Conditional expressions
-
-            elsif Nkind (Prev) = N_Expression_With_Actions
-              and then Nkind (Original_Node (Prev)) in
-                         N_If_Expression | N_Case_Expression
-            then
-               Add_Cond_Expression_Extra_Actual (Formal);
-
-            --  Internal constant generated to remove side effects (normally
-            --  from the expansion of dispatching calls).
-
-            --  First verify the actual is internal
-
-            elsif not Comes_From_Source (Prev)
-              and then not Is_Rewrite_Substitution (Prev)
-
-              --  Next check that the actual is a constant
-
-              and then Nkind (Prev) = N_Identifier
-              and then Ekind (Entity (Prev)) = E_Constant
-              and then Nkind (Parent (Entity (Prev))) = N_Object_Declaration
-            then
-               --  Generate the accessibility level based on the expression in
-               --  the constant's declaration.
-
-               declare
-                  Ent : Entity_Id := Entity (Prev);
-
-               begin
-                  --  Handle deferred constants
-
-                  if Present (Full_View (Ent)) then
-                     Ent := Full_View (Ent);
-                  end if;
-
-                  Add_Extra_Actual
-                    (Expr => Accessibility_Level
-                               (Expr            => Expression (Parent (Ent)),
-                                Level           => Dynamic_Level,
-                                Allow_Alt_Model => False),
-                     EF   => Extra_Accessibility (Formal));
-               end;
-
-            --  Normal case
-
-            else
-               Add_Extra_Actual
-                 (Expr => Accessibility_Level
-                            (Expr            => Prev,
-                             Level           => Dynamic_Level,
-                             Allow_Alt_Model => False),
-                  EF   => Extra_Accessibility (Formal));
-            end if;
-         end if;
-
          --  Perform the check of 4.6(49) that prevents a null value from being
          --  passed as an actual to an access parameter. Note that the check
          --  is elided in the common cases of passing an access attribute or
@@ -4525,66 +5001,11 @@ package body Exp_Ch6 is
          --  This label is required when skipping extra actual generation for
          --  Unchecked_Union parameters.
 
-         <<Skip_Extra_Actual_Generation>>
-
          Param_Count := Param_Count + 1;
          Next_Actual (Actual);
          Next_Formal (Formal);
       end loop;
 
-      --  If we are calling an Ada 2012 function which needs to have the
-      --  "accessibility level determined by the point of call" (AI05-0234)
-      --  passed in to it, then pass it in.
-
-      if Ekind (Subp) in E_Function | E_Operator | E_Subprogram_Type
-        and then
-          Present (Extra_Accessibility_Of_Result (Ultimate_Alias (Subp)))
-      then
-         declare
-            Extra_Form : Node_Id := Empty;
-            Level      : Node_Id := Empty;
-
-         begin
-            --  Detect cases where the function call has been internally
-            --  generated by examining the original node and return library
-            --  level - taking care to avoid ignoring function calls expanded
-            --  in prefix notation.
-
-            if Nkind (Original_Node (Call_Node)) not in N_Function_Call
-                                                      | N_Selected_Component
-                                                      | N_Indexed_Component
-            then
-               Level := Make_Integer_Literal
-                          (Loc, Scope_Depth (Standard_Standard));
-
-            --  Otherwise get the level normally based on the call node
-
-            else
-               Level := Accessibility_Level
-                          (Expr            => Call_Node,
-                           Level           => Dynamic_Level,
-                           Allow_Alt_Model => False);
-            end if;
-
-            --  It may be possible that we are re-expanding an already
-            --  expanded call when are are dealing with dispatching ???
-
-            if No (Parameter_Associations (Call_Node))
-              or else Nkind (Last (Parameter_Associations (Call_Node)))
-                        /= N_Parameter_Association
-              or else not Is_Accessibility_Actual
-                              (Last (Parameter_Associations (Call_Node)))
-            then
-               Extra_Form := Extra_Accessibility_Of_Result
-                               (Ultimate_Alias (Subp));
-
-               Add_Extra_Actual
-                 (Expr => Level,
-                  EF   => Extra_Form);
-            end if;
-         end;
-      end if;
-
       --  If we are expanding the RHS of an assignment we need to check if tag
       --  propagation is needed. You might expect this processing to be in
       --  Analyze_Assignment but has to be done earlier (bottom-up) because the
@@ -4597,27 +5018,34 @@ package body Exp_Ch6 is
       then
          declare
             Ass : Node_Id := Empty;
+            Par : Node_Id := Parent (Call_Node);
 
          begin
-            if Nkind (Parent (Call_Node)) = N_Assignment_Statement then
-               Ass := Parent (Call_Node);
+            --  Search for the LHS of an enclosing assignment statement to a
+            --  classwide type object (if present) and propagate the tag to
+            --  this function call.
+
+            while Nkind (Par) in N_Case_Expression
+                               | N_Case_Expression_Alternative
+                               | N_Explicit_Dereference
+                               | N_If_Expression
+                               | N_Qualified_Expression
+                               | N_Unchecked_Type_Conversion
+            loop
+               if Nkind (Par) = N_Case_Expression_Alternative then
+                  Par := Parent (Par);
+               end if;
 
-            elsif Nkind (Parent (Call_Node)) = N_Qualified_Expression
-              and then Nkind (Parent (Parent (Call_Node))) =
-                                                  N_Assignment_Statement
-            then
-               Ass := Parent (Parent (Call_Node));
+               exit when not Is_Tag_Indeterminate (Par);
 
-            elsif Nkind (Parent (Call_Node)) = N_Explicit_Dereference
-              and then Nkind (Parent (Parent (Call_Node))) =
-                                                  N_Assignment_Statement
-            then
-               Ass := Parent (Parent (Call_Node));
-            end if;
+               Par := Parent (Par);
+            end loop;
 
-            if Present (Ass)
-              and then Is_Class_Wide_Type (Etype (Name (Ass)))
+            if Nkind (Par) = N_Assignment_Statement
+              and then Is_Class_Wide_Type (Etype (Name (Par)))
             then
+               Ass := Par;
+
                --  Move the error messages below to sem???
 
                if Is_Access_Type (Etype (Call_Node)) then
@@ -4630,6 +5058,12 @@ package body Exp_Ch6 is
                          Call_Node, Root_Type (Etype (Name (Ass))));
                   else
                      Propagate_Tag (Name (Ass), Call_Node);
+
+                     --  Remember that the tag has been propagated to avoid
+                     --  propagating it again, as part of the (bottom-up)
+                     --  analysis of the enclosing assignment.
+
+                     Set_Tag_Propagated (Name (Ass));
                   end if;
 
                elsif Etype (Call_Node) /= Root_Type (Etype (Name (Ass))) then
@@ -4640,6 +5074,12 @@ package body Exp_Ch6 is
 
                else
                   Propagate_Tag (Name (Ass), Call_Node);
+
+                  --  Remember that the tag has been propagated to avoid
+                  --  propagating it again, as part of the (bottom-up)
+                  --  analysis of the enclosing assignment.
+
+                  Set_Tag_Propagated (Name (Ass));
                end if;
 
                --  The call will be rewritten as a dispatching call, and
@@ -4778,38 +5218,12 @@ package body Exp_Ch6 is
       then
          null;
 
-      --  During that loop we gathered the extra actuals (the ones that
-      --  correspond to Extra_Formals), so now they can be appended.
-
-      elsif Is_Non_Empty_List (Extra_Actuals) then
-         declare
-            Num_Extra_Actuals : constant Nat := List_Length (Extra_Actuals);
-
-         begin
-            while Is_Non_Empty_List (Extra_Actuals) loop
-               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
-            end loop;
-
-            --  Add dummy extra BIP actuals if we are calling a function that
-            --  inherited the BIP extra actuals but does not require them.
-
-            if Nkind (Call_Node) = N_Function_Call
-              and then Is_Function_Call_With_BIP_Formals (Call_Node)
-              and then not Is_Build_In_Place_Function_Call (Call_Node)
-            then
-               Add_Dummy_Build_In_Place_Actuals (Subp,
-                 Num_Added_Extra_Actuals => Num_Extra_Actuals);
-            end if;
-         end;
-
-      --  Add dummy extra BIP actuals if we are calling a function that
-      --  inherited the BIP extra actuals but does not require them.
+      elsif not Defer_Extra_Actuals then
+         Create_Extra_Formals (Subp);
 
-      elsif Nkind (Call_Node) = N_Function_Call
-        and then Is_Function_Call_With_BIP_Formals (Call_Node)
-        and then not Is_Build_In_Place_Function_Call (Call_Node)
-      then
-         Add_Dummy_Build_In_Place_Actuals (Subp);
+         if Extra_Formals_Known (Subp) then
+            Create_Extra_Actuals (N);
+         end if;
       end if;
 
       --  At this point we have all the actuals, so this is the point at which
@@ -5227,6 +5641,10 @@ package body Exp_Ch6 is
          --  also Build_Renamed_Body) cannot be expanded here because this may
          --  give rise to order-of-elaboration issues for the types of the
          --  parameters of the subprogram, if any.
+         --
+         --  Expand_Inlined_Call procedure does not support the frontend
+         --  inlining of calls that return unconstrained types used as actuals
+         --  or in return statements.
 
          elsif Present (Unit_Declaration_Node (Subp))
            and then Nkind (Unit_Declaration_Node (Subp)) =
@@ -5235,6 +5653,8 @@ package body Exp_Ch6 is
            and then
              Nkind (Body_To_Inline (Unit_Declaration_Node (Subp))) not in
                                                                        N_Entity
+           and then Nkind (Parent (N)) /= N_Function_Call
+           and then Nkind (Parent (N)) /= N_Simple_Return_Statement
          then
             Expand_Inlined_Call (Call_Node, Subp, Orig_Subp);
 
@@ -7159,6 +7579,16 @@ package body Exp_Ch6 is
       then
          Apply_CW_Accessibility_Check (Exp, Scope_Id);
 
+      --  Check that result's access discrims (if any) do not designate
+      --  entities that the function result could outlive. See preceding
+      --  comment about extended return statements and thunks.
+
+      elsif Has_Anonymous_Access_Discriminant (Exp_Typ)
+        and then not Comes_From_Extended_Return_Statement (N)
+        and then not Is_Thunk (Scope_Id)
+      then
+         Apply_Access_Discrims_Accessibility_Check (Exp, Scope_Id);
+
       --  Ada 2012 (AI05-0073): If the result subtype of the function is
       --  defined by an access_definition designating a specific tagged
       --  type T, a check is made that the result value is null or the tag
@@ -7669,7 +8099,7 @@ package body Exp_Ch6 is
               Get_Class_Wide_Pragma (Id, Pragma_Precondition);
 
          begin
-            if No (Prag) or else Is_Ignored (Prag) then
+            if No (Prag) or else Is_Ignored_In_Codegen (Prag) then
                return;
             end if;
 
@@ -8557,6 +8987,8 @@ package body Exp_Ch6 is
       Rewrite (Allocator, New_Occurrence_Of (Return_Obj_Access, Loc));
 
       Analyze_And_Resolve (Allocator, Acc_Type);
+
+      pragma Assert (Returns_By_Ref (Function_Id));
       pragma Assert (Check_Number_Of_Actuals (Func_Call, Function_Id));
       pragma Assert (Check_BIP_Actuals (Func_Call, Function_Id));
    end Make_Build_In_Place_Call_In_Allocator;
@@ -8662,6 +9094,7 @@ package body Exp_Ch6 is
 
          Set_Is_Expanded_Build_In_Place_Call (Func_Call);
 
+         pragma Assert (Returns_By_Ref (Function_Id));
          pragma Assert (Check_Number_Of_Actuals (Func_Call, Function_Id));
          pragma Assert (Check_BIP_Actuals (Func_Call, Function_Id));
       end if;
@@ -8763,6 +9196,8 @@ package body Exp_Ch6 is
       Insert_After_And_Analyze (Ptr_Typ_Decl, Obj_Decl);
 
       Rewrite (Assign, Make_Null_Statement (Loc));
+
+      pragma Assert (Returns_By_Ref (Func_Id));
       pragma Assert (Check_Number_Of_Actuals (Func_Call, Func_Id));
       pragma Assert (Check_BIP_Actuals (Func_Call, Func_Id));
    end Make_Build_In_Place_Call_In_Assignment;
@@ -9187,6 +9622,7 @@ package body Exp_Ch6 is
          end if;
       end if;
 
+      pragma Assert (Returns_By_Ref (Function_Id));
       pragma Assert (Check_Number_Of_Actuals (Func_Call, Function_Id));
       pragma Assert (Check_BIP_Actuals (Func_Call, Function_Id));
    end Make_Build_In_Place_Call_In_Object_Declaration;
@@ -9824,35 +10260,16 @@ package body Exp_Ch6 is
             =>
                declare
                   Call_Node : Node_Id renames Nod;
-                  Subp      : Entity_Id;
+                  Subp      : constant Entity_Id := Get_Called_Entity (Nod);
 
                begin
-                  --  Call using access to subprogram with explicit dereference
-
-                  if Nkind (Name (Call_Node)) = N_Explicit_Dereference then
-                     Subp := Etype (Name (Call_Node));
-
-                  --  Prefix notation calls
-
-                  elsif Nkind (Name (Call_Node)) = N_Selected_Component then
-                     Subp := Entity (Selector_Name (Name (Call_Node)));
-
-                  --  Call to member of entry family, where Name is an indexed
-                  --  component, with the prefix being a selected component
-                  --  giving the task and entry family name, and the index
-                  --  being the entry index.
-
-                  elsif Nkind (Name (Call_Node)) = N_Indexed_Component then
-                     Subp :=
-                       Entity (Selector_Name (Prefix (Name (Call_Node))));
+                  pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
 
-                  --  Normal case
+                  --  Build-in-place function calls return their result by
+                  --  reference.
 
-                  else
-                     Subp := Entity (Name (Call_Node));
-                  end if;
-
-                  pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
+                  pragma Assert (not Is_Build_In_Place_Function (Subp)
+                    or else Returns_By_Ref (Subp));
                end;
 
             --  Skip generic bodies
diff --git a/gcc/ada/exp_ch6.ads b/gcc/ada/exp_ch6.ads
index 118d994..5919627 100644
--- a/gcc/ada/exp_ch6.ads
+++ b/gcc/ada/exp_ch6.ads
@@ -101,7 +101,20 @@ package Exp_Ch6 is
    --  Adds Extra_Actual as a named parameter association for the formal
    --  Extra_Formal in Subprogram_Call.
 
+   procedure Create_Extra_Actuals (Call_Node : Node_Id);
+   --  Create the extra actuals of the given call and add them to its
+   --  actual parameters list.
+
+   procedure Apply_Access_Discrims_Accessibility_Check
+     (Exp : Node_Id; Func : Entity_Id);
+   --  Exp is an expression being returned from a function Func.
+   --  If the result type of the function has access discriminants, insert
+   --  checks that the accessibility level of each entity designated by an
+   --  access discriminant of the result is not deeper than the level of the
+   --  master of the call.
+
    procedure Apply_CW_Accessibility_Check (Exp : Node_Id; Func : Entity_Id);
+   --  Exp is an expression being returned from a function Func.
    --  Ada 2005 (AI95-344): If the result type is class-wide, insert a check
    --  that the level of the return expression's underlying type is not deeper
    --  than the level of the master enclosing the function. Always generate the
diff --git a/gcc/ada/exp_ch7.adb b/gcc/ada/exp_ch7.adb
index e4daf4b..dd864b7 100644
--- a/gcc/ada/exp_ch7.adb
+++ b/gcc/ada/exp_ch7.adb
@@ -59,6 +59,7 @@ with Sinfo.Nodes;    use Sinfo.Nodes;
 with Sinfo.Utils;    use Sinfo.Utils;
 with Sem;            use Sem;
 with Sem_Aux;        use Sem_Aux;
+with Sem_Ch6;        use Sem_Ch6;
 with Sem_Ch7;        use Sem_Ch7;
 with Sem_Ch8;        use Sem_Ch8;
 with Sem_Res;        use Sem_Res;
@@ -2331,6 +2332,8 @@ package body Exp_Ch7 is
 
                Ensure_Freeze_Node (Fin_Id);
                Insert_After (Fin_Spec, Freeze_Node (Fin_Id));
+               Mutate_Ekind (Fin_Id, E_Procedure);
+               Freeze_Extra_Formals (Fin_Id);
                Set_Is_Frozen (Fin_Id);
 
                Append_To (Stmts, Fin_Body);
@@ -3586,18 +3589,22 @@ package body Exp_Ch7 is
 
    procedure Build_Record_Deep_Procs (Typ : Entity_Id) is
    begin
-      Set_TSS (Typ,
-        Make_Deep_Proc
-          (Prim  => Initialize_Case,
-           Typ   => Typ,
-           Stmts => Make_Deep_Record_Body (Initialize_Case, Typ)));
-
-      if not Is_Inherently_Limited_Type (Typ) then
-         Set_TSS (Typ,
-           Make_Deep_Proc
-             (Prim  => Adjust_Case,
-              Typ   => Typ,
-              Stmts => Make_Deep_Record_Body (Adjust_Case, Typ)));
+      if Has_Controlled_Component (Typ) then
+         Set_TSS
+           (Typ,
+            Make_Deep_Proc
+              (Prim  => Initialize_Case,
+               Typ   => Typ,
+               Stmts => Make_Deep_Record_Body (Initialize_Case, Typ)));
+
+         if not Is_Inherently_Limited_Type (Typ) then
+            Set_TSS
+              (Typ,
+               Make_Deep_Proc
+                 (Prim  => Adjust_Case,
+                  Typ   => Typ,
+                  Stmts => Make_Deep_Record_Body (Adjust_Case, Typ)));
+         end if;
       end if;
 
       --  Do not generate Deep_Finalize and Finalize_Address if finalization is
@@ -5598,7 +5605,10 @@ package body Exp_Ch7 is
 
       --  Deal with untagged derivation of private views
 
-      if Present (Utyp) and then Is_Untagged_Derivation (Typ) then
+      if Present (Utyp)
+        and then Is_Untagged_Derivation (Typ)
+        and then Is_Implicit_Full_View (Utyp)
+      then
          Utyp := Underlying_Type (Root_Type (Base_Type (Typ)));
          Ref  := Unchecked_Convert_To (Utyp, Ref);
          Set_Assignment_OK (Ref);
@@ -6635,6 +6645,16 @@ package body Exp_Ch7 is
       --       Raised : Boolean := False;
       --
       --    begin
+      --       begin
+      --          <Destructor_Proc> (V);  --  If applicable
+      --       exception
+      --          when others =>
+      --             if not Raised then
+      --                Raised := True;
+      --                Save_Occurrence (E, Get_Current_Excep.all.all);
+      --             end if;
+      --       end;
+      --
       --       if F then
       --          begin
       --             Finalize (V);  --  If applicable
@@ -6690,6 +6710,8 @@ package body Exp_Ch7 is
       --
       --       begin
       --          Deep_Finalize (V._parent, False);  --  If applicable
+      --  or
+      --          Deep_Finalize (Parent_Type (V), False); -- Untagged case
       --       exception
       --          when Id : others =>
       --             if not Raised then
@@ -7094,7 +7116,7 @@ package body Exp_Ch7 is
          --  or the type is not controlled.
 
          if Is_Empty_List (Bod_Stmts) then
-            Append_To (Bod_Stmts, Make_Null_Statement (Loc));
+            Append_New_To (Bod_Stmts, Make_Null_Statement (Loc));
 
             return Bod_Stmts;
 
@@ -7581,9 +7603,13 @@ package body Exp_Ch7 is
 
          --    Deep_Finalize (Obj._parent, False);
 
-         if Is_Tagged_Type (Typ) and then Is_Derived_Type (Typ) then
+         if Is_Derived_Type (Typ) then
             declare
-               Par_Typ  : constant Entity_Id := Parent_Field_Type (Typ);
+               Tagd     : constant Boolean := Is_Tagged_Type (Typ);
+               Par_Typ  : constant Entity_Id :=
+                 (if Tagd
+                  then Parent_Field_Type (Typ)
+                  else Etype (Base_Type (Typ)));
                Call     : Node_Id;
                Fin_Stmt : Node_Id;
 
@@ -7592,10 +7618,16 @@ package body Exp_Ch7 is
                   Call :=
                     Make_Final_Call
                       (Obj_Ref   =>
-                         Make_Selected_Component (Loc,
-                           Prefix        => Make_Identifier (Loc, Name_V),
-                           Selector_Name =>
-                             Make_Identifier (Loc, Name_uParent)),
+                         (if Tagd
+                          then
+                            Make_Selected_Component
+                              (Loc,
+                               Prefix        => Make_Identifier (Loc, Name_V),
+                               Selector_Name =>
+                                 Make_Identifier (Loc, Name_uParent))
+                          else
+                            Convert_To
+                              (Par_Typ, Make_Identifier (Loc, Name_V))),
                        Typ       => Par_Typ,
                        Skip_Self => True);
 
@@ -7611,6 +7643,21 @@ package body Exp_Ch7 is
                   --               Get_Current_Excep.all.all);
                   --          end if;
                   --    end;
+                  --
+                  --  in the tagged case. In the untagged case, which arises
+                  --  with the Destructor aspect, generate:
+                  --
+                  --    begin
+                  --       Deep_Finalize (Parent_Type (V), False);
+
+                  --    exception
+                  --       when Id : others =>
+                  --          if not Raised then
+                  --             Raised := True;
+                  --             Save_Occurrence (E,
+                  --               Get_Current_Excep.all.all);
+                  --          end if;
+                  --    end;
 
                   if Present (Call) then
                      Fin_Stmt := Call;
@@ -7656,7 +7703,7 @@ package body Exp_Ch7 is
                         --  than before, the extension components. That might
                         --  be more intuitive (as discussed in preceding
                         --  comment), but it is not required.
-                        Prepend_To (Bod_Stmts, Fin_Stmt);
+                        Prepend_New_To (Bod_Stmts, Fin_Stmt);
                      end if;
                   end if;
                end if;
@@ -7707,12 +7754,58 @@ package body Exp_Ch7 is
                                  (Finalizer_Data))));
                   end if;
 
-                  Prepend_To (Bod_Stmts,
+                  Prepend_New_To (Bod_Stmts,
                     Make_If_Statement (Loc,
                       Condition       => Make_Identifier (Loc, Name_F),
                       Then_Statements => New_List (Fin_Stmt)));
                end if;
             end;
+
+            declare
+               ASN : constant Opt_N_Aspect_Specification_Id :=
+                 Get_Rep_Item (Typ, Name_Destructor, False);
+
+               Stmt : Node_Id;
+               Proc : Entity_Id;
+            begin
+               if Present (ASN) then
+                  --  Generate:
+                  --    begin
+                  --       <Destructor_Proc> (V);
+
+                  --    exception
+                  --       when others =>
+                  --          if not Raised then
+                  --             Raised := True;
+                  --             Save_Occurrence (E,
+                  --               Get_Current_Excep.all.all);
+                  --          end if;
+                  --    end;
+
+                  Proc := Entity (Expression (ASN));
+                  Stmt :=
+                    Make_Procedure_Call_Statement
+                      (Loc,
+                       Name                   => New_Occurrence_Of (Proc, Loc),
+                       Parameter_Associations =>
+                         New_List (Make_Identifier (Loc, Name_V)));
+                  if Exceptions_OK then
+                     Stmt :=
+                       Make_Block_Statement
+                         (Loc,
+                          Handled_Statement_Sequence =>
+                            Make_Handled_Sequence_Of_Statements
+                              (Loc,
+                               Statements         => New_List (Stmt),
+                               Exception_Handlers =>
+                                 New_List
+                                   (Build_Exception_Handler
+                                      (Finalizer_Data))));
+                  end if;
+
+                  Prepend_New_To (Bod_Stmts, Stmt);
+               end if;
+            end;
          end if;
 
          --  At this point either all finalization statements have been
@@ -7906,7 +7999,7 @@ package body Exp_Ch7 is
       if Is_Untagged_Derivation (Typ) then
          if Is_Protected_Type (Typ) then
             Utyp := Corresponding_Record_Type (Root_Type (Base_Type (Typ)));
-         else
+         elsif Is_Implicit_Full_View (Utyp) then
             Utyp := Underlying_Type (Root_Type (Base_Type (Typ)));
 
             if Is_Protected_Type (Utyp) then
@@ -7966,7 +8059,7 @@ package body Exp_Ch7 is
          return Empty;
 
       elsif Skip_Self then
-         if Has_Controlled_Component (Utyp) then
+         if Has_Controlled_Component (Utyp) or else Has_Destructor (Utyp) then
             if Is_Tagged_Type (Utyp) then
                Fin_Id := Find_Optional_Prim_Op (Utyp, TSS_Deep_Finalize);
             else
@@ -7979,6 +8072,7 @@ package body Exp_Ch7 is
       elsif Is_Class_Wide_Type (Typ)
         or else Is_Interface (Typ)
         or else Has_Controlled_Component (Utyp)
+        or else Has_Destructor (Utyp)
       then
          if Is_Tagged_Type (Utyp) then
             Fin_Id := Find_Optional_Prim_Op (Utyp, TSS_Deep_Finalize);
@@ -8476,7 +8570,10 @@ package body Exp_Ch7 is
 
       --  Deal with untagged derivation of private views
 
-      if Is_Untagged_Derivation (Typ) and then not Is_Conc then
+      if Is_Untagged_Derivation (Typ)
+        and then not Is_Conc
+        and then Is_Implicit_Full_View (Utyp)
+      then
          Utyp := Underlying_Type (Root_Type (Base_Type (Typ)));
          Ref  := Unchecked_Convert_To (Utyp, Ref);
 
@@ -9444,9 +9541,16 @@ package body Exp_Ch7 is
    procedure Wrap_Transient_Expression (N : Node_Id) is
       Loc  : constant Source_Ptr := Sloc (N);
       Expr : Node_Id             := Relocate_Node (N);
-      Temp : constant Entity_Id  := Make_Temporary (Loc, 'E', N);
       Typ  : constant Entity_Id  := Etype (N);
 
+      Temp : constant Entity_Id  := Make_Temporary (Loc, 'E',
+                                      Related_Node => Expr);
+      --  We link the temporary with its relocated expression to facilitate
+      --  locating the expression in the expanded code; this simplifies the
+      --  implementation of the function that searchs in the expanded code
+      --  for a function call that has been wrapped in a transient block
+      --  (see Get_Relocated_Function_Call).
+
    begin
       --  Generate:
 
diff --git a/gcc/ada/exp_ch9.adb b/gcc/ada/exp_ch9.adb
index 9cfc6b5..c979cf6 100644
--- a/gcc/ada/exp_ch9.adb
+++ b/gcc/ada/exp_ch9.adb
@@ -4691,11 +4691,11 @@ package body Exp_Ch9 is
 
       --  The availability of the activation chain entity does not ensure
       --  that we have tasks to activate because it may have been declared
-      --  by the frontend to pass a required extra formal to a build-in-place
+      --  by the front end to pass a required extra formal to a build-in-place
       --  subprogram call. If we are within the scope of a protected type and
       --  pragma Detect_Blocking is active we can assume that no tasks will be
       --  activated; if tasks are created in a protected object and this pragma
-      --  is active then the frontend emits a warning and Program_Error is
+      --  is active then the front end emits a warning and Program_Error is
       --  raised at runtime.
 
       elsif Detect_Blocking and then Within_Protected_Type (Current_Scope) then
@@ -8094,12 +8094,18 @@ package body Exp_Ch9 is
    --  access type. Finally the Entry_Component of each formal is set to
    --  reference the corresponding record component.
 
-   procedure Expand_N_Entry_Declaration (N : Node_Id) is
+   procedure Expand_N_Entry_Declaration
+     (N            : Node_Id;
+      Was_Deferred : Boolean := False)
+   is
+      use Deferred_Extra_Formals_Support;
+
       Loc        : constant Source_Ptr := Sloc (N);
       Entry_Ent  : constant Entity_Id  := Defining_Identifier (N);
       Components : List_Id;
       Formal     : Node_Id;
       Ftype      : Entity_Id;
+      First_Decl : Node_Id;
       Last_Decl  : Node_Id;
       Component  : Entity_Id;
       Ctype      : Entity_Id;
@@ -8108,7 +8114,21 @@ package body Exp_Ch9 is
       Acc_Ent    : Entity_Id;
 
    begin
+      --  No action if the addition of the extra formals was deferred,
+      --  since it means that the underlying type of some formal is not
+      --  available, and hence we cannot build the record type that will
+      --  hold all the parameter values.
+
+      if Present (First_Formal (Entry_Ent))
+        and then not Extra_Formals_Known (Entry_Ent)
+        and then not Is_Unsupported_Extra_Formals_Entity (Entry_Ent)
+      then
+         pragma Assert (Is_Deferred_Extra_Formals_Entity (Entry_Ent));
+         return;
+      end if;
+
       Formal := First_Formal (Entry_Ent);
+      First_Decl := N;
       Last_Decl := N;
 
       --  Most processing is done only if parameters are present
@@ -8184,6 +8204,24 @@ package body Exp_Ch9 is
                  Subtype_Indication => New_Occurrence_Of (Rec_Ent, Loc)));
 
          Insert_After (Last_Decl, Decl);
+         Last_Decl := Decl;
+
+         --  Analyze all the inserted declarations. This is required when
+         --  the entry has formals and the addition of its extra formals
+         --  was deferred; otherwise their analysis will be performed as
+         --  as part of the regular flow of the front end at the end of
+         --  analysis of the enclosing task/protected type declaration.
+
+         if Was_Deferred then
+            Push_Scope (Scope (Entry_Ent));
+
+            while First_Decl /= Last_Decl loop
+               Next (First_Decl);
+               Analyze (First_Decl);
+            end loop;
+
+            End_Scope;
+         end if;
       end if;
    end Expand_N_Entry_Declaration;
 
diff --git a/gcc/ada/exp_ch9.ads b/gcc/ada/exp_ch9.ads
index cae6cb3..6811141 100644
--- a/gcc/ada/exp_ch9.ads
+++ b/gcc/ada/exp_ch9.ads
@@ -227,9 +227,16 @@ package Exp_Ch9 is
    procedure Expand_N_Delay_Until_Statement      (N : Node_Id);
    procedure Expand_N_Entry_Body                 (N : Node_Id);
    procedure Expand_N_Entry_Call_Statement       (N : Node_Id);
-   procedure Expand_N_Entry_Declaration          (N : Node_Id);
    procedure Expand_N_Protected_Body             (N : Node_Id);
 
+   procedure Expand_N_Entry_Declaration
+     (N            : Node_Id;
+      Was_Deferred : Boolean := False);
+   --  Expands an entry declaration, building a record type to hold all the
+   --  parameter values. Was_Deferred is True when this expansion was deferred
+   --  because the underlying type of some formal was not available to build
+   --  the record.
+
    procedure Expand_N_Protected_Type_Declaration (N : Node_Id);
    --  Expands protected type declarations. This results, among other things,
    --  in the declaration of a record type for the representation of protected
diff --git a/gcc/ada/exp_disp.adb b/gcc/ada/exp_disp.adb
index 080a2e1..1c09e20 100644
--- a/gcc/ada/exp_disp.adb
+++ b/gcc/ada/exp_disp.adb
@@ -801,7 +801,7 @@ package body Exp_Disp is
 
         --  No action needed if the dispatching call has been already expanded
 
-        or else Is_Expanded_Dispatching_Call (Name (Call_Node))
+        or else Is_Expanded_Dispatching_Call (Call_Node)
       then
          return;
       end if;
@@ -926,6 +926,8 @@ package body Exp_Disp is
          New_Formal  : Entity_Id;
          Last_Formal : Entity_Id := Empty;
 
+         use Deferred_Extra_Formals_Support;
+
       begin
          if Present (Old_Formal) then
             New_Formal := New_Copy (Old_Formal);
@@ -962,51 +964,21 @@ package body Exp_Disp is
          end if;
 
          --  Now that the explicit formals have been duplicated, any extra
-         --  formals needed by the subprogram must be duplicated; we know
-         --  that extra formals are available because they were added when
-         --  the tagged type was frozen (see Expand_Freeze_Record_Type).
+         --  formals needed by the subprogram must be added; we know that
+         --  extra formals are available because they were added when the
+         --  tagged type was frozen (see Expand_Freeze_Record_Type).
 
          pragma Assert (Is_Frozen (Typ));
 
-         --  Warning: The addition of the extra formals cannot be performed
-         --  here invoking Create_Extra_Formals since we must ensure that all
-         --  the extra formals of the pointer type and the target subprogram
-         --  match (and for functions that return a tagged type the profile of
-         --  the built subprogram type always returns a class-wide type, which
-         --  may affect the addition of some extra formals).
-
-         if Present (Last_Formal)
-           and then Present (Extra_Formal (Last_Formal))
-         then
-            Old_Formal := Extra_Formal (Last_Formal);
-            New_Formal := New_Copy (Old_Formal);
-            Set_Scope (New_Formal, Subp_Typ);
-
-            Set_Extra_Formal (Last_Formal, New_Formal);
-            Set_Extra_Formals (Subp_Typ, New_Formal);
-
-            if Ekind (Subp) = E_Function
-              and then Present (Extra_Accessibility_Of_Result (Subp))
-              and then Extra_Accessibility_Of_Result (Subp) = Old_Formal
-            then
-               Set_Extra_Accessibility_Of_Result (Subp_Typ, New_Formal);
-            end if;
-
-            Old_Formal := Extra_Formal (Old_Formal);
-            while Present (Old_Formal) loop
-               Set_Extra_Formal (New_Formal, New_Copy (Old_Formal));
-               New_Formal := Extra_Formal (New_Formal);
-               Set_Scope (New_Formal, Subp_Typ);
+         if Extra_Formals_Known (Subp) then
+            Create_Extra_Formals (Subp_Typ);
 
-               if Ekind (Subp) = E_Function
-                 and then Present (Extra_Accessibility_Of_Result (Subp))
-                 and then Extra_Accessibility_Of_Result (Subp) = Old_Formal
-               then
-                  Set_Extra_Accessibility_Of_Result (Subp_Typ, New_Formal);
-               end if;
+         --  Extra formals were previously deferred
 
-               Old_Formal := Extra_Formal (Old_Formal);
-            end loop;
+         else
+            pragma Assert (Is_Deferred_Extra_Formals_Entity (Subp));
+            Register_Deferred_Extra_Formals_Entity (Subp_Typ);
+            Register_Deferred_Extra_Formals_Call (Call_Node, Current_Scope);
          end if;
       end;
 
@@ -1237,6 +1209,8 @@ package body Exp_Disp is
       --  the generation of spurious warnings under ZFP run-time.
 
       Analyze_And_Resolve (Call_Node, Call_Typ, Suppress => All_Checks);
+
+      Set_Is_Expanded_Dispatching_Call (Call_Node);
    end Expand_Dispatching_Call;
 
    ---------------------------------
@@ -2378,17 +2352,6 @@ package body Exp_Disp is
         and then not Restriction_Active (No_Dispatching_Calls);
    end Has_DT;
 
-   ----------------------------------
-   -- Is_Expanded_Dispatching_Call --
-   ----------------------------------
-
-   function Is_Expanded_Dispatching_Call (N : Node_Id) return Boolean is
-   begin
-      return Nkind (N) in N_Subprogram_Call
-        and then Nkind (Name (N)) = N_Explicit_Dereference
-        and then Is_Dispatch_Table_Entity (Etype (Name (N)));
-   end Is_Expanded_Dispatching_Call;
-
    -------------------------------------
    -- Is_Predefined_Dispatching_Alias --
    -------------------------------------
@@ -4630,8 +4593,7 @@ package body Exp_Disp is
       Name_TSD          : constant Name_Id :=
                             New_External_Name (Tname, 'B', Suffix_Index => -1);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       AI                 : Elmt_Id;
@@ -6563,7 +6525,7 @@ package body Exp_Disp is
       Register_CG_Node (Typ);
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
 
       return Result;
    end Make_DT;
@@ -8345,13 +8307,15 @@ package body Exp_Disp is
                         Defining_Unit_Name       => IP,
                         Parameter_Specifications => Parms)));
 
-               Set_Init_Proc (Typ, IP);
-               Set_Is_Imported    (IP);
-               Set_Is_Constructor (IP);
-               Set_Interface_Name (IP, Interface_Name (E));
-               Set_Convention     (IP, Convention_CPP);
-               Set_Is_Public      (IP);
-               Set_Has_Completion (IP);
+               Set_Init_Proc   (Typ, IP);
+               Set_Is_Imported      (IP);
+               Set_Is_Constructor   (IP);
+               Set_Interface_Name   (IP, Interface_Name (E));
+               Set_Convention       (IP, Convention_CPP);
+               Set_Is_Public        (IP);
+               Set_Has_Completion   (IP);
+               Mutate_Ekind         (IP, E_Procedure);
+               Freeze_Extra_Formals (IP);
 
             --  Case 2: Constructor of a tagged type
 
@@ -8484,6 +8448,8 @@ package body Exp_Disp is
 
                   Discard_Node (IP_Body);
                   Set_Init_Proc (Typ, IP);
+                  Mutate_Ekind (IP, E_Procedure);
+                  Freeze_Extra_Formals (IP);
                end;
             end if;
 
@@ -8549,6 +8515,8 @@ package body Exp_Disp is
 
             Discard_Node (IP_Body);
             Set_Init_Proc (Typ, IP);
+            Mutate_Ekind (IP, E_Procedure);
+            Freeze_Extra_Formals (IP);
          end;
       end if;
 
diff --git a/gcc/ada/exp_disp.ads b/gcc/ada/exp_disp.ads
index 3cba8ca..76f5923 100644
--- a/gcc/ada/exp_disp.ads
+++ b/gcc/ada/exp_disp.ads
@@ -236,9 +236,6 @@ package Exp_Disp is
    function Has_CPP_Constructors (Typ : Entity_Id) return Boolean;
    --  Returns true if the type has CPP constructors
 
-   function Is_Expanded_Dispatching_Call (N : Node_Id) return Boolean;
-   --  Returns true if N is the expanded code of a dispatching call
-
    function Make_DT (Typ : Entity_Id) return List_Id;
    --  Expand the declarations for the Dispatch Table of Typ
 
diff --git a/gcc/ada/exp_prag.adb b/gcc/ada/exp_prag.adb
index 340f2dc..7ec963a 100644
--- a/gcc/ada/exp_prag.adb
+++ b/gcc/ada/exp_prag.adb
@@ -134,7 +134,9 @@ package body Exp_Prag is
       --      Analyze_xxx_In_Decl_Part). The second part of the analysis will
       --      not happen if the pragma is rewritten.
 
-      if Assertion_Expression_Pragma (Prag_Id) and then Is_Ignored (N) then
+      if Assertion_Expression_Pragma (Prag_Id)
+        and then Is_Ignored_In_Codegen (N)
+      then
          return;
 
       --  Rewrite the pragma into a null statement when it is ignored using
@@ -143,7 +145,7 @@ package body Exp_Prag is
 
       elsif Should_Ignore_Pragma_Sem (N)
         or else (Prag_Id = Pragma_Default_Scalar_Storage_Order
-                  and then Ignore_Rep_Clauses)
+                 and then Ignore_Rep_Clauses)
       then
          Rewrite (N, Make_Null_Statement (Sloc (N)));
          return;
@@ -480,7 +482,7 @@ package body Exp_Prag is
    begin
       --  Nothing to do if pragma is ignored
 
-      if Is_Ignored (N) then
+      if Is_Ignored_In_Codegen (N) then
          return;
       end if;
 
@@ -1837,7 +1839,7 @@ package body Exp_Prag is
       --  Do nothing if pragma is not enabled. If pragma is disabled, it has
       --  already been rewritten as a Null statement.
 
-      if Is_Ignored (CCs) then
+      if Is_Ignored_In_Codegen (CCs) then
          return;
 
       --  Guard against malformed contract cases
@@ -2538,7 +2540,7 @@ package body Exp_Prag is
       --  Nothing to do when the pragma is ignored because its semantics are
       --  suppressed.
 
-      if Is_Ignored (IC_Prag) then
+      if Is_Ignored_In_Codegen (IC_Prag) then
          return;
 
       --  Nothing to do when the pragma or its argument are illegal because
@@ -3001,7 +3003,7 @@ package body Exp_Prag is
       --  Also do this in CodePeer mode, because the expanded code is too
       --  complicated for CodePeer to analyse.
 
-      if Is_Ignored (N)
+      if Is_Ignored_In_Codegen (N)
         or else Chars (Last_Var) = Name_Structural
         or else CodePeer_Mode
       then
@@ -3391,7 +3393,7 @@ package body Exp_Prag is
       --  Do nothing if pragma is not present or is disabled.
       --  Also ignore structural variants for execution.
 
-      if Is_Ignored (Prag)
+      if Is_Ignored_In_Codegen (Prag)
         or else Chars (Nlists.Last (Choices (Last_Variant))) = Name_Structural
       then
          return;
diff --git a/gcc/ada/exp_put_image.adb b/gcc/ada/exp_put_image.adb
index 40b2a65..ce3390b 100644
--- a/gcc/ada/exp_put_image.adb
+++ b/gcc/ada/exp_put_image.adb
@@ -77,8 +77,28 @@ package body Exp_Put_Image is
    --  reference). The Loc parameter is used as the Sloc of the created entity.
 
    function Put_Image_Base_Type (E : Entity_Id) return Entity_Id;
-   --  Returns the base type, except for an array type whose whose first
-   --  subtype is constrained, in which case it returns the first subtype.
+   --  For an array type whose whose first subtype is constrained, return
+   --  the first subtype. For the internal representation type corresponding
+   --  to a mutably tagged type, return the mutably tagged type. Otherwise,
+   --  return the base type. Similar to Exp_Strm.Stream_Base_Type.
+
+   procedure Put_Specific_Type_Name_Qualifier
+              (Loc               : Source_Ptr;
+               Stms              : List_Id;
+               Tagged_Obj        : Node_Id;
+               Buffer_Name       : Node_Id;
+               Is_Interface_Type : Boolean);
+   --  Append to the given statement list calls to add into the
+   --  buffer the name of the given object's tag and then a "'".
+
+   function Put_String_Exp_To_Buffer
+     (Loc         : Source_Ptr;
+      String_Exp  : Node_Id;
+      Buffer_Name : Node_Id;
+      Wide_Wide   : Boolean := False) return Node_Id;
+   --  Generate a call to evaluate a String (or Wide_Wide_String, depending
+   --  on the Wide_Wide Boolean parameter) expression and output it into
+   --  the buffer.
 
    -------------------------------------
    -- Build_Array_Put_Image_Procedure --
@@ -189,7 +209,7 @@ package body Exp_Put_Image is
       Ndim : constant Pos        := Number_Dimensions (Typ);
       Ctyp : constant Entity_Id  := Component_Type (Typ);
 
-      Stm         : Node_Id;
+      Stms        : List_Id := New_List;
       Exl         : constant List_Id := New_List;
       PI_Entity   : Entity_Id;
 
@@ -220,15 +240,36 @@ package body Exp_Put_Image is
          Append_To (Exl, Make_Identifier (Loc, New_External_Name ('L', Dim)));
       end loop;
 
-      Stm :=
-        Make_Attribute_Reference (Loc,
-          Prefix => New_Occurrence_Of (Put_Image_Base_Type (Ctyp), Loc),
-          Attribute_Name => Name_Put_Image,
-          Expressions => New_List (
-            Make_Identifier (Loc, Name_S),
-            Make_Indexed_Component (Loc,
-              Prefix      => Make_Identifier (Loc, Name_V),
-              Expressions => Exl)));
+      declare
+         Ctype_For_Call : constant Entity_Id := Put_Image_Base_Type (Ctyp);
+         Indexed_Comp   : constant Node_Id :=
+           Make_Indexed_Component (Loc,
+             Prefix      => Make_Identifier (Loc, Name_V),
+             Expressions => Exl);
+      begin
+         if Is_Mutably_Tagged_Type (Ctype_For_Call) then
+            pragma Assert (not Is_Mutably_Tagged_Type (Component_Type (Typ)));
+
+            Make_Mutably_Tagged_Conversion (Indexed_Comp,
+                                            Typ => Ctype_For_Call);
+
+            pragma Assert (Is_Mutably_Tagged_Type (Etype (Indexed_Comp)));
+
+            Put_Specific_Type_Name_Qualifier (Loc,
+               Stms              => Stms,
+               Tagged_Obj        => Indexed_Comp,
+               Buffer_Name       => Make_Identifier (Loc, Name_S),
+               Is_Interface_Type => False);
+         end if;
+
+         Append_To (Stms,
+           Make_Attribute_Reference (Loc,
+             Prefix => New_Occurrence_Of (Ctype_For_Call, Loc),
+             Attribute_Name => Name_Put_Image,
+             Expressions => New_List (
+               Make_Identifier (Loc, Name_S),
+               Indexed_Comp)));
+      end;
 
       --  The corresponding attribute for the component type of the array might
       --  be user-defined, and frozen after the array type. In that case,
@@ -245,46 +286,42 @@ package body Exp_Put_Image is
       --  Loop through the dimensions, innermost first, generating a loop for
       --  each dimension.
 
-      declare
-         Stms : List_Id := New_List (Stm);
-      begin
-         for Dim in reverse 1 .. Ndim loop
-            declare
-               New_Stms : constant List_Id := New_List;
-               Between_Proc : RE_Id;
-            begin
-               --  For a one-dimensional array of elementary type, use
-               --  RE_Simple_Array_Between. The same applies to the last
-               --  dimension of a multidimensional array.
+      for Dim in reverse 1 .. Ndim loop
+         declare
+            New_Stms : constant List_Id := New_List;
+            Between_Proc : RE_Id;
+         begin
+            --  For a one-dimensional array of elementary type, use
+            --  RE_Simple_Array_Between. The same applies to the last
+            --  dimension of a multidimensional array.
 
-               if Is_Elementary_Type (Ctyp) and then Dim = Ndim then
-                  Between_Proc := RE_Simple_Array_Between;
-               else
-                  Between_Proc := RE_Array_Between;
-               end if;
+            if Is_Elementary_Type (Ctyp) and then Dim = Ndim then
+               Between_Proc := RE_Simple_Array_Between;
+            else
+               Between_Proc := RE_Array_Between;
+            end if;
 
-               Append_To (New_Stms,
-                 Make_Procedure_Call_Statement (Loc,
-                   Name => New_Occurrence_Of (RTE (RE_Array_Before), Loc),
-                   Parameter_Associations => New_List
-                     (Make_Identifier (Loc, Name_S))));
+            Append_To (New_Stms,
+              Make_Procedure_Call_Statement (Loc,
+                Name => New_Occurrence_Of (RTE (RE_Array_Before), Loc),
+                Parameter_Associations => New_List
+                  (Make_Identifier (Loc, Name_S))));
 
-               Append_To
-                 (New_Stms,
-                  Wrap_In_Loop (Stms, Dim, Indices (Dim), Between_Proc));
+            Append_To
+              (New_Stms,
+               Wrap_In_Loop (Stms, Dim, Indices (Dim), Between_Proc));
 
-               Append_To (New_Stms,
-                 Make_Procedure_Call_Statement (Loc,
-                   Name => New_Occurrence_Of (RTE (RE_Array_After), Loc),
-                   Parameter_Associations => New_List
-                     (Make_Identifier (Loc, Name_S))));
+            Append_To (New_Stms,
+              Make_Procedure_Call_Statement (Loc,
+                Name => New_Occurrence_Of (RTE (RE_Array_After), Loc),
+                Parameter_Associations => New_List
+                  (Make_Identifier (Loc, Name_S))));
 
-               Stms := New_Stms;
-            end;
-         end loop;
+            Stms := New_Stms;
+         end;
+      end loop;
 
-         Build_Put_Image_Proc (Loc, Typ, Decl, Pnam, Stms);
-      end;
+      Build_Put_Image_Proc (Loc, Typ, Decl, Pnam, Stms);
    end Build_Array_Put_Image_Procedure;
 
    -------------------------------------
@@ -379,7 +416,8 @@ package body Exp_Put_Image is
          begin
             --  We have built a dispatching call to handle calls to
             --  descendants (since they are not available through rtsfind).
-            --  Further details available in the body of Put_String_Exp.
+            --  Further details available in the body of
+            --  Put_String_Exp_To_Buffer.
 
             return Put_Call;
          end;
@@ -691,19 +729,33 @@ package body Exp_Put_Image is
       ---------------------------
 
       procedure Append_Component_Attr (Clist : List_Id; C : Entity_Id) is
-         Component_Typ : constant Entity_Id :=
-           Put_Image_Base_Type
-             (Get_Corresponding_Mutably_Tagged_Type_If_Present (Etype (C)));
+         Component_Typ : constant Entity_Id := Put_Image_Base_Type (Etype (C));
+         Selected_Comp : constant Node_Id :=
+           Make_Selected_Component (Loc,
+             Prefix        => Make_Identifier (Loc, Name_V),
+             Selector_Name => New_Occurrence_Of (C, Loc));
       begin
+         if Is_Mutably_Tagged_Type (Component_Typ) then
+            pragma Assert (not Is_Mutably_Tagged_Type (Etype (C)));
+
+            Make_Mutably_Tagged_Conversion (Selected_Comp,
+                                            Typ => Component_Typ);
+
+            pragma Assert (Is_Mutably_Tagged_Type (Etype (Selected_Comp)));
+
+            Put_Specific_Type_Name_Qualifier (Loc,
+               Stms              => Clist,
+               Tagged_Obj        => Selected_Comp,
+               Buffer_Name       => Make_Identifier (Loc, Name_S),
+               Is_Interface_Type => False);
+         end if;
+
          Append_To (Clist,
            Make_Attribute_Reference (Loc,
              Prefix         => New_Occurrence_Of (Component_Typ, Loc),
              Attribute_Name => Name_Put_Image,
-             Expressions    => New_List (
-               Make_Identifier (Loc, Name_S),
-               Make_Selected_Component (Loc,
-                 Prefix        => Make_Identifier (Loc, Name_V),
-                 Selector_Name => New_Occurrence_Of (C, Loc)))));
+             Expressions    => New_List (Make_Identifier (Loc, Name_S),
+                                         Selected_Comp)));
       end Append_Component_Attr;
 
       -------------------------------
@@ -1303,105 +1355,20 @@ package body Exp_Put_Image is
                 New_Occurrence_Of (Sink_Entity, Loc))));
       Actions : List_Id;
 
-      function Put_String_Exp (String_Exp : Node_Id;
-                               Wide_Wide  : Boolean := False) return Node_Id;
-      --  Generate a call to evaluate a String (or Wide_Wide_String, depending
-      --  on the Wide_Wide Boolean parameter) expression and output it into
-      --  the buffer.
-
-      --------------------
-      -- Put_String_Exp --
-      --------------------
-
-      function Put_String_Exp (String_Exp : Node_Id;
-                               Wide_Wide  : Boolean := False) return Node_Id is
-         Put_Id : constant RE_Id :=
-           (if Wide_Wide then RE_Wide_Wide_Put else RE_Put_UTF_8);
-
-         --  We could build a nondispatching call here, but to make
-         --  that work we'd have to change Rtsfind spec to make available
-         --  corresponding callees out of Ada.Strings.Text_Buffers.Unbounded
-         --  (as opposed to from Ada.Strings.Text_Buffers). Seems simpler to
-         --  introduce a type conversion and leave it to the optimizer to
-         --  eliminate the dispatching. This does not *introduce* any problems
-         --  if a no-dispatching-allowed restriction is in effect, since we
-         --  are already in the middle of generating a call to T'Class'Image.
-
-         Sink_Exp : constant Node_Id :=
-           Make_Type_Conversion (Loc,
-             Subtype_Mark =>
-               New_Occurrence_Of
-                 (Class_Wide_Type (RTE (RE_Root_Buffer_Type)), Loc),
-             Expression   => New_Occurrence_Of (Sink_Entity, Loc));
-      begin
-         return
-           Make_Procedure_Call_Statement (Loc,
-             Name => New_Occurrence_Of (RTE (Put_Id), Loc),
-             Parameter_Associations => New_List (Sink_Exp, String_Exp));
-      end Put_String_Exp;
-
-      --  Local variables
-
-      Tag_Node : Node_Id;
-
    --  Start of processing for Build_Image_Call
 
    begin
       if Is_Class_Wide_Type (U_Type) then
+         Actions := New_List (Sink_Decl);
 
-         --  For interface types we must generate code to displace the pointer
-         --  to the object to reference the base of the underlying object.
-
-         --  Generate:
-         --    To_Tag_Ptr (Image_Prefix'Address).all
-
-         --  Note that Image_Prefix'Address is recursively expanded into a
-         --  call to Ada.Tags.Base_Address (Image_Prefix'Address).
-
-         if Is_Interface (U_Type) then
-            Tag_Node :=
-              Make_Explicit_Dereference (Loc,
-                Unchecked_Convert_To (RTE (RE_Tag_Ptr),
-                  Make_Attribute_Reference (Loc,
-                    Prefix => Duplicate_Subexpr (Image_Prefix),
-                    Attribute_Name => Name_Address)));
+         Put_Specific_Type_Name_Qualifier (Loc,
+           Stms              => Actions,
+           Tagged_Obj        => Image_Prefix,
+           Buffer_Name       => New_Occurrence_Of (Sink_Entity, Loc),
+           Is_Interface_Type => Is_Interface (U_Type));
 
-         --  Common case
-
-         else
-            Tag_Node :=
-              Make_Attribute_Reference (Loc,
-                Prefix         => Duplicate_Subexpr (Image_Prefix),
-                Attribute_Name => Name_Tag);
-         end if;
-
-         --  Generate qualified-expression syntax; qualification name comes
-         --  from calling Ada.Tags.Wide_Wide_Expanded_Name.
-
-         declare
-            --  The copy of Image_Prefix will be evaluated before the
-            --  original, which is ok if no side effects are involved.
-
-            pragma Assert (Side_Effect_Free (Image_Prefix));
-
-            Specific_Type_Name : constant Node_Id :=
-              Put_String_Exp
-                (Make_Function_Call (Loc,
-                   Name => New_Occurrence_Of
-                             (RTE (RE_Wide_Wide_Expanded_Name), Loc),
-                   Parameter_Associations => New_List (Tag_Node)),
-                 Wide_Wide => True);
-
-            Qualification : constant Node_Id :=
-              Put_String_Exp (Make_String_Literal (Loc, "'"));
-         begin
-            Actions := New_List
-                         (Sink_Decl,
-                          Specific_Type_Name,
-                          Qualification,
-                          Put_Im,
-                          Result_Decl);
-         end;
+         Append_To (Actions, Put_Im);
+         Append_To (Actions, Result_Decl);
       else
          Actions := New_List (Sink_Decl, Put_Im, Result_Decl);
       end if;
@@ -1485,9 +1452,89 @@ package body Exp_Put_Image is
          return E;
       elsif Is_Private_Type (Base_Type (E)) and not Is_Private_Type (E) then
          return Implementation_Base_Type (E);
+      elsif Is_Mutably_Tagged_CW_Equivalent_Type (E) then
+         return Get_Corresponding_Mutably_Tagged_Type_If_Present (E);
       else
          return Base_Type (E);
       end if;
    end Put_Image_Base_Type;
 
+   --------------------------------------
+   -- Put_Specific_Type_Name_Qualifier --
+   --------------------------------------
+
+   procedure Put_Specific_Type_Name_Qualifier
+     (Loc               : Source_Ptr;
+      Stms              : List_Id;
+      Tagged_Obj        : Node_Id;
+      Buffer_Name       : Node_Id;
+      Is_Interface_Type : Boolean)
+   is
+      Tag_Node : Node_Id;
+   begin
+      if Is_Interface_Type then
+         Tag_Node :=
+           Make_Explicit_Dereference (Loc,
+             Unchecked_Convert_To (RTE (RE_Tag_Ptr),
+               Make_Attribute_Reference (Loc,
+                 Prefix => New_Copy_Tree (Tagged_Obj),
+                 Attribute_Name => Name_Address)));
+      else
+         Tag_Node :=
+           Make_Attribute_Reference (Loc,
+             Prefix         => New_Copy_Tree (Tagged_Obj),
+             Attribute_Name => Name_Tag);
+      end if;
+
+      Append_To (Stms,
+                 Put_String_Exp_To_Buffer (Loc,
+                   String_Exp =>
+                     Make_Function_Call (Loc,
+                       Name => New_Occurrence_Of
+                                 (RTE (RE_Wide_Wide_Expanded_Name), Loc),
+                       Parameter_Associations => New_List (Tag_Node)),
+                   Buffer_Name => Buffer_Name,
+                   Wide_Wide => True));
+
+      Append_To (Stms,
+                 Put_String_Exp_To_Buffer (Loc,
+                   String_Exp  => Make_String_Literal (Loc, "'"),
+                   Buffer_Name => New_Copy_Tree (Buffer_Name)));
+   end Put_Specific_Type_Name_Qualifier;
+
+   ------------------------------
+   -- Put_String_Exp_To_Buffer --
+   ------------------------------
+
+   function Put_String_Exp_To_Buffer
+     (Loc         : Source_Ptr;
+      String_Exp  : Node_Id;
+      Buffer_Name : Node_Id;
+      Wide_Wide   : Boolean := False) return Node_Id
+   is
+      Put_Id : constant RE_Id :=
+        (if Wide_Wide then RE_Wide_Wide_Put else RE_Put_UTF_8);
+
+      --  We could build a nondispatching call here, but to make
+      --  that work we'd have to change Rtsfind spec to make available
+      --  corresponding callees out of Ada.Strings.Text_Buffers.Unbounded
+      --  (as opposed to from Ada.Strings.Text_Buffers). Seems simpler to
+      --  introduce a type conversion and leave it to the optimizer to
+      --  eliminate the dispatching. This does not *introduce* any problems
+      --  if a no-dispatching-allowed restriction is in effect, since we
+      --  are already in the middle of generating a call to T'Class'Image.
+
+      Sink_Exp : constant Node_Id :=
+        Make_Type_Conversion (Loc,
+          Subtype_Mark =>
+            New_Occurrence_Of
+              (Class_Wide_Type (RTE (RE_Root_Buffer_Type)), Loc),
+          Expression   => Buffer_Name);
+   begin
+      return
+        Make_Procedure_Call_Statement (Loc,
+          Name => New_Occurrence_Of (RTE (Put_Id), Loc),
+          Parameter_Associations => New_List (Sink_Exp, String_Exp));
+   end Put_String_Exp_To_Buffer;
+
 end Exp_Put_Image;
diff --git a/gcc/ada/exp_spark.adb b/gcc/ada/exp_spark.adb
index 6e1c86a..0f92034 100644
--- a/gcc/ada/exp_spark.adb
+++ b/gcc/ada/exp_spark.adb
@@ -73,6 +73,10 @@ package body Exp_SPARK is
    procedure Expand_SPARK_N_Attribute_Reference (N : Node_Id);
    --  Perform attribute-reference-specific expansion
 
+   procedure Expand_SPARK_N_Continue_Statement (N : Node_Id);
+   --  Expand continue statements which are resolved as procedure calls, into
+   --  said procedure calls. Real continue statements are left as-is.
+
    procedure Expand_SPARK_N_Delta_Aggregate (N : Node_Id);
    --  Perform delta-aggregate-specific expansion
 
@@ -191,6 +195,9 @@ package body Exp_SPARK is
 
          --  In SPARK mode, no other constructs require expansion
 
+         when N_Continue_Statement =>
+            Expand_SPARK_N_Continue_Statement (N);
+
          when others =>
             null;
       end case;
@@ -435,6 +442,23 @@ package body Exp_SPARK is
       end if;
    end Expand_SPARK_Delta_Or_Update;
 
+   ---------------------------------------
+   -- Expand_SPARK_N_Continue_Statement --
+   ---------------------------------------
+
+   procedure Expand_SPARK_N_Continue_Statement (N : Node_Id) is
+      X : constant Node_Id := Call_Or_Target_Loop (N);
+   begin
+      if No (X) then
+         return;
+      end if;
+
+      if Nkind (X) = N_Procedure_Call_Statement then
+         Replace (N, X);
+         Analyze (N);
+      end if;
+   end Expand_SPARK_N_Continue_Statement;
+
    ------------------------------
    -- Expand_SPARK_N_Aggregate --
    ------------------------------
@@ -1104,8 +1128,7 @@ package body Exp_SPARK is
       Wrapper_Decl_List : List_Id;
       Wrapper_Body_List : List_Id := No_List;
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
    begin
@@ -1229,7 +1252,7 @@ package body Exp_SPARK is
          end if;
       end if;
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end SPARK_Freeze_Type;
 
 end Exp_SPARK;
diff --git a/gcc/ada/exp_strm.adb b/gcc/ada/exp_strm.adb
index 250efd2..5e1c913 100644
--- a/gcc/ada/exp_strm.adb
+++ b/gcc/ada/exp_strm.adb
@@ -29,6 +29,7 @@ with Einfo.Entities; use Einfo.Entities;
 with Einfo.Utils;    use Einfo.Utils;
 with Elists;         use Elists;
 with Exp_Util;       use Exp_Util;
+with Mutably_Tagged; use Mutably_Tagged;
 with Namet;          use Namet;
 with Nlists;         use Nlists;
 with Nmake;          use Nmake;
@@ -101,13 +102,10 @@ package body Exp_Strm is
    --  Loc parameter is used as the Sloc of the created entity.
 
    function Stream_Base_Type (E : Entity_Id) return Entity_Id;
-   --  Stream attributes work on the basis of the base type except for the
-   --  array case. For the array case, we do not go to the base type, but
-   --  to the first subtype if it is constrained. This avoids problems with
-   --  incorrect conversions in the packed array case. Stream_Base_Type is
-   --  exactly this function (returns the base type, unless we have an array
-   --  type whose first subtype is constrained, in which case it returns the
-   --  first subtype).
+   --  For an array type whose whose first subtype is constrained, return
+   --  the first subtype. For the internal representation type corresponding
+   --  to a mutably tagged type, return the mutably tagged type. Otherwise,
+   --  return the base type. Similar to Exp_Put_Image.Put_Image_Base_Type.
 
    --------------------------------
    -- Build_Array_Input_Function --
@@ -1502,6 +1500,7 @@ package body Exp_Strm is
 
       function Make_Field_Attribute (C : Entity_Id) return Node_Id is
          Field_Typ : constant Entity_Id := Stream_Base_Type (Etype (C));
+         Selected  : Node_Id;
 
          TSS_Names : constant array (Name_Input .. Name_Write) of
                        TSS_Name_Type :=
@@ -1524,15 +1523,23 @@ package body Exp_Strm is
             return Make_Null_Statement (Loc);
          end if;
 
+         Selected := Make_Selected_Component (Loc,
+                       Prefix        => Make_Identifier (Loc, Name_V),
+                       Selector_Name => New_Occurrence_Of (C, Loc));
+
+         if Is_Mutably_Tagged_CW_Equivalent_Type (Etype (C)) then
+            Make_Mutably_Tagged_Conversion
+              (Selected,
+               Typ => Get_Corresponding_Mutably_Tagged_Type_If_Present
+                        (Etype (C)));
+         end if;
+
          return
            Make_Attribute_Reference (Loc,
              Prefix         => New_Occurrence_Of (Field_Typ, Loc),
              Attribute_Name => Nam,
-             Expressions    => New_List (
-               Make_Identifier (Loc, Name_S),
-               Make_Selected_Component (Loc,
-                 Prefix        => Make_Identifier (Loc, Name_V),
-                 Selector_Name => New_Occurrence_Of (C, Loc))));
+             Expressions    => New_List (Make_Identifier (Loc, Name_S),
+                                         Selected));
       end Make_Field_Attribute;
 
       ---------------------------
@@ -1808,6 +1815,10 @@ package body Exp_Strm is
 
    function Stream_Base_Type (E : Entity_Id) return Entity_Id is
    begin
+      if Is_Class_Wide_Equivalent_Type (E) then
+         return Corresponding_Mutably_Tagged_Type (E);
+      end if;
+
       if Is_Array_Type (E)
         and then Is_First_Subtype (E)
       then
diff --git a/gcc/ada/exp_util.adb b/gcc/ada/exp_util.adb
index 4f98779..e9ec7b7 100644
--- a/gcc/ada/exp_util.adb
+++ b/gcc/ada/exp_util.adb
@@ -1903,7 +1903,7 @@ package body Exp_Util is
       begin
          --  The DIC pragma is ignored, nothing left to do
 
-         if Is_Ignored (DIC_Prag) then
+         if Is_Ignored_In_Codegen (DIC_Prag) then
             null;
 
          --  Otherwise the DIC expression must be checked at run time.
@@ -2311,8 +2311,7 @@ package body Exp_Util is
 
       Loc : constant Source_Ptr := Sloc (Typ);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       DIC_Prag     : Node_Id;
@@ -2558,7 +2557,7 @@ package body Exp_Util is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Build_DIC_Procedure_Body;
 
    -------------------------------------
@@ -2575,8 +2574,7 @@ package body Exp_Util is
    is
       Loc : constant Source_Ptr := Sloc (Typ);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       DIC_Prag  : Node_Id;
@@ -2783,7 +2781,7 @@ package body Exp_Util is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Build_DIC_Procedure_Declaration;
 
    ------------------------------------
@@ -3237,7 +3235,7 @@ package body Exp_Util is
       begin
          --  The invariant is ignored, nothing left to do
 
-         if Is_Ignored (Prag) then
+         if Is_Ignored_In_Codegen (Prag) then
             null;
 
          --  Otherwise the invariant is checked. Build a pragma Check to verify
@@ -3709,8 +3707,7 @@ package body Exp_Util is
 
       --  Local variables
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Dummy        : Entity_Id;
@@ -4058,7 +4055,7 @@ package body Exp_Util is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Build_Invariant_Procedure_Body;
 
    -------------------------------------------
@@ -4075,8 +4072,7 @@ package body Exp_Util is
    is
       Loc : constant Source_Ptr := Sloc (Typ);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Proc_Decl : Node_Id;
@@ -4292,7 +4288,7 @@ package body Exp_Util is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Build_Invariant_Procedure_Declaration;
 
    ------------------------
@@ -6189,7 +6185,7 @@ package body Exp_Util is
          if Is_Protected_Type (Btyp) then
             Utyp := Corresponding_Record_Type (Root_Type (Btyp));
 
-         else
+         elsif Is_Implicit_Full_View (Utyp) then
             Utyp := Underlying_Type (Root_Type (Btyp));
 
             if Is_Protected_Type (Utyp) then
@@ -10640,8 +10636,7 @@ package body Exp_Util is
    is
       Loc : constant Source_Ptr := Sloc (Expr);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Call         : Node_Id;
@@ -10685,7 +10680,7 @@ package body Exp_Util is
           Name                   => New_Occurrence_Of (Func_Id, Loc),
           Parameter_Associations => Param_Assocs);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
 
       return Call;
    end Make_Predicate_Call;
@@ -11730,34 +11725,6 @@ package body Exp_Util is
       end if;
    end Matching_Standard_Type;
 
-   -----------------------------
-   -- May_Generate_Large_Temp --
-   -----------------------------
-
-   --  At the current time, the only types that we return False for (i.e. where
-   --  we decide we know they cannot generate large temps) are ones where we
-   --  know the size is 256 bits or less at compile time, and we are still not
-   --  doing a thorough job on arrays and records.
-
-   function May_Generate_Large_Temp (Typ : Entity_Id) return Boolean is
-   begin
-      if not Size_Known_At_Compile_Time (Typ) then
-         return False;
-      end if;
-
-      if Known_Esize (Typ) and then Esize (Typ) <= 256 then
-         return False;
-      end if;
-
-      if Is_Array_Type (Typ)
-        and then Present (Packed_Array_Impl_Type (Typ))
-      then
-         return May_Generate_Large_Temp (Packed_Array_Impl_Type (Typ));
-      end if;
-
-      return True;
-   end May_Generate_Large_Temp;
-
    ---------------------------------------
    -- Move_To_Initialization_Statements --
    ---------------------------------------
@@ -13760,11 +13727,12 @@ package body Exp_Util is
    --  The above requirements should be documented in Sinfo ???
 
    function Safe_Unchecked_Type_Conversion (Exp : Node_Id) return Boolean is
+      Pexp : constant Node_Id := Parent (Exp);
+
       Otyp   : Entity_Id;
       Ityp   : Entity_Id;
       Oalign : Uint;
       Ialign : Uint;
-      Pexp   : constant Node_Id := Parent (Exp);
 
    begin
       --  If the expression is the RHS of an assignment or object declaration
@@ -13782,18 +13750,12 @@ package body Exp_Util is
          return True;
 
       --  If the expression is the prefix of an N_Selected_Component we should
-      --  also be OK because GCC knows to look inside the conversion except if
-      --  the type is discriminated. We assume that we are OK anyway if the
-      --  type is not set yet or if it is controlled since we can't afford to
-      --  introduce a temporary in this case.
+      --  also be OK because GCC knows to look inside the conversion.
 
       elsif Nkind (Pexp) = N_Selected_Component
         and then Prefix (Pexp) = Exp
       then
-         return No (Etype (Pexp))
-           or else not Is_Type (Etype (Pexp))
-           or else not Has_Discriminants (Etype (Pexp))
-           or else Is_Constrained (Etype (Pexp));
+         return True;
       end if;
 
       --  Set the output type, this comes from Etype if it is set, otherwise we
@@ -13866,14 +13828,7 @@ package body Exp_Util is
       --  known size, but we can't consider them that way here, because we are
       --  talking about the actual size of the object.
 
-      --  We also make sure that in addition to the size being known, we do not
-      --  have a case which might generate an embarrassingly large temp in
-      --  stack checking mode.
-
       elsif Size_Known_At_Compile_Time (Otyp)
-        and then
-          (not Stack_Checking_Enabled
-            or else not May_Generate_Large_Temp (Otyp))
         and then not (Is_Record_Type (Otyp) and then not Is_Constrained (Otyp))
       then
          return True;
diff --git a/gcc/ada/exp_util.ads b/gcc/ada/exp_util.ads
index b8b7525..4226fcc 100644
--- a/gcc/ada/exp_util.ads
+++ b/gcc/ada/exp_util.ads
@@ -1064,16 +1064,6 @@ package Exp_Util is
    --  typically return Standard_Short_Integer. For fixed-point types, this
    --  will return integer types of the corresponding size.
 
-   function May_Generate_Large_Temp (Typ : Entity_Id) return Boolean;
-   --  Determines if the given type, Typ, may require a large temporary of the
-   --  kind that causes back-end trouble if stack checking is enabled. The
-   --  result is True only the size of the type is known at compile time and
-   --  large, where large is defined heuristically by the body of this routine.
-   --  The purpose of this routine is to help avoid generating troublesome
-   --  temporaries that interfere with stack checking mechanism. Note that the
-   --  caller has to check whether stack checking is actually enabled in order
-   --  to guide the expansion (typically of a function call).
-
    procedure Move_To_Initialization_Statements (Decl, Stop : Node_Id);
    --  Decl is an N_Object_Declaration node and Stop is a node past Decl in
    --  the same list. Move all the nodes on the list between Decl and Stop
diff --git a/gcc/ada/expander.adb b/gcc/ada/expander.adb
index 3d7b0d7..25f4950 100644
--- a/gcc/ada/expander.adb
+++ b/gcc/ada/expander.adb
@@ -84,8 +84,7 @@ package body Expander is
    --  Ghost mode.
 
    procedure Expand (N : Node_Id) is
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
    begin
@@ -559,7 +558,7 @@ package body Expander is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Expand;
 
    ---------------------------
diff --git a/gcc/ada/freeze.adb b/gcc/ada/freeze.adb
index be2115a..2ebffff 100644
--- a/gcc/ada/freeze.adb
+++ b/gcc/ada/freeze.adb
@@ -765,6 +765,9 @@ package body Freeze is
       --  in fact constrained by non-static discriminant values. Could be made
       --  more precise ???
 
+      function Value_Known (Exp : Node_Id) return Boolean;
+      --  Return True if the value of expression Exp is known at compile time
+
       --------------------
       -- Set_Small_Size --
       --------------------
@@ -880,13 +883,13 @@ package body Freeze is
                      High := Type_High_Bound (Etype (Index));
                   end if;
 
-                  if not Compile_Time_Known_Value (Low)
-                    or else not Compile_Time_Known_Value (High)
-                    or else Etype (Index) = Any_Type
-                  then
+                  if Etype (Index) = Any_Type then
                      return False;
 
-                  else
+                  elsif Compile_Time_Known_Value (Low)
+                    and then Compile_Time_Known_Value (High)
+                  then
+
                      Dim := Expr_Value (High) - Expr_Value (Low) + 1;
 
                      if Dim > Uint_0 then
@@ -894,6 +897,12 @@ package body Freeze is
                      else
                         Size := Uint_0;
                      end if;
+
+                  elsif Value_Known (Low) and then Value_Known (High) then
+                     Size := Uint_0;
+
+                  else
+                     return False;
                   end if;
 
                   Next_Index (Index);
@@ -1160,6 +1169,70 @@ package body Freeze is
          return True;
       end Static_Discriminated_Components;
 
+      -----------------
+      -- Value_Known --
+      -----------------
+
+      function Value_Known (Exp : Node_Id) return Boolean is
+      begin
+         --  This is the immediate case
+
+         if Compile_Time_Known_Value (Exp) then
+            return True;
+         end if;
+
+         --  The value may be known only to the back end, the typical example
+         --  being the alignment or the various sizes of composite types; in
+         --  the latter case, we may mutually recurse with Size_Known.
+
+         case Nkind (Exp) is
+            when N_Attribute_Reference =>
+               declare
+                  P : constant Node_Id := Prefix (Exp);
+
+               begin
+                  if not Is_Entity_Name (P)
+                    or else not Is_Type (Entity (P))
+                  then
+                     return False;
+                  end if;
+
+                  case Get_Attribute_Id (Attribute_Name (Exp)) is
+                     when Attribute_Alignment =>
+                        return True;
+
+                     when Attribute_Component_Size =>
+                        return Size_Known (Component_Type (Entity (P)));
+
+                     when Attribute_Object_Size
+                        | Attribute_Size
+                        | Attribute_Value_Size
+                     =>
+                        return Size_Known (Entity (P));
+
+                     when others =>
+                        return False;
+                  end case;
+               end;
+
+            when N_Binary_Op =>
+               return Value_Known (Left_Opnd (Exp))
+                 and then Value_Known (Right_Opnd (Exp));
+
+            when N_Qualified_Expression
+               | N_Type_Conversion
+               | N_Unchecked_Type_Conversion
+            =>
+               return Value_Known (Expression (Exp));
+
+            when N_Unary_Op =>
+               return Value_Known (Right_Opnd (Exp));
+
+            when others =>
+               return False;
+         end case;
+      end Value_Known;
+
    --  Start of processing for Check_Compile_Time_Size
 
    begin
@@ -2805,8 +2878,7 @@ package body Freeze is
    is
       Loc : constant Source_Ptr := Sloc (N);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Atype  : Entity_Id;
@@ -4740,6 +4812,8 @@ package body Freeze is
                  and then Convention (F_Type) = Convention_Ada
                  and then not Has_Warnings_Off (F_Type)
                  and then not Has_Size_Clause (F_Type)
+                 and then Present (Esize (F_Type))
+                 and then Esize (F_Type) = 8
                then
                   Error_Msg_N
                     ("& is an 8-bit Ada Boolean?x?", Formal);
@@ -7158,6 +7232,35 @@ package body Freeze is
             end if;
 
             Inherit_Aspects_At_Freeze_Point (E);
+
+            --  Destructor legality check
+
+            if Present (Primitive_Operations (E)) then
+               declare
+                  Subp             : Entity_Id;
+                  Parent_Operation : Entity_Id;
+
+                  Elmt : Elmt_Id := First_Elmt (Primitive_Operations (E));
+
+               begin
+                  while Present (Elmt) loop
+                     Subp := Node (Elmt);
+
+                     if Present (Overridden_Operation (Subp)) then
+                        Parent_Operation := Overridden_Operation (Subp);
+
+                        if Ekind (Parent_Operation) = E_Procedure
+                          and then Is_Destructor (Parent_Operation)
+                        then
+                           Error_Msg_N ("cannot override destructor", Subp);
+                        end if;
+                     end if;
+
+                     Next_Elmt (Elmt);
+                  end loop;
+               end;
+            end if;
+
          end if;
 
          --  Case of array type
@@ -8057,6 +8160,7 @@ package body Freeze is
          if Ekind (E) = E_Anonymous_Access_Subprogram_Type
            and then Ekind (Designated_Type (E)) = E_Subprogram_Type
          then
+            Create_Extra_Formals (Designated_Type (E));
             Layout_Type (Etype (Designated_Type (E)));
          end if;
 
@@ -8255,12 +8359,12 @@ package body Freeze is
       --  and Per-Object Expressions" will suppress the insertion, and the
       --  freeze node will be dropped on the floor.
 
-      if Saved_GM = Ignore
-        and then Ghost_Mode /= Ignore
-        and then Present (Ignored_Ghost_Region)
+      if Saved_Ghost_Config.Ghost_Mode = Ignore
+        and then Ghost_Config.Ghost_Mode /= Ignore
+        and then Present (Ghost_Config.Ignored_Ghost_Region)
       then
          Insert_Actions
-           (Assoc_Node   => Ignored_Ghost_Region,
+           (Assoc_Node   => Ghost_Config.Ignored_Ghost_Region,
             Ins_Actions  => Result,
             Spec_Expr_OK => True);
 
@@ -8268,7 +8372,7 @@ package body Freeze is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
 
       return Result;
    end Freeze_Entity;
@@ -10320,6 +10424,8 @@ package body Freeze is
 
       --  Local variables
 
+      use Deferred_Extra_Formals_Support;
+
       F      : Entity_Id;
       Retype : Entity_Id;
 
@@ -10420,8 +10526,11 @@ package body Freeze is
             Create_Extra_Formals (E);
 
             pragma Assert
-              ((Ekind (E) = E_Subprogram_Type
-                  and then Extra_Formals_OK (E))
+              ((Extra_Formals_Known (E)
+                 or else Is_Deferred_Extra_Formals_Entity (E))
+               or else
+                 (Ekind (E) = E_Subprogram_Type
+                   and then Extra_Formals_OK (E))
                or else
                  (Is_Subprogram (E)
                    and then Extra_Formals_OK (E)
@@ -10450,6 +10559,10 @@ package body Freeze is
       else
          Set_Mechanisms (E);
 
+         if not Extra_Formals_Known (E) then
+            Freeze_Extra_Formals (E);
+         end if;
+
          --  For foreign conventions, warn about return of unconstrained array
 
          if Ekind (E) = E_Function then
@@ -10505,6 +10618,11 @@ package body Freeze is
          end if;
       end if;
 
+      --  Check formals matching in thunks
+
+      pragma Assert (not Is_Thunk (E)
+        or else Extra_Formals_Match_OK (Thunk_Entity (E), E));
+
       --  Pragma Inline_Always is disallowed for dispatching subprograms
       --  because the address of such subprograms is saved in the dispatch
       --  table to support dispatching calls, and dispatching calls cannot
diff --git a/gcc/ada/frontend.adb b/gcc/ada/frontend.adb
index 564f153..92bc3c6 100644
--- a/gcc/ada/frontend.adb
+++ b/gcc/ada/frontend.adb
@@ -477,7 +477,7 @@ begin
             --  executable. This action must be performed very late because it
             --  heavily alters the tree.
 
-            if Operating_Mode = Generate_Code or else GNATprove_Mode then
+            if Operating_Mode = Generate_Code and not CodePeer_Mode then
                Remove_Ignored_Ghost_Code;
             end if;
 
diff --git a/gcc/ada/gcc-interface/Make-lang.in b/gcc/ada/gcc-interface/Make-lang.in
index 1c93816..bbbd697 100644
--- a/gcc/ada/gcc-interface/Make-lang.in
+++ b/gcc/ada/gcc-interface/Make-lang.in
@@ -1096,7 +1096,7 @@ check-ada-subtargets: check-acats-subtargets check-gnat-subtargets
 # No ada-specific selftests
 selftest-ada:
 
-ACATSDIR = $(TESTSUITEDIR)/ada/acats-2
+ACATSDIR = $(TESTSUITEDIR)/ada/acats-4
 ACATSCMD = run_acats.sh
 
 check_acats_numbers0:=1 2 3 4 5 6 7 8 9
diff --git a/gcc/ada/gcc-interface/Makefile.in b/gcc/ada/gcc-interface/Makefile.in
index 3557b46..d456ac1 100644
--- a/gcc/ada/gcc-interface/Makefile.in
+++ b/gcc/ada/gcc-interface/Makefile.in
@@ -314,16 +314,16 @@ Makefile: ../config.status $(srcdir)/ada/gcc-interface/Makefile.in $(srcdir)/ada
 GNATLINK_OBJS = gnatlink.o \
  a-except.o ali.o alloc.o butil.o casing.o csets.o debug.o fmap.o fname.o \
  gnatvsn.o hostparm.o indepsw.o interfac.o i-c.o i-cstrin.o namet.o opt.o \
- osint.o output.o rident.o s-exctab.o s-secsta.o s-stalib.o s-stoele.o \
- sdefault.o snames.o stylesw.o switch.o system.o table.o targparm.o \
- types.o validsw.o widechar.o
+ osint.o output.o rident.o s-excmac.o s-exctab.o s-secsta.o s-stalib.o \
+ s-stoele.o sdefault.o snames.o stylesw.o switch.o system.o table.o \
+ targparm.o types.o validsw.o widechar.o
 
 GNATMAKE_OBJS = a-except.o ali.o ali-util.o aspects.o s-casuti.o alloc.o \
  atree.o binderr.o butil.o casing.o csets.o debug.o elists.o einfo.o errout.o \
  erroutc.o errutil.o err_vars.o fmap.o fname.o fname-uf.o fname-sf.o \
  gnatmake.o gnatvsn.o hostparm.o interfac.o i-c.o i-cstrin.o krunch.o lib.o \
  make.o makeusg.o make_util.o namet.o nlists.o opt.o osint.o osint-m.o \
- output.o restrict.o rident.o s-exctab.o s-cautns.o \
+ output.o restrict.o rident.o s-cautns.o s-excmac.o s-exctab.o \
  s-secsta.o s-stalib.o s-stoele.o scans.o scng.o sdefault.o sfn_scan.o \
  s-purexc.o s-htable.o scil_ll.o sem_aux.o sinfo.o sinput.o sinput-c.o \
  snames.o stand.o stringt.o styleg.o stylesw.o system.o validsw.o \
@@ -840,35 +840,6 @@ gnatlib-shared:
 	     PICFLAG_FOR_TARGET="$(PICFLAG_FOR_TARGET)" \
 	     $(GNATLIB_SHARED)
 
-# When building a SJLJ runtime for VxWorks, we need to ensure that the extra
-# linker options needed for ZCX are not passed to prevent the inclusion of
-# useless objects and potential troubles from the presence of extra symbols
-# and references in some configurations.  The inhibition is performed by
-# commenting the pragma instead of deleting the line, as the latter might
-# result in getting multiple blank lines, hence possible style check errors.
-gnatlib-sjlj:
-	$(MAKE) $(FLAGS_TO_PASS) \
-	     EH_MECHANISM="" \
-	     MULTISUBDIR="$(MULTISUBDIR)" \
-	     THREAD_KIND="$(THREAD_KIND)" \
-	     LN_S="$(LN_S)" \
-	     ../stamp-gnatlib1-$(RTSDIR)
-	sed \
-	  -e 's/Frontend_Exceptions.*/Frontend_Exceptions       : constant Boolean := True;/' \
-	  -e 's/ZCX_By_Default.*/ZCX_By_Default            : constant Boolean := False;/' \
-	  $(RTSDIR)/system.ads > $(RTSDIR)/s.ads
-	$(MV) $(RTSDIR)/s.ads $(RTSDIR)/system.ads
-	$(MAKE) $(FLAGS_TO_PASS) \
-	     EH_MECHANISM="" \
-	     GNATLIBFLAGS="$(GNATLIBFLAGS)" \
-	     GNATLIBCFLAGS="$(GNATLIBCFLAGS)" \
-	     GNATLIBCFLAGS_FOR_C="$(GNATLIBCFLAGS_FOR_C)" \
-	     FORCE_DEBUG_ADAFLAGS="$(FORCE_DEBUG_ADAFLAGS)" \
-	     MULTISUBDIR="$(MULTISUBDIR)" \
-	     THREAD_KIND="$(THREAD_KIND)" \
-	     LN_S="$(LN_S)" \
-	     gnatlib
-
 gnatlib-zcx:
 	$(MAKE) $(FLAGS_TO_PASS) \
 	     EH_MECHANISM="-gcc" \
@@ -877,7 +848,6 @@ gnatlib-zcx:
 	     LN_S="$(LN_S)" \
 	     ../stamp-gnatlib1-$(RTSDIR)
 	sed \
-	  -e 's/Frontend_Exceptions.*/Frontend_Exceptions       : constant Boolean := False;/' \
 	  -e 's/ZCX_By_Default.*/ZCX_By_Default            : constant Boolean := True;/' \
 	  $(RTSDIR)/system.ads > $(RTSDIR)/s.ads
 	$(MV) $(RTSDIR)/s.ads $(RTSDIR)/system.ads
diff --git a/gcc/ada/gcc-interface/decl.cc b/gcc/ada/gcc-interface/decl.cc
index 903ec84..86cbf5b 100644
--- a/gcc/ada/gcc-interface/decl.cc
+++ b/gcc/ada/gcc-interface/decl.cc
@@ -6421,6 +6421,33 @@ gnat_to_gnu_subprog_type (Entity_Id gnat_subprog, bool definition,
 	 since structures are incomplete for the back-end.  */
       else if (Convention (gnat_subprog) != Convention_Stubbed)
 	{
+	  /* If we have two entries that may be returned in integer registers,
+	     the larger has power-of-2 size and the smaller is integer, then
+	     extend the smaller to this power-of-2 size to get a return type
+	     with power-of-2 size and no holes, again to speed up accesses.  */
+	  if (list_length (gnu_cico_field_list) == 2
+	      && gnu_cico_only_integral_type)
+	    {
+	      tree typ1 = TREE_TYPE (gnu_cico_field_list);
+	      tree typ2 = TREE_TYPE (DECL_CHAIN (gnu_cico_field_list));
+	      if (TREE_CODE (typ1) == INTEGER_TYPE
+		  && integer_pow2p (TYPE_SIZE (typ2))
+		  && compare_tree_int (TYPE_SIZE (typ2),
+				       MAX_FIXED_MODE_SIZE) <= 0
+		  && tree_int_cst_lt (TYPE_SIZE (typ1), TYPE_SIZE (typ2)))
+		TREE_TYPE (gnu_cico_field_list)
+		  = gnat_type_for_size (TREE_INT_CST_LOW (TYPE_SIZE (typ2)),
+					TYPE_UNSIGNED (typ1));
+	      else if (TREE_CODE (typ2) == INTEGER_TYPE
+		       && integer_pow2p (TYPE_SIZE (typ1))
+		       && compare_tree_int (TYPE_SIZE (typ1),
+					    MAX_FIXED_MODE_SIZE) <= 0
+		       && tree_int_cst_lt (TYPE_SIZE (typ2), TYPE_SIZE (typ1)))
+		TREE_TYPE (DECL_CHAIN (gnu_cico_field_list))
+		  = gnat_type_for_size (TREE_INT_CST_LOW (TYPE_SIZE (typ1)),
+					TYPE_UNSIGNED (typ2));
+	    }
+
 	  finish_record_type (gnu_cico_return_type,
 			      nreverse (gnu_cico_field_list),
 			      0, false);
diff --git a/gcc/ada/gcc-interface/misc.cc b/gcc/ada/gcc-interface/misc.cc
index 128040e..7711f8b 100644
--- a/gcc/ada/gcc-interface/misc.cc
+++ b/gcc/ada/gcc-interface/misc.cc
@@ -271,7 +271,7 @@ gnat_post_options (const char **pfilename ATTRIBUTE_UNUSED)
 
   /* No caret by default for Ada.  */
   if (!OPTION_SET_P (flag_diagnostics_show_caret))
-    global_dc->m_source_printing.enabled = false;
+    global_dc->get_source_printing_options ().enabled = false;
 
   /* Copy global settings to local versions.  */
   gnat_encodings = global_options.x_gnat_encodings;
@@ -292,7 +292,7 @@ gnat_post_options (const char **pfilename ATTRIBUTE_UNUSED)
 /* Here is the function to handle the compiler error processing in GCC.  */
 
 static void
-internal_error_function (diagnostic_context *context, const char *msgid,
+internal_error_function (diagnostics::context *context, const char *msgid,
 			 va_list *ap)
 {
   char *buffer, *p, *loc;
diff --git a/gcc/ada/gcc-interface/trans.cc b/gcc/ada/gcc-interface/trans.cc
index e02804b..fd1d39c 100644
--- a/gcc/ada/gcc-interface/trans.cc
+++ b/gcc/ada/gcc-interface/trans.cc
@@ -1510,7 +1510,7 @@ Pragma_to_gnu (Node_Id gnat_node)
 	const location_t location = input_location;
 	struct cl_option_handlers handlers;
 	unsigned int option_index;
-	diagnostic_t kind;
+	enum diagnostics::kind kind;
 	bool imply;
 
 	gnat_temp = First (Pragma_Argument_Associations (gnat_node));
@@ -1521,12 +1521,12 @@ Pragma_to_gnu (Node_Id gnat_node)
 	    switch (id)
 	      {
 	      case Pragma_Warning_As_Error:
-		kind = DK_ERROR;
+		kind = diagnostics::kind::error;
 		imply = false;
 		break;
 
 	      case Pragma_Warnings:
-		kind = DK_WARNING;
+		kind = diagnostics::kind::warning;
 		imply = true;
 		break;
 
@@ -1543,11 +1543,11 @@ Pragma_to_gnu (Node_Id gnat_node)
 	    switch (Chars (Expression (gnat_temp)))
 	      {
 		case Name_Off:
-		  kind = DK_IGNORED;
+		  kind = diagnostics::kind::ignored;
 		  break;
 
 		case Name_On:
-		  kind = DK_WARNING;
+		  kind = diagnostics::kind::warning;
 		  break;
 
 		default:
@@ -1569,7 +1569,7 @@ Pragma_to_gnu (Node_Id gnat_node)
 		gnat_expr = Empty;
 
 		/* For pragma Warnings (Off), we save the current state...  */
-		if (kind == DK_IGNORED)
+		if (kind == diagnostics::kind::ignored)
 		  diagnostic_push_diagnostics (global_dc, location);
 
 		/* ...so that, for pragma Warnings (On), we do not enable all
@@ -4049,7 +4049,7 @@ Subprogram_Body_to_gnu (Node_Id gnat_node)
 	    tree gnu_decl;
 
 	    /* Skip any entries that have been already filled in; they must
-	       correspond to In Out parameters.  */
+	       correspond to In Out parameters or previous Out parameters.  */
 	    while (gnu_cico_entry && TREE_VALUE (gnu_cico_entry))
 	      gnu_cico_entry = TREE_CHAIN (gnu_cico_entry);
 
@@ -4059,11 +4059,22 @@ Subprogram_Body_to_gnu (Node_Id gnat_node)
 	    if (DECL_BY_REF_P (gnu_decl))
 	      gnu_decl = build_unary_op (INDIRECT_REF, NULL_TREE, gnu_decl);
 
-	    /* Do any needed references for padded types.  */
-	    TREE_VALUE (gnu_cico_entry)
-	      = convert (TREE_TYPE (TREE_PURPOSE (gnu_cico_entry)), gnu_decl);
+	    TREE_VALUE (gnu_cico_entry) = gnu_decl;
 	  }
+
+      /* Finally, ensure type consistency between TREE_PURPOSE and TREE_VALUE
+	 so that the assignment of the latter to the former can be done.  */
+      tree gnu_cico_entry = gnu_cico_list;
+      while (gnu_cico_entry)
+	{
+	  if (!VOID_TYPE_P (TREE_VALUE (gnu_cico_entry)))
+	    TREE_VALUE (gnu_cico_entry)
+	      = convert (TREE_TYPE (TREE_PURPOSE (gnu_cico_entry)),
+			 TREE_VALUE (gnu_cico_entry));
+	  gnu_cico_entry = TREE_CHAIN (gnu_cico_entry);
+	}
     }
+
   else
     vec_safe_push (gnu_return_label_stack, NULL_TREE);
 
@@ -4161,9 +4172,13 @@ Subprogram_Body_to_gnu (Node_Id gnat_node)
 	    }
 	}
 
-      /* Otherwise, if this is a procedure or a function which does not return
-	 by invisible reference, we can do a direct block-copy out.  */
-      else
+      /* Otherwise, if this is a procedure or a function that does not return
+	 by invisible reference, we can do a direct block-copy out, but we do
+	 not need to do it for a null initialization procedure when the _Init
+	 parameter is not passed in since we would copy uninitialized bits.  */
+      else if (!(Is_Null_Init_Proc (gnat_subprog)
+		 && list_length (gnu_cico_list) == 1
+		 && TREE_CODE (TREE_VALUE (gnu_cico_list)) == VAR_DECL))
 	{
 	  tree gnu_retval;
 
@@ -8461,7 +8476,8 @@ gnat_to_gnu (Node_Id gnat_node)
 	      oconstraints[i] = constraint;
 
 	      if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-					   &allows_mem, &allows_reg, &fake))
+					   &allows_mem, &allows_reg, &fake,
+					   nullptr))
 		{
 		  /* If the operand is going to end up in memory,
 		     mark it addressable.  Note that we don't test
@@ -8489,9 +8505,9 @@ gnat_to_gnu (Node_Id gnat_node)
 	      constraint
 		= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (tail)));
 
-	      if (parse_input_constraint (&constraint, i, ninputs, noutputs,
-					  0, oconstraints,
-					  &allows_mem, &allows_reg))
+	      if (parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
+					  oconstraints, &allows_mem,
+					  &allows_reg, nullptr))
 		{
 		  /* If the operand is going to end up in memory,
 		     mark it addressable.  */
@@ -8737,7 +8753,7 @@ gnat_to_gnu (Node_Id gnat_node)
 
   /* Set the location information on the result if it's not a simple name
      or something that contains a simple name, for example a tag, because
-     we don"t want all the references to get the location of the first use.
+     we don't want all the references to get the location of the first use.
      Note that we may have no result if we tried to build a CALL_EXPR node
      to a procedure with no side-effects and optimization is enabled.  */
   else if (kind != N_Identifier
diff --git a/gcc/ada/gcc-interface/utils.cc b/gcc/ada/gcc-interface/utils.cc
index 23737c3..f501915 100644
--- a/gcc/ada/gcc-interface/utils.cc
+++ b/gcc/ada/gcc-interface/utils.cc
@@ -1225,7 +1225,6 @@ make_packable_type (tree type, bool in_record, unsigned int max_align)
      Note that we rely on the pointer equality created here for
      TYPE_NAME to look through conversions in various places.  */
   TYPE_NAME (new_type) = TYPE_NAME (type);
-  TYPE_PACKED (new_type) = 1;
   TYPE_JUSTIFIED_MODULAR_P (new_type) = TYPE_JUSTIFIED_MODULAR_P (type);
   TYPE_CONTAINS_TEMPLATE_P (new_type) = TYPE_CONTAINS_TEMPLATE_P (type);
   TYPE_REVERSE_STORAGE_ORDER (new_type) = TYPE_REVERSE_STORAGE_ORDER (type);
@@ -1240,6 +1239,8 @@ make_packable_type (tree type, bool in_record, unsigned int max_align)
       new_size = ceil_pow2 (size);
       new_align = MIN (new_size, BIGGEST_ALIGNMENT);
       SET_TYPE_ALIGN (new_type, new_align);
+      /* build_aligned_type needs to be able to adjust back the alignment.  */
+      TYPE_PACKED (new_type) = 0;
     }
   else
     {
@@ -1261,6 +1262,7 @@ make_packable_type (tree type, bool in_record, unsigned int max_align)
       if (max_align > 0 && new_align > max_align)
 	new_align = max_align;
       SET_TYPE_ALIGN (new_type, MIN (align, new_align));
+      TYPE_PACKED (new_type) = 1;
     }
 
   TYPE_USER_ALIGN (new_type) = 1;
@@ -4508,8 +4510,8 @@ build_unc_object_type_from_ptr (tree thin_fat_ptr_type, tree object_type,
 void
 update_pointer_to (tree old_type, tree new_type)
 {
-  tree ptr = TYPE_POINTER_TO (old_type);
-  tree ref = TYPE_REFERENCE_TO (old_type);
+  const tree old_ptr = TYPE_POINTER_TO (old_type);
+  const tree old_ref = TYPE_REFERENCE_TO (old_type);
   tree t;
 
   /* If this is the main variant, process all the other variants first.  */
@@ -4518,7 +4520,7 @@ update_pointer_to (tree old_type, tree new_type)
       update_pointer_to (t, new_type);
 
   /* If no pointers and no references, we are done.  */
-  if (!ptr && !ref)
+  if (!old_ptr && !old_ref)
     return;
 
   /* Merge the old type qualifiers in the new type.
@@ -4552,12 +4554,13 @@ update_pointer_to (tree old_type, tree new_type)
   if (TREE_CODE (new_type) != UNCONSTRAINED_ARRAY_TYPE)
     {
       tree new_ptr, new_ref;
+      tree ptr, ref;
 
       /* If pointer or reference already points to new type, nothing to do.
 	 This can happen as update_pointer_to can be invoked multiple times
 	 on the same couple of types because of the type variants.  */
-      if ((ptr && TREE_TYPE (ptr) == new_type)
-	  || (ref && TREE_TYPE (ref) == new_type))
+      if ((old_ptr && TREE_TYPE (old_ptr) == new_type)
+	  || (old_ref && TREE_TYPE (old_ref) == new_type))
 	return;
 
       /* Chain PTR and its variants at the end.  */
@@ -4566,13 +4569,13 @@ update_pointer_to (tree old_type, tree new_type)
 	{
 	  while (TYPE_NEXT_PTR_TO (new_ptr))
 	    new_ptr = TYPE_NEXT_PTR_TO (new_ptr);
-	  TYPE_NEXT_PTR_TO (new_ptr) = ptr;
+	  TYPE_NEXT_PTR_TO (new_ptr) = old_ptr;
 	}
       else
-	TYPE_POINTER_TO (new_type) = ptr;
+	TYPE_POINTER_TO (new_type) = old_ptr;
 
       /* Now adjust them.  */
-      for (; ptr; ptr = TYPE_NEXT_PTR_TO (ptr))
+      for (ptr = old_ptr; ptr; ptr = TYPE_NEXT_PTR_TO (ptr))
 	for (t = TYPE_MAIN_VARIANT (ptr); t; t = TYPE_NEXT_VARIANT (t))
 	  {
 	    TREE_TYPE (t) = new_type;
@@ -4587,13 +4590,13 @@ update_pointer_to (tree old_type, tree new_type)
 	{
 	  while (TYPE_NEXT_REF_TO (new_ref))
 	    new_ref = TYPE_NEXT_REF_TO (new_ref);
-	  TYPE_NEXT_REF_TO (new_ref) = ref;
+	  TYPE_NEXT_REF_TO (new_ref) = old_ref;
 	}
       else
-	TYPE_REFERENCE_TO (new_type) = ref;
+	TYPE_REFERENCE_TO (new_type) = old_ref;
 
       /* Now adjust them.  */
-      for (; ref; ref = TYPE_NEXT_REF_TO (ref))
+      for (ref = old_ref; ref; ref = TYPE_NEXT_REF_TO (ref))
 	for (t = TYPE_MAIN_VARIANT (ref); t; t = TYPE_NEXT_VARIANT (t))
 	  {
 	    TREE_TYPE (t) = new_type;
@@ -4612,20 +4615,20 @@ update_pointer_to (tree old_type, tree new_type)
     {
       tree new_ptr = TYPE_POINTER_TO (new_type);
 
-      gcc_assert (TYPE_IS_FAT_POINTER_P (ptr));
+      gcc_assert (TYPE_IS_FAT_POINTER_P (old_ptr));
 
       /* If PTR already points to NEW_TYPE, nothing to do.  This can happen
 	 since update_pointer_to can be invoked multiple times on the same
 	 couple of types because of the type variants.  */
-      if (TYPE_UNCONSTRAINED_ARRAY (ptr) == new_type)
+      if (TYPE_UNCONSTRAINED_ARRAY (old_ptr) == new_type)
 	return;
 
       update_pointer_to
-	(TREE_TYPE (TREE_TYPE (TYPE_FIELDS (ptr))),
+	(TREE_TYPE (TREE_TYPE (TYPE_FIELDS (old_ptr))),
 	 TREE_TYPE (TREE_TYPE (TYPE_FIELDS (new_ptr))));
 
       update_pointer_to
-	(TREE_TYPE (TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (ptr)))),
+	(TREE_TYPE (TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (old_ptr)))),
 	 TREE_TYPE (TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (new_ptr)))));
 
       update_pointer_to (TYPE_OBJECT_RECORD_TYPE (old_type),
diff --git a/gcc/ada/gen_il-fields.ads b/gcc/ada/gen_il-fields.ads
index 2d16e12..a1e284f 100644
--- a/gcc/ada/gen_il-fields.ads
+++ b/gcc/ada/gen_il-fields.ads
@@ -255,6 +255,7 @@ package Gen_IL.Fields is
       Is_Entry_Barrier_Function,
       Is_Expanded_Build_In_Place_Call,
       Is_Expanded_Constructor_Call,
+      Is_Expanded_Dispatching_Call,
       Is_Expanded_Prefixed_Call,
       Is_Folded_In_Parser,
       Is_Generic_Contract_Pragma,
@@ -400,6 +401,7 @@ package Gen_IL.Fields is
       Suppress_Loop_Warnings,
       Synchronized_Present,
       Tagged_Present,
+      Tag_Propagated,
       Target,
       Call_Or_Target_Loop,
       Target_Type,
@@ -539,6 +541,7 @@ package Gen_IL.Fields is
       Extra_Constrained,
       Extra_Formal,
       Extra_Formals,
+      Extra_Formals_Known,
       Finalization_Collection,
       Finalization_Master_Node,
       Finalize_Storage_Only,
@@ -572,6 +575,7 @@ package Gen_IL.Fields is
       Has_Delayed_Aspects,
       Has_Delayed_Freeze,
       Has_Delayed_Rep_Aspects,
+      Has_Destructor,
       Has_Discriminants,
       Has_Dispatch_Table,
       Has_Dynamic_Predicate_Aspect,
@@ -699,6 +703,7 @@ package Gen_IL.Fields is
       Is_CPP_Class,
       Is_CUDA_Kernel,
       Is_Descendant_Of_Address,
+      Is_Destructor,
       Is_DIC_Procedure,
       Is_Discrim_SO_Function,
       Is_Discriminant_Check_Function,
@@ -730,6 +735,7 @@ package Gen_IL.Fields is
       Is_Ignored_Ghost_Entity,
       Is_Immediately_Visible,
       Is_Implementation_Defined,
+      Is_Implicit_Full_View,
       Is_Imported,
       Is_Independent,
       Is_Initial_Condition_Procedure,
@@ -850,6 +856,7 @@ package Gen_IL.Fields is
       Original_Protected_Subprogram,
       Original_Record_Component,
       Overlays_Constant,
+      Overridden_Inherited_Operation,
       Overridden_Operation,
       Package_Instantiation,
       Packed_Array_Impl_Type,
diff --git a/gcc/ada/gen_il-gen-gen_entities.adb b/gcc/ada/gen_il-gen-gen_entities.adb
index 8cbed8a..0fedfbc 100644
--- a/gcc/ada/gen_il-gen-gen_entities.adb
+++ b/gcc/ada/gen_il-gen-gen_entities.adb
@@ -467,6 +467,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Full_View, Node_Id),
         Sm (Has_Completion_In_Body, Flag),
         Sm (Has_Constrained_Partial_View, Flag, Base_Type_Only),
+        Sm (Has_Destructor, Flag, Base_Type_Only),
         Sm (Has_Discriminants, Flag),
         Sm (Has_Dispatch_Table, Flag,
             Pre => "Is_Tagged_Type (N)"),
@@ -502,6 +503,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Is_Fixed_Lower_Bound_Array_Subtype, Flag),
         Sm (Is_Fixed_Lower_Bound_Index_Subtype, Flag),
         Sm (Is_Generic_Actual_Type, Flag),
+        Sm (Is_Implicit_Full_View, Flag),
         Sm (Is_Mutably_Tagged_Type, Flag),
         Sm (Is_Non_Static_Subtype, Flag),
         Sm (Is_Private_Composite, Flag),
@@ -935,11 +937,13 @@ begin -- Gen_IL.Gen.Gen_Entities
        (Sm (Access_Subprogram_Wrapper, Node_Id),
         Sm (Extra_Accessibility_Of_Result, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (Needs_No_Actuals, Flag)));
 
    Ab (Overloadable_Kind, Entity_Kind,
        (Sm (Renamed_Or_Alias, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (Is_Abstract_Subprogram, Flag),
         Sm (Is_Primitive, Flag),
         Sm (Needs_No_Actuals, Flag),
@@ -953,6 +957,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Enumeration_Rep_Expr, Node_Id),
         Sm (Esize, Uint),
         Sm (Alignment, Unat),
+        Sm (Overridden_Inherited_Operation, Node_Id),
         Sm (Interface_Name, Node_Id)));
 
    Ab (Subprogram_Kind, Overloadable_Kind,
@@ -981,6 +986,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Is_Machine_Code_Subprogram, Flag),
         Sm (Last_Entity, Node_Id),
         Sm (Linker_Section_Pragma, Node_Id),
+        Sm (Overridden_Inherited_Operation, Node_Id),
         Sm (Overridden_Operation, Node_Id),
         Sm (Protected_Body_Subprogram, Node_Id),
         Sm (No_Raise, Flag),
@@ -1050,6 +1056,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Is_Asynchronous, Flag),
         Sm (Is_Called, Flag),
         Sm (Is_CUDA_Kernel, Flag),
+        Sm (Is_Destructor, Flag),
         Sm (Is_DIC_Procedure, Flag),
         Sm (Is_Generic_Actual_Subprogram, Flag),
         Sm (Is_Initial_Condition_Procedure, Flag),
@@ -1125,6 +1132,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Entry_Accepted, Flag),
         Sm (Entry_Parameters_Type, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (First_Entity, Node_Id),
         Sm (Has_Out_Or_In_Out_Parameter, Flag),
         Sm (Ignore_SPARK_Mode_Pragmas, Flag),
@@ -1326,6 +1334,7 @@ begin -- Gen_IL.Gen.Gen_Entities
        (Sm (Anonymous_Collections, Elist_Id),
         Sm (Contract, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (First_Entity, Node_Id),
         Sm (Ignore_SPARK_Mode_Pragmas, Flag),
         Sm (Interface_Name, Node_Id),
diff --git a/gcc/ada/gen_il-gen-gen_nodes.adb b/gcc/ada/gen_il-gen-gen_nodes.adb
index f4e7917..412565f 100644
--- a/gcc/ada/gen_il-gen-gen_nodes.adb
+++ b/gcc/ada/gen_il-gen-gen_nodes.adb
@@ -149,6 +149,7 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Is_Controlling_Actual, Flag),
         Sm (Is_Overloaded, Flag),
         Sm (Is_Static_Expression, Flag),
+        Sm (Is_Expanded_Dispatching_Call, Flag),
         Sm (Must_Not_Freeze, Flag),
         Sm (Raises_Constraint_Error, Flag)));
 
@@ -181,7 +182,8 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Is_Elaboration_Warnings_OK_Node, Flag),
         Sm (Is_SPARK_Mode_On_Node, Flag),
         Sm (Original_Discriminant, Node_Id),
-        Sm (Redundant_Use, Flag)));
+        Sm (Redundant_Use, Flag),
+        Sm (Tag_Propagated, Flag)));
 
    Cc (N_Operator_Symbol, N_Direct_Name,
        (Sy (Strval, String_Id)));
@@ -346,7 +348,8 @@ begin -- Gen_IL.Gen.Gen_Nodes
        (Sy (Prefix, Node_Id),
         Sm (Actual_Designated_Subtype, Node_Id),
         Sm (Atomic_Sync_Required, Flag),
-        Sm (Has_Dereference_Action, Flag)));
+        Sm (Has_Dereference_Action, Flag),
+        Sm (Tag_Propagated, Flag)));
 
    Cc (N_Expression_With_Actions, N_Subexpr,
        (Sy (Actions, List_Id, Default_No_List),
@@ -463,6 +466,7 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Do_Length_Check, Flag),
         Sm (Do_Overflow_Check, Flag),
         Sm (Float_Truncate, Flag),
+        Sm (Tag_Propagated, Flag),
         Sm (Rounded_Result, Flag)));
 
    Cc (N_Unchecked_Type_Conversion, N_Subexpr,
@@ -905,6 +909,7 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Cleanup_Actions, List_Id),
         Sm (Exception_Junk, Flag),
         Sm (Is_Abort_Block, Flag),
+        Sm (Is_Expanded_Dispatching_Call, Flag),
         Sm (Is_Initialization_Block, Flag),
         Sm (Is_Task_Master, Flag)));
 
diff --git a/gcc/ada/gen_il-internals.adb b/gcc/ada/gen_il-internals.adb
index 8d0dfc7..3fa8b94 100644
--- a/gcc/ada/gen_il-internals.adb
+++ b/gcc/ada/gen_il-internals.adb
@@ -277,6 +277,8 @@ package body Gen_IL.Internals is
             return "DT_Offset_To_Top_Func";
          when DT_Position =>
             return "DT_Position";
+         when Extra_Formals_Known =>
+            return "Extra_Formals_Known";
          when Forwards_OK =>
             return "Forwards_OK";
          when Has_First_Controlling_Parameter_Aspect =>
diff --git a/gcc/ada/ghost.adb b/gcc/ada/ghost.adb
index 6f648f2..f9c2853 100644
--- a/gcc/ada/ghost.adb
+++ b/gcc/ada/ghost.adb
@@ -447,7 +447,7 @@ package body Ghost is
          --  The context is Ghost when it appears within a Ghost package or
          --  subprogram.
 
-         if Ghost_Mode > None then
+         if Ghost_Config.Ghost_Mode > None then
             return True;
 
          --  Routine Expand_Record_Extension creates a parent subtype without
@@ -719,7 +719,7 @@ package body Ghost is
       --  The context is ghost when it appears within a Ghost package or
       --  subprogram.
 
-      if Ghost_Mode > None then
+      if Ghost_Config.Ghost_Mode > None then
          return;
 
       --  The context is ghost if Formal is explicitly marked as ghost
@@ -1130,22 +1130,22 @@ package body Ghost is
       --  The context is already within an ignored Ghost region. Maintain the
       --  start of the outermost ignored Ghost region.
 
-      if Present (Ignored_Ghost_Region) then
+      if Present (Ghost_Config.Ignored_Ghost_Region) then
          null;
 
       --  The current region is the outermost ignored Ghost region. Save its
       --  starting node.
 
       elsif Present (N) and then Mode = Ignore then
-         Ignored_Ghost_Region := N;
+         Ghost_Config.Ignored_Ghost_Region := N;
 
       --  Otherwise the current region is not ignored, nothing to save
 
       else
-         Ignored_Ghost_Region := Empty;
+         Ghost_Config.Ignored_Ghost_Region := Empty;
       end if;
 
-      Ghost_Mode := Mode;
+      Ghost_Config.Ghost_Mode := Mode;
    end Install_Ghost_Region;
 
    procedure Install_Ghost_Region (Mode : Name_Id; N : Node_Id) is
@@ -1504,10 +1504,10 @@ package body Ghost is
       --  A body declared within a Ghost region is automatically Ghost
       --  (SPARK RM 6.9(2)).
 
-      elsif Ghost_Mode = Check then
+      elsif Ghost_Config.Ghost_Mode = Check then
          Policy := Name_Check;
 
-      elsif Ghost_Mode = Ignore then
+      elsif Ghost_Config.Ghost_Mode = Ignore then
          Policy := Name_Ignore;
 
       --  Inherit the "ghostness" of the previous declaration when the body
@@ -1553,10 +1553,10 @@ package body Ghost is
       --  A completion elaborated in a Ghost region is automatically Ghost
       --  (SPARK RM 6.9(2)).
 
-      if Ghost_Mode = Check then
+      if Ghost_Config.Ghost_Mode = Check then
          Policy := Name_Check;
 
-      elsif Ghost_Mode = Ignore then
+      elsif Ghost_Config.Ghost_Mode = Ignore then
          Policy := Name_Ignore;
 
       --  The completion becomes Ghost when its initial declaration is also
@@ -1603,10 +1603,10 @@ package body Ghost is
       --  A declaration elaborated in a Ghost region is automatically Ghost
       --  (SPARK RM 6.9(2)).
 
-      elsif Ghost_Mode = Check then
+      elsif Ghost_Config.Ghost_Mode = Check then
          Policy := Name_Check;
 
-      elsif Ghost_Mode = Ignore then
+      elsif Ghost_Config.Ghost_Mode = Ignore then
          Policy := Name_Ignore;
 
       --  A child package or subprogram declaration becomes Ghost when its
@@ -1698,10 +1698,10 @@ package body Ghost is
       --  An instantiation declaration within a Ghost region is automatically
       --  Ghost (SPARK RM 6.9(2)).
 
-      elsif Ghost_Mode = Check then
+      elsif Ghost_Config.Ghost_Mode = Check then
          Policy := Name_Check;
 
-      elsif Ghost_Mode = Ignore then
+      elsif Ghost_Config.Ghost_Mode = Ignore then
          Policy := Name_Ignore;
 
       --  Inherit the "ghostness" of the generic unit, but the current Ghost
@@ -2018,10 +2018,9 @@ package body Ghost is
    -- Restore_Ghost_Region --
    --------------------------
 
-   procedure Restore_Ghost_Region (Mode : Ghost_Mode_Type; N : Node_Id) is
+   procedure Restore_Ghost_Region (Config : Ghost_Config_Type) is
    begin
-      Ghost_Mode           := Mode;
-      Ignored_Ghost_Region := N;
+      Ghost_Config := Config;
    end Restore_Ghost_Region;
 
    --------------------
diff --git a/gcc/ada/ghost.ads b/gcc/ada/ghost.ads
index 3863e50..62c809c 100644
--- a/gcc/ada/ghost.ads
+++ b/gcc/ada/ghost.ads
@@ -243,7 +243,7 @@ package Ghost is
    --  WARNING: this is a separate front end pass, care should be taken to keep
    --  it optimized.
 
-   procedure Restore_Ghost_Region (Mode : Ghost_Mode_Type; N : Node_Id);
+   procedure Restore_Ghost_Region (Config : Ghost_Config_Type);
    pragma Inline (Restore_Ghost_Region);
    --  Restore a Ghost region to a previous state described by mode Mode and
    --  ignored region start node N. This routine must be used in conjunction
diff --git a/gcc/ada/gnat1drv.adb b/gcc/ada/gnat1drv.adb
index 52063c8..ee2c329 100644
--- a/gcc/ada/gnat1drv.adb
+++ b/gcc/ada/gnat1drv.adb
@@ -503,11 +503,6 @@ procedure Gnat1drv is
 
          Operating_Mode := Check_Semantics;
 
-         --  Enable assertions, since they give valuable extra information for
-         --  formal verification.
-
-         Assertions_Enabled := True;
-
          --  Disable validity checks, since it generates code raising
          --  exceptions for invalid data, which confuses GNATprove. Invalid
          --  data is directly detected by GNATprove's flow analysis.
diff --git a/gcc/ada/gnat_rm.texi b/gcc/ada/gnat_rm.texi
index 79fb225..5d7bedc 100644
--- a/gcc/ada/gnat_rm.texi
+++ b/gcc/ada/gnat_rm.texi
@@ -19,7 +19,7 @@
 
 @copying
 @quotation
-GNAT Reference Manual , Jun 27, 2025
+GNAT Reference Manual , Jul 24, 2025
 
 AdaCore
 
@@ -931,6 +931,7 @@ Experimental Language Extensions
 * External_Initialization Aspect:: 
 * Finally construct:: 
 * Continue statement:: 
+* Destructors:: 
 
 Storage Model
 
@@ -7548,13 +7549,33 @@ Syntax:
 pragma Short_Circuit_And_Or;
 @end example
 
-This configuration pragma causes any occurrence of the AND operator applied to
-operands of type Standard.Boolean to be short-circuited (i.e. the AND operator
-is treated as if it were AND THEN). Or is similarly treated as OR ELSE. This
-may be useful in the context of certification protocols requiring the use of
-short-circuited logical operators. If this configuration pragma occurs locally
-within the file being compiled, it applies only to the file being compiled.
+This configuration pragma causes the predefined AND and OR operators of
+type Standard.Boolean to have short-circuit semantics. That is, they
+behave like AND THEN and OR ELSE; the right-hand side is not evaluated
+if the left-hand side determines the result. This may be useful in the
+context of certification protocols requiring the use of short-circuited
+logical operators.
+
 There is no requirement that all units in a partition use this option.
+However, mixing of short-circuit and non-short-circuit semantics can be
+confusing. Therefore, the recommended use is to put the pragma in a
+configuration file that applies to the whole program. Alternatively, if
+you have a legacy library that should not use this pragma, you can put
+it in a separate library project that does not use the pragma.
+In any case, fine-grained mixing of the different semantics is not
+recommended. If pragma @code{Short_Circuit_And_Or} is specified, then it
+is illegal to rename the predefined Boolean AND and OR, or to pass
+them to generic formal functions; this corresponds to the fact that
+AND THEN and OR ELSE cannot be renamed nor passed as generic formal
+functions.
+
+Note that this pragma has no effect on other logical operators –
+predefined operators of modular types, array-of-boolean types and types
+derived from Standard.Boolean, nor user-defined operators.
+
+See also the pragma @code{Unevaluated_Use_Of_Old} and the restriction
+@code{No_Direct_Boolean_Operators}, which may be useful in conjunction
+with @code{Short_Circuit_And_Or}.
 
 @node Pragma Short_Descriptors,Pragma Side_Effects,Pragma Short_Circuit_And_Or,Implementation Defined Pragmas
 @anchor{gnat_rm/implementation_defined_pragmas pragma-short-descriptors}@anchor{ed}
@@ -13051,9 +13072,9 @@ pragma Export (C, Last_Chance_Handler,
                "__gnat_last_chance_handler");
 @end example
 
-The parameter is a C null-terminated string representing a message to be
-associated with the exception (typically the source location of the raise
-statement generated by the compiler). The Line parameter when nonzero
+The @code{Source_Location} parameter is a C null-terminated string representing a
+message to be associated with the exception (typically the source location of
+the raise statement generated by the compiler). The Line parameter when nonzero
 represents the line number in the source program where the raise occurs.
 
 @node No_Exception_Propagation,No_Exception_Registration,No_Exception_Handlers,Partition-Wide Restrictions
@@ -31216,6 +31237,7 @@ Features activated via @code{-gnatX0} or
 * External_Initialization Aspect:: 
 * Finally construct:: 
 * Continue statement:: 
+* Destructors:: 
 
 @end menu
 
@@ -32568,7 +32590,7 @@ Abort/ATC (asynchronous transfer of control) cannot interrupt a finally block, n
 execution, that is the finally block must be executed in full even if the containing task is
 aborted, or if the control is transferred out of the block.
 
-@node Continue statement,,Finally construct,Experimental Language Extensions
+@node Continue statement,Destructors,Finally construct,Experimental Language Extensions
 @anchor{gnat_rm/gnat_language_extensions continue-statement}@anchor{472}
 @subsection Continue statement
 
@@ -32586,8 +32608,78 @@ statement in the sequence of statements of the specified loop_statement.
 Note that @code{continue} is a keyword but it is not a reserved word. This is a
 configuration that does not exist in standard Ada.
 
+@node Destructors,,Continue statement,Experimental Language Extensions
+@anchor{gnat_rm/gnat_language_extensions destructors}@anchor{473}
+@subsection Destructors
+
+
+The @code{Destructor} aspect can be applied to any record type, tagged or not.
+It must denote a primitive of the type that is a procedure with one parameter
+of the type and of mode @code{in out}:
+
+@example
+type T is record
+   ...
+end record with Destructor => Foo;
+
+procedure Foo (X : in out T);
+@end example
+
+This is equivalent to the following code that uses @code{Finalizable}:
+
+@example
+type T is record
+   ...
+end record with Finalizable => (Finalize => Foo);
+
+procedure Foo (X : in out T);
+@end example
+
+Unlike @code{Finalizable}, however, @code{Destructor} can be specified on a derived
+type. And when it is, the effect of the aspect combines with the destructors of
+the parent type. Take, for example:
+
+@example
+type T1 is record
+   ...
+end record with Destructor => Foo;
+
+procedure Foo (X : in out T1);
+
+type T2 is new T1 with Destructor => Bar;
+
+procedure Bar (X : in out T2);
+@end example
+
+Here, when an object of type @code{T2} is finalized, a call to @code{Bar}
+will be performed and it will be followed by a call to @code{Foo}.
+
+The @code{Destructor} aspect comes with a legality rule: if a primitive procedure
+of a type is denoted by a @code{Destructor} aspect specification, it is illegal to
+override this procedure in a derived type. For example, the following is illegal:
+
+@example
+type T1 is record
+   ...
+end record with Destructor => Foo;
+
+procedure Foo (X : in out T1);
+
+type T2 is new T1;
+
+overriding
+procedure Foo (X : in out T2); -- Error here
+@end example
+
+It is possible to specify @code{Destructor} on the completion of a private type,
+but there is one more restriction in that case: the denoted primitive must
+be private to the enclosing package. This is necessary due to the previously
+mentioned legality rule, to prevent breaking the privacy of the type when
+imposing that rule on outside types that derive from the private view of the
+type.
+
 @node Security Hardening Features,Obsolescent Features,GNAT language extensions,Top
-@anchor{gnat_rm/security_hardening_features doc}@anchor{473}@anchor{gnat_rm/security_hardening_features id1}@anchor{474}@anchor{gnat_rm/security_hardening_features security-hardening-features}@anchor{15}
+@anchor{gnat_rm/security_hardening_features doc}@anchor{474}@anchor{gnat_rm/security_hardening_features id1}@anchor{475}@anchor{gnat_rm/security_hardening_features security-hardening-features}@anchor{15}
 @chapter Security Hardening Features
 
 
@@ -32609,7 +32701,7 @@ change.
 @end menu
 
 @node Register Scrubbing,Stack Scrubbing,,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features register-scrubbing}@anchor{475}
+@anchor{gnat_rm/security_hardening_features register-scrubbing}@anchor{476}
 @section Register Scrubbing
 
 
@@ -32645,7 +32737,7 @@ programming languages, see @cite{Using the GNU Compiler Collection (GCC)}.
 @c Stack Scrubbing:
 
 @node Stack Scrubbing,Hardened Conditionals,Register Scrubbing,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features stack-scrubbing}@anchor{476}
+@anchor{gnat_rm/security_hardening_features stack-scrubbing}@anchor{477}
 @section Stack Scrubbing
 
 
@@ -32789,7 +32881,7 @@ Bar_Callable_Ptr.
 @c Hardened Conditionals:
 
 @node Hardened Conditionals,Hardened Booleans,Stack Scrubbing,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features hardened-conditionals}@anchor{477}
+@anchor{gnat_rm/security_hardening_features hardened-conditionals}@anchor{478}
 @section Hardened Conditionals
 
 
@@ -32879,7 +32971,7 @@ be used with other programming languages supported by GCC.
 @c Hardened Booleans:
 
 @node Hardened Booleans,Control Flow Redundancy,Hardened Conditionals,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features hardened-booleans}@anchor{478}
+@anchor{gnat_rm/security_hardening_features hardened-booleans}@anchor{479}
 @section Hardened Booleans
 
 
@@ -32940,7 +33032,7 @@ and more details on that attribute, see @cite{Using the GNU Compiler Collection
 @c Control Flow Redundancy:
 
 @node Control Flow Redundancy,,Hardened Booleans,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features control-flow-redundancy}@anchor{479}
+@anchor{gnat_rm/security_hardening_features control-flow-redundancy}@anchor{47a}
 @section Control Flow Redundancy
 
 
@@ -33108,7 +33200,7 @@ see @cite{Using the GNU Compiler Collection (GCC)}.  These options
 can be used with other programming languages supported by GCC.
 
 @node Obsolescent Features,Compatibility and Porting Guide,Security Hardening Features,Top
-@anchor{gnat_rm/obsolescent_features doc}@anchor{47a}@anchor{gnat_rm/obsolescent_features id1}@anchor{47b}@anchor{gnat_rm/obsolescent_features obsolescent-features}@anchor{16}
+@anchor{gnat_rm/obsolescent_features doc}@anchor{47b}@anchor{gnat_rm/obsolescent_features id1}@anchor{47c}@anchor{gnat_rm/obsolescent_features obsolescent-features}@anchor{16}
 @chapter Obsolescent Features
 
 
@@ -33127,7 +33219,7 @@ compatibility purposes.
 @end menu
 
 @node pragma No_Run_Time,pragma Ravenscar,,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id2}@anchor{47c}@anchor{gnat_rm/obsolescent_features pragma-no-run-time}@anchor{47d}
+@anchor{gnat_rm/obsolescent_features id2}@anchor{47d}@anchor{gnat_rm/obsolescent_features pragma-no-run-time}@anchor{47e}
 @section pragma No_Run_Time
 
 
@@ -33140,7 +33232,7 @@ preferred usage is to use an appropriately configured run-time that
 includes just those features that are to be made accessible.
 
 @node pragma Ravenscar,pragma Restricted_Run_Time,pragma No_Run_Time,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id3}@anchor{47e}@anchor{gnat_rm/obsolescent_features pragma-ravenscar}@anchor{47f}
+@anchor{gnat_rm/obsolescent_features id3}@anchor{47f}@anchor{gnat_rm/obsolescent_features pragma-ravenscar}@anchor{480}
 @section pragma Ravenscar
 
 
@@ -33149,7 +33241,7 @@ The pragma @code{Ravenscar} has exactly the same effect as pragma
 is part of the new Ada 2005 standard.
 
 @node pragma Restricted_Run_Time,pragma Task_Info,pragma Ravenscar,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id4}@anchor{480}@anchor{gnat_rm/obsolescent_features pragma-restricted-run-time}@anchor{481}
+@anchor{gnat_rm/obsolescent_features id4}@anchor{481}@anchor{gnat_rm/obsolescent_features pragma-restricted-run-time}@anchor{482}
 @section pragma Restricted_Run_Time
 
 
@@ -33159,7 +33251,7 @@ preferred since the Ada 2005 pragma @code{Profile} is intended for
 this kind of implementation dependent addition.
 
 @node pragma Task_Info,package System Task_Info s-tasinf ads,pragma Restricted_Run_Time,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id5}@anchor{482}@anchor{gnat_rm/obsolescent_features pragma-task-info}@anchor{483}
+@anchor{gnat_rm/obsolescent_features id5}@anchor{483}@anchor{gnat_rm/obsolescent_features pragma-task-info}@anchor{484}
 @section pragma Task_Info
 
 
@@ -33185,7 +33277,7 @@ in the spec of package System.Task_Info in the runtime
 library.
 
 @node package System Task_Info s-tasinf ads,,pragma Task_Info,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features package-system-task-info}@anchor{484}@anchor{gnat_rm/obsolescent_features package-system-task-info-s-tasinf-ads}@anchor{485}
+@anchor{gnat_rm/obsolescent_features package-system-task-info}@anchor{485}@anchor{gnat_rm/obsolescent_features package-system-task-info-s-tasinf-ads}@anchor{486}
 @section package System.Task_Info (@code{s-tasinf.ads})
 
 
@@ -33195,7 +33287,7 @@ to support the @code{Task_Info} pragma. The predefined Ada package
 standard replacement for GNAT’s @code{Task_Info} functionality.
 
 @node Compatibility and Porting Guide,GNU Free Documentation License,Obsolescent Features,Top
-@anchor{gnat_rm/compatibility_and_porting_guide doc}@anchor{486}@anchor{gnat_rm/compatibility_and_porting_guide compatibility-and-porting-guide}@anchor{17}@anchor{gnat_rm/compatibility_and_porting_guide id1}@anchor{487}
+@anchor{gnat_rm/compatibility_and_porting_guide doc}@anchor{487}@anchor{gnat_rm/compatibility_and_porting_guide compatibility-and-porting-guide}@anchor{17}@anchor{gnat_rm/compatibility_and_porting_guide id1}@anchor{488}
 @chapter Compatibility and Porting Guide
 
 
@@ -33217,7 +33309,7 @@ applications developed in other Ada environments.
 @end menu
 
 @node Writing Portable Fixed-Point Declarations,Compatibility with Ada 83,,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id2}@anchor{488}@anchor{gnat_rm/compatibility_and_porting_guide writing-portable-fixed-point-declarations}@anchor{489}
+@anchor{gnat_rm/compatibility_and_porting_guide id2}@anchor{489}@anchor{gnat_rm/compatibility_and_porting_guide writing-portable-fixed-point-declarations}@anchor{48a}
 @section Writing Portable Fixed-Point Declarations
 
 
@@ -33339,7 +33431,7 @@ If you follow this scheme you will be guaranteed that your fixed-point
 types will be portable.
 
 @node Compatibility with Ada 83,Compatibility between Ada 95 and Ada 2005,Writing Portable Fixed-Point Declarations,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-ada-83}@anchor{48a}@anchor{gnat_rm/compatibility_and_porting_guide id3}@anchor{48b}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-ada-83}@anchor{48b}@anchor{gnat_rm/compatibility_and_porting_guide id3}@anchor{48c}
 @section Compatibility with Ada 83
 
 
@@ -33367,7 +33459,7 @@ following subsections treat the most likely issues to be encountered.
 @end menu
 
 @node Legal Ada 83 programs that are illegal in Ada 95,More deterministic semantics,,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id4}@anchor{48c}@anchor{gnat_rm/compatibility_and_porting_guide legal-ada-83-programs-that-are-illegal-in-ada-95}@anchor{48d}
+@anchor{gnat_rm/compatibility_and_porting_guide id4}@anchor{48d}@anchor{gnat_rm/compatibility_and_porting_guide legal-ada-83-programs-that-are-illegal-in-ada-95}@anchor{48e}
 @subsection Legal Ada 83 programs that are illegal in Ada 95
 
 
@@ -33467,7 +33559,7 @@ the fix is usually simply to add the @code{(<>)} to the generic declaration.
 @end itemize
 
 @node More deterministic semantics,Changed semantics,Legal Ada 83 programs that are illegal in Ada 95,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id5}@anchor{48e}@anchor{gnat_rm/compatibility_and_porting_guide more-deterministic-semantics}@anchor{48f}
+@anchor{gnat_rm/compatibility_and_porting_guide id5}@anchor{48f}@anchor{gnat_rm/compatibility_and_porting_guide more-deterministic-semantics}@anchor{490}
 @subsection More deterministic semantics
 
 
@@ -33495,7 +33587,7 @@ which open select branches are executed.
 @end itemize
 
 @node Changed semantics,Other language compatibility issues,More deterministic semantics,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide changed-semantics}@anchor{490}@anchor{gnat_rm/compatibility_and_porting_guide id6}@anchor{491}
+@anchor{gnat_rm/compatibility_and_porting_guide changed-semantics}@anchor{491}@anchor{gnat_rm/compatibility_and_porting_guide id6}@anchor{492}
 @subsection Changed semantics
 
 
@@ -33537,7 +33629,7 @@ covers only the restricted range.
 @end itemize
 
 @node Other language compatibility issues,,Changed semantics,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id7}@anchor{492}@anchor{gnat_rm/compatibility_and_porting_guide other-language-compatibility-issues}@anchor{493}
+@anchor{gnat_rm/compatibility_and_porting_guide id7}@anchor{493}@anchor{gnat_rm/compatibility_and_porting_guide other-language-compatibility-issues}@anchor{494}
 @subsection Other language compatibility issues
 
 
@@ -33570,7 +33662,7 @@ include @code{pragma Interface} and the floating point type attributes
 @end itemize
 
 @node Compatibility between Ada 95 and Ada 2005,Implementation-dependent characteristics,Compatibility with Ada 83,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-between-ada-95-and-ada-2005}@anchor{494}@anchor{gnat_rm/compatibility_and_porting_guide id8}@anchor{495}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-between-ada-95-and-ada-2005}@anchor{495}@anchor{gnat_rm/compatibility_and_porting_guide id8}@anchor{496}
 @section Compatibility between Ada 95 and Ada 2005
 
 
@@ -33642,7 +33734,7 @@ can declare a function returning a value from an anonymous access type.
 @end itemize
 
 @node Implementation-dependent characteristics,Compatibility with Other Ada Systems,Compatibility between Ada 95 and Ada 2005,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id9}@anchor{496}@anchor{gnat_rm/compatibility_and_porting_guide implementation-dependent-characteristics}@anchor{497}
+@anchor{gnat_rm/compatibility_and_porting_guide id9}@anchor{497}@anchor{gnat_rm/compatibility_and_porting_guide implementation-dependent-characteristics}@anchor{498}
 @section Implementation-dependent characteristics
 
 
@@ -33665,7 +33757,7 @@ transition from certain Ada 83 compilers.
 @end menu
 
 @node Implementation-defined pragmas,Implementation-defined attributes,,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id10}@anchor{498}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-pragmas}@anchor{499}
+@anchor{gnat_rm/compatibility_and_porting_guide id10}@anchor{499}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-pragmas}@anchor{49a}
 @subsection Implementation-defined pragmas
 
 
@@ -33687,7 +33779,7 @@ avoiding compiler rejection of units that contain such pragmas; they are not
 relevant in a GNAT context and hence are not otherwise implemented.
 
 @node Implementation-defined attributes,Libraries,Implementation-defined pragmas,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id11}@anchor{49a}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-attributes}@anchor{49b}
+@anchor{gnat_rm/compatibility_and_porting_guide id11}@anchor{49b}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-attributes}@anchor{49c}
 @subsection Implementation-defined attributes
 
 
@@ -33701,7 +33793,7 @@ Ada 83, GNAT supplies the attributes @code{Bit}, @code{Machine_Size} and
 @code{Type_Class}.
 
 @node Libraries,Elaboration order,Implementation-defined attributes,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id12}@anchor{49c}@anchor{gnat_rm/compatibility_and_porting_guide libraries}@anchor{49d}
+@anchor{gnat_rm/compatibility_and_porting_guide id12}@anchor{49d}@anchor{gnat_rm/compatibility_and_porting_guide libraries}@anchor{49e}
 @subsection Libraries
 
 
@@ -33730,7 +33822,7 @@ be preferable to retrofit the application using modular types.
 @end itemize
 
 @node Elaboration order,Target-specific aspects,Libraries,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide elaboration-order}@anchor{49e}@anchor{gnat_rm/compatibility_and_porting_guide id13}@anchor{49f}
+@anchor{gnat_rm/compatibility_and_porting_guide elaboration-order}@anchor{49f}@anchor{gnat_rm/compatibility_and_porting_guide id13}@anchor{4a0}
 @subsection Elaboration order
 
 
@@ -33766,7 +33858,7 @@ pragmas either globally (as an effect of the `-gnatE' switch) or locally
 @end itemize
 
 @node Target-specific aspects,,Elaboration order,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id14}@anchor{4a0}@anchor{gnat_rm/compatibility_and_porting_guide target-specific-aspects}@anchor{4a1}
+@anchor{gnat_rm/compatibility_and_porting_guide id14}@anchor{4a1}@anchor{gnat_rm/compatibility_and_porting_guide target-specific-aspects}@anchor{4a2}
 @subsection Target-specific aspects
 
 
@@ -33779,10 +33871,10 @@ on the robustness of the original design.  Moreover, Ada 95 (and thus
 Ada 2005, Ada 2012, and Ada 2022) are sometimes
 incompatible with typical Ada 83 compiler practices regarding implicit
 packing, the meaning of the Size attribute, and the size of access values.
-GNAT’s approach to these issues is described in @ref{4a2,,Representation Clauses}.
+GNAT’s approach to these issues is described in @ref{4a3,,Representation Clauses}.
 
 @node Compatibility with Other Ada Systems,Representation Clauses,Implementation-dependent characteristics,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-other-ada-systems}@anchor{4a3}@anchor{gnat_rm/compatibility_and_porting_guide id15}@anchor{4a4}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-other-ada-systems}@anchor{4a4}@anchor{gnat_rm/compatibility_and_porting_guide id15}@anchor{4a5}
 @section Compatibility with Other Ada Systems
 
 
@@ -33825,7 +33917,7 @@ far beyond this minimal set, as described in the next section.
 @end itemize
 
 @node Representation Clauses,Compatibility with HP Ada 83,Compatibility with Other Ada Systems,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id16}@anchor{4a5}@anchor{gnat_rm/compatibility_and_porting_guide representation-clauses}@anchor{4a2}
+@anchor{gnat_rm/compatibility_and_porting_guide id16}@anchor{4a6}@anchor{gnat_rm/compatibility_and_porting_guide representation-clauses}@anchor{4a3}
 @section Representation Clauses
 
 
@@ -33918,7 +34010,7 @@ with thin pointers.
 @end itemize
 
 @node Compatibility with HP Ada 83,,Representation Clauses,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-hp-ada-83}@anchor{4a6}@anchor{gnat_rm/compatibility_and_porting_guide id17}@anchor{4a7}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-hp-ada-83}@anchor{4a7}@anchor{gnat_rm/compatibility_and_porting_guide id17}@anchor{4a8}
 @section Compatibility with HP Ada 83
 
 
@@ -33948,7 +34040,7 @@ extension of package System.
 @end itemize
 
 @node GNU Free Documentation License,Index,Compatibility and Porting Guide,Top
-@anchor{share/gnu_free_documentation_license doc}@anchor{4a8}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{4a9}
+@anchor{share/gnu_free_documentation_license doc}@anchor{4a9}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{4aa}
 @chapter GNU Free Documentation License
 
 
diff --git a/gcc/ada/gsocket.h b/gcc/ada/gsocket.h
index 79836e8..5a553d1 100644
--- a/gcc/ada/gsocket.h
+++ b/gcc/ada/gsocket.h
@@ -64,6 +64,12 @@
 #include <ioLib.h>
 #include <hostLib.h>
 
+#if __has_include ("strings.h")
+/* On VxWorks6, FD_ZERO uses bzero, but since it's not a standard header, don't
+   require it.  */
+#include "strings.h"
+#endif
+
 #define SHUT_RD		0
 #define SHUT_WR		1
 #define SHUT_RDWR	2
diff --git a/gcc/ada/inline.adb b/gcc/ada/inline.adb
index e8eeebd..5e2f033 100644
--- a/gcc/ada/inline.adb
+++ b/gcc/ada/inline.adb
@@ -3797,6 +3797,17 @@ package body Inline is
                           and then Is_Unc;
       end if;
 
+      --  Inlining function calls returning an object of unconstrained type as
+      --  function actuals or in a return statement is not supported: a
+      --  temporary variable will be declared of unconstrained type without
+      --  initializing expression.
+
+      pragma Assert
+        (not Uses_Back_End
+           or else Nkind (Parent (N)) not in
+             N_Function_Call | N_Simple_Return_Statement
+           or else not Is_Unc);
+
       --  Check for an illegal attempt to inline a recursive procedure. If the
       --  subprogram has parameters this is detected when trying to supply a
       --  binding for parameters that already have one. For parameterless
diff --git a/gcc/ada/lib-writ.adb b/gcc/ada/lib-writ.adb
index b7a7f12..fb7c416 100644
--- a/gcc/ada/lib-writ.adb
+++ b/gcc/ada/lib-writ.adb
@@ -905,7 +905,7 @@ package body Lib.Writ is
             --  Do not generate a with line for an ignored Ghost unit because
             --  the unit does not have an ALI file.
 
-            if Is_Ignored_Ghost_Entity (Cunit_Entity (Unum)) then
+            if Is_Ignored_Ghost_Entity_In_Codegen (Cunit_Entity (Unum)) then
                goto Next_With_Line;
             end if;
 
diff --git a/gcc/ada/lib-xref.adb b/gcc/ada/lib-xref.adb
index 145d314..aa9ae57 100644
--- a/gcc/ada/lib-xref.adb
+++ b/gcc/ada/lib-xref.adb
@@ -1729,7 +1729,7 @@ package body Lib.Xref is
             --  entity because neither the entity nor its references will
             --  appear in the final tree.
 
-            if Is_Ignored_Ghost_Entity (Ent) then
+            if Is_Ignored_Ghost_Entity_In_Codegen (Ent) then
                goto Orphan_Continue;
             end if;
 
@@ -2190,7 +2190,7 @@ package body Lib.Xref is
                --  entity because neither the entity nor its references will
                --  appear in the final tree.
 
-               if Is_Ignored_Ghost_Entity (Ent) then
+               if Is_Ignored_Ghost_Entity_In_Codegen (Ent) then
                   goto Continue;
                end if;
 
diff --git a/gcc/ada/libgnarl/s-taskin.ads b/gcc/ada/libgnarl/s-taskin.ads
index d68e199..dbf2e7b 100644
--- a/gcc/ada/libgnarl/s-taskin.ads
+++ b/gcc/ada/libgnarl/s-taskin.ads
@@ -390,7 +390,7 @@ package System.Tasking is
    System_Domain : Dispatching_Domain_Access;
    --  All processors belong to default system dispatching domain at start up.
    --  We use a pointer which creates the actual variable for the reasons
-   --  explained bellow in Dispatching_Domain_Tasks.
+   --  explained below in Dispatching_Domain_Tasks.
 
    Dispatching_Domains_Frozen : Boolean := False;
    --  True when the main procedure has been called. Hence, no new dispatching
diff --git a/gcc/ada/libgnat/a-calend.adb b/gcc/ada/libgnat/a-calend.adb
index 4f89a40..0be5673 100644
--- a/gcc/ada/libgnat/a-calend.adb
+++ b/gcc/ada/libgnat/a-calend.adb
@@ -1068,19 +1068,28 @@ is
          tv_nsec : out Long_Integer)
       is
          pragma Unsuppress (Overflow_Check);
-         Secs      : Duration;
-         Nano_Secs : Duration;
 
       begin
-         --  Seconds extraction, avoid potential rounding errors
-
-         Secs   := D - 0.5;
-         tv_sec := Long_Long_Integer (Secs);
-
-         --  Nanoseconds extraction
+         if D = 0.0 then
+            tv_sec  := 0;
+            tv_nsec := 0;
+
+         elsif D < 0.0 then
+            tv_sec := Long_Long_Integer (D + 0.5);
+            if D = Duration (tv_sec) then
+               tv_nsec := 0;
+            else
+               tv_nsec := Long_Integer ((D - Duration (tv_sec)) * Nano + 0.5);
+            end if;
 
-         Nano_Secs := D - Duration (tv_sec);
-         tv_nsec := Long_Integer (Nano_Secs * Nano);
+         else
+            tv_sec := Long_Long_Integer (D - 0.5);
+            if D = Duration (tv_sec) then
+               tv_nsec := 0;
+            else
+               tv_nsec := Long_Integer ((D - Duration (tv_sec)) * Nano - 0.5);
+            end if;
+         end if;
       end To_Struct_Timespec_64;
 
       ------------------
diff --git a/gcc/ada/libgnat/a-cbhama.adb b/gcc/ada/libgnat/a-cbhama.adb
index ee6584d..b2d7964 100644
--- a/gcc/ada/libgnat/a-cbhama.adb
+++ b/gcc/ada/libgnat/a-cbhama.adb
@@ -368,7 +368,7 @@ is
    -- Empty --
    -----------
 
-   function Empty (Capacity : Count_Type) return Map is
+   function Empty (Capacity : Count_Type := 10) return Map is
    begin
       return Result : Map (Capacity, 0) do
          null;
diff --git a/gcc/ada/libgnat/a-cbhama.ads b/gcc/ada/libgnat/a-cbhama.ads
index 6ffc815..c741b40 100644
--- a/gcc/ada/libgnat/a-cbhama.ads
+++ b/gcc/ada/libgnat/a-cbhama.ads
@@ -71,7 +71,7 @@ is
    --  Map objects declared without an initialization expression are
    --  initialized to the value Empty_Map.
 
-   function Empty (Capacity : Count_Type) return Map;
+   function Empty (Capacity : Count_Type := 10) return Map;
 
    No_Element : constant Cursor;
    --  Cursor objects declared without an initialization expression are
diff --git a/gcc/ada/libgnat/g-calend.adb b/gcc/ada/libgnat/g-calend.adb
index e410c3e..a2bc77c 100644
--- a/gcc/ada/libgnat/g-calend.adb
+++ b/gcc/ada/libgnat/g-calend.adb
@@ -344,6 +344,8 @@ package body GNAT.Calendar is
       sec   : aliased C.Extensions.long_long;
       usec  : aliased C.long;
 
+      pragma Unsuppress (Overflow_Check);
+
    begin
       timeval_to_duration (T, sec'Access, usec'Access);
       pragma Annotate (CodePeer, Modified, sec);
@@ -369,13 +371,28 @@ package body GNAT.Calendar is
       sec    : C.Extensions.long_long;
       usec   : C.long;
 
+      pragma Unsuppress (Overflow_Check);
+
    begin
       if D = 0.0 then
          sec  := 0;
          usec := 0;
+
+      elsif D < 0.0 then
+         sec := C.Extensions.long_long (D + 0.5);
+         if D = Duration (sec) then
+            usec := 0;
+         else
+            usec := C.long ((D - Duration (sec)) * Micro + 0.5);
+         end if;
+
       else
-         sec  := C.Extensions.long_long (D - 0.5);
-         usec := C.long ((D - Duration (sec)) * Micro - 0.5);
+         sec := C.Extensions.long_long (D - 0.5);
+         if D = Duration (sec) then
+            usec := 0;
+         else
+            usec := C.long ((D - Duration (sec)) * Micro - 0.5);
+         end if;
       end if;
 
       duration_to_timeval (sec, usec, Result'Access);
diff --git a/gcc/ada/libgnat/g-socket.adb b/gcc/ada/libgnat/g-socket.adb
index 5042dac..0fed791 100644
--- a/gcc/ada/libgnat/g-socket.adb
+++ b/gcc/ada/libgnat/g-socket.adb
@@ -3059,12 +3059,11 @@ package body GNAT.Sockets is
       --  Normal case where we do round down
 
       else
-         S  := time_t (Val - 0.5);
-         uS := suseconds_t (1_000_000 * (Val - Selector_Duration (S)) - 0.5);
-
-         if uS = -1 then
-            --  It happen on integer duration
+         S := time_t (Val - 0.5);
+         if Val = Timeval_Duration (S) then
             uS := 0;
+         else
+            uS := suseconds_t ((Val - Timeval_Duration (S)) * 1_000_000 - 0.5);
          end if;
       end if;
 
diff --git a/gcc/ada/libgnat/system-linux-loongarch.ads b/gcc/ada/libgnat/system-linux-loongarch.ads
index 77a2139..683b7a4 100644
--- a/gcc/ada/libgnat/system-linux-loongarch.ads
+++ b/gcc/ada/libgnat/system-linux-loongarch.ads
@@ -139,7 +139,6 @@ private
    Always_Compatible_Rep     : constant Boolean := False;
    Suppress_Standard_Library : constant Boolean := False;
    Use_Ada_Main_Program_Name : constant Boolean := False;
-   Frontend_Exceptions       : constant Boolean := False;
    ZCX_By_Default            : constant Boolean := True;
 
 end System;
diff --git a/gcc/ada/mutably_tagged.adb b/gcc/ada/mutably_tagged.adb
index 153d168..b04ba92 100644
--- a/gcc/ada/mutably_tagged.adb
+++ b/gcc/ada/mutably_tagged.adb
@@ -40,6 +40,7 @@ with Sinfo.Nodes;    use Sinfo.Nodes;
 with Sinfo.Utils;    use Sinfo.Utils;
 with Stringt;        use Stringt;
 with Tbuild;         use Tbuild;
+with Uintp;          use Uintp;
 
 package body Mutably_Tagged is
 
@@ -205,21 +206,41 @@ package body Mutably_Tagged is
       Mut_Tag_Typ : Entity_Id) return Node_Id
    is
       Loc : constant Source_Ptr := Sloc (New_Typ);
+
+      CW_Size : constant Uint := RM_Size (Mut_Tag_Typ);
+
+      function To_Mixed_Case (S : String) return String;
+      --  convert string to mixed case
+
+      -------------------
+      -- To_Mixed_Case --
+      -------------------
+
+      function To_Mixed_Case (S : String) return String is
+         Buf : Bounded_String;
+      begin
+         Append (Buf, S);
+         Set_Casing (Buf, Mixed_Case);
+         return +Buf;
+      end To_Mixed_Case;
+
+   --  Start of processing for Make_CW_Size_Compile_Check
+
    begin
-      --  Generate a string literal for New_Typ's name which is needed for
-      --  printing within the Compile_Time_Error.
+      --  Build a Compile_Time_Error pragma in order to defer the
+      --  (compile-time) size check until after the back end has
+      --  determined sizes.
+      --
+      --  It would be nice if we could somehow include the value of
+      --  New_Type'Size in the error message, but it is not clear how to
+      --  accomplish that with the current FE/BE interfaces.
+
+      --  Get New_Typ's name (in mixed case) into the name buffer;
+      --  this is used immediately afterwards in the Make_Pragma call.
 
       Get_Decoded_Name_String (Chars (New_Typ));
       Set_Casing (Mixed_Case);
 
-      --  Build a pragma Compile_Time_Error to force the backend to
-      --  preform appropriate sizing checks.
-
-      --  Generate:
-      --    pragma Compile_Time_Error
-      --             (New_Typ'Size < Mut_Tag_Typ'Size,
-      --              "class size for by-reference type ""New_Typ"" too small")
-
       return
         Make_Pragma (Loc,
           Chars                        => Name_Compile_Time_Error,
@@ -233,19 +254,18 @@ package body Mutably_Tagged is
                       Prefix         =>
                         New_Occurrence_Of (New_Typ, Loc)),
                   Right_Opnd =>
-                    Make_Integer_Literal (Loc,
-                      RM_Size (Mut_Tag_Typ))))),
+                    Make_Integer_Literal (Loc, CW_Size)))),
              Make_Pragma_Argument_Association (Loc,
                Expression =>
-
-                 --  Is it possible to print the size of New_Typ via
-                 --  Validate_Compile_Time_Warning_Or_Error after the back-end
-                 --  has run to generate the error message manually ???
-
                  Make_String_Literal (Loc,
-                   "class size for by-reference type """
-                   & To_String (String_From_Name_Buffer)
-                   & """ too small"))));
+                   To_String (String_From_Name_Buffer)
+                     & "'Size exceeds "
+                     & To_Mixed_Case (
+                         To_String (Fully_Qualified_Name_String
+                                     (Find_Specific_Type (Mut_Tag_Typ),
+                                      Append_NUL => False)))
+                     & "'Size'Class limit of "
+                     & UI_Image (CW_Size)))));
    end Make_CW_Size_Compile_Check;
 
    ------------------------------------
diff --git a/gcc/ada/opt.adb b/gcc/ada/opt.adb
index d2291a9..bd74215 100644
--- a/gcc/ada/opt.adb
+++ b/gcc/ada/opt.adb
@@ -204,14 +204,7 @@ package body Opt is
             SPARK_Mode_Pragma        := SPARK_Mode_Pragma_Config;
 
          else
-            --  In GNATprove mode assertions should be always enabled, even
-            --  when analysing internal units.
-
-            if GNATprove_Mode then
-               pragma Assert (Assertions_Enabled);
-               null;
-
-            elsif GNAT_Mode_Config then
+            if GNAT_Mode_Config then
                Assertions_Enabled    := Assertions_Enabled_Config;
             else
                Assertions_Enabled    := False;
diff --git a/gcc/ada/opt.ads b/gcc/ada/opt.ads
index e595b08..73f9fe8 100644
--- a/gcc/ada/opt.ads
+++ b/gcc/ada/opt.ads
@@ -746,9 +746,20 @@ package Opt is
    --  Possible legal modes that can be set by aspect/pragma Ghost as well as
    --  value None, which indicates that no such aspect/pragma applies.
 
-   Ghost_Mode : Ghost_Mode_Type := None;
+   type Ghost_Config_Type is record
+      Ghost_Mode : Ghost_Mode_Type := None;
+      --  The current Ghost mode in effect
+
+      Ignored_Ghost_Region : Node_Id := Empty;
+      --  The start of the current ignored Ghost region. This value must always
+      --  reflect the starting node of the outermost ignored Ghost region. If a
+      --  nested ignored Ghost region is entered, the value must remain
+      --  unchanged.
+   end record;
+
+   Ghost_Config : Ghost_Config_Type;
    --  GNAT
-   --  The current Ghost mode in effect
+   --  All relevant Ghost mode settings
 
    Global_Discard_Names : Boolean := False;
    --  GNAT, GNATBIND
@@ -810,12 +821,6 @@ package Opt is
    --  use of -gnateu, causing subsequent unrecognized switches to result in
    --  a warning rather than an error.
 
-   Ignored_Ghost_Region : Node_Id := Empty;
-   --  GNAT
-   --  The start of the current ignored Ghost region. This value must always
-   --  reflect the starting node of the outermost ignored Ghost region. If a
-   --  nested ignored Ghost region is entered, the value must remain unchanged.
-
    Implicit_Packing : Boolean := False;
    --  GNAT
    --  If set True, then a Size attribute clause on an array is allowed to
diff --git a/gcc/ada/par-ch12.adb b/gcc/ada/par-ch12.adb
index b539a29..5fb6f8c 100644
--- a/gcc/ada/par-ch12.adb
+++ b/gcc/ada/par-ch12.adb
@@ -420,32 +420,17 @@ package body Ch12 is
 
    procedure P_Formal_Object_Declarations (Decls : List_Id) is
       Decl_Node        : Node_Id;
-      Ident            : Pos;
       Not_Null_Present : Boolean := False;
-      Num_Idents       : Pos;
       Scan_State       : Saved_Scan_State;
 
-      Idents : array (Pos range 1 .. 4096) of Entity_Id;
-      --  This array holds the list of defining identifiers. The upper bound
-      --  of 4096 is intended to be essentially infinite, and we do not even
-      --  bother to check for it being exceeded.
+      Def_Ids : Defining_Identifiers;
+      Ident   : Pos;
 
    begin
-      Idents (1) := P_Defining_Identifier (C_Comma_Colon);
-      Num_Idents := 1;
-      while Comma_Present loop
-         Num_Idents := Num_Idents + 1;
-         Idents (Num_Idents) := P_Defining_Identifier (C_Comma_Colon);
-      end loop;
-
+      P_Def_Ids (Def_Ids);
       T_Colon;
 
-      --  If there are multiple identifiers, we repeatedly scan the
-      --  type and initialization expression information by resetting
-      --  the scan pointer (so that we get completely separate trees
-      --  for each occurrence).
-
-      if Num_Idents > 1 then
+      if Def_Ids.Num_Idents > 1 then
          Save_Scan_State (Scan_State);
       end if;
 
@@ -454,7 +439,7 @@ package body Ch12 is
       Ident := 1;
       Ident_Loop : loop
          Decl_Node := New_Node (N_Formal_Object_Declaration, Token_Ptr);
-         Set_Defining_Identifier (Decl_Node, Idents (Ident));
+         Set_Defining_Identifier (Decl_Node, Def_Ids.Idents (Ident));
          P_Mode (Decl_Node);
 
          Not_Null_Present := P_Null_Exclusion;  --  Ada 2005 (AI-423)
@@ -488,13 +473,13 @@ package body Ch12 is
             Set_Prev_Ids (Decl_Node, True);
          end if;
 
-         if Ident < Num_Idents then
+         if Ident < Def_Ids.Num_Idents then
             Set_More_Ids (Decl_Node, True);
          end if;
 
          Append (Decl_Node, Decls);
 
-         exit Ident_Loop when Ident = Num_Idents;
+         exit Ident_Loop when Ident = Def_Ids.Num_Idents;
          Ident := Ident + 1;
          Restore_Scan_State (Scan_State);
       end loop Ident_Loop;
diff --git a/gcc/ada/par-ch3.adb b/gcc/ada/par-ch3.adb
index fe727d7..a685812 100644
--- a/gcc/ada/par-ch3.adb
+++ b/gcc/ada/par-ch3.adb
@@ -1302,19 +1302,13 @@ package body Ch3 is
       Ident_Sloc       : Source_Ptr;
       Scan_State       : Saved_Scan_State;
       List_OK          : Boolean := True;
-      Ident            : Nat;
       Init_Expr        : Node_Id;
       Init_Loc         : Source_Ptr;
       Con_Loc          : Source_Ptr;
       Not_Null_Present : Boolean := False;
 
-      Idents : array (Int range 1 .. 4096) of Entity_Id;
-      --  Used to save identifiers in the identifier list. The upper bound
-      --  of 4096 is expected to be infinite in practice, and we do not even
-      --  bother to check if this upper bound is exceeded.
-
-      Num_Idents : Nat := 1;
-      --  Number of identifiers stored in Idents
+      Def_Ids : Defining_Identifiers;
+      Ident   : Pos;
 
       function Identifier_Starts_Statement return Boolean;
       --  Called with Token being an identifier that might start a declaration
@@ -1389,10 +1383,9 @@ package body Ch3 is
 
       procedure No_List is
       begin
-         if Num_Idents > 1 then
+         if Def_Ids.Num_Idents > 1 then
             Error_Msg_N
-              ("identifier list not allowed for RENAMES",
-               Idents (2));
+              ("identifier list not allowed for RENAMES", Def_Ids.Idents (2));
          end if;
 
          List_OK := False;
@@ -1443,7 +1436,7 @@ package body Ch3 is
 
       Ident_Sloc := Token_Ptr;
       Save_Scan_State (Scan_State); -- at first identifier
-      Idents (1) := P_Defining_Identifier (C_Comma_Colon);
+      Append (Def_Ids, P_Defining_Identifier (C_Comma_Colon));
 
       --  If we have a colon after the identifier, then we can assume that
       --  this is in fact a valid identifier declaration and can steam ahead.
@@ -1455,8 +1448,7 @@ package body Ch3 is
 
       elsif Token = Tok_Comma then
          while Comma_Present loop
-            Num_Idents := Num_Idents + 1;
-            Idents (Num_Idents) := P_Defining_Identifier (C_Comma_Colon);
+            Append (Def_Ids, P_Defining_Identifier (C_Comma_Colon));
          end loop;
 
          Save_Scan_State (Scan_State); -- at colon
@@ -1510,7 +1502,7 @@ package body Ch3 is
          Decl_Node :=
            New_Node (N_Object_Renaming_Declaration, Ident_Sloc);
          Set_Name (Decl_Node, P_Name);
-         Set_Defining_Identifier (Decl_Node, Idents (1));
+         Set_Defining_Identifier (Decl_Node, Def_Ids.Idents (1));
 
          P_Aspect_Specifications (Decl_Node, Semicolon => False);
 
@@ -1917,7 +1909,7 @@ package body Ch3 is
             end if;
          end if;
 
-         Set_Defining_Identifier (Decl_Node, Idents (Ident));
+         Set_Defining_Identifier (Decl_Node, Def_Ids.Idents (Ident));
          P_Aspect_Specifications (Decl_Node, Semicolon => False);
 
          --  Allow initialization expression to follow aspects (note that in
@@ -1945,17 +1937,17 @@ package body Ch3 is
          T_Semicolon;
 
          if List_OK then
-            if Ident < Num_Idents then
-               Set_More_Ids (Decl_Node, True);
-            end if;
-
             if Ident > 1 then
                Set_Prev_Ids (Decl_Node, True);
             end if;
+
+            if Ident < Def_Ids.Num_Idents then
+               Set_More_Ids (Decl_Node, True);
+            end if;
          end if;
 
          Append (Decl_Node, Decls);
-         exit Ident_Loop when Ident = Num_Idents;
+         exit Ident_Loop when Ident = Def_Ids.Num_Idents;
          Restore_Scan_State (Scan_State);
          T_Colon;
          Ident := Ident + 1;
@@ -3191,14 +3183,7 @@ package body Ch3 is
       Specification_List : List_Id;
       Ident_Sloc         : Source_Ptr;
       Scan_State         : Saved_Scan_State;
-      Num_Idents         : Nat;
       Not_Null_Present   : Boolean;
-      Ident              : Nat;
-
-      Idents : array (Int range 1 .. 4096) of Entity_Id;
-      --  This array holds the list of defining identifiers. The upper bound
-      --  of 4096 is intended to be essentially infinite, and we do not even
-      --  bother to check for it being exceeded.
 
    begin
       if Token = Tok_Left_Paren then
@@ -3207,97 +3192,91 @@ package body Ch3 is
          P_Pragmas_Misplaced;
 
          Specification_Loop : loop
+            declare
+               Def_Ids : Defining_Identifiers;
+               Ident   : Pos;
+            begin
+               Ident_Sloc := Token_Ptr;
+               P_Def_Ids (Def_Ids);
 
-            Ident_Sloc := Token_Ptr;
-            Idents (1) := P_Defining_Identifier (C_Comma_Colon);
-            Num_Idents := 1;
-
-            while Comma_Present loop
-               Num_Idents := Num_Idents + 1;
-               Idents (Num_Idents) := P_Defining_Identifier (C_Comma_Colon);
-            end loop;
-
-            --  If there are multiple identifiers, we repeatedly scan the
-            --  type and initialization expression information by resetting
-            --  the scan pointer (so that we get completely separate trees
-            --  for each occurrence).
+               if Def_Ids.Num_Idents > 1 then
+                  Save_Scan_State (Scan_State);
+               end if;
 
-            if Num_Idents > 1 then
-               Save_Scan_State (Scan_State);
-            end if;
+               T_Colon;
 
-            T_Colon;
+               --  Loop through defining identifiers in list
 
-            --  Loop through defining identifiers in list
+               Ident := 1;
+               Ident_Loop : loop
+                  Specification_Node :=
+                    New_Node (N_Discriminant_Specification, Ident_Sloc);
+                  Set_Defining_Identifier
+                    (Specification_Node, Def_Ids.Idents (Ident));
+                  Not_Null_Present :=  --  Ada 2005 (AI-231, AI-447)
+                    P_Null_Exclusion (Allow_Anonymous_In_95 => True);
 
-            Ident := 1;
-            Ident_Loop : loop
-               Specification_Node :=
-                 New_Node (N_Discriminant_Specification, Ident_Sloc);
-               Set_Defining_Identifier (Specification_Node, Idents (Ident));
-               Not_Null_Present :=  --  Ada 2005 (AI-231, AI-447)
-                 P_Null_Exclusion (Allow_Anonymous_In_95 => True);
+                  if Token = Tok_Access then
+                     if Ada_Version = Ada_83 then
+                        Error_Msg_SC
+                          ("(Ada 83) access discriminant not allowed!");
+                     end if;
 
-               if Token = Tok_Access then
-                  if Ada_Version = Ada_83 then
-                     Error_Msg_SC
-                       ("(Ada 83) access discriminant not allowed!");
-                  end if;
+                     Set_Discriminant_Type
+                       (Specification_Node,
+                        P_Access_Definition (Not_Null_Present));
 
-                  Set_Discriminant_Type
-                    (Specification_Node,
-                     P_Access_Definition (Not_Null_Present));
+                  --  Catch ouf-of-order keywords
 
-               --  Catch ouf-of-order keywords
+                  elsif Token = Tok_Constant then
+                     Scan;
 
-               elsif Token = Tok_Constant then
-                  Scan;
+                     if Token = Tok_Access then
+                        Error_Msg_SC -- CODEFIX
+                          ("ACCESS must come before CONSTANT");
+                        Set_Discriminant_Type
+                          (Specification_Node,
+                           P_Access_Definition (Not_Null_Present));
 
-                  if Token = Tok_Access then
-                     Error_Msg_SC -- CODEFIX
-                       ("ACCESS must come before CONSTANT");
-                     Set_Discriminant_Type
-                       (Specification_Node,
-                        P_Access_Definition (Not_Null_Present));
+                     else
+                        Error_Msg_SC ("misplaced CONSTANT");
+                     end if;
 
                   else
-                     Error_Msg_SC ("misplaced CONSTANT");
+                     Set_Discriminant_Type
+                       (Specification_Node, P_Subtype_Mark);
+                     No_Constraint;
+                     Set_Null_Exclusion_Present  -- Ada 2005 (AI-231)
+                       (Specification_Node, Not_Null_Present);
                   end if;
 
-               else
-                  Set_Discriminant_Type
-                    (Specification_Node, P_Subtype_Mark);
-                  No_Constraint;
-                  Set_Null_Exclusion_Present  -- Ada 2005 (AI-231)
-                    (Specification_Node, Not_Null_Present);
-               end if;
-
-               Set_Expression
-                 (Specification_Node, Init_Expr_Opt (True));
+                  Set_Expression
+                    (Specification_Node, Init_Expr_Opt (True));
 
-               if Token = Tok_With then
-                  P_Aspect_Specifications
-                    (Specification_Node, Semicolon => False);
-               end if;
+                  if Token = Tok_With then
+                     P_Aspect_Specifications
+                       (Specification_Node, Semicolon => False);
+                  end if;
 
-               if Ident > 1 then
-                  Set_Prev_Ids (Specification_Node, True);
-               end if;
+                  if Ident > 1 then
+                     Set_Prev_Ids (Specification_Node, True);
+                  end if;
 
-               if Ident < Num_Idents then
-                  Set_More_Ids (Specification_Node, True);
-               end if;
+                  if Ident < Def_Ids.Num_Idents then
+                     Set_More_Ids (Specification_Node, True);
+                  end if;
 
-               Append (Specification_Node, Specification_List);
-               exit Ident_Loop when Ident = Num_Idents;
-               Ident := Ident + 1;
-               Restore_Scan_State (Scan_State);
-               T_Colon;
-            end loop Ident_Loop;
+                  Append (Specification_Node, Specification_List);
+                  exit Ident_Loop when Ident = Def_Ids.Num_Idents;
+                  Ident := Ident + 1;
+                  Restore_Scan_State (Scan_State);
+                  T_Colon;
+               end loop Ident_Loop;
 
-            exit Specification_Loop when Token /= Tok_Semicolon;
-            Scan; -- past ;
-            P_Pragmas_Misplaced;
+               exit Specification_Loop when Token /= Tok_Semicolon;
+               Scan; -- past ;
+               P_Pragmas_Misplaced;
+            end;
          end loop Specification_Loop;
 
          T_Right_Paren;
@@ -3770,14 +3749,10 @@ package body Ch3 is
       Decl_Node        : Node_Id := Empty;  -- initialize to prevent warning
       Scan_State       : Saved_Scan_State;
       Not_Null_Present : Boolean := False;
-      Num_Idents       : Nat;
-      Ident            : Nat;
       Ident_Sloc       : Source_Ptr;
 
-      Idents : array (Int range 1 .. 4096) of Entity_Id;
-      --  This array holds the list of defining identifiers. The upper bound
-      --  of 4096 is intended to be essentially infinite, and we do not even
-      --  bother to check for it being exceeded.
+      Def_Ids : Defining_Identifiers;
+      Ident   : Pos;
 
    begin
       if Token /= Tok_Identifier then
@@ -3788,20 +3763,9 @@ package body Ch3 is
 
       Ident_Sloc := Token_Ptr;
       Check_Bad_Layout;
-      Idents (1) := P_Defining_Identifier (C_Comma_Colon);
-      Num_Idents := 1;
-
-      while Comma_Present loop
-         Num_Idents := Num_Idents + 1;
-         Idents (Num_Idents) := P_Defining_Identifier (C_Comma_Colon);
-      end loop;
-
-      --  If there are multiple identifiers, we repeatedly scan the
-      --  type and initialization expression information by resetting
-      --  the scan pointer (so that we get completely separate trees
-      --  for each occurrence).
+      P_Def_Ids (Def_Ids);
 
-      if Num_Idents > 1 then
+      if Def_Ids.Num_Idents > 1 then
          Save_Scan_State (Scan_State);
       end if;
 
@@ -3817,7 +3781,7 @@ package body Ch3 is
 
          begin
             Decl_Node := New_Node (N_Component_Declaration, Ident_Sloc);
-            Set_Defining_Identifier (Decl_Node, Idents (Ident));
+            Set_Defining_Identifier (Decl_Node, Def_Ids.Idents (Ident));
 
             if Token = Tok_Constant then
                Error_Msg_SC ("constant component not permitted");
@@ -3876,7 +3840,7 @@ package body Ch3 is
                Set_Prev_Ids (Decl_Node, True);
             end if;
 
-            if Ident < Num_Idents then
+            if Ident < Def_Ids.Num_Idents then
                Set_More_Ids (Decl_Node, True);
             end if;
 
@@ -3890,7 +3854,7 @@ package body Ch3 is
                end if;
          end;
 
-         exit Ident_Loop when Ident = Num_Idents;
+         exit Ident_Loop when Ident = Def_Ids.Num_Idents;
          Ident := Ident + 1;
          Restore_Scan_State (Scan_State);
          T_Colon;
diff --git a/gcc/ada/par-ch6.adb b/gcc/ada/par-ch6.adb
index 0f7765b..2465108 100644
--- a/gcc/ada/par-ch6.adb
+++ b/gcc/ada/par-ch6.adb
@@ -1384,20 +1384,16 @@ package body Ch6 is
       Specification_List : List_Id;
       Specification_Node : Node_Id;
       Scan_State         : Saved_Scan_State;
-      Num_Idents         : Nat;
-      Ident              : Nat;
       Ident_Sloc         : Source_Ptr;
       Not_Null_Present   : Boolean := False;
       Not_Null_Sloc      : Source_Ptr;
 
-      Idents : array (Int range 1 .. 4096) of Entity_Id;
-      --  This array holds the list of defining identifiers. The upper bound
-      --  of 4096 is intended to be essentially infinite, and we do not even
-      --  bother to check for it being exceeded.
-
    begin
       Specification_List := New_List;
       Specification_Loop : loop
+         declare
+            Def_Ids : Defining_Identifiers;
+            Ident   : Pos;
          begin
             if Token = Tok_Pragma then
                Error_Msg_SC ("pragma not allowed in formal part");
@@ -1406,8 +1402,7 @@ package body Ch6 is
 
             Ignore (Tok_Left_Paren);
             Ident_Sloc := Token_Ptr;
-            Idents (1) := P_Defining_Identifier (C_Comma_Colon);
-            Num_Idents := 1;
+            Append (Def_Ids, P_Defining_Identifier (C_Comma_Colon));
 
             Ident_Loop : loop
                exit Ident_Loop when Token = Tok_Colon;
@@ -1457,8 +1452,7 @@ package body Ch6 is
                --  Here if a comma is present, or to be assumed
 
                T_Comma;
-               Num_Idents := Num_Idents + 1;
-               Idents (Num_Idents) := P_Defining_Identifier (C_Comma_Colon);
+               Append (Def_Ids, P_Defining_Identifier (C_Comma_Colon));
             end loop Ident_Loop;
 
             --  Fall through the loop on encountering a colon, or deciding
@@ -1466,12 +1460,7 @@ package body Ch6 is
 
             T_Colon;
 
-            --  If there are multiple identifiers, we repeatedly scan the
-            --  type and initialization expression information by resetting
-            --  the scan pointer (so that we get completely separate trees
-            --  for each occurrence).
-
-            if Num_Idents > 1 then
+            if Def_Ids.Num_Idents > 1 then
                Save_Scan_State (Scan_State);
             end if;
 
@@ -1482,7 +1471,8 @@ package body Ch6 is
             Ident_List_Loop : loop
                Specification_Node :=
                  New_Node (N_Parameter_Specification, Ident_Sloc);
-               Set_Defining_Identifier (Specification_Node, Idents (Ident));
+               Set_Defining_Identifier
+                 (Specification_Node, Def_Ids.Idents (Ident));
 
                --  Scan possible ALIASED for Ada 2012 (AI-142)
 
@@ -1574,12 +1564,12 @@ package body Ch6 is
                   Set_Prev_Ids (Specification_Node, True);
                end if;
 
-               if Ident < Num_Idents then
+               if Ident < Def_Ids.Num_Idents then
                   Set_More_Ids (Specification_Node, True);
                end if;
 
                Append (Specification_Node, Specification_List);
-               exit Ident_List_Loop when Ident = Num_Idents;
+               exit Ident_List_Loop when Ident = Def_Ids.Num_Idents;
                Ident := Ident + 1;
                Restore_Scan_State (Scan_State);
             end loop Ident_List_Loop;
diff --git a/gcc/ada/par-load.adb b/gcc/ada/par-load.adb
index 96fa7e8..4a97f14 100644
--- a/gcc/ada/par-load.adb
+++ b/gcc/ada/par-load.adb
@@ -83,7 +83,7 @@ procedure Load is
    --  withed units and the second round handles Ada 2005 limited-withed units.
    --  This is required to allow the low-level circuitry that detects circular
    --  dependencies of units the correct notification of errors (see comment
-   --  bellow). This variable is used to indicate that the second round is
+   --  below). This variable is used to indicate that the second round is
    --  required.
 
    function Same_File_Name_Except_For_Case
diff --git a/gcc/ada/par-util.adb b/gcc/ada/par-util.adb
index 78a76b3..6a6afd0 100644
--- a/gcc/ada/par-util.adb
+++ b/gcc/ada/par-util.adb
@@ -34,6 +34,22 @@ with GNAT.Spelling_Checker; use GNAT.Spelling_Checker;
 separate (Par)
 package body Util is
 
+   ------------
+   -- Append --
+   ------------
+
+   procedure Append
+     (Def_Ids : in out Defining_Identifiers; Def_Id : Entity_Id)
+   is
+   begin
+      if Def_Ids.Num_Idents >= Defining_Identifiers_Array'Last then
+         raise Program_Error;
+      end if;
+
+      Def_Ids.Num_Idents := Def_Ids.Num_Idents + 1;
+      Def_Ids.Idents (Def_Ids.Num_Idents) := Def_Id;
+   end Append;
+
    ---------------------
    -- Bad_Spelling_Of --
    ---------------------
@@ -691,6 +707,19 @@ package body Util is
       end if;
    end No_Constraint;
 
+   ---------------
+   -- P_Def_Ids --
+   ---------------
+
+   procedure P_Def_Ids (Def_Ids : out Defining_Identifiers) is
+      pragma Assert (Def_Ids.Num_Idents = 0);
+   begin
+      loop
+         Append (Def_Ids, P_Defining_Identifier (C_Comma_Colon));
+         exit when not Comma_Present;
+      end loop;
+   end P_Def_Ids;
+
    ---------------------
    -- Pop_Scope_Stack --
    ---------------------
diff --git a/gcc/ada/par.adb b/gcc/ada/par.adb
index e11ec7e..99bbed2 100644
--- a/gcc/ada/par.adb
+++ b/gcc/ada/par.adb
@@ -227,6 +227,69 @@ function Par (Configuration_Pragmas : Boolean) return List_Id is
    --   that there is a missing body, but it seems more reasonable to let the
    --   later semantic checking discover this.
 
+   --------------------------------------------
+   -- Handling IS Used in Place of Semicolon --
+   --------------------------------------------
+
+   --  This is a somewhat trickier situation, and we can't catch it in all
+   --  cases, but we do our best to detect common situations resulting from
+   --  a "cut and paste" operation which forgets to change the IS to semicolon.
+   --  Consider the following example:
+
+   --    package body X is
+   --      procedure A;
+   --      procedure B is -- Error: IS should be semicolon
+   --      procedure C;
+   --      ...
+   --      procedure D is
+   --      begin
+   --         ...
+   --      end;
+   --    begin
+   --      ...
+   --    end; -- end of B?
+
+   --  The trouble is that the section of text from PROCEDURE B through the
+   --  END; marked "-- end of B?" constitutes a valid procedure body, and the
+   --  danger is that we find out far too late that something is wrong.
+
+   --  We have two approaches to helping to control this situation. First we
+   --  make every attempt to avoid swallowing the last END; if we can be sure
+   --  that some error will result from doing so. In particular, we won't
+   --  accept the END; unless it is exactly correct (in particular it must not
+   --  have incorrect name tokens), and we won't accept it if it is immediately
+   --  followed by end of file, WITH or SEPARATE (tokens that unmistakeably
+   --  signal the start of a compilation unit, and which therefore allow us to
+   --  reserve the END; for the outer level.) For more details on this aspect
+   --  of the handling, see package Par.Endh.
+
+   --  If we can avoid eating up the END; then the result in the absence of
+   --  any additional steps would be to post a missing END referring back to
+   --  the subprogram with the bogus IS. Similarly, if the enclosing package
+   --  has no BEGIN, then the result is a missing BEGIN message, which again
+   --  refers back to the subprogram header.
+
+   --  Such an error message is not too bad, but it's not ideal, because
+   --  the declarations following the IS have been absorbed into the wrong
+   --  scope. In the above case, this could result for example in a bogus
+   --  complaint that the body of D was missing from the package.
+
+   --  To catch at least some of these cases, we take the following additional
+   --  steps. First, a subprogram body is marked as having a suspicious IS if
+   --  the declaration line is followed by a line that starts with a symbol
+   --  that can start a declaration in the same column, or to the left of the
+   --  column in which the FUNCTION or PROCEDURE starts (normal style is to
+   --  indent any declarations that really belong a subprogram). If such a
+   --  subprogram encounters a missing BEGIN or missing END, then we decide
+   --  that the IS should have been a semicolon, and the subprogram body node
+   --  is marked (by setting the Bad_Is_Detected flag true. Note that we do
+   --  not do this for library level procedures, only for nested procedures,
+   --  since for library level procedures, we must have a body.
+
+   --  The processing for a declarative part checks to see if the last
+   --  declaration scanned is marked in this way, and if it is, the tree
+   --  is modified to reflect the IS being interpreted as a semicolon.
+
    ----------------------------------------------------
    -- Handling of Reserved Words Used as Identifiers --
    ----------------------------------------------------
@@ -294,71 +357,6 @@ function Par (Configuration_Pragmas : Boolean) return List_Id is
       C_Vertical_Bar_Arrow);
       --  Consider as identifier if followed by | or =>
 
-   --------------------------------------------
-   -- Handling IS Used in Place of Semicolon --
-   --------------------------------------------
-
-   --  This is a somewhat trickier situation, and we can't catch it in all
-   --  cases, but we do our best to detect common situations resulting from
-   --  a "cut and paste" operation which forgets to change the IS to semicolon.
-   --  Consider the following example:
-
-   --    package body X is
-   --      procedure A;
-   --      procedure B is
-   --      procedure C;
-   --      ...
-   --      procedure D is
-   --      begin
-   --         ...
-   --      end;
-   --    begin
-   --      ...
-   --    end;
-
-   --  The trouble is that the section of text from PROCEDURE B through END;
-   --  constitutes a valid procedure body, and the danger is that we find out
-   --  far too late that something is wrong (indeed most compilers will behave
-   --  uncomfortably on the above example).
-
-   --  We have two approaches to helping to control this situation. First we
-   --  make every attempt to avoid swallowing the last END; if we can be sure
-   --  that some error will result from doing so. In particular, we won't
-   --  accept the END; unless it is exactly correct (in particular it must not
-   --  have incorrect name tokens), and we won't accept it if it is immediately
-   --  followed by end of file, WITH or SEPARATE (all tokens that unmistakeably
-   --  signal the start of a compilation unit, and which therefore allow us to
-   --  reserve the END; for the outer level.) For more details on this aspect
-   --  of the handling, see package Par.Endh.
-
-   --  If we can avoid eating up the END; then the result in the absence of
-   --  any additional steps would be to post a missing END referring back to
-   --  the subprogram with the bogus IS. Similarly, if the enclosing package
-   --  has no BEGIN, then the result is a missing BEGIN message, which again
-   --  refers back to the subprogram header.
-
-   --  Such an error message is not too bad (it's already a big improvement
-   --  over what many parsers do), but it's not ideal, because the declarations
-   --  following the IS have been absorbed into the wrong scope. In the above
-   --  case, this could result for example in a bogus complaint that the body
-   --  of D was missing from the package.
-
-   --  To catch at least some of these cases, we take the following additional
-   --  steps. First, a subprogram body is marked as having a suspicious IS if
-   --  the declaration line is followed by a line which starts with a symbol
-   --  that can start a declaration in the same column, or to the left of the
-   --  column in which the FUNCTION or PROCEDURE starts (normal style is to
-   --  indent any declarations which really belong a subprogram). If such a
-   --  subprogram encounters a missing BEGIN or missing END, then we decide
-   --  that the IS should have been a semicolon, and the subprogram body node
-   --  is marked (by setting the Bad_Is_Detected flag true. Note that we do
-   --  not do this for library level procedures, only for nested procedures,
-   --  since for library level procedures, we must have a body.
-
-   --  The processing for a declarative part checks to see if the last
-   --  declaration scanned is marked in this way, and if it is, the tree
-   --  is modified to reflect the IS being interpreted as a semicolon.
-
    ---------------------------------------------------
    -- Parser Type Definitions and Control Variables --
    ---------------------------------------------------
@@ -1450,6 +1448,47 @@ function Par (Configuration_Pragmas : Boolean) return List_Id is
       --  the Node N (which is a Defining_Identifier node with the Chars field
       --  set) is a renaming of an entity in package Standard.
 
+      -----------------------------------
+      -- Multiple defining identifiers --
+      -----------------------------------
+
+      --  RM-3.3.1(7) says:
+      --
+      --    Any declaration that includes a defining_identifier_list with
+      --    more than one defining_identifier is equivalent to a series of
+      --    declarations each containing one defining_identifier from the list,
+      --    with the rest of the text of the declaration copied for each
+      --    declaration in the series, in the same order as the list.
+      --
+      --  We parse such declarations by first calling P_Def_Ids (see below).
+      --  Then, if there are multiple identifiers, we repeatedly scan the
+      --  type and initialization expression information by resetting the
+      --  scan pointer (so that we get completely separate trees for each
+      --  occurrence).
+
+      --  Defining_Identifiers is a sequence of identifiers parsed by
+      --  P_Def_Ids. Idents holds the identifiers, and Num_Idents
+      --  points to the last-used array elements. The upper bound
+      --  is intended to be essentially infinite, so we don't bother
+      --  giving a good error message when it is exceeded -- we
+      --  simply raise an exception.
+
+      type Defining_Identifiers_Array is
+        array (Pos range 1 .. 4096) of Entity_Id;
+
+      type Defining_Identifiers is record
+         Num_Idents : Nat := 0;
+         Idents     : Defining_Identifiers_Array;
+      end record;
+
+      procedure Append
+        (Def_Ids : in out Defining_Identifiers; Def_Id : Entity_Id);
+      --  Append one defining identifier onto Def_Ids.
+
+      procedure P_Def_Ids (Def_Ids : out Defining_Identifiers);
+      --  Parse a defining_identifier_list, appending the identifiers
+      --  onto Def_Ids, which should be initially empty.
+
    end Util;
 
    --------------
diff --git a/gcc/ada/rtsfind.adb b/gcc/ada/rtsfind.adb
index 86713ff..f47aacc 100644
--- a/gcc/ada/rtsfind.adb
+++ b/gcc/ada/rtsfind.adb
@@ -1030,8 +1030,7 @@ package body Rtsfind is
       U        : RT_Unit_Table_Record renames RT_Unit_Table (U_Id);
       Priv_Par : constant Elist_Id := New_Elmt_List;
       Lib_Unit : Node_Id;
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_ISMP : constant Boolean        :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
       Saved_SM  : constant SPARK_Mode_Type := SPARK_Mode;
@@ -1099,7 +1098,7 @@ package body Rtsfind is
       procedure Restore_SPARK_Context is
       begin
          Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-         Restore_Ghost_Region (Saved_GM, Saved_IGR);
+         Restore_Ghost_Region (Saved_Ghost_Config);
          Restore_SPARK_Mode   (Saved_SM, Saved_SMP);
       end Restore_SPARK_Context;
 
@@ -1289,7 +1288,7 @@ package body Rtsfind is
 
       declare
          LibUnit  : constant Node_Id         := Unit (Cunit (U.Unum));
-         Saved_GM : constant Ghost_Mode_Type := Ghost_Mode;
+         Saved_GM : constant Ghost_Mode_Type := Ghost_Config.Ghost_Mode;
          Clause   : Node_Id;
          Withn    : Node_Id;
 
@@ -1308,13 +1307,13 @@ package body Rtsfind is
          --  later, after ignored ghost code is converted to a null
          --  statement.
 
-         Ghost_Mode := None;
+         Ghost_Config.Ghost_Mode := None;
          Withn :=
            Make_With_Clause (Standard_Location,
              Name =>
                Make_Unit_Name
                  (U, Defining_Unit_Name (Specification (LibUnit))));
-         Ghost_Mode := Saved_GM;
+         Ghost_Config.Ghost_Mode := Saved_GM;
 
          Set_Corresponding_Spec  (Withn, U.Entity);
          Set_First_Name          (Withn);
@@ -1627,7 +1626,9 @@ package body Rtsfind is
       --  is pulled within an ignored Ghost context because all this code will
       --  disappear.
 
-      if U_Id = System_Secondary_Stack and then Ghost_Mode /= Ignore then
+      if U_Id = System_Secondary_Stack
+        and then Ghost_Config.Ghost_Mode /= Ignore
+      then
          Sec_Stack_Used := True;
       end if;
 
diff --git a/gcc/ada/s-oscons-tmplt.c b/gcc/ada/s-oscons-tmplt.c
index 97537ef..7a5e987 100644
--- a/gcc/ada/s-oscons-tmplt.c
+++ b/gcc/ada/s-oscons-tmplt.c
@@ -1976,7 +1976,7 @@ CND(CLOCK_THREAD_CPUTIME_ID, "Thread CPU clock")
 #if defined(__linux__) || defined(__FreeBSD__) \
  || (defined(_AIX) && defined(_AIXVERSION_530)) \
  || defined(__DragonFly__) || defined(__QNX__) \
- || defined (__vxworks)
+ || (defined (__vxworks) && /* VxWorks7 */ defined (_VSB_CONFIG_FILE))
 /** On these platforms use system provided monotonic clock instead of
  ** the default CLOCK_REALTIME. We then need to set up cond var attributes
  ** appropriately (see thread.c).
@@ -1985,6 +1985,10 @@ CND(CLOCK_THREAD_CPUTIME_ID, "Thread CPU clock")
  ** pthread_cond_timedwait (and does not have pthread_condattr_setclock),
  ** hence the conditionalization on AIX version above). _AIXVERSION_530
  ** is defined in AIX 5.3 and more recent versions.
+ **
+ ** VxWorks6 lacks pthread_condattr_setclock, so define CLOCK_RT_Ada to
+ ** CLOCK_REALTIME to get the dummy definition of __gnat_pthread_condattr_setup
+ ** in libgnarl/thread.c.
  **/
 # define CLOCK_RT_Ada "CLOCK_MONOTONIC"
 
diff --git a/gcc/ada/sem.adb b/gcc/ada/sem.adb
index e168d62..944ece1 100644
--- a/gcc/ada/sem.adb
+++ b/gcc/ada/sem.adb
@@ -104,8 +104,7 @@ package body Sem is
    --  Ghost mode.
 
    procedure Analyze (N : Node_Id) is
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
    begin
@@ -842,7 +841,7 @@ package body Sem is
          Expand_SPARK_Potential_Renaming (N);
       end if;
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze;
 
    --  Version with check(s) suppressed
@@ -1440,8 +1439,7 @@ package body Sem is
       --  the Ghost mode.
 
       procedure Do_Analyze is
-         Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-         Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+         Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
          Saved_ISMP : constant Boolean        :=
                         Ignore_SPARK_Mode_Pragmas_In_Instance;
          --  Save Ghost and SPARK mode-related data to restore on exit
@@ -1489,7 +1487,7 @@ package body Sem is
          Style_Max_Line_Length := Saved_ML;
          Style_Check_Max_Line_Length := Saved_CML;
 
-         Restore_Ghost_Region (Saved_GM, Saved_IGR);
+         Restore_Ghost_Region (Saved_Ghost_Config);
          Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
       end Do_Analyze;
 
diff --git a/gcc/ada/sem_attr.adb b/gcc/ada/sem_attr.adb
index 9602944..78b6318 100644
--- a/gcc/ada/sem_attr.adb
+++ b/gcc/ada/sem_attr.adb
@@ -5092,7 +5092,7 @@ package body Sem_Attr is
          --  early transformation also avoids the generation of a useless loop
          --  entry constant.
 
-         if Present (Encl_Prag) and then Is_Ignored (Encl_Prag) then
+         if Present (Encl_Prag) and then Is_Ignored_In_Codegen (Encl_Prag) then
             Rewrite (N, Relocate_Node (P));
             Preanalyze_And_Resolve (N);
 
@@ -9359,6 +9359,20 @@ package body Sem_Attr is
       when Attribute_First =>
          Set_Bounds;
 
+         --  In GNATprove mode we only fold array attributes when prefix is
+         --  static (because that's required by the Ada rules) or at least can
+         --  be evaluated without checks (because GNATprove would miss them).
+
+         if GNATprove_Mode
+            and then
+              not (Static
+                   or else (Is_Entity_Name (P) and then Is_Type (Entity (P)))
+                   or else Statically_Names_Object (P)
+                   or else Ekind (P_Type) = E_String_Literal_Subtype)
+         then
+            return;
+         end if;
+
          if Compile_Time_Known_Value (Lo_Bound) then
             if Is_Real_Type (P_Type) then
                Fold_Ureal (N, Expr_Value_R (Lo_Bound), Static);
@@ -9572,6 +9586,20 @@ package body Sem_Attr is
       when Attribute_Last =>
          Set_Bounds;
 
+         --  In GNATprove mode we only fold array attributes when prefix is
+         --  static (because that's required by the Ada rules) or at least can
+         --  be evaluated without checks (because GNATprove would miss them).
+
+         if GNATprove_Mode
+            and then
+              not (Static
+                   or else (Is_Entity_Name (P) and then Is_Type (Entity (P)))
+                   or else Statically_Names_Object (P)
+                   or else Ekind (P_Type) = E_String_Literal_Subtype)
+         then
+            return;
+         end if;
+
          if Compile_Time_Known_Value (Hi_Bound) then
             if Is_Real_Type (P_Type) then
                Fold_Ureal (N, Expr_Value_R (Hi_Bound), Static);
@@ -9655,6 +9683,20 @@ package body Sem_Attr is
 
          Set_Bounds;
 
+         --  In GNATprove mode we only fold array attributes when prefix is
+         --  static (because that's required by the Ada rules) or at least can
+         --  be evaluated without checks (because GNATprove would miss them).
+
+         if GNATprove_Mode
+            and then
+              not (Static
+                   or else (Is_Entity_Name (P) and then Is_Type (Entity (P)))
+                   or else Statically_Names_Object (P)
+                   or else Ekind (P_Type) = E_String_Literal_Subtype)
+         then
+            return;
+         end if;
+
          --  For two compile time values, we can compute length
 
          if Compile_Time_Known_Value (Lo_Bound)
diff --git a/gcc/ada/sem_aux.adb b/gcc/ada/sem_aux.adb
index bb1624d..0aa74e3 100644
--- a/gcc/ada/sem_aux.adb
+++ b/gcc/ada/sem_aux.adb
@@ -25,7 +25,6 @@
 
 with Atree;          use Atree;
 with Einfo;          use Einfo;
-with Einfo.Entities; use Einfo.Entities;
 with Einfo.Utils;    use Einfo.Utils;
 with Nlists;         use Nlists;
 with Sinfo;          use Sinfo;
@@ -454,16 +453,28 @@ package body Sem_Aux is
       Id  : Entity_Id;
 
    begin
+      --  Call using access to subprogram with explicit dereference
+
       if Nkind (Nam) = N_Explicit_Dereference then
          Id := Etype (Nam);
          pragma Assert (Ekind (Id) = E_Subprogram_Type);
 
+      --  Case of call to simple entry, where the Name is a selected component
+      --  whose prefix is the task or protected record, and whose selector name
+      --  is the entry name.
+
       elsif Nkind (Nam) = N_Selected_Component then
          Id := Entity (Selector_Name (Nam));
 
+      --  Case of call to member of entry family, where Name is an indexed
+      --  component, with the prefix being a selected component giving the
+      --  task and entry family name, and the index being the entry index.
+
       elsif Nkind (Nam) = N_Indexed_Component then
          Id := Entity (Selector_Name (Prefix (Nam)));
 
+      --  Normal case
+
       else
          Id := Entity (Nam);
       end if;
@@ -1546,6 +1557,81 @@ package body Sem_Aux is
       return E;
    end Ultimate_Alias;
 
+   ---------------------------
+   -- Unique_Component_Name --
+   ---------------------------
+
+   function Unique_Component_Name
+     (Component : Record_Field_Kind_Id) return Name_Id
+   is
+      Homographic_Component_Count : Pos := 1;
+      Hcc                         : Pos renames Homographic_Component_Count;
+      Enclosing_Type              : Entity_Id :=
+        Underlying_Type (Base_Type (Scope (Component)));
+   begin
+      if Ekind (Enclosing_Type) = E_Record_Type
+        and then Is_Tagged_Type (Enclosing_Type)
+        and then Has_Private_Ancestor (Enclosing_Type)
+      then
+         --  traverse ancestors to determine Hcc value
+         loop
+            declare
+               Type_Decl : constant Node_Id :=
+                 Parent (Underlying_Type (Base_Type (Enclosing_Type)));
+               Type_Def : constant Node_Id := Type_Definition (Type_Decl);
+            begin
+               exit when Nkind (Type_Def) /= N_Derived_Type_Definition;
+               Enclosing_Type :=
+                 Underlying_Type (Base_Type (Etype (Enclosing_Type)));
+
+               declare
+                  Ancestor_Comp : Opt_Record_Field_Kind_Id :=
+                    First_Component_Or_Discriminant (Enclosing_Type);
+               begin
+                  while Present (Ancestor_Comp) loop
+                     if Chars (Ancestor_Comp) = Chars (Component) then
+                        Hcc := Hcc + 1;
+                        exit; -- exit not required, but might as well
+                     end if;
+                     Next_Component_Or_Discriminant (Ancestor_Comp);
+                  end loop;
+               end;
+            end;
+         end loop;
+      end if;
+
+      if Hcc = 1 then
+         --  the usual case
+         return Chars (Component);
+      else
+         declare
+            Buff : Bounded_String;
+         begin
+            Append (Buff, Chars (Component));
+
+            Append (Buff, "__");
+            --  A double underscore in an identifier is legal in C, not in Ada.
+            --  Returning a result that is not a legal Ada identifier
+            --  ensures that we won't have problems with collisions.
+            --  If we have a component named Foo and we just append a
+            --  number (without any underscores), that new name might match
+            --  the name of another component (which would be bad).
+            --  The result of this function is intended for use as an
+            --  identifier in generated C code, so it needs to be a
+            --  legal C identifer.
+
+            Append (Buff, Hcc);
+            --  Should we instead append Hcc - 1 here? This is a human
+            --  readability question. If parent type and extension each
+            --  have a Foo component, do we want the name returned for the
+            --  second Foo to be "foo__2" or "foo__1" ? Does it matter?
+            --  Either way, the name returned for the first Foo will be "foo".
+
+            return Name_Find (Buff);
+         end;
+      end if;
+   end Unique_Component_Name;
+
    --------------------------
    -- Unit_Declaration_Node --
    --------------------------
diff --git a/gcc/ada/sem_aux.ads b/gcc/ada/sem_aux.ads
index 5a63002..1a298a9 100644
--- a/gcc/ada/sem_aux.ads
+++ b/gcc/ada/sem_aux.ads
@@ -31,6 +31,7 @@
 --  require more than minimal semantic knowledge.
 
 with Alloc;
+with Einfo.Entities; use Einfo.Entities;
 with Namet; use Namet;
 with Table;
 with Types; use Types;
@@ -105,7 +106,7 @@ package Sem_Aux is
    --  this is equivalent to First_Entity. The exception arises for tagged
    --  types, where the tag itself is prepended to the front of the entity
    --  chain, so the First_Discriminant function steps past the tag if it is
-   --  present.  When called on a private type with unknown discriminants, the
+   --  present. When called on a private type with unknown discriminants, the
    --  function always returns Empty.
 
    --  WARNING: There is a matching C declaration of this subprogram in fe.h
@@ -405,6 +406,19 @@ package Sem_Aux is
    --  Return the last entity in the chain of aliased entities of Prim. If Prim
    --  has no alias return Prim.
 
+   function Unique_Component_Name
+     (Component : Record_Field_Kind_Id) return Name_Id;
+   --  Usually, a record type cannot have two components with the same name.
+   --  But in the case of a component declared in an extension of a tagged
+   --  private (or private extension) parent type, it is possible that some
+   --  ancestor type also has a (non-visible) component with the same name.
+   --  In the common case, this function simply returns the Chars attribute
+   --  of its argument.
+   --  But in the multiple-components-with-the-same-name case, it appends
+   --  a uniquifying suffix. The result in this case will not be a
+   --  syntactically valid Ada identifier, but it will be a syntactically
+   --  valid C identifier.
+
    function Unit_Declaration_Node (Unit_Id : Entity_Id) return Node_Id;
    --  Unit_Id is the simple name of a program unit, this function returns the
    --  corresponding xxx_Declaration node for the entity. Also applies to the
diff --git a/gcc/ada/sem_ch12.adb b/gcc/ada/sem_ch12.adb
index e80aea5..1ba76dc 100644
--- a/gcc/ada/sem_ch12.adb
+++ b/gcc/ada/sem_ch12.adb
@@ -4900,8 +4900,7 @@ package body Sem_Ch12 is
       Loc        : constant Source_Ptr := Sloc (N);
       Is_Abbrev  : constant Boolean    :=
                      Is_Abbreviated_Instance (Defining_Entity (N));
-      Saved_GM   : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR  : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_ISMP : constant Boolean         :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
       Saved_SM   : constant SPARK_Mode_Type := SPARK_Mode;
@@ -5680,7 +5679,7 @@ package body Sem_Ch12 is
       end if;
 
       Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
       Restore_SPARK_Mode   (Saved_SM, Saved_SMP);
       Style_Check := Saved_Style_Check;
 
@@ -5695,7 +5694,7 @@ package body Sem_Ch12 is
          end if;
 
          Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-         Restore_Ghost_Region (Saved_GM, Saved_IGR);
+         Restore_Ghost_Region (Saved_Ghost_Config);
          Restore_SPARK_Mode   (Saved_SM, Saved_SMP);
          Style_Check := Saved_Style_Check;
    end Analyze_Package_Instantiation;
@@ -6340,8 +6339,7 @@ package body Sem_Ch12 is
 
       --  Local variables
 
-      Saved_GM   : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR  : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_ISMP : constant Boolean         :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
       Saved_SM   : constant SPARK_Mode_Type := SPARK_Mode;
@@ -6736,7 +6734,7 @@ package body Sem_Ch12 is
       end if;
 
       Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
       Restore_SPARK_Mode   (Saved_SM, Saved_SMP);
 
    exception
@@ -6750,7 +6748,7 @@ package body Sem_Ch12 is
          end if;
 
          Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-         Restore_Ghost_Region (Saved_GM, Saved_IGR);
+         Restore_Ghost_Region (Saved_Ghost_Config);
          Restore_SPARK_Mode   (Saved_SM, Saved_SMP);
    end Analyze_Subprogram_Instantiation;
 
@@ -7574,6 +7572,12 @@ package body Sem_Ch12 is
                     or else not Same_Instantiated_Function (E1, E2));
             end if;
 
+         --  No check is needed if this is the body of a subprogram that is
+         --  implicitly created in the case of class-wide predefined functions.
+
+         elsif Ekind (E1) = E_Subprogram_Body then
+            null;
+
          else
             raise Program_Error;
          end if;
@@ -12868,8 +12872,7 @@ package body Sem_Ch12 is
       --  the package body.
 
       Saved_CS   : constant Config_Switches_Type     := Save_Config_Switches;
-      Saved_GM   : constant Ghost_Mode_Type          := Ghost_Mode;
-      Saved_IGR  : constant Node_Id                  := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_ISMP : constant Boolean                  :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
       Saved_LSST : constant Suppress_Stack_Entry_Ptr :=
@@ -13399,7 +13402,7 @@ package body Sem_Ch12 is
 
       Expander_Mode_Restore;
       Restore_Config_Switches (Saved_CS);
-      Restore_Ghost_Region    (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region    (Saved_Ghost_Config);
       Restore_SPARK_Mode      (Saved_SM, Saved_SMP);
       Restore_Warnings        (Saved_Warn);
    end Instantiate_Package_Body;
@@ -13430,8 +13433,7 @@ package body Sem_Ch12 is
       --  the subprogram body.
 
       Saved_CS   : constant Config_Switches_Type     := Save_Config_Switches;
-      Saved_GM   : constant Ghost_Mode_Type          := Ghost_Mode;
-      Saved_IGR  : constant Node_Id                  := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_ISMP : constant Boolean                  :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
       Saved_LSST : constant Suppress_Stack_Entry_Ptr :=
@@ -13734,7 +13736,7 @@ package body Sem_Ch12 is
 
       Expander_Mode_Restore;
       Restore_Config_Switches (Saved_CS);
-      Restore_Ghost_Region    (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region    (Saved_Ghost_Config);
       Restore_SPARK_Mode      (Saved_SM, Saved_SMP);
       Restore_Warnings        (Saved_Warn);
    end Instantiate_Subprogram_Body;
@@ -14371,8 +14373,21 @@ package body Sem_Ch12 is
          elsif
            Scope (Scope (Base_Type (Etype (A_Gen_T)))) = Scope (A_Gen_T)
          then
-            Ancestor :=
-              Get_Instance_Of (Base_Type (Etype (A_Gen_T)));
+            declare
+               Formal_Ancestor : constant Entity_Id :=
+                 Base_Type (Etype (A_Gen_T));
+            begin
+               Ancestor := Get_Instance_Of (Formal_Ancestor);
+
+               --  Handle (rare) case where Get_Instance_Of found nothing in
+               --  the map.
+
+               if Ancestor = Formal_Ancestor then
+                  Ancestor :=
+                    Get_Instance_Of
+                      (Base_Type (Etype (Get_Instance_Of (A_Gen_T))));
+               end if;
+            end;
 
          --  The type may be a local derivation, or a type extension of a
          --  previous formal, or of a formal of a parent package.
diff --git a/gcc/ada/sem_ch13.adb b/gcc/ada/sem_ch13.adb
index 1e88ef4..31735e4 100644
--- a/gcc/ada/sem_ch13.adb
+++ b/gcc/ada/sem_ch13.adb
@@ -337,6 +337,13 @@ package body Sem_Ch13 is
    --  Resolve each one of the arguments specified in the specification of
    --  aspect Finalizable.
 
+   function Resolve_Finalization_Procedure
+     (N   : Node_Id;
+      Typ : Entity_Id) return Boolean;
+   --  Resolve a procedure argument specified in the specification of one of
+   --  the finalization aspects, i.e. Finalizable and Destructor. Returns True
+   --  if successful, False otherwise.
+
    procedure Resolve_Iterable_Operation
      (N      : Node_Id;
       Cursor : Entity_Id;
@@ -4634,6 +4641,7 @@ package body Sem_Ch13 is
 
                when Aspect_Designated_Storage_Model =>
                   if not All_Extensions_Allowed then
+                     Error_Msg_Name_1 := Nam;
                      Error_Msg_GNAT_Extension ("aspect %", Loc);
                      goto Continue;
 
@@ -4646,6 +4654,20 @@ package body Sem_Ch13 is
                      goto Continue;
                   end if;
 
+               when Aspect_Destructor =>
+                  if not All_Extensions_Allowed then
+                     Error_Msg_Name_1 := Nam;
+                     Error_Msg_GNAT_Extension ("aspect %", Loc);
+                     goto Continue;
+
+                  elsif not Is_Type (E) then
+                     Error_Msg_N ("can only be specified for a type", Aspect);
+                     goto Continue;
+                  end if;
+
+                  Set_Has_Destructor (E);
+                  Set_Is_Controlled_Active (E);
+
                when Aspect_Storage_Model_Type =>
                   if not All_Extensions_Allowed then
                      Error_Msg_Name_1 := Nam;
@@ -4777,7 +4799,7 @@ package body Sem_Ch13 is
                     and then not Is_Ignored_Ghost_Entity (E)
                   then
                      if A_Id = Aspect_Pre then
-                        if Is_Ignored (Aspect) then
+                        if Is_Ignored_In_Codegen (Aspect) then
                            Set_Ignored_Class_Preconditions (E,
                              New_Copy_Tree (Expr));
                         else
@@ -4791,7 +4813,7 @@ package body Sem_Ch13 is
                      elsif No (Class_Postconditions (E))
                        and then No (Ignored_Class_Postconditions (E))
                      then
-                        if Is_Ignored (Aspect) then
+                        if Is_Ignored_In_Codegen (Aspect) then
                            Set_Ignored_Class_Postconditions (E,
                              New_Copy_Tree (Expr));
                         else
@@ -5063,6 +5085,14 @@ package body Sem_Ch13 is
                         Check_Expr_Is_OK_Static_Expression (Expr, Any_Boolean);
                      end if;
 
+                     --  Record the No_Task_Parts aspects as a rep item so it
+                     --  can be consistently looked up on the full view of the
+                     --  type.
+
+                     if Is_Private_Type (E) then
+                        Record_Rep_Item (E, Aspect);
+                     end if;
+
                      goto Continue;
 
                   --  Ada 2022 (AI12-0075): static expression functions
@@ -8861,6 +8891,43 @@ package body Sem_Ch13 is
             Num_Repped_Components   : Nat := 0;
             Num_Unrepped_Components : Nat := 0;
 
+            function Unchecked_Union_Pragma_Pending return Boolean;
+            --  Return True in the corner case of an Unchecked_Union pragma
+            --  occuring after the record representation clause (which
+            --  means that Is_Unchecked_Union will return False for Rectype,
+            --  even though it would return True if called later after the
+            --  pragma is analyzed).
+
+            ------------------------------------
+            -- Unchecked_Union_Pragma_Pending --
+            ------------------------------------
+
+            function Unchecked_Union_Pragma_Pending return Boolean is
+               Decl_List_Element : Node_Id := N;
+               Pragma_Arg : Node_Id;
+            begin
+               while Present (Decl_List_Element) loop
+                  if Nkind (Decl_List_Element) = N_Pragma
+                    and then Get_Pragma_Id (Decl_List_Element) =
+                             Pragma_Unchecked_Union
+                    and then not Is_Empty_List (Pragma_Argument_Associations
+                                                  (Decl_List_Element))
+                  then
+                     Pragma_Arg := Get_Pragma_Arg
+                                     (First (Pragma_Argument_Associations
+                                               (Decl_List_Element)));
+                     if Nkind (Pragma_Arg) = N_Identifier
+                       and then Chars (Pragma_Arg) = Chars (Rectype)
+                     then
+                        return True;
+                     end if;
+                  end if;
+
+                  Next (Decl_List_Element);
+               end loop;
+               return False;
+            end Unchecked_Union_Pragma_Pending;
+
          begin
             --  First count number of repped and unrepped components
 
@@ -8899,8 +8966,10 @@ package body Sem_Ch13 is
                     --  Ignore discriminant in unchecked union, since it is
                     --  not there, and cannot have a component clause.
 
-                    and then (not Is_Unchecked_Union (Rectype)
-                               or else Ekind (Comp) /= E_Discriminant)
+                    and then (Ekind (Comp) /= E_Discriminant
+                              or else not (Is_Unchecked_Union (Rectype)
+                                           or else
+                                             Unchecked_Union_Pragma_Pending))
                   then
                      Error_Msg_Sloc := Sloc (Comp);
                      Error_Msg_NE
@@ -10213,8 +10282,7 @@ package body Sem_Ch13 is
    procedure Build_Predicate_Function (Typ : Entity_Id; N : Node_Id) is
       Loc : constant Source_Ptr := Sloc (Typ);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Expr : Node_Id;
@@ -10380,7 +10448,7 @@ package body Sem_Ch13 is
             --  which is needed to generate the corresponding predicate
             --  function.
 
-            if Is_Ignored_Ghost_Pragma (Prag) then
+            if Is_Ignored_Ghost_Pragma_In_Codegen (Prag) then
                Add_Condition (New_Occurrence_Of (Standard_True, Sloc (Prag)));
 
             else
@@ -10421,7 +10489,8 @@ package body Sem_Ch13 is
 
                   --  "and"-in the Arg2 condition to evolving expression
 
-                  if not Is_Ignored_Ghost_Pragma (Prag) then
+                  if not Is_Ignored_Ghost_Pragma_In_Codegen (Prag)
+                  then
                      Add_Condition (Arg2_Copy);
                   end if;
                end;
@@ -11021,7 +11090,7 @@ package body Sem_Ch13 is
          end;
       end if;
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
 
       if Restore_Scope then
          Pop_Scope;
@@ -11041,8 +11110,7 @@ package body Sem_Ch13 is
    is
       Loc : constant Source_Ptr := Sloc (Typ);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Func_Decl : Node_Id;
@@ -11123,7 +11191,7 @@ package body Sem_Ch13 is
       Insert_After (Parent (Typ), Func_Decl);
       Analyze (Func_Decl);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
 
       return Func_Decl;
    end Build_Predicate_Function_Declaration;
@@ -11201,6 +11269,13 @@ package body Sem_Ch13 is
    --  Start of processing for Check_Aspect_At_End_Of_Declarations
 
    begin
+      --  Indicate that the expression comes from an aspect specification,
+      --  which is used in subsequent analysis even if expansion is off.
+
+      if Present (End_Decl_Expr) then
+         Set_Parent (End_Decl_Expr, ASN);
+      end if;
+
       --  In an instance we do not perform the consistency check between freeze
       --  point and end of declarations, because it was done already in the
       --  analysis of the generic. Furthermore, the delayed analysis of an
@@ -11230,6 +11305,7 @@ package body Sem_Ch13 is
       --  the one available at at the freeze point.
 
       elsif A_Id in Aspect_Constructor
+                  | Aspect_Destructor
                   | Aspect_Input
                   | Aspect_Output
                   | Aspect_Read
@@ -11292,13 +11368,6 @@ package body Sem_Ch13 is
             end if;
          end if;
 
-         --  Indicate that the expression comes from an aspect specification,
-         --  which is used in subsequent analysis even if expansion is off.
-
-         if Present (End_Decl_Expr) then
-            Set_Parent (End_Decl_Expr, ASN);
-         end if;
-
          --  In a generic context the original aspect expressions have not
          --  been preanalyzed, so do it now. There are no conformance checks
          --  to perform in this case. As before, we have to make components
@@ -11406,24 +11475,16 @@ package body Sem_Ch13 is
    ----------------------------------
 
    procedure Check_Aspect_At_Freeze_Point (ASN : Node_Id) is
-      Ident : constant Node_Id := Identifier (ASN);
-      --  Identifier (use Entity field to save expression)
-
       Expr : constant Node_Id := Expression (ASN);
-      --  For cases where using Entity (Identifier) doesn't work
 
-      A_Id : constant Aspect_Id := Get_Aspect_Id (Chars (Ident));
+      A_Id : constant Aspect_Id := Get_Aspect_Id (Chars (Identifier (ASN)));
 
       T : Entity_Id := Empty;
       --  Type required for preanalyze call
 
    begin
-      --  On entry to this procedure, Entity (Ident) contains a copy of the
-      --  original expression from the aspect, saved for this purpose.
-
-      --  On exit from this procedure Entity (Ident) is unchanged, still
-      --  containing that copy, but Expression (Ident) is a preanalyzed copy
-      --  of the expression, preanalyzed just after the freeze point.
+      --  On exit from this procedure, Expression (ASN) is a copy of the
+      --  original expression, preanalyzed just after the freeze point.
 
       --  Make a copy of the expression to be preanalyzed
 
@@ -11702,6 +11763,67 @@ package body Sem_Ch13 is
             Analyze (Expression (ASN));
             return;
 
+         when Aspect_Destructor =>
+            if not Is_Record_Type (Entity (ASN)) then
+               Error_Msg_N
+                 ("aspect Destructor can only be specified for a "
+                  & "record type",
+                  ASN);
+               return;
+            end if;
+
+            Set_Has_Destructor (Entity (ASN));
+            Set_Is_Controlled_Active (Entity (ASN));
+
+            Analyze (Expression (ASN));
+
+            if not Resolve_Finalization_Procedure
+                     (Expression (ASN), Entity (ASN))
+            then
+               Error_Msg_N
+                 ("destructor must be local procedure whose only formal "
+                  & "parameter has mode `IN OUT` and is of the type the "
+                  & "destructor is for",
+                  Expression (ASN));
+            end if;
+
+            Set_Is_Destructor (Entity (Expression (ASN)));
+
+            declare
+               Proc  : constant Entity_Id := Entity (Expression (ASN));
+               Overr : constant Opt_N_Entity_Id :=
+                 Overridden_Inherited_Operation (Proc);
+               Orig  : constant Entity_Id :=
+                 (if Present (Overr) then Overr else Proc);
+
+               Decl : constant Node_Id :=
+                 Parent
+                   (if Nkind (Parent (Orig)) = N_Procedure_Specification
+                    then Parent (Orig)
+                    else Orig);
+
+               Encl : constant Node_Id := Parent (Decl);
+
+               Is_Private : constant Boolean :=
+                 Nkind (Encl) = N_Package_Specification
+                 and then Is_List_Member (Decl)
+                 and then List_Containing (Decl) = Private_Declarations (Encl);
+
+            begin
+
+               if Has_Private_Declaration (Entity (ASN))
+                 and then not Aspect_On_Partial_View (ASN)
+                 and then not Is_Private
+               then
+                  Error_Msg_N
+                    ("aspect Destructor on full view cannot denote public "
+                     & "primitive",
+                     ASN);
+               end if;
+            end;
+
+            return;
+
          when Aspect_Storage_Model_Type =>
 
             --  The aggregate argument of Storage_Model_Type is optional, and
@@ -15855,6 +15977,8 @@ package body Sem_Ch13 is
          --  We may freeze Subp_Id immediately since Ent has just been frozen.
          --  This will help to shield us from potential late freezing issues.
 
+         Mutate_Ekind (Subp_Id, E_Procedure);
+         Freeze_Extra_Formals (Subp_Id);
          Set_Is_Frozen (Subp_Id);
 
       else
@@ -17302,6 +17426,35 @@ package body Sem_Ch13 is
       Typ : Entity_Id;
       Nam : Name_Id)
    is
+   begin
+      if Nam = Name_Relaxed_Finalization then
+         Resolve (N, Any_Boolean);
+
+         if Is_OK_Static_Expression (N) then
+            Set_Has_Relaxed_Finalization (Typ, Is_True (Static_Boolean (N)));
+
+         else
+            Flag_Non_Static_Expr
+              ("expression of aspect Finalizable must be static!", N);
+         end if;
+
+         return;
+      end if;
+
+      if Resolve_Finalization_Procedure (N, Typ) then
+         return;
+      end if;
+
+      Error_Msg_N
+        ("finalizable primitive must be local procedure whose only formal " &
+         "parameter has mode `IN OUT` and is of the finalizable type", N);
+   end Resolve_Finalizable_Argument;
+
+   function Resolve_Finalization_Procedure
+     (N   : Node_Id;
+      Typ : Entity_Id)
+      return Boolean
+   is
       function Is_Finalizable_Primitive (E : Entity_Id) return Boolean;
       --  Check whether E is a finalizable primitive for Typ
 
@@ -17319,29 +17472,15 @@ package body Sem_Ch13 is
            and then No (Next_Formal (First_Formal (E)));
       end Is_Finalizable_Primitive;
 
-   --  Start of processing for Resolve_Finalizable_Argument
+   --  Start of processing for Resolve_Finalization_Procedure
 
    begin
-      if Nam = Name_Relaxed_Finalization then
-         Resolve (N, Any_Boolean);
-
-         if Is_OK_Static_Expression (N) then
-            Set_Has_Relaxed_Finalization (Typ, Is_True (Static_Boolean (N)));
-
-         else
-            Flag_Non_Static_Expr
-              ("expression of aspect Finalizable must be static!", N);
-         end if;
-
-         return;
-      end if;
-
       if not Is_Entity_Name (N) then
          null;
 
       elsif not Is_Overloaded (N) then
          if Is_Finalizable_Primitive (Entity (N)) then
-            return;
+            return True;
          end if;
 
       else
@@ -17357,7 +17496,7 @@ package body Sem_Ch13 is
             while Present (It.Typ) loop
                if Is_Finalizable_Primitive (It.Nam) then
                   Set_Entity (N, It.Nam);
-                  return;
+                  return True;
                end if;
 
                Get_Next_Interp (I, It);
@@ -17365,10 +17504,8 @@ package body Sem_Ch13 is
          end;
       end if;
 
-      Error_Msg_N
-        ("finalizable primitive must be local procedure whose only formal " &
-         "parameter has mode `IN OUT` and is of the finalizable type", N);
-   end Resolve_Finalizable_Argument;
+      return False;
+   end Resolve_Finalization_Procedure;
 
    --------------------------------
    -- Resolve_Iterable_Operation --
diff --git a/gcc/ada/sem_ch3.adb b/gcc/ada/sem_ch3.adb
index 5354d82..9f69e4f 100644
--- a/gcc/ada/sem_ch3.adb
+++ b/gcc/ada/sem_ch3.adb
@@ -3020,6 +3020,8 @@ package body Sem_Ch3 is
    -----------------------------------
 
    procedure Analyze_Full_Type_Declaration (N : Node_Id) is
+      use Deferred_Extra_Formals_Support;
+
       Def    : constant Node_Id   := Type_Definition (N);
       Def_Id : constant Entity_Id := Defining_Identifier (N);
       T      : Entity_Id;
@@ -3558,6 +3560,16 @@ package body Sem_Ch3 is
          end if;
       end if;
 
+      --  If we have some subprogram, subprogram type, or entry, with deferred
+      --  addition of its extra formals (because the underlying type of this
+      --  type was not previously available), then try creating now its extra
+      --  formals. Create also the extra actuals of deferred calls to entities
+      --  with deferred extra formals.
+
+      if Has_Deferred_Extra_Formals (T) then
+         Add_Deferred_Extra_Params (T);
+      end if;
+
       if Ekind (T) = E_Record_Type
         and then Is_Large_Unconstrained_Definite (T)
         and then not Is_Limited_Type (T)
@@ -4374,8 +4386,7 @@ package body Sem_Ch3 is
 
       --  Local variables
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Prev_Entity       : Entity_Id := Empty;
@@ -5463,7 +5474,7 @@ package body Sem_Ch3 is
          Check_No_Hidden_State (Id);
       end if;
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Object_Declaration;
 
    ---------------------------
@@ -6819,7 +6830,7 @@ package body Sem_Ch3 is
       --  that the element type is constrained.
 
       if Is_Mutably_Tagged_Type (Element_Type) then
-         Set_Component_Type (T,
+         Set_Component_Type (Base_Type (T),
            Class_Wide_Equivalent_Type (Element_Type));
 
       elsif not Is_Definite_Subtype (Element_Type) then
@@ -8440,15 +8451,17 @@ package body Sem_Ch3 is
          Set_Has_Private_Declaration (Full_Der);
          Set_Has_Private_Declaration (Derived_Type);
 
-         Set_Scope                (Full_Der, Scope (Derived_Type));
-         Set_Is_First_Subtype     (Full_Der, Is_First_Subtype (Derived_Type));
-         Set_Has_Size_Clause      (Full_Der, False);
-         Set_Has_Alignment_Clause (Full_Der, False);
-         Set_Has_Delayed_Freeze   (Full_Der);
-         Set_Is_Frozen            (Full_Der, False);
-         Set_Freeze_Node          (Full_Der, Empty);
-         Set_Depends_On_Private   (Full_Der, Has_Private_Component (Full_Der));
-         Set_Is_Public            (Full_Der, Is_Public (Derived_Type));
+         Set_Scope                 (Full_Der, Scope (Derived_Type));
+         Set_Is_First_Subtype      (Full_Der, Is_First_Subtype (Derived_Type));
+         Set_Has_Size_Clause       (Full_Der, False);
+         Set_Has_Alignment_Clause  (Full_Der, False);
+         Set_Has_Delayed_Freeze    (Full_Der);
+         Set_Is_Frozen             (Full_Der, False);
+         Set_Freeze_Node           (Full_Der, Empty);
+         Set_Depends_On_Private
+           (Full_Der, Has_Private_Component (Full_Der));
+         Set_Is_Public             (Full_Der, Is_Public (Derived_Type));
+         Set_Is_Implicit_Full_View (Full_Der);
 
          --  The convention on the base type may be set in the private part
          --  and not propagated to the subtype until later, so we obtain the
@@ -9646,6 +9659,8 @@ package body Sem_Ch3 is
            (New_Decl, Parent_Base, New_Base,
             Is_Completion => False, Derive_Subps => False);
 
+         Set_Is_Implicit_Full_View (New_Base);
+
          --  ??? This needs re-examination to determine whether the
          --  following call can simply be replaced by a call to Analyze.
 
@@ -21279,8 +21294,8 @@ package body Sem_Ch3 is
       --  On entry, the current scope is the composite type.
 
       --  The discriminants are initially entered into the scope of the type
-      --  via Enter_Name with the default Ekind of E_Void to prevent premature
-      --  use, as explained at the end of this procedure.
+      --  via Enter_Name with Is_Not_Self_Hidden set to False to prevent
+      --  premature use, as explained at the end of this procedure.
 
       Discr := First (Discriminant_Specifications (N));
       while Present (Discr) loop
@@ -21553,12 +21568,12 @@ package body Sem_Ch3 is
       --  expressions of a discriminant part if the specification of the
       --  discriminant is itself given in the discriminant part. (RM 3.7.1)
 
-      --  To detect this, the discriminant names are entered initially with an
-      --  Ekind of E_Void (which is the default Ekind given by Enter_Name). Any
-      --  attempt to use a void entity (for example in an expression that is
-      --  type-checked) produces the error message: premature usage. Now after
-      --  completing the semantic analysis of the discriminant part, we can set
-      --  the Ekind of all the discriminants appropriately.
+      --  To detect this, the discriminant names are entered initially with
+      --  Is_Not_Self_Hidden set to False. Any attempt to use a self-hidden
+      --  entity (for example in an expression that is type-checked) produces
+      --  the error message: premature usage. Now after completing the semantic
+      --  analysis of the discriminant part, we can set Is_Not_Self_Hidden on
+      --  all the discriminants appropriately.
 
       Discr := First (Discriminant_Specifications (N));
       Discr_Number := Uint_1;
@@ -21691,8 +21706,7 @@ package body Sem_Ch3 is
 
       --  Local variables
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Full_Indic  : Node_Id;
@@ -22385,7 +22399,7 @@ package body Sem_Ch3 is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Process_Full_View;
 
    -----------------------------------
diff --git a/gcc/ada/sem_ch4.adb b/gcc/ada/sem_ch4.adb
index dc81467..018c8a0 100644
--- a/gcc/ada/sem_ch4.adb
+++ b/gcc/ada/sem_ch4.adb
@@ -5603,6 +5603,10 @@ package body Sem_Ch4 is
                   if No (Act_Decl) then
                      Set_Etype (N, Etype (Comp));
 
+                     if Is_Mutably_Tagged_CW_Equivalent_Type (Etype (N)) then
+                        Make_Mutably_Tagged_Conversion (N);
+                     end if;
+
                   else
                      --  If discriminants were present in the component
                      --  declaration, they have been replaced by the
@@ -10406,11 +10410,14 @@ package body Sem_Ch4 is
          --  may be candidates, so that Try_Primitive_Operations can examine
          --  them if no real primitive is found.
 
-         function Is_Private_Overriding (Op : Entity_Id) return Boolean;
+         function Is_Callable_Private_Overriding
+           (Op : Entity_Id) return Boolean;
          --  An operation that overrides an inherited operation in the private
          --  part of its package may be hidden, but if the inherited operation
-         --  is visible a direct call to it will dispatch to the private one,
-         --  which is therefore a valid candidate.
+         --  that it overrides is visible, then a direct call to it will
+         --  dispatch to the private one, which is therefore a valid candidate.
+         --  Returns True if the operation can be called from outside the
+         --  enclosing package.
 
          function Names_Match
            (Obj_Type : Entity_Id;
@@ -10581,11 +10588,13 @@ package body Sem_Ch4 is
             return Op_List;
          end Extended_Primitive_Ops;
 
-         ---------------------------
-         -- Is_Private_Overriding --
-         ---------------------------
+         ------------------------------------
+         -- Is_Callable_Private_Overriding --
+         ------------------------------------
 
-         function Is_Private_Overriding (Op : Entity_Id) return Boolean is
+         function Is_Callable_Private_Overriding
+           (Op : Entity_Id) return Boolean
+         is
             Visible_Op : Entity_Id;
 
          begin
@@ -10607,7 +10616,10 @@ package body Sem_Ch4 is
                   --  have found what we're looking for.
 
                   if not Is_Hidden (Visible_Op)
-                    or else not Is_Hidden (Overridden_Operation (Op))
+                    or else
+                      (Present (Overridden_Inherited_Operation (Op))
+                        and then not Is_Hidden
+                                       (Overridden_Inherited_Operation (Op)))
                   then
                      return True;
                   end if;
@@ -10617,7 +10629,7 @@ package body Sem_Ch4 is
             end loop;
 
             return False;
-         end Is_Private_Overriding;
+         end Is_Callable_Private_Overriding;
 
          -----------------
          -- Names_Match --
@@ -10692,6 +10704,7 @@ package body Sem_Ch4 is
 
               or else
                 (Has_Unknown_Discriminants (Typ)
+                  and then Is_Record_Type (Base_Type (Obj_Type))
                   and then Typ = Underlying_Record_View (Base_Type (Obj_Type)))
 
                --  Prefix can be dereferenced
@@ -10759,13 +10772,15 @@ package body Sem_Ch4 is
 
                  --  Do not consider hidden primitives unless the type is in an
                  --  open scope or we are within an instance, where visibility
-                 --  is known to be correct, or else if this is an overriding
-                 --  operation in the private part for an inherited operation.
+                 --  is known to be correct, or else if this is an operation
+                 --  declared in the private part that overrides a visible
+                 --  inherited operation.
 
                  or else (Is_Hidden (Prim_Op)
                            and then not Is_Immediately_Visible (Obj_Type)
                            and then not In_Instance
-                           and then not Is_Private_Overriding (Prim_Op))
+                           and then
+                             not Is_Callable_Private_Overriding (Prim_Op))
                then
                   goto Continue;
                end if;
diff --git a/gcc/ada/sem_ch5.adb b/gcc/ada/sem_ch5.adb
index e1d6be4..9e4936b 100644
--- a/gcc/ada/sem_ch5.adb
+++ b/gcc/ada/sem_ch5.adb
@@ -385,8 +385,7 @@ package body Sem_Ch5 is
 
       --  Local variables
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       T1 : Entity_Id;
@@ -807,7 +806,14 @@ package body Sem_Ch5 is
 
       if Is_Tag_Indeterminate (Rhs) then
          if Is_Class_Wide_Type (T1) then
-            Propagate_Tag (Lhs, Rhs);
+
+            --  No need to propagate the tag when the RHS has function calls
+            --  that already propagated it (see Expand_Call_Helper), or if
+            --  some error was reported analyzing RHS.
+
+            if not (Error_Posted (Rhs) or else Tag_Propagated (Lhs)) then
+               Propagate_Tag (Lhs, Rhs);
+            end if;
 
          elsif Nkind (Rhs) = N_Function_Call
            and then Is_Entity_Name (Name (Rhs))
@@ -1186,7 +1192,7 @@ package body Sem_Ch5 is
       Analyze_Dimension (N);
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
 
       --  If the right-hand side contains target names, expansion has been
       --  disabled to prevent expansion that might move target names out of
@@ -2101,7 +2107,7 @@ package body Sem_Ch5 is
       --  A label declared within a Ghost region becomes Ghost (SPARK RM
       --  6.9(2)).
 
-      if Ghost_Mode > None then
+      if Ghost_Config.Ghost_Mode > None then
          Set_Is_Ghost_Entity (Id);
       end if;
    end Analyze_Implicit_Label_Declaration;
diff --git a/gcc/ada/sem_ch6.adb b/gcc/ada/sem_ch6.adb
index 48dcf8e..b7ddc4b 100644
--- a/gcc/ada/sem_ch6.adb
+++ b/gcc/ada/sem_ch6.adb
@@ -1372,8 +1372,7 @@ package body Sem_Ch6 is
       Loc  : constant Source_Ptr := Sloc (N);
       Spec : constant Node_Id    := Specification (N);
 
-      Saved_GM   : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR  : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_ISMP : constant Boolean         :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
       --  Save the Ghost and SPARK mode-related data to restore on exit
@@ -1529,7 +1528,7 @@ package body Sem_Ch6 is
 
    <<Leave>>
       Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Null_Procedure;
 
    -----------------------------
@@ -1624,8 +1623,7 @@ package body Sem_Ch6 is
       Loc     : constant Source_Ptr := Sloc (N);
       P       : constant Node_Id    := Name (N);
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Actual : Node_Id;
@@ -1890,7 +1888,7 @@ package body Sem_Ch6 is
       end if;
 
    <<Leave>>
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Procedure_Call;
 
    ------------------------------
@@ -3608,8 +3606,7 @@ package body Sem_Ch6 is
 
       --  Local variables
 
-      Saved_GM   : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR  : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_EA   : constant Boolean         := Expander_Active;
       Saved_ISMP : constant Boolean         :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
@@ -3836,7 +3833,7 @@ package body Sem_Ch6 is
       --  user entities, as internally generated entitities might still need
       --  to be expanded (e.g. those generated for types).
 
-      if Present (Ignored_Ghost_Region)
+      if Present (Ghost_Config.Ignored_Ghost_Region)
         and then Comes_From_Source (Body_Id)
       then
          Expander_Active := False;
@@ -3864,9 +3861,14 @@ package body Sem_Ch6 is
          Spec_Id := Build_Internal_Protected_Declaration (N);
       end if;
 
-      --  If a separate spec is present, then deal with freezing issues
+      --  Separate spec is not present
 
-      if Present (Spec_Id) then
+      if No (Spec_Id) then
+         Create_Extra_Formals (Body_Id);
+
+      --  Separate spec is present; deal with freezing issues
+
+      else
          Spec_Decl := Unit_Declaration_Node (Spec_Id);
          Verify_Overriding_Indicator;
 
@@ -3882,6 +3884,8 @@ package body Sem_Ch6 is
            and then not Has_BIP_Formals (Spec_Id)
          then
             Create_Extra_Formals (Spec_Id);
+            pragma Assert (not Expander_Active
+              or else Extra_Formals_Known (Spec_Id));
             Compute_Returns_By_Ref (Spec_Id);
          end if;
 
@@ -5015,12 +5019,12 @@ package body Sem_Ch6 is
       end if;
 
    <<Leave>>
-      if Present (Ignored_Ghost_Region) then
+      if Present (Ghost_Config.Ignored_Ghost_Region) then
          Expander_Active := Saved_EA;
       end if;
 
       Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Subprogram_Body_Helper;
 
    ------------------------------------
@@ -8564,14 +8568,13 @@ package body Sem_Ch6 is
       --  without coordinating with CodePeer, which makes use of these to
       --  provide better messages.
 
+      --  A and B denote extra formals for unchecked unions equality. See
+      --  exp_ch3.Build_Variant_Record_Equality.
       --  O denotes the Constrained bit.
       --  L denotes the accessibility level.
       --  BIP_xxx denotes an extra formal for a build-in-place function. See
       --  the full list in exp_ch6.BIP_Formal_Kind.
 
-      function Has_Extra_Formals (E : Entity_Id) return Boolean;
-      --  Determines if E has its extra formals
-
       function Might_Need_BIP_Task_Actuals (E : Entity_Id) return Boolean;
       --  Determines if E is a function or an access to a function returning a
       --  limited tagged type object. On dispatching primitives this predicate
@@ -8610,14 +8613,6 @@ package body Sem_Ch6 is
          EF : Entity_Id;
 
       begin
-         --  A little optimization. Never generate an extra formal for the
-         --  _init operand of an initialization procedure, since it could
-         --  never be used.
-
-         if Chars (Formal) = Name_uInit then
-            return Empty;
-         end if;
-
          EF := Make_Defining_Identifier (Sloc (Assoc_Entity),
                  Chars => New_External_Name (Chars (Assoc_Entity),
                                              Suffix => Suffix));
@@ -8643,25 +8638,22 @@ package body Sem_Ch6 is
          return EF;
       end Add_Extra_Formal;
 
-      -----------------------
-      -- Has_Extra_Formals --
-      -----------------------
-
-      function Has_Extra_Formals (E : Entity_Id) return Boolean is
-      begin
-         return Present (Extra_Formals (E))
-           or else
-             (Ekind (E) = E_Function
-                and then Present (Extra_Accessibility_Of_Result (E)));
-      end Has_Extra_Formals;
-
       ---------------------------------
       -- Might_Need_BIP_Task_Actuals --
       ---------------------------------
 
       function Might_Need_BIP_Task_Actuals (E : Entity_Id) return Boolean is
          Subp_Id  : Entity_Id;
-         Func_Typ : Entity_Id;
+         Original : Entity_Id;
+         Root     : Entity_Id;
+
+         function Has_No_Task_Parts_Enabled (E : Entity_Id) return Boolean
+         is (Has_Enabled_Aspect (E, Aspect_No_Task_Parts));
+
+         function Collect_Ancestors_With_No_Task_Parts is new
+           Collect_Types_In_Hierarchy (Predicate => Has_No_Task_Parts_Enabled);
+
+      --  Start of processing for Might_Need_BIP_Task_Actuals
 
       begin
          if Global_No_Tasking or else No_Run_Time_Mode then
@@ -8689,21 +8681,29 @@ package body Sem_Ch6 is
          then
             Subp_Id := Protected_Body_Subprogram (E);
 
-         else
+         --  For access-to-subprogram types we look at the return type of the
+         --  subprogram type itself, as it cannot be overridden or inherited.
+
+         elsif Ekind (E) = E_Subprogram_Type then
             Subp_Id := E;
-         end if;
 
-         --  We check the root type of the return type since the same
-         --  decision must be taken for all descendants overriding a
-         --  dispatching operation.
+         --  Otherwise, we need to return the same value we would return for
+         --  the original corresponding operation of the root of the aliased
+         --  chain.
+
+         else
+            Subp_Id := Original_Corresponding_Operation (Ultimate_Alias (E));
+         end if;
 
-         Func_Typ := Root_Type (Underlying_Type (Etype (Subp_Id)));
+         Original := Underlying_Type (Etype (Subp_Id));
+         Root := Underlying_Type (Root_Type (Original));
 
          return Ekind (Subp_Id) in E_Function | E_Subprogram_Type
-           and then not Has_Foreign_Convention (Func_Typ)
-           and then Is_Tagged_Type (Func_Typ)
-           and then Is_Limited_Type (Func_Typ)
-           and then not Has_Aspect (Func_Typ, Aspect_No_Task_Parts);
+           and then Is_Inherently_Limited_Type (Original)
+           and then not Has_Foreign_Convention (Root)
+           and then Is_Tagged_Type (Root)
+           and then Is_Empty_Elmt_List
+             (Collect_Ancestors_With_No_Task_Parts (Original));
       end Might_Need_BIP_Task_Actuals;
 
       -------------------------------------
@@ -8792,10 +8792,12 @@ package body Sem_Ch6 is
                --  we have no direct way to climb to the corresponding parent
                --  subprogram but this internal entity has the extra formals
                --  (if any) required for the purpose of checking the extra
-               --  formals of Subp_Id.
+               --  formals of Subp_Id because its extra formals are shared
+               --  with its parent subprogram (see Sem_Ch3.Derive_Subprogram).
 
                else
                   pragma Assert (not Comes_From_Source (Ovr_E));
+                  Freeze_Extra_Formals (Ovr_E);
                end if;
 
             --  Use as our reference entity the ultimate renaming of the
@@ -8818,10 +8820,14 @@ package body Sem_Ch6 is
 
       --  Local variables
 
-      Formal_Type      : Entity_Id;
-      May_Have_Alias   : Boolean;
+      use Deferred_Extra_Formals_Support;
+
+      Can_Be_Deferred  : constant Boolean :=
+                           not Is_Unsupported_Extra_Formals_Entity (E);
       Alias_Formal     : Entity_Id := Empty;
       Alias_Subp       : Entity_Id := Empty;
+      Formal_Type      : Entity_Id;
+      May_Have_Alias   : Boolean;
       Parent_Formal    : Entity_Id := Empty;
       Parent_Subp      : Entity_Id := Empty;
       Ref_E            : Entity_Id;
@@ -8832,10 +8838,18 @@ package body Sem_Ch6 is
       pragma Assert (Is_Subprogram_Or_Entry (E)
         or else Ekind (E) in E_Subprogram_Type);
 
+      --  No action needed if extra formals were already handled. This
+      --  situation may arise because of a previous call to create the
+      --  extra formals, and also for subprogram types created as part
+      --  of dispatching calls (see Expand_Dispatching_Call).
+
+      if Extra_Formals_Known (E) then
+         return;
+
       --  We never generate extra formals if expansion is not active because we
       --  don't need them unless we are generating code.
 
-      if not Expander_Active then
+      elsif not Expander_Active then
          return;
 
       --  Enumeration literals have no extra formal; this case occurs when
@@ -8844,25 +8858,38 @@ package body Sem_Ch6 is
       elsif Ekind (E) = E_Function
         and then Ekind (Ultimate_Alias (E)) = E_Enumeration_Literal
       then
+         Freeze_Extra_Formals (E);
          return;
 
-      --  Extra formals of Initialization procedures are added by the function
-      --  Exp_Ch3.Init_Formals
+      --  Extra formals of init procs are added by Exp_Ch3.Init_Formals and
+      --  Set_CPP_Constructors when they are built, but we must handle here
+      --  aliased init procs.
 
       elsif Is_Init_Proc (E) then
+         pragma Assert (Present (Alias (E)));
+         pragma Assert (Extra_Formals_Known (Ultimate_Alias (E)));
+         Freeze_Extra_Formals (E);
          return;
 
       --  No need to generate extra formals in thunks whose target has no extra
       --  formals, but we can have two of them chained (interface and stack).
 
-      elsif Is_Thunk (E) and then No (Extra_Formals (Thunk_Target (E))) then
+      elsif Is_Thunk (E)
+        and then Extra_Formals_Known (Thunk_Target (E))
+        and then No (Extra_Formals (Thunk_Target (E)))
+      then
+         Freeze_Extra_Formals (E);
          return;
 
-      --  If Extra_Formals were already created, don't do it again. This
-      --  situation may arise for subprogram types created as part of
-      --  dispatching calls (see Expand_Dispatching_Call).
+      --  Handle alias of unchecked union equality with frozen extra formals
 
-      elsif Has_Extra_Formals (E) then
+      elsif Is_Overloadable (E)
+        and then Present (Alias (E))
+        and then Extra_Formals_Known (Ultimate_Alias (E))
+        and then Is_Unchecked_Union_Equality (Ultimate_Alias (E))
+      then
+         Set_Extra_Formals (E, Extra_Formals (Ultimate_Alias (E)));
+         Freeze_Extra_Formals (E);
          return;
 
       --  Extra formals of renamings of generic actual subprograms and
@@ -8880,6 +8907,8 @@ package body Sem_Ch6 is
            = Is_Generic_Instance (Ultimate_Alias (E)));
 
          Create_Extra_Formals (Ultimate_Alias (E));
+         pragma Assert (not Expander_Active
+           or else Extra_Formals_Known (Ultimate_Alias (E)));
 
          --  Share the extra formals
 
@@ -8891,17 +8920,72 @@ package body Sem_Ch6 is
          end if;
 
          pragma Assert (Extra_Formals_OK (E));
+         Freeze_Extra_Formals (E);
          return;
       end if;
 
-      --  Locate the last formal; required by Add_Extra_Formal.
+      --  Check if the addition of the extra formals must be deferred
 
       Formal := First_Formal (E);
       while Present (Formal) loop
-         Last_Extra := Formal;
+         if No (Underlying_Type (Etype (Formal)))
+           and then Can_Be_Deferred
+         then
+            Register_Deferred_Extra_Formals_Entity (E);
+            return;
+         end if;
+
          Next_Formal (Formal);
       end loop;
 
+      if Ekind (E) in E_Function
+                    | E_Subprogram_Type
+        and then No (Underlying_Type (Etype (E)))
+        and then Can_Be_Deferred
+      then
+         Register_Deferred_Extra_Formals_Entity (E);
+         return;
+      end if;
+
+      --  Here we start adding the extra formals
+
+      --  We we know that either the underlying type of all the formals and
+      --  returned results of E are known, or this is an special case where
+      --  some underlying type is still not available.
+
+      --  In the former case, we can already mark functions that return their
+      --  result by reference; in the latter case, we can mark them only if the
+      --  underlying return type is available (and it will be marked later).
+
+      if not Is_Unsupported_Extra_Formals_Entity (E)
+        or else (Ekind (E) in E_Function | E_Subprogram_Type
+                   and then Present (Underlying_Type (Etype (E))))
+      then
+         Compute_Returns_By_Ref (E);
+      end if;
+
+      --  Locate the last formal (required by Add_Extra_Formal)
+
+      if Present (First_Formal (E))
+        and then Is_Unchecked_Union (Etype (First_Formal (E)))
+        and then Present (Extra_Formals (E))
+        and then Has_Suffix (Extra_Formals (E), 'A')
+      then
+         --  An unchecked union equality has two extra formals per discriminant
+
+         First_Extra := Extra_Formals (E);
+         Last_Extra  := First_Extra;
+         while Present (Last_Extra) loop
+            pragma Assert (Has_Suffix (Last_Extra, 'A'));
+            Last_Extra := Extra_Formal (Last_Extra);
+
+            pragma Assert (Has_Suffix (Last_Extra, 'B'));
+            Last_Extra := Extra_Formal (Last_Extra);
+         end loop;
+      else
+         Last_Extra := Last_Formal (E);
+      end if;
+
       --  We rely on three entities to ensure consistency of extra formals of
       --  entity E:
       --
@@ -8961,6 +9045,7 @@ package body Sem_Ch6 is
         or else (Present (Alias_Subp)
                    and then Has_Foreign_Convention (Alias_Subp))
       then
+         Freeze_Extra_Formals (E);
          return;
       end if;
 
@@ -9039,14 +9124,44 @@ package body Sem_Ch6 is
          --  Here we establish our priority for deciding on the extra
          --  formals: 1) Parent primitive 2) Aliased primitive 3) Identity
 
-         if Present (Parent_Formal) then
-            Formal_Type := Etype (Parent_Formal);
+         --  Common case: the underlying type of all the formals is known
+         --  to be available.
+
+         if Can_Be_Deferred then
+            if Present (Parent_Formal) then
+               Formal_Type := Underlying_Type (Etype (Parent_Formal));
+            elsif Present (Alias_Formal) then
+               Formal_Type := Underlying_Type (Etype (Alias_Formal));
+            else
+               Formal_Type := Underlying_Type (Etype (Formal));
+            end if;
+
+            pragma Assert (Present (Formal_Type));
 
-         elsif Present (Alias_Formal) then
-            Formal_Type := Etype (Alias_Formal);
+         --  Special case: The underlying type of some formal is not available.
+         --  We use the underlying type when present. More work needed here???
 
          else
-            Formal_Type := Etype (Formal);
+            if Present (Parent_Formal) then
+               Formal_Type := Etype (Parent_Formal);
+
+               if Present (Underlying_Type (Formal_Type)) then
+                  Formal_Type := Underlying_Type (Formal_Type);
+               end if;
+
+            elsif Present (Alias_Formal) then
+               Formal_Type := Etype (Alias_Formal);
+
+               if Present (Underlying_Type (Formal_Type)) then
+                  Formal_Type := Underlying_Type (Formal_Type);
+               end if;
+            else
+               Formal_Type := Etype (Formal);
+
+               if Present (Underlying_Type (Formal_Type)) then
+                  Formal_Type := Underlying_Type (Formal_Type);
+               end if;
+            end if;
          end if;
 
          --  Create extra formal for supporting the attribute 'Constrained.
@@ -9093,12 +9208,13 @@ package body Sem_Ch6 is
               and then (Is_Definite_Subtype (Formal_Type)
                          or else Is_Mutably_Tagged_Type (Formal_Type))
               and then (Ada_Version < Ada_2012
-                         or else No (Underlying_Type (Formal_Type))
+                         or else
+                           (not Can_Be_Deferred
+                             and then No (Underlying_Type (Formal_Type)))
                          or else not
                            (Is_Limited_Type (Formal_Type)
                              and then
-                               Is_Tagged_Type
-                                 (Underlying_Type (Formal_Type))))
+                               Is_Tagged_Type (Formal_Type)))
             then
                Set_Extra_Constrained
                  (Formal, Add_Extra_Formal (Formal, Standard_Boolean, E, "O"));
@@ -9337,6 +9453,8 @@ package body Sem_Ch6 is
          Set_Extra_Formals (Alias (E), Extra_Formals (E));
       end if;
 
+      Freeze_Extra_Formals (E);
+
       pragma Assert (No (Alias_Subp)
         or else Extra_Formals_Match_OK (E, Alias_Subp));
 
@@ -9651,6 +9769,19 @@ package body Sem_Ch6 is
                   return False;
                end if;
 
+            --  Extra formals (A and B) of Unchecked_Unions (see Build_Variant_
+            --  Record_Equality)
+
+            elsif Has_Suffix (Formal_1, 'A') then
+               if not Has_Suffix (Formal_2, 'A') then
+                  return False;
+               end if;
+
+            elsif Has_Suffix (Formal_1, 'B') then
+               if not Has_Suffix (Formal_2, 'B') then
+                  return False;
+               end if;
+
             elsif BIP_Suffix_Kind (Formal_1) /= BIP_Suffix_Kind (Formal_2) then
                return False;
             end if;
@@ -10003,6 +10134,16 @@ package body Sem_Ch6 is
       return Empty;
    end Find_Corresponding_Spec;
 
+   --------------------------
+   -- Freeze_Extra_Formals --
+   --------------------------
+
+   procedure Freeze_Extra_Formals (E : Entity_Id) is
+   begin
+      pragma Assert (not Extra_Formals_Known (E));
+      Set_Extra_Formals_Known (E);
+   end Freeze_Extra_Formals;
+
    ----------------------
    -- Fully_Conformant --
    ----------------------
@@ -10622,6 +10763,10 @@ package body Sem_Ch6 is
       Formal : Entity_Id := First_Formal_With_Extras (E);
 
    begin
+      --  It makes no sense to perform this check if the extra formals
+      --  have not been added.
+      pragma Assert (Extra_Formals_Known (E));
+
       while Present (Formal) loop
          if Is_Build_In_Place_Entity (Formal) then
             return True;
@@ -12133,36 +12278,51 @@ package body Sem_Ch6 is
         and then Present (Find_Dispatching_Type (Alias (S)))
         and then Is_Interface (Find_Dispatching_Type (Alias (S)))
       then
-         --  For private types, when the full-view is processed we propagate to
-         --  the full view the non-overridden entities whose attribute "alias"
-         --  references an interface primitive. These entities were added by
-         --  Derive_Subprograms to ensure that interface primitives are
-         --  covered.
-
-         --  Inside_Freeze_Actions is non zero when S corresponds with an
-         --  internal entity that links an interface primitive with its
-         --  covering primitive through attribute Interface_Alias (see
-         --  Add_Internal_Interface_Entities).
-
-         if Inside_Freezing_Actions = 0
-           and then Is_Package_Or_Generic_Package (Current_Scope)
-           and then In_Private_Part (Current_Scope)
-           and then Parent_Kind (E) = N_Private_Extension_Declaration
-           and then Nkind (Parent (S)) = N_Full_Type_Declaration
-           and then Full_View (Defining_Identifier (Parent (E)))
-                      = Defining_Identifier (Parent (S))
-           and then Alias (E) = Alias (S)
-         then
-            Check_Operation_From_Private_View (S, E);
-            Set_Is_Dispatching_Operation (S);
+         declare
+            Private_Operation_Exported_By_Visible_Part : constant Boolean :=
+              Is_Package_Or_Generic_Package (Current_Scope)
+              and then In_Private_Part (Current_Scope)
+              and then Parent_Kind (E) = N_Private_Extension_Declaration
+              and then Nkind (Parent (S)) = N_Full_Type_Declaration
+              and then Full_View (Defining_Identifier (Parent (E)))
+                         = Defining_Identifier (Parent (S));
+
+         begin
+            --  For private types, when the full view is processed we propagate
+            --  to the full view the nonoverridden entities whose attribute
+            --  "alias" references an interface primitive. These entities were
+            --  added by Derive_Subprograms to ensure that interface primitives
+            --  are covered.
+
+            --  Inside_Freeze_Actions is nonzero when S corresponds to an
+            --  internal entity that links an interface primitive with its
+            --  covering primitive through attribute Interface_Alias (see
+            --  Add_Internal_Interface_Entities).
+
+            if Inside_Freezing_Actions = 0
+              and then Private_Operation_Exported_By_Visible_Part
+              and then Alias (E) = Alias (S)
+            then
+               Check_Operation_From_Private_View (S, E);
+               Set_Is_Dispatching_Operation (S);
 
-         --  Common case
+            --  Common case
 
-         else
-            Enter_Overloaded_Entity (S);
-            Check_Dispatching_Operation (S, Empty);
-            Check_For_Primitive_Subprogram (Is_Primitive_Subp);
-         end if;
+            else
+               Enter_Overloaded_Entity (S);
+               Check_Dispatching_Operation (S, Empty);
+               Check_For_Primitive_Subprogram (Is_Primitive_Subp);
+            end if;
+
+            if Private_Operation_Exported_By_Visible_Part
+              and then Type_Conformant (E, S)
+            then
+               --  Record the actual inherited subprogram that's being
+               --  overridden.
+
+               Set_Overridden_Inherited_Operation (S, E);
+            end if;
+         end;
 
          return;
       end if;
@@ -12601,6 +12761,26 @@ package body Sem_Ch6 is
                           and then not Is_Dispatch_Table_Wrapper (S)))
                   then
                      Set_Overridden_Operation    (S, Alias (E));
+
+                     --  Record the actual inherited subprogram that's being
+                     --  overridden. In the case where a subprogram declared
+                     --  in a private part overrides an inherited subprogram
+                     --  that itself is also declared in the private part,
+                     --  and that subprogram in turns overrides a subprogram
+                     --  declared in a package visible part (inherited via
+                     --  a private extension), we record the visible subprogram
+                     --  as the overridden one, so that we can determine
+                     --  visibility properly for prefixed calls to the
+                     --  subprogram made from outside the package. (See
+                     --  Try_Primitive_Operation in Sem_Ch4.)
+
+                     if Present (Overridden_Inherited_Operation (E)) then
+                        Set_Overridden_Inherited_Operation
+                          (S, Overridden_Inherited_Operation (E));
+                     else
+                        Set_Overridden_Inherited_Operation (S, E);
+                     end if;
+
                      Inherit_Subprogram_Contract (S, Alias (E));
 
                      Set_Is_Ada_2022_Only (S, Is_Ada_2022_Only (Alias (E)));
@@ -12760,6 +12940,530 @@ package body Sem_Ch6 is
          end if;
    end New_Overloaded_Entity;
 
+   ------------------------------------
+   -- Deferred_Extra_Formals_Support --
+   ------------------------------------
+
+   package body Deferred_Extra_Formals_Support is
+      Calls_List       : Elist_Id := No_Elist;
+      Calls_Scope_List : Elist_Id := No_Elist;
+      --  Calls to subprograms or entries with some unknown underlying type
+      --  in their parameters or result type, and the scope where each call
+      --  is performed.
+
+      Entities_List    : Elist_Id := No_Elist;
+      --  Subprograms, entries, and subprogram types with some unknown
+      --  underlying type in their formals or result type.
+
+      Types_List       : Elist_Id := No_Elist;
+      --  Types with no underlying type
+
+      function Underlying_Types_Available (E : Entity_Id) return Boolean;
+      --  Determines if the underlying type of all the formals and result
+      --  type of the given subprogram, subprogram type, or entry are
+      --  available.
+
+      -------------------------------
+      -- Add_Deferred_Extra_Params --
+      -------------------------------
+
+      procedure Add_Deferred_Extra_Params (Typ : Entity_Id) is
+
+         procedure Check_Registered_Calls;
+         --  Check all the registered calls; for each registered call that
+         --  has the underlying type of all the parameters and result types
+         --  of the called entity available, call Create_Extra_Actuals, and
+         --  unregister the call.
+
+         procedure Check_Registered_Entities;
+         --  Check all the registered entities (subprograms, entries and
+         --  subprogram types); for each registered entity E that has all
+         --  its underlying types available, call Create_Extra_Formals,
+         --  and unregister E.
+
+         ----------------------------
+         -- Check_Registered_Calls --
+         ----------------------------
+
+         procedure Check_Registered_Calls is
+
+            function Get_Relocated_Function_Call (N : Node_Id) return Node_Id;
+            --  Given a node N that references a function call that has been
+            --  relocated to remove possible side effects of the call (see
+            --  Remove_Side_Effects) or to wrap the call in a transient scope
+            --  (see Wrap_Transient_Expression), search and return the function
+            --  call. Notice that this function does not use the Original_Node
+            --  field of N; it searchs for the actual call associated with N
+            --  in the expanded code (since we need to add to such call its
+            --  missing extra actuals).
+
+            ---------------------------------
+            -- Get_Relocated_Function_Call --
+            ---------------------------------
+
+            function Get_Relocated_Function_Call (N : Node_Id) return Node_Id
+            is
+               Current_Node : Node_Id;
+               Decl         : Node_Id;
+               Id           : Entity_Id;
+
+            begin
+               Current_Node := N;
+
+               while Nkind (Current_Node) /= N_Function_Call loop
+                  case Nkind (Current_Node) is
+                     when N_Identifier =>
+                        Id   := Entity (Current_Node);
+                        Decl := Parent (Id);
+
+                        if Nkind (Decl) = N_Object_Renaming_Declaration then
+                           Current_Node := Name (Decl);
+
+                        else
+                           pragma Assert (Nkind (Decl) = N_Object_Declaration);
+
+                           if Present (Expression (Decl)) then
+                              Current_Node := Expression (Decl);
+
+                           elsif Present (BIP_Initialization_Call (Id)) then
+                              Decl := BIP_Initialization_Call (Id);
+                              pragma Assert (Present (Expression (Decl)));
+                              Current_Node := Expression (Decl);
+
+                           elsif Present (Related_Expression (Id)) then
+                              Current_Node := Related_Expression (Id);
+
+                           else
+                              pragma Assert (False);
+                              raise Program_Error;
+                           end if;
+                        end if;
+
+                     when N_Explicit_Dereference | N_Reference =>
+                        Current_Node := Prefix (Current_Node);
+
+                     when others =>
+                        pragma Assert (False);
+                        raise Program_Error;
+                  end case;
+               end loop;
+
+               return Current_Node;
+            end Get_Relocated_Function_Call;
+
+            --  Local variables
+
+            Call_Node        : Node_Id;
+            Call_Id          : Entity_Id;
+            Elmt_Call        : Elmt_Id;
+            Elmt_Scope       : Elmt_Id;
+            Remove_Call      : Boolean;
+            Scop_Id          : Entity_Id;
+
+         --  Start of processing for Check_Registered_Calls
+
+         begin
+            --  Perform a single traversal of both lists simultaneously,
+            --  since they have the same number of elements with a 1-to-1
+            --  relationship.
+
+            Elmt_Scope := First_Elmt (Calls_Scope_List);
+            Elmt_Call  := First_Elmt (Calls_List);
+
+            while Present (Elmt_Scope) loop
+               Scop_Id     := Node (Elmt_Scope);
+               Remove_Call := False;
+
+               --  Check the enclosing scope of the call: if the underlying
+               --  type of some formal or return type of the enclosing scope
+               --  of this call is not available then we must skip processing
+               --  this call.
+
+               if Underlying_Types_Available (Scop_Id) then
+                  Call_Node := Node (Elmt_Call);
+
+                  if Nkind (Call_Node) in N_Entry_Call_Statement
+                                        | N_Function_Call
+                                        | N_Procedure_Call_Statement
+                  then
+                     Call_Id := Get_Called_Entity (Call_Node);
+
+                  --  Handle expanded function calls that could have side
+                  --  effects.
+
+                  else
+                     pragma Assert
+                       (Nkind (Original_Node (Call_Node)) = N_Function_Call);
+
+                     Call_Node := Get_Relocated_Function_Call (Call_Node);
+                     Call_Id := Get_Called_Entity (Call_Node);
+                  end if;
+
+                  --  If the underlying types of all the formal and return
+                  --  types of this called entity are available then create
+                  --  its extra actuals and remove it from the list of
+                  --  registered calls.
+
+                  if Underlying_Types_Available (Call_Id) then
+
+                     --  Given that the call is placed in the body of an
+                     --  internally built subprogram, ensure that the extra
+                     --  formals of the enclosing scope are available before
+                     --  adding the extra actuals of this call.
+
+                     Create_Extra_Formals (Scop_Id);
+                     Create_Extra_Formals (Call_Id);
+
+                     pragma Assert (Extra_Formals_Known (Scop_Id));
+                     pragma Assert (Extra_Formals_Known (Call_Id));
+
+                     --  Mark functions that return a result by reference
+
+                     Compute_Returns_By_Ref (Scop_Id);
+                     Compute_Returns_By_Ref (Call_Id);
+
+                     Push_Scope (Scop_Id);
+                     Create_Extra_Actuals (Call_Node);
+                     Pop_Scope;
+
+                     Remove_Call := True;
+                  end if;
+               end if;
+
+               --  In order to safely remove these elements from their
+               --  containing lists, remember these elements before moving
+               --  to the next list elements.
+
+               if Remove_Call then
+                  declare
+                     Removed_Call  : constant Elmt_Id := Elmt_Call;
+                     Removed_Scope : constant Elmt_Id := Elmt_Scope;
+
+                  begin
+                     Next_Elmt (Elmt_Scope);
+                     Next_Elmt (Elmt_Call);
+
+                     Remove_Elmt (Calls_List, Removed_Call);
+                     Remove_Elmt (Calls_Scope_List, Removed_Scope);
+                  end;
+               else
+                  Next_Elmt (Elmt_Scope);
+                  Next_Elmt (Elmt_Call);
+               end if;
+
+            end loop;
+         end Check_Registered_Calls;
+
+         -------------------------------
+         -- Check_Registered_Entities --
+         -------------------------------
+
+         procedure Check_Registered_Entities is
+            Elmt       : Elmt_Id;
+            Found_Elmt : Elmt_Id;
+            Id         : Entity_Id;
+
+         begin
+            Elmt := First_Elmt (Entities_List);
+
+            while Present (Elmt) loop
+               Id := Node (Elmt);
+
+               --  If the underlying type of some formal or return type of this
+               --  entity is not available then skip this element.
+
+               if not Underlying_Types_Available (Id) then
+                  Next_Elmt (Elmt);
+
+               --  Otherwise, create its extra formals and remove it from the
+               --  list of entities that require adding the extra formals.
+
+               else
+                  --  In order to safely remove this element from the list,
+                  --  temporarily remember this element, and move to the next
+                  --  element.
+
+                  Found_Elmt := Elmt;
+                  Next_Elmt (Elmt);
+
+                  --  Create the extra formals, and mark functions that return
+                  --  by reference (not be done before if the underying return
+                  --  type was previously unknown).
+
+                  Create_Extra_Formals (Id);
+                  Compute_Returns_By_Ref (Id);
+
+                  Remove_Elmt (Entities_List, Found_Elmt);
+
+                  --  For deferred entries and entry families, the expansion of
+                  --  their entry declaration was deferred, and must be done
+                  --  now (after adding their extra formals).
+
+                  if Ekind (Id) in E_Entry | E_Entry_Family then
+                     Expand_N_Entry_Declaration (Parent (Id),
+                       Was_Deferred => True);
+                  end if;
+               end if;
+            end loop;
+         end Check_Registered_Entities;
+
+      --  Start of processing for Add_Deferred_Extra_Params
+
+      begin
+         pragma Assert (Present (Underlying_Type (Typ)));
+
+         if Present (Entities_List) then
+            Check_Registered_Entities;
+         end if;
+
+         if Present (Calls_List) then
+            Check_Registered_Calls;
+         end if;
+
+         Remove (Types_List, Typ);
+      end Add_Deferred_Extra_Params;
+
+      --------------------------------
+      -- Has_Deferred_Extra_Formals --
+      --------------------------------
+
+      function Has_Deferred_Extra_Formals (Typ : Entity_Id) return Boolean is
+      begin
+         return Contains (Types_List, Typ);
+      end Has_Deferred_Extra_Formals;
+
+      --------------------------------------
+      -- Is_Deferred_Extra_Formals_Entity --
+      --------------------------------------
+
+      function Is_Deferred_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean is
+      begin
+         return Contains (Entities_List, Id);
+      end Is_Deferred_Extra_Formals_Entity;
+
+      ---------------------------------------
+      -- Is_Unsupported_Extra_Actuals_Call --
+      ---------------------------------------
+
+      --  Similarly to Is_Unsupported_Extra_Formals_Entity, we cannot
+      --  determine if the extra formals are needed when the underlying
+      --  type of some formal or result type is not available, and we are
+      --  compiling the body of a subprogram or package. However, for calls
+      --  we must also handle internal calls generated by the compiler as
+      --  part of compiling a package spec. For example, internal calls
+      --  performed in thunks of secondary dispatch table entries.
+      --
+      --  Example
+      --  -------
+      --  package P is
+      --     type T is tagged null record;
+      --  end;
+      --
+      --  limited with P;
+      --  package Q is
+      --     type Iface is interface;
+      --     procedure Prim (Self : Iface; Current : P.T) is abstract;
+      --  end;
+      --
+      --  limited with P;
+      --  with Q;
+      --  package R is
+      --     type Root is tagged null record;
+      --     type DT is new Root and Q.Iface with null record;
+      --
+      --     procedure Prim (Self : DT; Current : P.T);
+      --  end;
+      --
+      --  The initialization of the secondary dispatch table of tagged type
+      --  DT has an internally generated thunk that displaces the pointer to
+      --  the object and calls the primitive Prim (and the underlying type
+      --  of type T is not available).
+
+      function Is_Unsupported_Extra_Actuals_Call
+        (Call_Node : Node_Id; Id : Entity_Id) return Boolean
+      is
+         Comp_Unit : constant Entity_Id :=
+                       Cunit_Entity (Get_Source_Unit (Call_Node));
+      begin
+         return not Underlying_Types_Available (Id)
+           and then Is_Compilation_Unit (Comp_Unit)
+           and then Ekind (Comp_Unit) in E_Package
+                                       | E_Package_Body
+                                       | E_Subprogram_Body;
+      end Is_Unsupported_Extra_Actuals_Call;
+
+      -----------------------------------------
+      -- Is_Unsupported_Extra_Formals_Entity --
+      -----------------------------------------
+
+      --  We cannot determine if the extra formals are needed when the
+      --  underlying type of some formal or result type is not available,
+      --  and we are compiling the body of a subprogram or package. The
+      --  scenery for this case is a package spec that has a limited_with_
+      --  clause on unit Q, and its body has no regular with-clause on Q
+      --  (AI05-0151-1/08).
+
+      function Is_Unsupported_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean
+      is
+         Comp_Unit : constant Entity_Id :=
+                       Cunit_Entity (Get_Source_Unit (Id));
+      begin
+         return not Underlying_Types_Available (Id)
+           and then Is_Compilation_Unit (Comp_Unit)
+           and then Ekind (Comp_Unit) in E_Package_Body
+                                       | E_Subprogram_Body;
+      end Is_Unsupported_Extra_Formals_Entity;
+
+      --------------------------------------------
+      -- Register_Deferred_Extra_Formals_Entity --
+      --------------------------------------------
+
+      procedure Register_Deferred_Extra_Formals_Entity (Id : Entity_Id) is
+
+         procedure Register_Type (Typ : Entity_Id);
+         --  Register the given type in Types_List; for types visible though
+         --  limited_with_clauses, register their non-limited view.
+
+         -------------------
+         -- Register_Type --
+         -------------------
+
+         procedure Register_Type (Typ : Entity_Id) is
+         begin
+            --  Handle entities visible through limited_with_clauses
+
+            if Has_Non_Limited_View (Typ) then
+               Append_Unique_Elmt (Non_Limited_View (Typ), Types_List);
+            else
+               Append_Unique_Elmt (Typ, Types_List);
+            end if;
+         end Register_Type;
+
+         --  Local variables
+
+         Formal : Entity_Id;
+
+      --  Start of processing for Register_Deferred_Extra_Formals_Entity
+
+      begin
+         pragma Assert (Is_Subprogram_Or_Entry (Id)
+           or else Ekind (Id) in E_Subprogram_Type);
+
+         if not Is_Deferred_Extra_Formals_Entity (Id) then
+            if No (Types_List) then
+               Types_List := New_Elmt_List;
+            end if;
+
+            if No (Entities_List) then
+               Entities_List := New_Elmt_List;
+            end if;
+
+            --  Register all the types of the subprogram profile that are not
+            --  fully known.
+
+            Formal := First_Formal (Id);
+            while Present (Formal) loop
+
+               if No (Underlying_Type (Etype (Formal))) then
+                  Register_Type (Etype (Formal));
+               end if;
+
+               Next_Formal (Formal);
+            end loop;
+
+            if Ekind (Id) in E_Function | E_Subprogram_Type
+              and then No (Underlying_Type (Etype (Id)))
+            then
+               Register_Type (Etype (Id));
+            end if;
+
+            --  Register this subprogram
+
+            Append_Elmt (Id, Entities_List);
+         end if;
+      end Register_Deferred_Extra_Formals_Entity;
+
+      ------------------------------------------
+      -- Register_Deferred_Extra_Formals_Call --
+      ------------------------------------------
+
+      procedure Register_Deferred_Extra_Formals_Call
+        (Call_Node : Node_Id;
+         Scope_Id  : Entity_Id) is
+      begin
+         pragma Assert (Nkind (Call_Node) in N_Subprogram_Call
+                                           | N_Entry_Call_Statement);
+         if No (Calls_List) then
+            Calls_List := New_Elmt_List;
+            Calls_Scope_List := New_Elmt_List;
+         end if;
+
+         --  Avoid registering any call twice; this may occur in dispatching
+         --  calls with deferred extra actuals because Expand_Call_Helper
+         --  registers the call and invokes Expand_Dispatching_Call (which
+         --  tries again to register the expanded call).
+
+         if not Contains (Calls_List, Call_Node) then
+            Append_Elmt (Call_Node, Calls_List);
+            Append_Elmt (Scope_Id, Calls_Scope_List);
+         end if;
+      end Register_Deferred_Extra_Formals_Call;
+
+      --------------------------------
+      -- Underlying_Types_Available --
+      --------------------------------
+
+      function Underlying_Types_Available (E : Entity_Id) return Boolean is
+         Formal     : Entity_Id;
+         Formal_Typ : Entity_Id;
+         Func_Typ   : Entity_Id;
+
+      begin
+         --  If the extra formals are available, then the nonlimited view
+         --  of all the types referenced in the profile are available.
+
+         if Extra_Formals_Known (E) then
+            return True;
+         end if;
+
+         --  Check the return type
+
+         if Ekind (E) in E_Function | E_Subprogram_Type then
+            Func_Typ := Etype (E);
+
+            if Has_Non_Limited_View (Func_Typ) then
+               Func_Typ := Non_Limited_View (Func_Typ);
+            end if;
+
+            if No (Underlying_Type (Func_Typ)) then
+               return False;
+            end if;
+         end if;
+
+         --  Check the type of the formals
+
+         Formal := First_Formal (E);
+         while Present (Formal) loop
+            Formal_Typ := Etype (Formal);
+
+            if Has_Non_Limited_View (Formal_Typ) then
+               Formal_Typ := Non_Limited_View (Formal_Typ);
+            end if;
+
+            if No (Underlying_Type (Formal_Typ)) then
+               return False;
+            end if;
+
+            Next_Formal (Formal);
+         end loop;
+
+         return True;
+      end Underlying_Types_Available;
+
+   end Deferred_Extra_Formals_Support;
+
    ---------------------
    -- Process_Formals --
    ---------------------
diff --git a/gcc/ada/sem_ch6.ads b/gcc/ada/sem_ch6.ads
index 7ebbcaa..4ef5b65 100644
--- a/gcc/ada/sem_ch6.ads
+++ b/gcc/ada/sem_ch6.ads
@@ -190,6 +190,14 @@ package Sem_Ch6 is
    --  Use the subprogram specification in the body to retrieve the previous
    --  subprogram declaration, if any.
 
+   procedure Freeze_Extra_Formals (E : Entity_Id);
+   --  Given a subprogram, subprogram type, or entry, flag E to indicate that
+   --  its extra formals (if any) are known (by setting Extra_Formals_Known).
+   --  This subprogram serves three purposes: (1) Document the places where
+   --  the extra formals are known, (2) Ensure that extra formals are added
+   --  only once, and (3) Provide a convenient place for setting a debugger
+   --  breakpoint to locate when extra formals are known.
+
    function Fully_Conformant (New_Id, Old_Id : Entity_Id) return Boolean;
    --  Determine whether two callable entities (subprograms, entries,
    --  literals) are fully conformant (RM 6.3.1(17))
@@ -299,4 +307,156 @@ package Sem_Ch6 is
    procedure Valid_Operator_Definition (Designator : Entity_Id);
    --  Verify that an operator definition has the proper number of formals
 
+   ------------------------------------
+   -- Deferred_Extra_Formals_Support --
+   ------------------------------------
+
+   --  This package provides support for deferring the addition of extra
+   --  formals to subprograms, entries, and subprogram types; it also provides
+   --  support for deferring the addition of extra actuals to direct calls to
+   --  subprograms and entries, and indirect calls through subprogram types.
+   --  The addition of the extra formals and actuals is deferred until the
+   --  underlying type of all the parameters and result types of registered
+   --  subprograms, entries, and subprogram types is known.
+
+   --  Functional Description
+   --  ----------------------
+   --
+   --  When Create_Extra_Formals identifies that the underlying type of
+   --  some parameter or result type of an entity E is not available, E is
+   --  registered by this package, and the addition of its extra formals is
+   --  deferred. As part of this registration, the types of all the params
+   --  and result types of E with no underlying type are also registered.
+   --
+   --  When Expand_Call_Helper identifies that the underlying type of some
+   --  parameter or result type of a called entity is not available, the call
+   --  is registered by Register_Deferred_Extra_Formals_Call, and the addition
+   --  of its extra actuals is deferred.
+   --
+   --  When the full type declaration of some registered type T is analyzed,
+   --  the subprogram Add_Deferred_Extra_Params is invoked; this subprogram
+   --  does the following actions:
+   --    1) Check all the registered entities (subprograms, entries, and
+   --       subprogram types); for each registered entity that has all its
+   --       underlying types available, call Create_Extra_Formals, and
+   --       unregister the entity.
+   --    2) Check all the registered calls; for each registered call that
+   --       has available the underlying type of all the parameters and result
+   --       types of the called entity, call Create_Extra_Actuals, and
+   --       unregister the call.
+   --    3) Unregister T.
+   --
+   --  Example 1
+   --  ---------
+   --  A package spec has a private type declaration T, and declarations of
+   --  expression functions and/or primitives with class-wide conditions
+   --  invoking primitives of type T before the full view of T is defined.
+   --
+   --  As part of processing the early freezing of the called subprograms
+   --  (and as part of processing the calls) the functions are registered as
+   --  subprograms with deferred extra formals, and the calls are registered
+   --  as calls with deferred extra actuals.
+   --
+   --  When the full type declaration of T is analyzed, extra formals are
+   --  added to all the registered subprograms, and extra actuals are added
+   --  to all the registered calls with deferred extra actuals.
+   --
+   --  Example 2
+   --  ---------
+   --  The specification of package P has a limited_with_clause on package Q,
+   --  and the type of the formals of subprograms defined in P are types
+   --  defined in Q.
+   --
+   --  When compiling the spec of P, similarly to the previous example,
+   --  subprograms with incomplete formals are registered as subprograms
+   --  with deferred extra formals; if the spec of P has calls to these
+   --  subprograms, then these calls are registered as calls with deferred
+   --  extra actuals. That is, when the analysis of package P completes,
+   --  deferred extra formals and actuals have not been added.
+   --
+   --  When another compilation unit is analyzed (including the body of
+   --  package P), and a regular with-clause on Q is processed, when the
+   --  full type declaration of deferred entities is analyzed, deferred
+   --  extra formals and deferred extra actuals are added.
+   --
+   --  This machinery relies on the GNAT Compilation Model; that is, when
+   --  we analyze the spec of P (for which we generally don't generate code),
+   --  it is safe to complete the compilation and still have entities with
+   --  deferred extra formals, and calls with deferred extra actuals.
+   --
+   --  The body package P generally has a regular with-clause on package Q.
+   --  Hence, when we compile the body of package P, the implicit dependence
+   --  on its package spec causes the analysis of the spec of P (thus
+   --  registering deferred entities), followed by the analysis of context
+   --  clauses in the body of P. When the regular with-clause on package Q
+   --  is analyzed, we add the extra formals and extra actuals to deferred
+   --  entities. Thus, the generated code will have all the needed formals.
+   --
+   --  The (still) unsupported case is when the body of package P does not
+   --  have a regular with-clause on package Q (AI05-0151-1/08). This case
+   --  is left documented in the front-end sources by means of calls to
+   --  the following subprograms: Is_Unsupported_Extra_Formals_Entity, and
+   --  Is_Unsupported_Extra_Actuals_Call.
+
+   package Deferred_Extra_Formals_Support is
+
+      procedure Add_Deferred_Extra_Params (Typ : Entity_Id);
+      --  Check all the registered subprograms, entries, and subprogram types
+      --  with deferred addition of their extra formals; if the underlying
+      --  types of all their formals is available then add their extra formals.
+      --  Check also all the registered calls with deferred addition of their
+      --  extra actuals; add their extra actuals if the underlying types of all
+      --  their parameters and result types are available. Finally unregister
+      --  Typ from the list of types used for the deferral of extra formals/
+      --  actuals.
+
+      procedure Register_Deferred_Extra_Formals_Entity (Id : Entity_Id);
+      --  Register the given subprogram, entry, or subprogram type to defer the
+      --  addition of its extra formals.
+
+      procedure Register_Deferred_Extra_Formals_Call
+        (Call_Node : Node_Id;
+         Scope_Id  : Entity_Id);
+      --  Register the given call, performed from the given scope, to defer the
+      --  addition of its extra actuals.
+
+      function Has_Deferred_Extra_Formals (Typ : Entity_Id) return Boolean;
+      --  Return True if there some registered subprogram, subprogram type, or
+      --  entry with deferred extra formals that has some formal type or
+      --  result type of type Typ (i.e. which depends on the given type to
+      --  add its extra formals).
+
+      function Is_Deferred_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean;
+      --  Return True if Id is a subprogram, subprogram type, or entry that has
+      --  been registered to defer the addition of its extra formals.
+
+      function Is_Unsupported_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean;
+      --  Id is a subprogram, subprogram type, or entry. Return True if Id is
+      --  unsupported for deferring the addition of its extra formals; that is,
+      --  it is defined in a compilation unit that is a package body or a
+      --  subprogram body, and the underlying type of some of its parameters
+      --  or result type is not available.
+      --
+      --  The context for this case is an unsupported case of AI05-0151-1/08
+      --  that allows incomplete tagged types as parameter and result types.
+      --  More concretely, a type T is visible in a package spec through a
+      --  limited_with_clause, and the body of the package has no regular
+      --  with_clause. In such a case, the machinery for deferring the
+      --  addition of extra formals does not work because the underlying
+      --  type of the type is not seen during the compilation of the
+      --  package body.
+      --
+      --  The purpose of this function is to facilitate locating in the sources
+      --  the places where the front end performs the current (incomplete)
+      --  management of such case (to facilitate further work) ???
+
+      function Is_Unsupported_Extra_Actuals_Call
+        (Call_Node : Node_Id; Id : Entity_Id) return Boolean;
+      --  Same as previous function but applicable to a call to the given
+      --  entity Id.
+
+   end Deferred_Extra_Formals_Support;
+
 end Sem_Ch6;
diff --git a/gcc/ada/sem_ch7.adb b/gcc/ada/sem_ch7.adb
index c2e60aa..d28bafb 100644
--- a/gcc/ada/sem_ch7.adb
+++ b/gcc/ada/sem_ch7.adb
@@ -714,8 +714,7 @@ package body Sem_Ch7 is
 
       --  Local variables
 
-      Saved_GM   : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR  : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       Saved_EA   : constant Boolean         := Expander_Active;
       Saved_ISMP : constant Boolean         :=
                      Ignore_SPARK_Mode_Pragmas_In_Instance;
@@ -836,7 +835,7 @@ package body Sem_Ch7 is
       --  user entities, as internally generated entities might still need
       --  to be expanded (e.g. those generated for types).
 
-      if Present (Ignored_Ghost_Region)
+      if Present (Ghost_Config.Ignored_Ghost_Region)
         and then Comes_From_Source (Body_Id)
       then
          Expander_Active := False;
@@ -1149,12 +1148,12 @@ package body Sem_Ch7 is
          end if;
       end if;
 
-      if Present (Ignored_Ghost_Region) then
+      if Present (Ghost_Config.Ignored_Ghost_Region) then
          Expander_Active := Saved_EA;
       end if;
 
       Ignore_SPARK_Mode_Pragmas_In_Instance := Saved_ISMP;
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Package_Body_Helper;
 
    ---------------------------------
diff --git a/gcc/ada/sem_ch8.adb b/gcc/ada/sem_ch8.adb
index db892d0..e6ef658 100644
--- a/gcc/ada/sem_ch8.adb
+++ b/gcc/ada/sem_ch8.adb
@@ -4270,6 +4270,40 @@ package body Sem_Ch8 is
          Local_Restrict.Check_Actual_Subprogram_For_Instance
            (Actual_Subp_Name => Nam, Formal_Subp => Formal_Spec);
       end if;
+
+      --  If pragma Short_Circuit_And_Or is specified, then we give an error
+      --  for renaming an operator that is made short circuit.
+      --  For example, this is illegal:
+      --
+      --      function My_And (X, Y: Boolean) return Boolean renames "and";
+      --
+      --  if "and" denotes the usual predefined Boolean operator. Otherwise,
+      --  the semantics are confusing (sometimes short circuit, and sometimes
+      --  not, for calls to My_And). If we ever relax this rule, we will need
+      --  to clean up that run-time semantics.
+
+      if Short_Circuit_And_Or
+        and then Chars (Old_S) in Name_Op_And | Name_Op_Or
+        and then In_Extended_Main_Source_Unit (N)
+        and then Etype (Old_S) = Standard_Boolean
+        and then Is_Intrinsic_Subprogram (Old_S)
+      then
+         if Comes_From_Source (N) then
+            Error_Msg_N
+              ("pragma Short_Circuit_And_Or disallows renaming of " &
+               "operator", N);
+
+         --  Same error in case of an instantiation with My_And => "and"
+
+         elsif Present (Corresponding_Formal_Spec (N)) then
+            Error_Msg_N
+              ("pragma Short_Circuit_And_Or disallows passing of " &
+               "operator as a generic actual", N);
+
+         else
+            raise Program_Error;
+         end if;
+      end if;
    end Analyze_Subprogram_Renaming;
 
    -------------------------
@@ -5410,7 +5444,15 @@ package body Sem_Ch8 is
          elsif In_Open_Scopes (Scope (Base_Type (T))) then
             null;
 
-         elsif not Redundant_Use (Id) then
+         --  Turn off the use_type_clause on the type unless the clause is
+         --  redundant, or there's a previous use_type_clause. (The case where
+         --  a use_type_clause without "all" is followed by one with "all" in
+         --  a more nested scope is not considered redundant, necessitating
+         --  the test for a previous clause. One might expect the latter test
+         --  to suffice, but it turns out there are cases where Redundant_Use
+         --  is set, but Prev_Use_Clause is not set. ???)
+
+         elsif not Redundant_Use (Id) and then No (Prev_Use_Clause (N)) then
             Set_In_Use (T, False);
             Set_In_Use (Base_Type (T), False);
             Set_Current_Use_Clause (T, Empty);
diff --git a/gcc/ada/sem_ch9.adb b/gcc/ada/sem_ch9.adb
index e32612e..bf387d3 100644
--- a/gcc/ada/sem_ch9.adb
+++ b/gcc/ada/sem_ch9.adb
@@ -1700,6 +1700,12 @@ package body Sem_Ch9 is
          Process_Formals (Formals, N);
          Create_Extra_Formals (Def_Id);
          End_Scope;
+
+      --  If the entry has no formals, extra formals are definitely not
+      --  required.
+
+      else
+         Freeze_Extra_Formals (Def_Id);
       end if;
 
       if Ekind (Def_Id) = E_Entry then
diff --git a/gcc/ada/sem_disp.adb b/gcc/ada/sem_disp.adb
index 9d03eff..5a8bd58 100644
--- a/gcc/ada/sem_disp.adb
+++ b/gcc/ada/sem_disp.adb
@@ -2416,6 +2416,8 @@ package body Sem_Disp is
       Formal    : Entity_Id;
       Ctrl_Type : Entity_Id;
 
+   --  Start of processing for Find_Dispatching_Type
+
    begin
       if Ekind (Subp) in E_Function | E_Procedure
         and then Present (DTC_Entity (Subp))
@@ -3083,6 +3085,52 @@ package body Sem_Disp is
       then
          return Is_Tag_Indeterminate (Prefix (Orig_Node));
 
+      --  An if-expression is tag-indeterminate if all of the dependent
+      --  expressions are tag-indeterminate (RM 4.5.7 (17/3)).
+
+      elsif Nkind (Orig_Node) = N_If_Expression then
+         declare
+            Cond : constant Node_Id := First (Expressions (Orig_Node));
+            Expr : Node_Id := Next (Cond);
+
+         begin
+            if not Is_Tag_Indeterminate (Original_Node (Expr)) then
+               return False;
+            end if;
+
+            Next (Expr);
+
+            if Present (Expr)
+              and then not Is_Tag_Indeterminate (Original_Node (Expr))
+            then
+               return False;
+            end if;
+
+            return True;
+         end;
+
+      --  A case-expression is tag-indeterminate if all of the dependent
+      --  expressions are tag-indeterminate (RM 4.5.7 (17/3)).
+
+      elsif Nkind (Orig_Node) = N_Case_Expression then
+         declare
+            Alt  : Node_Id := First (Alternatives (Orig_Node));
+            Expr : Node_Id;
+
+         begin
+            while Present (Alt) loop
+               Expr := Expression (Alt);
+
+               if not Is_Tag_Indeterminate (Original_Node (Expr)) then
+                  return False;
+               end if;
+
+               Next (Alt);
+            end loop;
+
+            return True;
+         end;
+
       else
          return False;
       end if;
@@ -3243,6 +3291,7 @@ package body Sem_Disp is
       elsif Nkind (Actual) = N_Explicit_Dereference
         and then Nkind (Original_Node (Prefix (Actual))) = N_Function_Call
       then
+         pragma Assert (Is_Expanded_Dispatching_Call (Actual));
          return;
 
       --  When expansion is suppressed, an unexpanded call to 'Input can occur,
diff --git a/gcc/ada/sem_prag.adb b/gcc/ada/sem_prag.adb
index 2717c38..4fd5b65 100644
--- a/gcc/ada/sem_prag.adb
+++ b/gcc/ada/sem_prag.adb
@@ -436,8 +436,7 @@ package body Sem_Prag is
       Arg1      : constant Node_Id   :=
         First (Pragma_Argument_Associations (N));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Errors        : Nat;
@@ -492,7 +491,7 @@ package body Sem_Prag is
             End_Scope;
          end if;
 
-         Restore_Ghost_Region (Saved_GM, Saved_IGR);
+         Restore_Ghost_Region (Saved_Ghost_Config);
       end if;
 
       Set_Is_Analyzed_Pragma (N);
@@ -607,8 +606,7 @@ package body Sem_Prag is
 
       CCases : constant Node_Id := Expression (Get_Argument (N, Spec_Id));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       CCase         : Node_Id;
@@ -695,7 +693,7 @@ package body Sem_Prag is
 
       Set_Is_Analyzed_Pragma (N);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Contract_Cases_In_Decl_Part;
 
    ----------------------------------
@@ -2464,8 +2462,7 @@ package body Sem_Prag is
       Exceptional_Contracts : constant Node_Id :=
         Expression (Get_Argument (N, Spec_Id));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Exceptional_Contract : Node_Id;
@@ -2556,7 +2553,7 @@ package body Sem_Prag is
 
       Set_Is_Analyzed_Pragma (N);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Exceptional_Cases_In_Decl_Part;
 
    -------------------------------------
@@ -2772,8 +2769,7 @@ package body Sem_Prag is
       Exit_Contracts : constant Node_Id :=
         Expression (Get_Argument (N, Spec_Id));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Exit_Contract : Node_Id;
@@ -2863,7 +2859,7 @@ package body Sem_Prag is
 
       Set_Is_Analyzed_Pragma (N);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Exit_Cases_In_Decl_Part;
 
    --------------------------------------------
@@ -3688,8 +3684,7 @@ package body Sem_Prag is
       Pack_Id   : constant Entity_Id := Defining_Entity (Pack_Decl);
       Expr      : constant Node_Id   := Expression (Get_Argument (N, Pack_Id));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
    begin
@@ -3713,7 +3708,7 @@ package body Sem_Prag is
       Preanalyze_And_Resolve (Expr, Standard_Boolean);
       Set_Is_Analyzed_Pragma (N);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Initial_Condition_In_Decl_Part;
 
    --------------------------------------
@@ -5766,7 +5761,7 @@ package body Sem_Prag is
 
             begin
                if Pname = Name_Pre_Class then
-                  if Is_Ignored (N) then
+                  if Is_Ignored_In_Codegen (N) then
                      Set_Ignored_Class_Preconditions (Subp_Id,
                        New_Copy_Tree (Expr));
                   else
@@ -5774,7 +5769,7 @@ package body Sem_Prag is
                   end if;
 
                else
-                  if Is_Ignored (N) then
+                  if Is_Ignored_In_Codegen (N) then
                      Set_Ignored_Class_Postconditions (Subp_Id,
                        New_Copy_Tree (Expr));
                   else
@@ -12987,7 +12982,9 @@ package body Sem_Prag is
                   --  An abstract state declared within a Ghost region becomes
                   --  Ghost (SPARK RM 6.9(2)).
 
-                  if Ghost_Mode > None or else Is_Ghost_Entity (Pack_Id) then
+                  if Ghost_Config.Ghost_Mode > None
+                    or else Is_Ghost_Entity (Pack_Id)
+                  then
                      Set_Is_Ghost_Entity (State_Id);
                   end if;
 
@@ -14302,7 +14299,7 @@ package body Sem_Prag is
                      --  cannot occur within a Ghost subprogram or package
                      --  (SPARK RM 6.9(16)).
 
-                     if Ghost_Mode > None then
+                     if Ghost_Config.Ghost_Mode > None then
                         Error_Pragma
                           ("pragma % cannot appear within ghost subprogram or "
                            & "package");
@@ -14871,25 +14868,15 @@ package body Sem_Prag is
                   Set_Is_Ignored (N, False);
 
                else
-                  --  In CodePeer mode and GNATprove mode, we need to
-                  --  consider all assertions, unless they are disabled,
-                  --  because transformations of the AST may depend on
-                  --  assertions being checked.
+                  Set_Is_Checked (N, False);
+                  Set_Is_Ignored (N, True);
 
-                  if CodePeer_Mode or GNATprove_Mode then
-                     Set_Is_Checked (N, True);
-                     Set_Is_Ignored (N, False);
-                  else
-                     Set_Is_Checked (N, False);
-                     Set_Is_Ignored (N, True);
-                  end if;
                end if;
             end Handle_Dynamic_Predicate_Check;
 
             --  Local variables
 
-            Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-            Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+            Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
             --  Save the Ghost-related attributes to restore on exit
 
             Cname : Name_Id;
@@ -15047,7 +15034,7 @@ package body Sem_Prag is
             --  False at compile time, and we do not want to delete this
             --  warning when we delete the if statement.
 
-            if Expander_Active and Is_Ignored (N) then
+            if Expander_Active and Is_Ignored_In_Codegen (N) then
                Eloc := Sloc (Expr);
 
                Rewrite (N,
@@ -15100,7 +15087,7 @@ package body Sem_Prag is
                In_Assertion_Expr := In_Assertion_Expr - 1;
             end if;
 
-            Restore_Ghost_Region (Saved_GM, Saved_IGR);
+            Restore_Ghost_Region (Saved_Ghost_Config);
          end Check;
 
          --------------------------
@@ -16246,10 +16233,10 @@ package body Sem_Prag is
             Cond :=
               New_Occurrence_Of
                 (Boolean_Literals
-                  (Expander_Active and then not Is_Ignored (N)),
+                  (Expander_Active and then not Is_Ignored_In_Codegen (N)),
                  Loc);
 
-            if not Is_Ignored (N) then
+            if not Is_Ignored_In_Codegen (N) then
                Set_SCO_Pragma_Enabled (Loc);
             end if;
 
@@ -18720,7 +18707,7 @@ package body Sem_Prag is
                   --  region (SPARK RM 6.9(6)).
 
                   if Is_False (Expr_Value (Expr))
-                    and then Ghost_Mode > None
+                     and then Ghost_Config.Ghost_Mode > None
                   then
                      Error_Pragma
                        ("pragma % with value False cannot appear in enabled "
@@ -28323,8 +28310,7 @@ package body Sem_Prag is
 
       Expr : constant Node_Id := Expression (Get_Argument (N, Spec_Id));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Errors        : Nat;
@@ -28417,7 +28403,7 @@ package body Sem_Prag is
       Check_Postcondition_Use_In_Inlined_Subprogram (N, Spec_Id);
       Set_Is_Analyzed_Pragma (N);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Pre_Post_Condition_In_Decl_Part;
 
    ---------------------------------------
@@ -28437,8 +28423,7 @@ package body Sem_Prag is
       Arg1      : constant Node_Id   :=
         First (Pragma_Argument_Associations (N));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Errors        : Nat;
@@ -28561,7 +28546,7 @@ package body Sem_Prag is
 
          Check_Postcondition_Use_In_Inlined_Subprogram (N, Spec_Id);
 
-         Restore_Ghost_Region (Saved_GM, Saved_IGR);
+         Restore_Ghost_Region (Saved_Ghost_Config);
       end if;
 
       Set_Is_Analyzed_Pragma (N);
@@ -31803,8 +31788,7 @@ package body Sem_Prag is
 
       Variants : constant Node_Id := Expression (Get_Argument (N, Spec_Id));
 
-      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
-      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
       --  Save the Ghost-related attributes to restore on exit
 
       Variant       : Node_Id;
@@ -31899,7 +31883,7 @@ package body Sem_Prag is
 
       Set_Is_Analyzed_Pragma (N);
 
-      Restore_Ghost_Region (Saved_GM, Saved_IGR);
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Analyze_Subprogram_Variant_In_Decl_Part;
 
    ------------------------------------
@@ -32195,20 +32179,8 @@ package body Sem_Prag is
                   when Name_Ignore
                      | Name_Off
                   =>
-                     --  In CodePeer mode and GNATprove mode, we need to
-                     --  consider all assertions, unless they are disabled.
-                     --  Force Is_Checked on ignored assertions, in particular
-                     --  because transformations of the AST may depend on
-                     --  assertions being checked (e.g. the translation of
-                     --  attribute 'Loop_Entry).
-
-                     if CodePeer_Mode or GNATprove_Mode then
-                        Set_Is_Checked (N, True);
-                        Set_Is_Ignored (N, False);
-                     else
-                        Set_Is_Checked (N, False);
-                        Set_Is_Ignored (N, True);
-                     end if;
+                     Set_Is_Checked (N, False);
+                     Set_Is_Ignored (N, True);
 
                   when Name_Check
                      | Name_On
@@ -34270,113 +34242,123 @@ package body Sem_Prag is
      (N     : Node_Id;
       Eloc  : Source_Ptr)
    is
-      Arg1  : constant Node_Id := First (Pragma_Argument_Associations (N));
-      Arg1x : constant Node_Id := Get_Pragma_Arg (Arg1);
-      Arg2  : constant Node_Id := Next (Arg1);
+      Arg1    : constant Node_Id := First (Pragma_Argument_Associations (N));
+      Arg1x   : constant Node_Id := Get_Pragma_Arg (Arg1);
+      Prag_Id : constant Pragma_Id := Get_Pragma_Id (N);
 
-      Pname   : constant Name_Id   := Pragma_Name_Unmapped (N);
-      Prag_Id : constant Pragma_Id := Get_Pragma_Id (Pname);
+      procedure Emit_Compile_Time_Message (Msg_Arg : Node_Id);
+      --  Emit the pragma a as diagnostic message. New_Line characters are
+      --  considered separators for those messages where the following lines
+      --  are considered as continuation messages for the same diagnostic.
 
-   begin
-      Analyze_And_Resolve (Arg1x, Standard_Boolean);
+      -------------------------------
+      -- Emit_Compile_Time_Message --
+      -------------------------------
 
-      if Compile_Time_Known_Value (Arg1x) then
-         if Is_True (Expr_Value (Arg1x)) then
+      procedure Emit_Compile_Time_Message (Msg_Arg : Node_Id) is
+         --  We have already verified that the Msg_Arg is a static
+         --  string expression. Its string value must be retrieved
+         --  explicitly if it is a declared constant, otherwise it has
+         --  been constant-folded previously.
+
+         Cent    : constant Entity_Id := Cunit_Entity (Current_Sem_Unit);
+         Str     : constant String_Id :=
+           Strval (Expr_Value_S (Get_Pragma_Arg (Msg_Arg)));
+         Str_Len : constant Nat := String_Length (Str);
+
+         Force : constant Boolean :=
+           Prag_Id = Pragma_Compile_Time_Warning
+           and then Is_Spec_Name (Unit_Name (Current_Sem_Unit))
+           and then (Ekind (Cent) /= E_Package
+                     or else not In_Private_Part (Cent));
+         --  Set True if this is the warning case, and we are in the
+         --  visible part of a package spec, or in a subprogram spec,
+         --  in which case we want to force the client to see the
+         --  warning, even though it is not in the main unit.
+
+         Msg_Ctrl : Bounded_String (6);
+         --  Control characters for the message.
+         --  The longest value contains 6 characters: "\<<~!!"
+
+         C    : Character;
+         CC   : Char_Code;
+         Cont : Boolean;
+         Ptr  : Nat;
 
-            --  We have already verified that the second argument is a static
-            --  string expression. Its string value must be retrieved
-            --  explicitly if it is a declared constant, otherwise it has
-            --  been constant-folded previously.
+      begin
+         --  Loop through segments of message separated by line feeds.
+         --  We output these segments as separate messages with
+         --  continuation marks for all but the first.
 
-            declare
-               Cent    : constant Entity_Id := Cunit_Entity (Current_Sem_Unit);
-               Str     : constant String_Id :=
-                           Strval (Expr_Value_S (Get_Pragma_Arg (Arg2)));
-               Str_Len : constant Nat       := String_Length (Str);
-
-               Force : constant Boolean :=
-                         Prag_Id = Pragma_Compile_Time_Warning
-                           and then Is_Spec_Name (Unit_Name (Current_Sem_Unit))
-                           and then (Ekind (Cent) /= E_Package
-                                      or else not In_Private_Part (Cent));
-               --  Set True if this is the warning case, and we are in the
-               --  visible part of a package spec, or in a subprogram spec,
-               --  in which case we want to force the client to see the
-               --  warning, even though it is not in the main unit.
-
-               C    : Character;
-               CC   : Char_Code;
-               Cont : Boolean;
-               Ptr  : Nat;
+         Cont := False;
+         Ptr := 1;
+         loop
+            Error_Msg_Strlen := 0;
+            Msg_Ctrl.Length := 0;
 
-            begin
-               --  Loop through segments of message separated by line feeds.
-               --  We output these segments as separate messages with
-               --  continuation marks for all but the first.
+            --  Loop to copy characters from argument to error message
+            --  string buffer.
 
-               Cont := False;
-               Ptr  := 1;
-               loop
-                  Error_Msg_Strlen := 0;
+            loop
+               exit when Ptr > Str_Len;
+               CC := Get_String_Char (Str, Ptr);
+               Ptr := Ptr + 1;
 
-                  --  Loop to copy characters from argument to error message
-                  --  string buffer.
+               --  Ignore wide chars ??? else store character
 
-                  loop
-                     exit when Ptr > Str_Len;
-                     CC := Get_String_Char (Str, Ptr);
-                     Ptr := Ptr + 1;
+               if In_Character_Range (CC) then
+                  C := Get_Character (CC);
+                  exit when C = ASCII.LF;
+                  Error_Msg_Strlen := Error_Msg_Strlen + 1;
+                  Error_Msg_String (Error_Msg_Strlen) := C;
+               end if;
+            end loop;
 
-                     --  Ignore wide chars ??? else store character
+            --  Here with one line ready to go
 
-                     if In_Character_Range (CC) then
-                        C := Get_Character (CC);
-                        exit when C = ASCII.LF;
-                        Error_Msg_Strlen := Error_Msg_Strlen + 1;
-                        Error_Msg_String (Error_Msg_Strlen) := C;
-                     end if;
-                  end loop;
+            Error_Msg_Warn := Prag_Id = Pragma_Compile_Time_Warning;
 
-                  --  Here with one line ready to go
+            if Cont then
+               Append (Msg_Ctrl, "\");
+            end if;
 
-                  Error_Msg_Warn := Prag_Id = Pragma_Compile_Time_Warning;
+            Append (Msg_Ctrl, "<<~");
 
-                  --  If this is a warning in a spec, then we want clients
-                  --  to see the warning, so mark the message with the
-                  --  special sequence !! to force the warning. In the case
-                  --  of a package spec, we do not force this if we are in
-                  --  the private part of the spec.
+            --  If this is a warning in a spec, then we want clients
+            --  to see the warning, so mark the message with the
+            --  special sequence !! to force the warning. In the case
+            --  of a package spec, we do not force this if we are in
+            --  the private part of the spec.
 
-                  if Force then
-                     if Cont = False then
-                        Error_Msg
-                           ("<<~!!", Eloc, N, Is_Compile_Time_Pragma => True);
-                        Cont := True;
-                     else
-                        Error_Msg
-                           ("\<<~!!", Eloc, N, Is_Compile_Time_Pragma => True);
-                     end if;
+            if Force then
+               Append (Msg_Ctrl, "!!");
+            end if;
 
-                  --  Error, rather than warning, or in a body, so we do not
-                  --  need to force visibility for client (error will be
-                  --  output in any case, and this is the situation in which
-                  --  we do not want a client to get a warning, since the
-                  --  warning is in the body or the spec private part).
+            --  Error, rather than warning, or in a body, so we do not
+            --  need to force visibility for client (error will be
+            --  output in any case, and this is the situation in which
+            --  we do not want a client to get a warning, since the
+            --  warning is in the body or the spec private part).
 
-                  else
-                     if Cont = False then
-                        Error_Msg
-                           ("<<~", Eloc, N, Is_Compile_Time_Pragma => True);
-                        Cont := True;
-                     else
-                        Error_Msg
-                           ("\<<~", Eloc, N, Is_Compile_Time_Pragma => True);
-                     end if;
-                  end if;
+            Error_Msg
+              (To_String (Msg_Ctrl), Eloc, N, Is_Compile_Time_Pragma => True);
 
-                  exit when Ptr > Str_Len;
-               end loop;
-            end;
+            --  The next lines are considered continuation messages
+
+            Cont := True;
+
+            exit when Ptr > Str_Len;
+         end loop;
+      end Emit_Compile_Time_Message;
+
+   --  Start of processing for Validate_Compile_Time_Warning_Or_Error
+
+   begin
+      Analyze_And_Resolve (Arg1x, Standard_Boolean);
+
+      if Compile_Time_Known_Value (Arg1x) then
+         if Is_True (Expr_Value (Arg1x)) then
+            Emit_Compile_Time_Message (Next (Arg1));
          end if;
 
       --  Arg1x is not known at compile time, so possibly issue an error
@@ -35101,7 +35083,17 @@ package body Sem_Prag is
          begin
             Set_Scope (T.Scope);
             Reset_Analyzed_Flags (T.Prag);
-            Validate_Compile_Time_Warning_Or_Error (T.Prag, T.Eloc);
+            if Nkind (T.Prag) = N_Pragma then
+               Validate_Compile_Time_Warning_Or_Error (T.Prag, T.Eloc);
+            else
+               pragma Assert (Nkind (Original_Node (T.Prag)) = N_Pragma);
+
+               --  The pragma was likely removed in ignored ghost code. Check
+               --  the original node instead.
+
+               Validate_Compile_Time_Warning_Or_Error
+                 (Original_Node (T.Prag), T.Eloc);
+            end if;
             Unset_Scope (T.Scope);
          end;
       end loop;
diff --git a/gcc/ada/sem_res.adb b/gcc/ada/sem_res.adb
index 96e8da6..29b7766 100644
--- a/gcc/ada/sem_res.adb
+++ b/gcc/ada/sem_res.adb
@@ -7270,7 +7270,9 @@ package body Sem_Res is
 
       if Restriction_Check_Required (No_Relative_Delay)
         and then Is_RTE (Nam, RE_Set_Handler)
-        and then Is_RTE (Etype (Next_Actual (First_Actual (N))), RE_Time_Span)
+        and then
+          Is_RTE
+            (Base_Type (Etype (Next_Actual (First_Actual (N)))), RE_Time_Span)
       then
          Check_Restriction (No_Relative_Delay, N);
       end if;
@@ -12463,16 +12465,6 @@ package body Sem_Res is
          Orig_N := Original_Node (Expression (Orig_N));
          Orig_T := Target_Typ;
 
-         --  If the node is part of a larger expression, the Target_Type
-         --  may not be the original type of the node if the context is a
-         --  condition. Recover original type to see if conversion is needed.
-
-         if Is_Boolean_Type (Orig_T)
-          and then Nkind (Parent (N)) in N_Op
-         then
-            Orig_T := Etype (Parent (N));
-         end if;
-
          --  If we have an entity name, then give the warning if the entity
          --  is the right type, or if it is a loop parameter covered by the
          --  original type (that's needed because loop parameters have an
@@ -12548,6 +12540,16 @@ package body Sem_Res is
             then
                null;
 
+            --  Do not warn if original source-level conversion was
+            --  between two different types.
+
+            elsif Nkind (Original_Node (N)) = N_Type_Conversion
+              and then
+                Base_Type (Etype (Subtype_Mark (Original_Node (N))))
+                  /= Base_Type (Etype (Expression (Original_Node (N))))
+            then
+               null;
+
             --  Here we give the redundant conversion warning. If it is an
             --  entity, give the name of the entity in the message. If not,
             --  just mention the expression.
diff --git a/gcc/ada/sem_util.adb b/gcc/ada/sem_util.adb
index 74de26a..d19b3b9 100644
--- a/gcc/ada/sem_util.adb
+++ b/gcc/ada/sem_util.adb
@@ -35,6 +35,7 @@ with Exp_Ch11;       use Exp_Ch11;
 with Exp_Util;       use Exp_Util;
 with Fname;          use Fname;
 with Freeze;         use Freeze;
+with Ghost;          use Ghost;
 with Itypes;         use Itypes;
 with Lib;            use Lib;
 with Lib.Xref;       use Lib.Xref;
@@ -1923,6 +1924,10 @@ package body Sem_Util is
    -- Build_Elaboration_Entity --
    ------------------------------
 
+   --  WARNING: This routine manages Ghost regions. Return statements must be
+   --  replaced by gotos which jump to the end of the routine and restore the
+   --  Ghost mode.
+
    procedure Build_Elaboration_Entity (N : Node_Id; Spec_Id : Entity_Id) is
       Loc      : constant Source_Ptr := Sloc (N);
       Decl     : Node_Id;
@@ -1956,6 +1961,11 @@ package body Sem_Util is
          end if;
       end Set_Package_Name;
 
+      --  Local variables
+
+      Saved_Ghost_Config : constant Ghost_Config_Type := Ghost_Config;
+      --  Save the Ghost-related attributes to restore on exit
+
    --  Start of processing for Build_Elaboration_Entity
 
    begin
@@ -2003,6 +2013,11 @@ package body Sem_Util is
          return;
       end if;
 
+      --  Elaboration entity is never a ghost object, regardless of the context
+      --  in which this routine is called.
+
+      Install_Ghost_Region (None, N);
+
       --  Here we need the elaboration entity
 
       --  Construct name of elaboration entity as xxx_E, where xxx is the unit
@@ -2043,6 +2058,8 @@ package body Sem_Util is
 
       Set_Has_Qualified_Name       (Elab_Ent);
       Set_Has_Fully_Qualified_Name (Elab_Ent);
+
+      Restore_Ghost_Region (Saved_Ghost_Config);
    end Build_Elaboration_Entity;
 
    --------------------------------
@@ -3688,6 +3705,7 @@ package body Sem_Util is
            Aspect_Aggregate,
            Aspect_Max_Entry_Queue_Length
            --  , Aspect_No_Controlled_Parts
+           --  , Aspect_No_Task_Parts
           );
 
       --  Note that none of these 8 aspects can be specified (for a type)
@@ -10043,16 +10061,19 @@ package body Sem_Util is
            and then not Has_Unknown_Discriminants (Utyp)
            and then not (Ekind (Utyp) = E_String_Literal_Subtype)
       then
-         --  Nothing to do if in spec expression (why not???)
+         --  If the type has no discriminants, there is no subtype to build,
+         --  even if the underlying type is discriminated.
 
-         if In_Spec_Expression then
+         if Is_Private_Type (Typ) and then not Has_Discriminants (Typ) then
             return Typ;
 
-         elsif Is_Private_Type (Typ) and then not Has_Discriminants (Typ) then
-
-            --  If the type has no discriminants, there is no subtype to
-            --  build, even if the underlying type is discriminated.
+         --  If we are performing preanalysis on a conjured-up copy of a name
+         --  (see calls to Preanalyze_Range in sem_ch5.adb) then we don't want
+         --  to freeze Atyp, now or ever. In this case, the tree we eventually
+         --  pass to the back end should contain no references to Atyp (and a
+         --  freeze node would contain such a reference).
 
+         elsif not (Expander_Active or GNATprove_Mode) then
             return Typ;
 
          --  Else build the actual subtype
@@ -10068,42 +10089,21 @@ package body Sem_Util is
 
             Atyp := Defining_Identifier (Decl);
 
-            --  If Build_Actual_Subtype generated a new declaration then use it
-
-            if Atyp /= Typ then
-
-               --  The actual subtype is an Itype, so analyze the declaration,
-               --  but do not attach it to the tree, to get the type defined.
-
-               Set_Parent (Decl, N);
-               Set_Is_Itype (Atyp);
-               Analyze (Decl, Suppress => All_Checks);
-               Set_Associated_Node_For_Itype (Atyp, N);
-               if Expander_Active then
-                  Set_Has_Delayed_Freeze (Atyp, False);
-
-                  --  We need to freeze the actual subtype immediately. This is
-                  --  needed because otherwise this Itype will not get frozen
-                  --  at all; it is always safe to freeze on creation because
-                  --  any associated types must be frozen at this point.
+            --  The actual subtype is an Itype, so analyze the declaration
+            --  after attaching it to the tree, to get the type defined.
 
-                  --  On the other hand, if we are performing preanalysis on
-                  --  a conjured-up copy of a name (see calls to
-                  --  Preanalyze_Range in sem_ch5.adb) then we don't want
-                  --  to freeze Atyp, now or ever. In this case, the tree
-                  --  we eventually pass to the back end should contain no
-                  --  references to Atyp (and a freeze node would contain
-                  --  such a reference). That's why Expander_Active is tested.
+            Set_Parent (Decl, N);
+            Set_Is_Itype (Atyp);
+            Analyze (Decl, Suppress => All_Checks);
+            Set_Associated_Node_For_Itype (Atyp, N);
 
-                  Freeze_Itype (Atyp, N);
-               end if;
-               return Atyp;
-
-            --  Otherwise we did not build a declaration, so return original
+            --  We need to freeze the actual subtype immediately. This is
+            --  needed because otherwise this Itype will not get frozen
+            --  at all; it is always safe to freeze on creation because
+            --  any associated types must be frozen at this point.
 
-            else
-               return Typ;
-            end if;
+            Freeze_Itype (Atyp, N);
+            return Atyp;
          end if;
 
       --  For all remaining cases, the actual subtype is the same as
@@ -12472,6 +12472,41 @@ package body Sem_Util is
       end if;
    end Is_Extended_Access_Type;
 
+   ----------------------------------------
+   -- Is_Ignored_Ghost_Entity_In_Codegen --
+   ----------------------------------------
+
+   function Is_Ignored_Ghost_Entity_In_Codegen (N : Entity_Id) return Boolean
+   is
+   begin
+      return
+        Is_Ignored_Ghost_Entity (N)
+        and then not GNATprove_Mode
+        and then not CodePeer_Mode;
+   end Is_Ignored_Ghost_Entity_In_Codegen;
+
+   ----------------------------------------
+   -- Is_Ignored_Ghost_Pragma_In_Codegen --
+   ----------------------------------------
+
+   function Is_Ignored_Ghost_Pragma_In_Codegen (N : Node_Id) return Boolean is
+   begin
+      return
+        Is_Ignored_Ghost_Pragma (N)
+        and then not GNATprove_Mode
+        and then not CodePeer_Mode;
+   end Is_Ignored_Ghost_Pragma_In_Codegen;
+
+   ---------------------------
+   -- Is_Ignored_In_Codegen --
+   ---------------------------
+
+   function Is_Ignored_In_Codegen (N : Node_Id) return Boolean is
+   begin
+      return
+        Is_Ignored (N) and then not GNATprove_Mode and then not CodePeer_Mode;
+   end Is_Ignored_In_Codegen;
+
    ---------------------------------
    -- Side_Effect_Free_Statements --
    ---------------------------------
@@ -15017,6 +15052,7 @@ package body Sem_Util is
                | Aspect_Iterator_Element
                | Aspect_Max_Entry_Queue_Length
                | Aspect_No_Controlled_Parts
+               | Aspect_No_Task_Parts
             =>
                return;
          end case;
@@ -16276,8 +16312,9 @@ package body Sem_Util is
                  Names_Match (Assign_Indexed_1, Assign_Indexed_2);
             end;
 
-         --  Checking for this aspect is performed elsewhere during freezing
-         when Aspect_No_Controlled_Parts =>
+         --  Checking for these aspects is performed elsewhere during freezing
+         when Aspect_No_Controlled_Parts
+            | Aspect_No_Task_Parts =>
             return True;
 
          --  scalar-valued aspects; compare (static) values.
@@ -21373,6 +21410,18 @@ package body Sem_Util is
       return False;
    end Is_Unchecked_Conversion_Instance;
 
+   ---------------------------------
+   -- Is_Unchecked_Union_Equality --
+   ---------------------------------
+
+   function Is_Unchecked_Union_Equality (Id : Entity_Id) return Boolean is
+   begin
+      return Ekind (Id) = E_Function
+        and then Present (First_Formal (Id))
+        and then Is_Unchecked_Union (Etype (First_Formal (Id)))
+        and then Id = TSS (Etype (First_Formal (Id)), TSS_Composite_Equality);
+   end Is_Unchecked_Union_Equality;
+
    -------------------------------
    -- Is_Universal_Numeric_Type --
    -------------------------------
@@ -22559,7 +22608,7 @@ package body Sem_Util is
          --  Mark the Ghost and SPARK mode in effect
 
          if Modes then
-            if Ghost_Mode = Ignore then
+            if Ghost_Config.Ghost_Mode = Ignore then
                Set_Is_Ignored_Ghost_Node (N);
             end if;
 
@@ -26424,16 +26473,6 @@ package body Sem_Util is
          end if;
       end if;
 
-      --  In CodePeer mode and GNATprove mode, we need to consider all
-      --  assertions, unless they are disabled. Force Name_Check on
-      --  ignored assertions.
-
-      if Kind in Name_Ignore | Name_Off
-        and then (CodePeer_Mode or GNATprove_Mode)
-      then
-         Kind := Name_Check;
-      end if;
-
       return Kind;
    end Policy_In_Effect;
 
@@ -26467,9 +26506,11 @@ package body Sem_Util is
 
    function Predicate_Enabled (Typ : Entity_Id) return Boolean is
    begin
-      return Present (Predicate_Function (Typ))
-        and then not Predicates_Ignored (Typ)
-        and then not Predicate_Checks_Suppressed (Empty);
+      return
+        Present (Predicate_Function (Typ))
+        and then (GNATprove_Mode
+                  or else (not Predicates_Ignored (Typ)
+                           and then not Predicate_Checks_Suppressed (Empty)));
    end Predicate_Enabled;
 
    ----------------------------------
@@ -26955,6 +26996,10 @@ package body Sem_Util is
       if Has_Relaxed_Finalization (From_Typ) then
          Set_Has_Relaxed_Finalization (Typ);
       end if;
+
+      if Deriv and then Has_Destructor (From_Typ) then
+         Set_Has_Destructor (Typ);
+      end if;
    end Propagate_Controlled_Flags;
 
    ------------------------------
diff --git a/gcc/ada/sem_util.ads b/gcc/ada/sem_util.ads
index efeafda..47fcc7d 100644
--- a/gcc/ada/sem_util.ads
+++ b/gcc/ada/sem_util.ads
@@ -1578,7 +1578,7 @@ package Sem_Util is
    --  underlying type).
 
    function Has_Suffix (E : Entity_Id; Suffix : Character) return Boolean;
-   --  Returns true if the last character of E is Suffix. Used in Assertions.
+   --  Returns true if the last character of E is Suffix.
 
    function Has_Tagged_Component (Typ : Entity_Id) return Boolean;
    --  Returns True if Typ is a composite type (array or record) that is either
@@ -2079,6 +2079,18 @@ package Sem_Util is
    --  . machine_emax = 2**14
    --  . machine_emin = 3 - machine_emax
 
+   function Is_Ignored_Ghost_Entity_In_Codegen (N : Node_Id) return Boolean;
+   --  True if N Is_Ignored_Ghost_Entity and GNATProve_mode and Codepeer_Mode
+   --  are not active.
+
+   function Is_Ignored_Ghost_Pragma_In_Codegen (N : Node_Id) return Boolean;
+   --  True if N Is_Ignored_Ghost_Pragma and GNATProve_mode and Codepeer_Mode
+   --  are not active.
+
+   function Is_Ignored_In_Codegen (N : Node_Id) return Boolean;
+   --  True if N Is_Ignored and GNATProve_mode and Codepeer_Mode are not
+   --  active.
+
    function Is_EVF_Expression (N : Node_Id) return Boolean;
    --  Determine whether node N denotes a reference to a formal parameter of
    --  a specific tagged type whose related subprogram is subject to pragma
@@ -2131,7 +2143,7 @@ package Sem_Util is
    --  object as per RM C.6(8).
 
    function Is_Inherited_Operation (E : Entity_Id) return Boolean;
-   --  E is a subprogram. Return True is E is an implicit operation inherited
+   --  E is a subprogram. Return True if E is an implicit operation inherited
    --  by a derived type declaration.
 
    function Is_Inlinable_Expression_Function (Subp : Entity_Id) return Boolean;
@@ -2196,7 +2208,7 @@ package Sem_Util is
    --  the encapsulated expression is nontrivial.
 
    function Is_Null_Extension
-    (T : Entity_Id; Ignore_Privacy : Boolean := False) return Boolean;
+     (T : Entity_Id; Ignore_Privacy : Boolean := False) return Boolean;
    --  Given a tagged type, returns True if argument is a type extension
    --  that introduces no new components (discriminant or nondiscriminant).
    --  Ignore_Privacy should be True for use in implementing dynamic semantics.
@@ -2449,6 +2461,10 @@ package Sem_Util is
    --  Determine whether an arbitrary entity denotes an instance of function
    --  Ada.Unchecked_Conversion.
 
+   function Is_Unchecked_Union_Equality (Id : Entity_Id) return Boolean;
+   --  Determine whether an arbitrary entity denotes the predefined equality
+   --  function of an Unchecked_Union type (see Build_Variant_Record_Equality).
+
    function Is_Unconstrained_Or_Tagged_Item (Item : Entity_Id) return Boolean;
    --  Subsidiary to Collect_Subprogram_Inputs_Outputs and the analysis of
    --  pragma Depends. Determine whether the type of dependency item Item is
@@ -2973,11 +2989,11 @@ package Sem_Util is
       Comp     : Boolean := False;
       Deriv    : Boolean := False);
    --  Set Disable_Controlled, Finalize_Storage_Only, Has_Controlled_Component,
-   --  Has_Relaxed_Finalization, and Is_Controlled_Active on Typ when the flags
-   --  are set on From_Typ. If Comp is True, From_Typ is assumed to be the type
-   --  of a component of Typ while, if Deriv is True, From_Typ is assumed to be
-   --  the parent type of Typ. This procedure can only set flags for Typ, and
-   --  never clear them.
+   --  Has_Destructor, Has_Relaxed_Finalization, and Is_Controlled_Active on
+   --  Typ when the flags are set on From_Typ. If Comp is True, From_Typ is
+   --  assumed to be the type of a component of Typ while, if Deriv is True,
+   --  From_Typ is assumed to be the parent type of Typ. This procedure can
+   --  only set flags for Typ, and never clear them.
 
    procedure Propagate_DIC_Attributes
      (Typ      : Entity_Id;
diff --git a/gcc/ada/sinfo.ads b/gcc/ada/sinfo.ads
index c63a97d..3d11d5c 100644
--- a/gcc/ada/sinfo.ads
+++ b/gcc/ada/sinfo.ads
@@ -402,16 +402,17 @@ package Sinfo is
    --       Has_Secondary_Private_View  set in generic units
 
    --    "plus fields for expression"
-   --       Paren_Count               number of parentheses levels
-   --       Etype                     type of the expression
-   --       Is_Overloaded             >1 type interpretation exists
-   --       Is_Static_Expression      set for static expression
-   --       Raises_Constraint_Error   evaluation raises CE
-   --       Must_Not_Freeze           set if must not freeze
-   --       Do_Range_Check            set if a range check needed
-   --       Has_Dynamic_Length_Check  set if length check inserted
-   --       Assignment_OK             set if modification is OK
-   --       Is_Controlling_Actual     set for controlling argument
+   --       Paren_Count                  number of parentheses levels
+   --       Etype                        type of the expression
+   --       Is_Overloaded                >1 type interpretation exists
+   --       Is_Static_Expression         set for static expression
+   --       Raises_Constraint_Error      evaluation raises CE
+   --       Must_Not_Freeze              set if must not freeze
+   --       Do_Range_Check               set if a range check needed
+   --       Has_Dynamic_Length_Check     set if length check inserted
+   --       Assignment_OK                set if modification is OK
+   --       Is_Controlling_Actual        set for controlling argument
+   --       Is_Expanded_Dispatching_Call set for expanded dispatching calls
 
    --  Note: see under (EXPRESSION) for further details on the use of
    --  the Paren_Count field to record the number of parentheses levels.
@@ -1664,6 +1665,10 @@ package Sinfo is
    --    actuals to support a build-in-place style of call have been added to
    --    the call.
 
+   --  Is_Expanded_Dispatching_Call
+   --    This flag is set in N_Block_Statement, and expression nodes to
+   --    indicate that it is an expanded dispatching call.
+
    --  Is_Expanded_Prefixed_Call
    --    This flag is set in N_Function_Call and N_Procedure_Call_Statement
    --    nodes to indicate that it is an expanded prefixed call.
@@ -2321,6 +2326,13 @@ package Sinfo is
    --    statement applies to. Finally, if Analyze_Continue_Statement detects
    --    an error, this field is set to Empty.
 
+   --  Tag_Propagated
+   --    This flag is set in N_Identifier, N_Explicit_Dereference, and N_Type_
+   --    Conversion nodes that are the LHS of an assignment statement. Used to
+   --    remember that the RHS of the assignment has tag indeterminate function
+   --    calls and the tag has been propagated to the calls (as part of the
+   --    bottom-up analysis of the RHS of the assignment statement).
+
    --  Target_Type
    --    Used in an N_Validate_Unchecked_Conversion node to point to the target
    --    type entity for the unchecked conversion instantiation which gigi must
@@ -2507,6 +2519,7 @@ package Sinfo is
       --  Has_Private_View (set in generic units)
       --  Has_Secondary_Private_View (set in generic units)
       --  Redundant_Use
+      --  Tag_Propagated
       --  Atomic_Sync_Required
       --  plus fields for expression
 
@@ -3820,6 +3833,7 @@ package Sinfo is
       --  Prefix
       --  Actual_Designated_Subtype
       --  Has_Dereference_Action
+      --  Tag_Propagated
       --  Atomic_Sync_Required
       --  plus fields for expression
 
@@ -4755,6 +4769,7 @@ package Sinfo is
       --  Conversion_OK
       --  Do_Overflow_Check
       --  Rounded_Result
+      --  Tag_Propagated
       --  plus fields for expression
 
       --  Note: if a range check is required, then the Do_Range_Check flag
@@ -5196,6 +5211,7 @@ package Sinfo is
       --  Has_Created_Identifier
       --  Is_Abort_Block
       --  Is_Asynchronous_Call_Block
+      --  Is_Expanded_Dispatching_Call
       --  Is_Initialization_Block
       --  Is_Task_Allocation_Block
       --  Is_Task_Master
diff --git a/gcc/ada/snames.ads-tmpl b/gcc/ada/snames.ads-tmpl
index f26515e..272e10b 100644
--- a/gcc/ada/snames.ads-tmpl
+++ b/gcc/ada/snames.ads-tmpl
@@ -151,6 +151,7 @@ package Snames is
    Name_Default_Value                  : constant Name_Id := N + $;
    Name_Default_Component_Value        : constant Name_Id := N + $;
    Name_Designated_Storage_Model       : constant Name_Id := N + $;
+   Name_Destructor                     : constant Name_Id := N + $;
    Name_Dimension                      : constant Name_Id := N + $;
    Name_Dimension_System               : constant Name_Id := N + $;
    Name_Disable_Controlled             : constant Name_Id := N + $;
diff --git a/gcc/ada/styleg.adb b/gcc/ada/styleg.adb
index 7b7f252..20945fb 100644
--- a/gcc/ada/styleg.adb
+++ b/gcc/ada/styleg.adb
@@ -499,7 +499,8 @@ package body Styleg is
                   if Is_Box_Comment
                     or else Style_Check_Comments_Spacing = 1
                   then
-                     Error_Space_Required (Scan_Ptr + 2);
+                     Error_Msg -- CODEFIX
+                       ("(style) space required?c?", Scan_Ptr + 2);
                   else
                      Error_Msg -- CODEFIX
                        ("(style) two spaces required?c?", Scan_Ptr + 2);
@@ -526,7 +527,8 @@ package body Styleg is
          --  box comment.
 
          elsif not Is_Box_Comment then
-            Error_Space_Required (Scan_Ptr + 3);
+            Error_Msg -- CODEFIX
+              ("(style) space required?c?", Scan_Ptr + 3);
          end if;
       end if;
    end Check_Comment;
diff --git a/gcc/ada/sysdep.c b/gcc/ada/sysdep.c
index 3dc76f9..afbb362 100644
--- a/gcc/ada/sysdep.c
+++ b/gcc/ada/sysdep.c
@@ -40,6 +40,12 @@
    - either they are defined as ENOENT (vx7r2);
    - or the corresponding system includes are not provided (Helix Cert).  */
 
+#if __has_include ("strings.h")
+/* On VxWorks6, FD_ZERO uses bzero, and index is also declared in strings.h,
+   but since it's not a standard header, don't require it.  */
+#include "strings.h"
+#endif
+
 #if __has_include ("dosFsLib.h")
 /* On helix-cert, this include is only provided for RTPs.  */
 #include "dosFsLib.h"
diff --git a/gcc/ada/table.adb b/gcc/ada/table.adb
index 37c4949..31891de 100644
--- a/gcc/ada/table.adb
+++ b/gcc/ada/table.adb
@@ -40,6 +40,9 @@ package body Table is
       Min : constant Int := Int (Table_Low_Bound);
       --  Subscript of the minimum entry in the currently allocated table
 
+      Max : Int := Min + (Table_Initial * Table_Factor) - 1;
+      --  Subscript of the maximum entry in the currently allocated table
+
       Length : Int := 0;
       --  Number of entries in currently allocated table. The value of zero
       --  ensures that we initially allocate the table.
diff --git a/gcc/ada/table.ads b/gcc/ada/table.ads
index 22e9172..623ce14 100644
--- a/gcc/ada/table.ads
+++ b/gcc/ada/table.ads
@@ -223,9 +223,6 @@ package Table is
       --  the official interfaces (since a modification to Last may require a
       --  reallocation of the table).
 
-      Max : Int;
-      --  Subscript of the maximum entry in the currently allocated table
-
       type Saved_Table is record
          Last_Val : Int;
          Max      : Int;
diff --git a/gcc/ada/tbuild.adb b/gcc/ada/tbuild.adb
index b89c408..52fdbfc 100644
--- a/gcc/ada/tbuild.adb
+++ b/gcc/ada/tbuild.adb
@@ -926,11 +926,11 @@ package body Tbuild is
       --  conversion of an unchecked conversion. Extra unchecked conversions
       --  make the .dg output less readable. We can't do this in cases
       --  involving bitfields, because the sizes might not match. The
-      --  "not Is_Scalar_Type" checks avoid such cases.
+      --  Is_Composite_Type checks avoid such cases.
 
       elsif Nkind (Expr) = N_Unchecked_Type_Conversion
-        and then not Is_Scalar_Type (Etype (Expr))
-        and then not Is_Scalar_Type (Typ)
+        and then Is_Composite_Type (Etype (Expr))
+        and then Is_Composite_Type (Typ)
       then
          Set_Subtype_Mark (Expr, New_Occurrence_Of (Typ, Loc));
          Result := Relocate_Node (Expr);
diff --git a/gcc/ada/treepr.adb b/gcc/ada/treepr.adb
index 375608d..857b926 100644
--- a/gcc/ada/treepr.adb
+++ b/gcc/ada/treepr.adb
@@ -1600,19 +1600,17 @@ package body Treepr is
          --  If this is a discrete expression whose value is known, print that
          --  value.
 
-         if Nkind (N) in N_Subexpr
+         if ((Is_Entity_Name (N) -- e.g. enumeration literal
+               and then Present (Entity (N)))
+              or else Nkind (N) in N_Integer_Literal
+                                 | N_Character_Literal
+                                 | N_Unchecked_Type_Conversion)
            and then Compile_Time_Known_Value (N)
            and then Present (Etype (N))
            and then Is_Discrete_Type (Etype (N))
          then
-            if Is_Entity_Name (N) -- e.g. enumeration literal
-              or else Nkind (N) in N_Integer_Literal
-                                 | N_Character_Literal
-                                 | N_Unchecked_Type_Conversion
-            then
-               Print_Str (" val = ");
-               UI_Write (Expr_Value (N));
-            end if;
+            Print_Str (" val = ");
+            UI_Write (Expr_Value (N));
          end if;
 
          if Nkind (N) in N_Entity then
diff --git a/gcc/analyzer/ChangeLog b/gcc/analyzer/ChangeLog
index d31cbbc..8e5d38e 100644
--- a/gcc/analyzer/ChangeLog
+++ b/gcc/analyzer/ChangeLog
@@ -1,3 +1,148 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* program-point.cc: Make diagnostics::context::m_source_printing
+	private.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* pending-diagnostic.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* program-point.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* program-point.cc: : Update for diagnostic_context becoming
+	diagnostics::context, and for diagnostic_source_print_policy
+	becoming diagnostics::source_print_policy.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* ana-state-to-diagnostic-state.cc: Update for move of
+	diagnostics output formats into namespace "diagnostics" as
+	"sinks".
+	* bounds-checking.cc: Likewise.
+	* call-details.cc: Likewise.
+	* checker-event.cc: Likewise.
+	* checker-event.h: Likewise.
+	* diagnostic-manager.cc: Likewise.
+	* diagnostic-manager.h: Likewise.
+	* infinite-loop.cc: Likewise.
+	* infinite-recursion.cc: Likewise.
+	* pending-diagnostic.h: Likewise.
+	* region-model.cc: Likewise.
+	* sm-taint.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* access-diagram.cc: Update for changes to diagnostic paths:
+	"diagnostic-path.h" moving to "diagnostics/paths.h",
+	"diagnostic-event-id.h" moving to "diagnostics/event-id.h",
+	diagnostic_event_id_t to diagnostics::paths::event_id_t,
+	diagnostic_path to diagnostics::paths::path, and
+	diagnostic_event to diagnostics::paths::event.
+	* access-diagram.h: Likewise.
+	* analyzer.cc: Likewise.
+	* bounds-checking.cc: Likewise.
+	* call-info.cc: Likewise.
+	* checker-event.cc: Likewise.
+	* checker-event.h: Likewise.
+	* checker-path.cc: Likewise.
+	* checker-path.h: Likewise.
+	* common.h: Likewise.
+	* diagnostic-manager.cc: Likewise.
+	* pending-diagnostic.cc: Likewise.
+	* pending-diagnostic.h: Likewise.
+	* program-point.cc: Likewise.
+	* program-state.cc: Likewise.
+	* region-model.cc: Likewise.
+	* sm-fd.cc: Likewise.
+	* sm-file.cc: Likewise.
+	* sm-malloc.cc: Likewise.
+	* sm-pattern-test.cc: Likewise.
+	* sm-sensitive.cc: Likewise.
+	* sm-signal.cc: Likewise.
+	* sm-taint.cc: Likewise.
+	* varargs.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* bounds-checking.cc: Update #include for move of
+	"diagnostic-diagram.h" to "diagnostics/diagram.h". Update for
+	move of diagnostic_diagram to diagnostics::diagram.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-manager.cc: Update for move of diagnostic_metadata to
+	diagnostics::metadata.
+	* kf.cc: Likewise.
+	* pending-diagnostic.h: Likewise; also, update #include for move
+	of "diagnostic-metadata.h" to "diagnostics/metadata.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* checker-event.h (checker_event::get_logical_location): Update
+	for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	(checker_event::m_logical_loc): Likewise.
+	* diagnostic-manager.cc
+	(diagnostic_manager::get_logical_location_manager): Likewise.
+	* diagnostic-manager.h
+	(diagnostic_manager::get_logical_location_manager): Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* ana-state-to-diagnostic-state.cc: Update #include for move of
+	"diagnostic-state-graphs.h" to "diagnostics/state-graphs.h".
+	* ana-state-to-diagnostic-state.h: Likewise.
+	* checker-event.cc: Likewise.
+	* checker-event.h: Update #include for move of
+	"diagnostic-digraphs.h" to "diagnostics/digraphs.h".
+	* program-state.cc: : Update #include for move of
+	"diagnostic-state-graphs.h" to "diagnostics/state-graphs.h".
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* region-model-asm.cc (region_model::on_asm_stmt): Pass null
+	pointer to parse_{input,output}_constraint().
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* ana-state-to-diagnostic-state.cc: Reimplement, replacing
+	XML-based implementation with one based on state graphs.
+	* ana-state-to-diagnostic-state.h: Likewise.
+	* checker-event.cc: Replace include of "xml.h" with include of
+	"diagnostic-state-graphs.h".
+	(checker_event::maybe_make_xml_state): Replace with...
+	(checker_event::maybe_make_diagnostic_state_graph): ...this.
+	* checker-event.h: Add include of "diagnostic-digraphs.h".
+	(checker_event::maybe_make_xml_state): Replace decl with...
+	(checker_event::maybe_make_diagnostic_state_graph): ...this.
+	* engine.cc (exploded_node::on_stmt_pre): Replace
+	"_analyzer_dump_xml" with "__analyzer_dump_sarif".
+	* program-state.cc: Replace include of "diagnostic-state.h" with
+	"diagnostic-state-graphs.h".
+	(program_state::dump_dot): Port from XML to state graphs.
+	* program-state.h: Drop reduntant forward decl of xml::document.
+	(program_state::make_xml): Replace decl with...
+	(program_state::make_diagnostic_state_graph): ...this.
+	(program_state::dump_xml_to_pp): Drop decl.
+	(program_state::dump_xml_to_file): Drop decl.
+	(program_state::dump_xml): Drop decl.
+	(program_state::dump_dump_sarif): New decl.
+	* sm-malloc.cc (get_dynalloc_state_for_state): New.
+	(malloc_state_machine::add_state_to_xml): Replace with...
+	(malloc_state_machine::add_state_to_state_graph): ...this.
+	* sm.cc (state_machine::add_state_to_xml): Replace with...
+	(state_machine::add_state_to_state_graph): ...this.
+	(state_machine::add_global_state_to_xml): Replace with...
+	(state_machine::add_global_state_to_state_graph): ...this.
+	* sm.h (class xml_state): Drop forward decl.
+	(class analyzer_state_graph): New forward decl.
+	(state_machine::add_state_to_xml): Replace decl with...
+	(state_machine::add_state_to_state_graph): ...this.
+	(state_machine::add_global_state_to_xml): Replace decl with...
+	(state_machine::add_global_state_to_state_graph): ...this.
+
 2025-06-30  David Malcolm  <dmalcolm@redhat.com>
 
 	* access-diagram.cc: Use nullptr rather than NULL where
diff --git a/gcc/analyzer/access-diagram.cc b/gcc/analyzer/access-diagram.cc
index 90b396c..166be08 100644
--- a/gcc/analyzer/access-diagram.cc
+++ b/gcc/analyzer/access-diagram.cc
@@ -1300,7 +1300,7 @@ class valid_region_spatial_item : public spatial_item
 {
 public:
   valid_region_spatial_item (const access_operation &op,
-			     diagnostic_event_id_t region_creation_event_id,
+			     diagnostics::paths::event_id_t region_creation_event_id,
 			     const theme &theme)
   : m_op (op),
     m_region_creation_event_id (region_creation_event_id),
@@ -1523,7 +1523,7 @@ public:
 
 private:
   const access_operation &m_op;
-  diagnostic_event_id_t m_region_creation_event_id;
+  diagnostics::paths::event_id_t m_region_creation_event_id;
   mutable const boundaries *m_boundaries;
   const svalue *m_existing_sval;
   std::unique_ptr<spatial_item> m_existing_sval_spatial_item;
@@ -2012,7 +2012,7 @@ class access_diagram_impl : public vbox_widget
 {
 public:
   access_diagram_impl (const access_operation &op,
-		       diagnostic_event_id_t region_creation_event_id,
+		       diagnostics::paths::event_id_t region_creation_event_id,
 		       style_manager &sm,
 		       const theme &theme,
 		       logger *logger)
@@ -2564,7 +2564,7 @@ private:
   }
 
   const access_operation &m_op;
-  diagnostic_event_id_t m_region_creation_event_id;
+  diagnostics::paths::event_id_t m_region_creation_event_id;
   style_manager &m_sm;
   const theme &m_theme;
   logger *m_logger;
@@ -2662,7 +2662,7 @@ direction_widget::paint_to_canvas (canvas &canvas)
    an access_diagram_impl.  */
 
 access_diagram::access_diagram (const access_operation &op,
-				diagnostic_event_id_t region_creation_event_id,
+				diagnostics::paths::event_id_t region_creation_event_id,
 				style_manager &sm,
 				const theme &theme,
 				logger *logger)
diff --git a/gcc/analyzer/access-diagram.h b/gcc/analyzer/access-diagram.h
index e31dd28..d558f53 100644
--- a/gcc/analyzer/access-diagram.h
+++ b/gcc/analyzer/access-diagram.h
@@ -152,7 +152,7 @@ class access_diagram : public text_art::wrapper_widget
 {
 public:
   access_diagram (const access_operation &op,
-		  diagnostic_event_id_t region_creation_event_id,
+		  diagnostics::paths::event_id_t region_creation_event_id,
 		  text_art::style_manager &sm,
 		  const text_art::theme &theme,
 		  logger *logger);
diff --git a/gcc/analyzer/ana-state-to-diagnostic-state.cc b/gcc/analyzer/ana-state-to-diagnostic-state.cc
index b85a2f1..996538c 100644
--- a/gcc/analyzer/ana-state-to-diagnostic-state.cc
+++ b/gcc/analyzer/ana-state-to-diagnostic-state.cc
@@ -1,4 +1,4 @@
-/* Converting ana::program_state to XML state documents.
+/* Creating diagnostic state graphs from ana::program_state.
    Copyright (C) 2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -23,8 +23,8 @@ along with GCC; see the file COPYING3.  If not see
 #define INCLUDE_SET
 #include "analyzer/common.h"
 
-#include "xml.h"
-#include "xml-printer.h"
+#include "diagnostics/state-graphs.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/region-model.h"
 #include "analyzer/program-state.h"
@@ -39,57 +39,47 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace ana {
 
+using namespace ::diagnostics::state_graphs;
+
 static void
-set_wi_attr (xml::element &e,
+set_wi_attr (state_node_ref state_node,
 	     const char *attr_name,
 	     const wide_int_ref &w,
 	     signop sgn)
 {
   pretty_printer pp;
   pp_wide_int (&pp, w, sgn);
-  e.set_attr (attr_name, pp_formatted_text (&pp));
+  state_node.set_attr (attr_name, pp_formatted_text (&pp));
 }
 
 static void
-set_type_attr (xml::element &e, const_tree type)
+set_type_attr (state_node_ref state_node, const_tree type)
 {
   gcc_assert (type);
   pretty_printer pp;
   pp_format_decoder (&pp) = default_tree_printer;
   pp_printf (&pp, "%T", type);
-  e.set_attr ("type", pp_formatted_text (&pp));
+  state_node.set_type (pp_formatted_text (&pp));
 }
 
 static void
-set_bits_attr (xml::element &e,
+set_bits_attr (state_node_ref state_node,
 	       bit_range bits)
 {
   pretty_printer pp;
   bits.dump_to_pp (&pp);
-  e.set_attr ("bits", pp_formatted_text (&pp));
+  state_node.set_attr ("bits", pp_formatted_text (&pp));
 }
 
-static void
-set_region_id_attr (xml::element &e,
-		    const region &reg)
-{
-  e.set_attr ("region_id", std::to_string (reg.get_id ()));
-}
+// class analyzer_state_graph : public diagnostics::digraphs::digraph
 
-// class xml_state : public xml::document
-
-xml_state::xml_state (const program_state &state,
-		      const extrinsic_state &ext_state)
-: xml::document (),
-  m_state (state),
+analyzer_state_graph::analyzer_state_graph (const program_state &state,
+					    const extrinsic_state &ext_state)
+: m_state (state),
   m_ext_state (ext_state),
   m_mgr (*ext_state.get_engine ()->get_model_manager ()),
-  m_root (nullptr)
+  m_next_id (0)
 {
-  auto root = std::make_unique<xml::element> ("state-diagram", false);
-  m_root = root.get ();
-  add_child (std::move (root));
-
   /* Find pointers to heap-allocated regions, and record their types,
      so that we have a user-friendly way of showing the memory
      (by field, rather than by byte offset).  */
@@ -117,14 +107,14 @@ xml_state::xml_state (const program_state &state,
   for (int i = state.m_region_model->get_stack_depth () - 1; i >= 0; --i)
     {
       const frame_region *reg = state.m_region_model->get_frame_at_index (i);
-      get_or_create_element (*reg);
+      get_or_create_state_node (*reg);
     }
 
   /* Create bound memory.  */
   for (auto iter : *state.m_region_model->get_store ())
     {
       const bool create_all = false; // "true" for verbose, for debugging
-      create_elements_for_binding_cluster (*iter.second, create_all);
+      create_state_nodes_for_binding_cluster (*iter.second, create_all);
     }
 
   /* TODO: Constraints.  */
@@ -137,52 +127,165 @@ xml_state::xml_state (const program_state &state,
       {
 	auto &sm = ext_state.get_sm (i);
 	for (const auto &iter : *smap)
-	  sm.add_state_to_xml (*this, *iter.first, iter.second.m_state);
+	  sm.add_state_to_state_graph (*this, *iter.first, iter.second.m_state);
 	if (auto s = smap->get_global_state ())
-	  sm.add_global_state_to_xml (*this, s);
+	  sm.add_global_state_to_state_graph (*this, s);
       }
   }
+
+  /* Process pending edges.  */
+  while (m_pending_edges.size () > 0)
+    {
+      pending_edge item = m_pending_edges.back ();
+      m_pending_edges.pop_back ();
+
+      /* Ensure we have a node for the dst region.  This
+	 could lead to additional pending edges.  */
+      auto dst_node = get_or_create_state_node (item.m_dst_reg);
+      add_edge (nullptr, item.m_src_node.m_node, dst_node.m_node);
+    }
 }
 
-xml::element &
-xml_state::get_or_create_element (const region &reg)
+state_node_ref
+analyzer_state_graph::get_or_create_state_node (const region &reg)
 {
-  auto existing = m_region_to_element_map.find (&reg);
-  if (existing != m_region_to_element_map.end ())
+  auto existing = m_region_to_state_node_map.find (&reg);
+  if (existing != m_region_to_state_node_map.end ())
     return *existing->second;
 
-  auto &e = create_and_add_element (reg);
-  m_region_to_element_map[&reg] = &e;
-  return e;
+  auto ref = create_and_add_state_node (reg);
+  m_region_to_state_node_map[&reg] = &ref.m_node;
+  return ref;
 }
 
-xml::element&
-xml_state::create_and_add_element (const region &reg)
+state_node_ref
+analyzer_state_graph::create_and_add_state_node (const region &reg)
 {
-  auto e = create_element (reg);
-  xml::element &result = *e;
+  auto node = create_state_node (reg);
+
+  state_node_ref result = *node;
   if (auto parent_reg = reg.get_parent_region ())
+    if (parent_reg->get_kind () != RK_ROOT)
+      {
+	auto parent_state_node = get_or_create_state_node (*parent_reg);
+	parent_state_node.m_node.add_child (std::move (node));
+	return result;
+      }
+  add_node (std::move (node));
+  return result;
+}
+
+std::string
+analyzer_state_graph::make_node_id (const char *prefix)
+{
+  return std::string (prefix) + "-" + std::to_string (m_next_id++);
+}
+
+std::string
+analyzer_state_graph::make_node_id (const region &reg)
+{
+  const char *prefix = nullptr;
+  switch (reg.get_kind ())
     {
-      auto parent_element = &get_or_create_element (*parent_reg);
-      parent_element->add_child (std::move (e));
+    case RK_ROOT:
+    default:
+      gcc_unreachable ();
+      break;
+
+    case RK_GLOBALS:
+      return "globals";
+    case RK_CODE:
+      return "code";
+    case RK_STACK:
+      return "stack";
+    case RK_HEAP:
+      return "heap";
+
+    case RK_FRAME:
+      prefix = "frame-region";
+      break;
+    case RK_FUNCTION:
+      prefix = "function-region";
+      break;
+    case RK_LABEL:
+      prefix = "label-region";
+      break;
+    case RK_THREAD_LOCAL:
+      prefix = "thread-local-region";
+      break;
+    case RK_SYMBOLIC:
+      prefix = "symbolic-region";
+      break;
+    case RK_DECL:
+      prefix = "decl-region";
+      break;
+    case RK_FIELD:
+      prefix = "field-region";
+      break;
+    case RK_ELEMENT:
+      prefix = "element-region";
+      break;
+    case RK_OFFSET:
+      prefix = "offset-region";
+      break;
+    case RK_SIZED:
+      prefix = "sized-region";
+      break;
+    case RK_CAST:
+      prefix = "cast-region";
+      break;
+    case RK_HEAP_ALLOCATED:
+      prefix = "heap-allocated-region";
+      break;
+    case RK_ALLOCA:
+      prefix = "alloca-region";
+      break;
+    case RK_STRING:
+      prefix = "string-region";
+      break;
+    case RK_BIT_RANGE:
+      prefix = "bit-range-region";
+      break;
+    case RK_VAR_ARG:
+      prefix = "var-arg-region";
+      break;
+    case RK_ERRNO:
+      prefix = "errno-region";
+      break;
+    case RK_PRIVATE:
+      prefix = "private-region";
+      break;
+    case RK_UNKNOWN:
+      prefix = "unknown-region";
+      break;
     }
-    else
-      m_root->add_child (std::move (e));
-  return result;
+  return std::string (prefix) + "-" + std::to_string (reg.get_id ());
+}
+
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::
+make_state_node (diagnostics::state_graphs::node_kind kind,
+		 std::string id)
+{
+  auto node = std::make_unique<diagnostics::digraphs::node> (*this, std::move (id));
+  state_node_ref node_ref (*node);
+  node_ref.set_node_kind (kind);
+  return node;
 }
 
-std::unique_ptr<xml::element>
-xml_state::make_memory_space_element (const char *label)
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::
+make_memspace_state_node (const region &reg,
+			  diagnostics::state_graphs::node_kind kind)
 {
-  auto e = std::make_unique<xml::element> ("memory-space", false);
-  e->set_attr ("label", label);
-  return e;
+  return make_state_node (kind, make_node_id (reg));
 }
 
-std::unique_ptr<xml::element>
-xml_state::create_element (const region &reg)
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::create_state_node (const region &reg)
 {
-  std::unique_ptr<xml::element> e;
+  std::unique_ptr<diagnostics::digraphs::node> node;
+
   switch (reg.get_kind ())
     {
     default:
@@ -190,110 +293,102 @@ xml_state::create_element (const region &reg)
 
     case RK_FRAME:
       {
-	e = std::make_unique<xml::element> ("stack-frame", false);
 	const frame_region &frame_reg
 	  = static_cast<const frame_region &> (reg);
+
+	node = make_state_node (diagnostics::state_graphs::node_kind::stack_frame,
+				make_node_id (reg));
+	node->set_logical_loc
+	  (m_logical_loc_mgr.key_from_tree (frame_reg.get_fndecl ()));
 	{
 	  pretty_printer pp;
 	  pp_format_decoder (&pp) = default_tree_printer;
 	  pp_printf (&pp, "%E", frame_reg.get_fndecl ());
-	  e->set_attr ("function", pp_formatted_text (&pp));
+	  node->set_attr (STATE_NODE_PREFIX, "function",
+			  pp_formatted_text (&pp));
 	}
       }
       break;
+
     case RK_GLOBALS:
-      e = make_memory_space_element ("Globals");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::globals);
       break;
     case RK_CODE:
-      e = make_memory_space_element ("Code");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::code);
       break;
     case RK_FUNCTION:
-      e = std::make_unique<xml::element> ("function", false);
-      // TODO
-      break;
-    case RK_LABEL:
-      e = std::make_unique<xml::element> ("label", false);
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::function);
       // TODO
       break;
+
     case RK_STACK:
-      e = std::make_unique<xml::element> ("stack", false);
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::stack);
       break;
     case RK_HEAP:
-      e = make_memory_space_element ("Heap");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::heap_);
       break;
     case RK_THREAD_LOCAL:
-      e = make_memory_space_element ("Thread-local");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::thread_local_);
       break;
     case RK_ROOT:
-      e = std::make_unique<xml::element> ("memory-regions", false);
+      gcc_unreachable ();
       break;
     case RK_SYMBOLIC:
-      e = std::make_unique<xml::element> ("symbolic-region", false);
-      // TODO
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::other);
       break;
+
     case RK_DECL:
       {
-	e = std::make_unique<xml::element> ("variable", false);
+	node = make_state_node (diagnostics::state_graphs::node_kind::variable,
+				make_node_id (reg));
 	const decl_region &decl_reg
 	  = static_cast<const decl_region &> (reg);
+	state_node_ref node_ref (*node);
 	{
 	  pretty_printer pp;
 	  pp_format_decoder (&pp) = default_tree_printer;
 	  pp_printf (&pp, "%E", decl_reg.get_decl ());
-	  e->set_attr ("name", pp_formatted_text (&pp));
+	  node_ref.set_name (pp_formatted_text (&pp));
 	}
-	set_type_attr (*e, TREE_TYPE (decl_reg.get_decl ()));
+	set_type_attr (*node, TREE_TYPE (decl_reg.get_decl ()));
       }
       break;
+
     case RK_FIELD:
-      e = std::make_unique<xml::element> ("field", false);
-      break;
     case RK_ELEMENT:
-      e = std::make_unique<xml::element> ("element", false);
+      /* These should be handled in populate_state_node_for_typed_region.  */
+      gcc_unreachable ();
       break;
+
+    case RK_LABEL:
     case RK_OFFSET:
-      e = std::make_unique<xml::element> ("offset-region", false);
-      // TODO
-      break;
     case RK_SIZED:
-      e = std::make_unique<xml::element> ("sized-region", false);
-      // TODO
-      break;
     case RK_CAST:
-      e = std::make_unique<xml::element> ("cast-region", false);
-      // TODO
-      break;
-    case RK_HEAP_ALLOCATED:
-      e = std::make_unique<xml::element> ("heap-buffer", false);
-      set_attr_for_dynamic_extents (reg, *e);
-      break;
-    case RK_ALLOCA:
-      e = std::make_unique<xml::element> ("alloca-buffer", false);
-      set_attr_for_dynamic_extents (reg, *e);
-      break;
     case RK_STRING:
-      e = std::make_unique<xml::element> ("string-region", false);
-      // TODO
-      break;
     case RK_BIT_RANGE:
-      e = std::make_unique<xml::element> ("RK_BIT_RANGE", false); // TODO
-      break;
     case RK_VAR_ARG:
-      e = std::make_unique<xml::element> ("RK_VAR_ARG", false); // TODO
-      break;
     case RK_ERRNO:
-      e = std::make_unique<xml::element> ("errno", false);
-      break;
     case RK_PRIVATE:
-      e = std::make_unique<xml::element> ("RK_PRIVATE", false); // TODO
-      break;
     case RK_UNKNOWN:
-      e = std::make_unique<xml::element> ("RK_UNKNOWN", false); // TODO
+      node = make_state_node (diagnostics::state_graphs::node_kind::other,
+				make_node_id (reg));
       break;
-    }
-  gcc_assert (e);
 
-  set_region_id_attr (*e, reg);
+    case RK_HEAP_ALLOCATED:
+    case RK_ALLOCA:
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::dynalloc_buffer);
+      set_attr_for_dynamic_extents (reg, *node);
+      break;
+    }
+  gcc_assert (node);
 
   if (reg.get_base_region () == &reg)
     if (!reg.get_type ())
@@ -303,16 +398,17 @@ xml_state::create_element (const region &reg)
 	if (search != m_types_for_untyped_regions.end ())
 	  {
 	    tree type_to_use = search->second;
-	    set_type_attr (*e, type_to_use);
+	    set_type_attr (*node, type_to_use);
 	  }
       }
 
-  return e;
+  return node;
 }
 
 void
-xml_state::create_elements_for_binding_cluster (const binding_cluster &cluster,
-						bool create_all)
+analyzer_state_graph::
+create_state_nodes_for_binding_cluster (const binding_cluster &cluster,
+					bool create_all)
 {
   /* TODO:
      - symbolic bindings
@@ -326,12 +422,12 @@ xml_state::create_elements_for_binding_cluster (const binding_cluster &cluster,
       if (auto conc_key = key->dyn_cast_concrete_binding ())
 	conc_bindings[conc_key->get_bit_range ()] = svalue;
       if (const region *reg = svalue->maybe_get_region ())
-	get_or_create_element (*reg);
+	get_or_create_state_node (*reg);
     }
 
-  auto &e = get_or_create_element (*cluster.get_base_region ());
+  auto ref = get_or_create_state_node (*cluster.get_base_region ());
 
-  e.add_child (create_element_for_conc_bindings (conc_bindings));
+  ref.m_node.add_child (create_state_node_for_conc_bindings (conc_bindings));
 
   const region *typed_reg = cluster.get_base_region ();
   if (!typed_reg->get_type ())
@@ -346,44 +442,45 @@ xml_state::create_elements_for_binding_cluster (const binding_cluster &cluster,
     }
 
   if (typed_reg->get_type ())
-    populate_element_for_typed_region (e,
-				       *typed_reg,
-				       conc_bindings,
-				       create_all);
+    populate_state_node_for_typed_region (ref,
+					  *typed_reg,
+					  conc_bindings,
+					  create_all);
   else
     {
       // TODO
     }
 }
 
-std::unique_ptr<xml::element>
-xml_state::create_element_for_conc_bindings (const concrete_bindings_t &conc_bindings)
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::create_state_node_for_conc_bindings (const concrete_bindings_t &conc_bindings)
 {
-  auto e = std::make_unique<xml::element> ("concrete-bindings", false);
+  auto node = make_state_node (diagnostics::state_graphs::node_kind::other,
+			       make_node_id ("concrete-bindings"));
   for (auto iter : conc_bindings)
     {
       const bit_range bits = iter.first;
       const svalue *sval = iter.second;
-      auto binding_element
-	= std::make_unique<xml::element> ("binding", false);
-      set_bits_attr (*binding_element, bits);
+      auto binding_state_node
+	= make_state_node (diagnostics::state_graphs::node_kind::other,
+			   make_node_id ("binding"));
+      set_bits_attr (*binding_state_node, bits);
       {
 	pretty_printer pp;
 	pp_format_decoder (&pp) = default_tree_printer;
 	sval->dump_to_pp (&pp, true);
-	binding_element->set_attr ("value", pp_formatted_text (&pp));
-	if (auto svalue_element = create_element_for_svalue (sval))
-	  binding_element->add_child (std::move (svalue_element));
+	binding_state_node->set_attr (STATE_NODE_PREFIX, "value",
+				      pp_formatted_text (&pp));
       }
-      e->add_child (std::move (binding_element));
+      node->add_child (std::move (binding_state_node));
     }
-  return e;
+  return node;
 }
 
 // Try to get the bit_range of REG within its base region
 bool
-xml_state::get_bit_range_within_base_region (const region &reg,
-					     bit_range &out)
+analyzer_state_graph::get_bit_range_within_base_region (const region &reg,
+							bit_range &out)
 {
   region_offset start_offset = reg.get_offset (&m_mgr);
   if (!start_offset.concrete_p ())
@@ -398,30 +495,28 @@ xml_state::get_bit_range_within_base_region (const region &reg,
 }
 
 void
-xml_state::populate_element_for_typed_region (xml::element &e,
-					      const region &reg,
-					      const concrete_bindings_t &conc_bindings,
-					      bool create_all)
+analyzer_state_graph::
+populate_state_node_for_typed_region (state_node_ref node,
+				      const region &reg,
+				      const concrete_bindings_t &conc_bindings,
+				      bool create_all)
 {
   const_tree reg_type = reg.get_type ();
   gcc_assert (reg_type);
-  set_type_attr (e, reg_type);
+  set_type_attr (node, reg_type);
 
   bit_range bits (0, 0);
   if (get_bit_range_within_base_region (reg, bits))
     {
-      set_bits_attr (e, bits);
+      set_bits_attr (node, bits);
 
       auto search = conc_bindings.find (bits);
       if (search != conc_bindings.end ())
 	{
 	  const svalue *bound_sval = search->second;
-	  if (auto svalue_element = create_element_for_svalue (bound_sval))
-	    {
-	      xml::printer xp (e);
-	      xp.push_tag ("value-of-region");
-	      xp.append (std::move (svalue_element));
-	    }
+	  node.set_json_attr ("value", bound_sval->to_json ());
+	  if (const region *dst_reg = bound_sval->maybe_get_region ())
+	      m_pending_edges.push_back ({node, *dst_reg});
 	}
     }
 
@@ -454,22 +549,23 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 	      = m_mgr.get_element_region (&reg,
 					  const_cast<tree> (element_type),
 					  sval_index);
-	    if (show_child_element_for_child_region_p (*child_reg,
+	    if (show_child_state_node_for_child_region_p (*child_reg,
 						       conc_bindings,
 						       create_all))
 	      {
-		// Here "element" is in the xml sense
-		auto child_element
-		  = std::make_unique<xml::element> ("element", false);
-		set_wi_attr (*child_element, "index", idx, UNSIGNED);
-		set_region_id_attr (*child_element, *child_reg);
+		auto child_state_node
+		  = make_state_node
+		      (diagnostics::state_graphs::node_kind::element,
+		       make_node_id (*child_reg));
+		set_wi_attr (*child_state_node, "index", idx, UNSIGNED);
+
 		// Recurse:
 		gcc_assert (element_type);
-		populate_element_for_typed_region (*child_element,
-						   *child_reg,
-						   conc_bindings,
-						   create_all);
-		e.add_child (std::move (child_element));
+		populate_state_node_for_typed_region (*child_state_node,
+						      *child_reg,
+						      conc_bindings,
+						      create_all);
+		node.m_node.add_child (std::move (child_state_node));
 	      }
 	  }
       }
@@ -485,15 +581,17 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 		const bit_range bits (0, item.m_bit_range.m_size_in_bits);
 		const region *child_reg
 		  = m_mgr.get_bit_range (&reg, NULL_TREE, bits);
-		if (show_child_element_for_child_region_p (*child_reg,
-							   conc_bindings,
-							   create_all))
+		if (show_child_state_node_for_child_region_p (*child_reg,
+							      conc_bindings,
+							      create_all))
 		  {
-		    auto child_element
-		      = std::make_unique<xml::element> ("padding", false);
-		    set_wi_attr (*child_element, "num_bits",
+		    auto child_state_node
+		      = make_state_node
+			  (diagnostics::state_graphs::node_kind::padding,
+			   make_node_id (*child_reg));
+		    set_wi_attr (*child_state_node, "num_bits",
 				 item.m_bit_range.m_size_in_bits, SIGNED);
-		    e.add_child (std::move (child_element));
+		    node.m_node.add_child (std::move (child_state_node));
 		  }
 	      }
 	    else
@@ -501,26 +599,28 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 		const region *child_reg
 		  = m_mgr.get_field_region (&reg,
 					    const_cast<tree> (item.m_field));
-		if (show_child_element_for_child_region_p (*child_reg,
+		if (show_child_state_node_for_child_region_p (*child_reg,
 							   conc_bindings,
 							   create_all))
 		  {
-		    auto child_element
-		      = std::make_unique<xml::element> ("field", false);
+		    auto child_state_node
+		      = make_state_node
+			  (diagnostics::state_graphs::node_kind::field,
+			   make_node_id (*child_reg));
 		    {
 		      pretty_printer pp;
 		      pp_format_decoder (&pp) = default_tree_printer;
 		      pp_printf (&pp, "%D", item.m_field);
-		      child_element->set_attr ("name",
-					       pp_formatted_text (&pp));
+		      child_state_node->set_attr (STATE_NODE_PREFIX, "name",
+						  pp_formatted_text (&pp));
 		    }
-		    set_region_id_attr (*child_element, *child_reg);
+
 		    // Recurse:
-		    populate_element_for_typed_region (*child_element,
+		    populate_state_node_for_typed_region (*child_state_node,
 						       *child_reg,
 						       conc_bindings,
 						       create_all);
-		    e.add_child (std::move (child_element));
+		    node.m_node.add_child (std::move (child_state_node));
 		  }
 	      }
 	  }
@@ -530,7 +630,8 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 }
 
 void
-xml_state::set_attr_for_dynamic_extents (const region &reg, xml::element &e)
+analyzer_state_graph::set_attr_for_dynamic_extents (const region &reg,
+						    state_node_ref node_ref)
 {
   const svalue *sval = m_state.m_region_model->get_dynamic_extents (&reg);
   if (sval)
@@ -541,13 +642,13 @@ xml_state::set_attr_for_dynamic_extents (const region &reg, xml::element &e)
 	pp_wide_int (&pp, wi::to_wide (cst), UNSIGNED);
       else
 	sval->dump_to_pp (&pp, true);
-      e.set_attr ("dynamic-extents", pp_formatted_text (&pp));
+      node_ref.set_attr ("dynamic-extents", pp_formatted_text (&pp));
     }
 }
 
 bool
-xml_state::
-show_child_element_for_child_region_p (const region &reg,
+analyzer_state_graph::
+show_child_state_node_for_child_region_p (const region &reg,
 				       const concrete_bindings_t &conc_bindings,
 				       bool create_all)
 {
@@ -569,216 +670,18 @@ show_child_element_for_child_region_p (const region &reg,
   return false;
 }
 
-std::unique_ptr<xml::element>
-xml_state::create_element_for_svalue (const svalue *sval)
+std::unique_ptr<diagnostics::digraphs::digraph>
+program_state::
+make_diagnostic_state_graph (const extrinsic_state &ext_state) const
 {
-  if (!sval)
-    return nullptr;
-
-  std::unique_ptr<xml::element> result;
-  switch (sval->get_kind ())
-    {
-    default:
-      gcc_unreachable ();
-    case SK_REGION:
-      {
-	const region_svalue *region_sval = (const region_svalue *)sval;
-	result
-	  = std::make_unique<xml::element> ("pointer-to-region", false);
-	set_region_id_attr (*result, *region_sval->get_pointee ());
-      }
-      break;
-    case SK_CONSTANT:
-      {
-	const constant_svalue *constant_sval = (const constant_svalue *)sval;
-	result = std::make_unique<xml::element> ("constant", false);
-	pretty_printer pp;
-	pp_format_decoder (&pp) = default_tree_printer;
-	pp_printf (&pp, "%E", constant_sval->get_constant ());
-	result->set_attr ("value", pp_formatted_text (&pp));
-      }
-      break;
-    case SK_UNKNOWN:
-      result = std::make_unique<xml::element> ("unknown", false);
-      break;
-    case SK_POISONED:
-      {
-	const poisoned_svalue *poisoned_sval = (const poisoned_svalue *)sval;
-	switch (poisoned_sval->get_poison_kind ())
-	  {
-	  default:
-	    gcc_unreachable ();
-	  case poison_kind::uninit:
-	    result = std::make_unique<xml::element> ("uninitialized", false);
-	    break;
-	  case poison_kind::freed:
-	    result = std::make_unique<xml::element> ("freed", false);
-	    break;
-	  case poison_kind::deleted:
-	    result = std::make_unique<xml::element> ("deleted", false);
-	    break;
-	  case poison_kind::popped_stack:
-	    result = std::make_unique<xml::element> ("popped-stack", false);
-	    break;
-	  }
-      }
-      break;
-    case SK_SETJMP:
-      {
-	//const setjmp_svalue *setjmp_sval = (const setjmp_svalue *)sval;
-	result = std::make_unique<xml::element> ("setjmp-buffer", false);
-	// TODO
-      }
-      break;
-    case SK_INITIAL:
-      {
-	const initial_svalue *initial_sval = (const initial_svalue *)sval;
-	result = std::make_unique<xml::element> ("initial-value-of", false);
-	set_region_id_attr (*result, *initial_sval->get_region ());
-      }
-      break;
-    case SK_UNARYOP:
-      {
-	const unaryop_svalue *unaryop_sval = (const unaryop_svalue *)sval;
-	result = std::make_unique<xml::element> ("unary-op", false);
-	result->set_attr ("op", get_tree_code_name (unaryop_sval->get_op ()));
-	result->add_child
-	  (create_element_for_svalue (unaryop_sval->get_arg ()));
-      }
-      break;
-    case SK_BINOP:
-      {
-	const binop_svalue *binop_sval = (const binop_svalue *)sval;
-	result = std::make_unique<xml::element> ("binary-op", false);
-	result->set_attr ("op", get_tree_code_name (binop_sval->get_op ()));
-	result->add_child (create_element_for_svalue (binop_sval->get_arg0 ()));
-	result->add_child (create_element_for_svalue (binop_sval->get_arg1 ()));
-      }
-      break;
-    case SK_SUB:
-      {
-	//const sub_svalue *sub_sval = (const sub_svalue *)sval;
-	result = std::make_unique<xml::element> ("subregion-value", false);
-	// TODO
-      }
-      break;
-    case SK_REPEATED:
-      {
-	const repeated_svalue *repeated_sval = (const repeated_svalue *)sval;
-	result = std::make_unique<xml::element> ("repeated-value", false);
-	result->add_child
-	  (create_element_for_svalue (repeated_sval->get_outer_size ()));
-	result->add_child
-	  (create_element_for_svalue (repeated_sval->get_inner_svalue ()));
-      }
-      break;
-    case SK_BITS_WITHIN:
-      {
-	const bits_within_svalue *bits_within_sval
-	  = (const bits_within_svalue *)sval;
-	result = std::make_unique<xml::element> ("bits-within", false);
-	set_bits_attr (*result, bits_within_sval->get_bits ());
-	result->add_child
-	  (create_element_for_svalue (bits_within_sval->get_inner_svalue ()));
-      }
-      break;
-    case SK_UNMERGEABLE:
-      {
-	const unmergeable_svalue *unmergeable_sval
-	  = (const unmergeable_svalue *)sval;
-	result = std::make_unique<xml::element> ("unmergeable", false);
-	result->add_child
-	  (create_element_for_svalue (unmergeable_sval->get_arg ()));
-      }
-      break;
-    case SK_PLACEHOLDER:
-      {
-	const placeholder_svalue *placeholder_sval
-	  = (const placeholder_svalue *)sval;
-	result = std::make_unique<xml::element> ("placeholder", false);
-	result->set_attr ("name", placeholder_sval->get_name ());
-      }
-      break;
-    case SK_WIDENING:
-      {
-	//const widening_svalue *widening_sval = (const widening_svalue *)sval;
-	result = std::make_unique<xml::element> ("iterating-value", false);
-	// TODO
-      }
-      break;
-    case SK_COMPOUND:
-      {
-	//const compound_svalue *compound_sval = (const compound_svalue *)sval;
-	result = std::make_unique<xml::element> ("compound-value", false);
-	// TODO
-      }
-      break;
-    case SK_CONJURED:
-      {
-	//const conjured_svalue *conjured_sval = (const conjured_svalue *)sval;
-	result = std::make_unique<xml::element> ("conjured-value", false);
-	// TODO
-      }
-      break;
-    case SK_ASM_OUTPUT:
-      {
-	/* const asm_output_svalue *asm_output_sval
-	   = (const asm_output_svalue *)sval; */
-	result = std::make_unique<xml::element> ("asm-output", false);
-	// TODO
-      }
-      break;
-    case SK_CONST_FN_RESULT:
-      {
-	/* const const_fn_result_svalue *const_fn_result_sval
-	   = (const const_fn_result_svalue *)sval; */
-	result = std::make_unique<xml::element> ("const-fn-result", false);
-	// TODO
-      }
-    }
-
-  if (result)
-    {
-      if (sval->get_type ())
-	set_type_attr (*result, sval->get_type ());
-
-      pretty_printer pp;
-      pp_format_decoder (&pp) = default_tree_printer;
-      sval->dump_to_pp (&pp, true);
-      result->set_attr ("dump-text", pp_formatted_text (&pp));
-    }
-
-  return result;
-}
-
-std::unique_ptr<xml::document>
-program_state::make_xml (const extrinsic_state &ext_state) const
-{
-  return std::make_unique<xml_state> (*this, ext_state);
-}
-
-void
-program_state::dump_xml_to_pp (const extrinsic_state &ext_state,
-			       pretty_printer *pp) const
-{
-  auto doc = make_xml (ext_state);
-  doc->write_as_xml (pp, 0, true);
-}
-
-void
-program_state::dump_xml_to_file (const extrinsic_state &ext_state,
-				 FILE *outf) const
-{
-  pretty_printer pp;
-  pp.set_output_stream (outf);
-  dump_xml_to_pp (ext_state, &pp);
-  pp_flush (&pp);
+  return std::make_unique<analyzer_state_graph> (*this, ext_state);
 }
 
 void
-program_state::dump_xml (const extrinsic_state &ext_state) const
+program_state::dump_sarif (const extrinsic_state &ext_state) const
 {
-  dump_xml_to_file (ext_state, stderr);
+  auto g = make_diagnostic_state_graph (ext_state);
+  g->dump ();
 }
 
 } // namespace ana
diff --git a/gcc/analyzer/ana-state-to-diagnostic-state.h b/gcc/analyzer/ana-state-to-diagnostic-state.h
index bd6aa46..3a5ccc1 100644
--- a/gcc/analyzer/ana-state-to-diagnostic-state.h
+++ b/gcc/analyzer/ana-state-to-diagnostic-state.h
@@ -1,4 +1,4 @@
-/* XML documents for dumping state in an easier-to-read form.
+/* Creating diagnostic state graphs from ana::program_state.
    Copyright (C) 2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -18,41 +18,52 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_ANALYZER_ANA_STATE_TO_XML_STATE_H
-#define GCC_ANALYZER_ANA_STATE_TO_XML_STATE_H
+#ifndef GCC_ANALYZER_ANA_STATE_TO_DIAGNOSTIC_STATE_H
+#define GCC_ANALYZER_ANA_STATE_TO_DIAGNOSTIC_STATE_H
 
-#include "xml.h"
+#include "diagnostics/state-graphs.h"
+#include "tree-logical-location.h"
 
 namespace ana {
 
-class xml_state : public xml::document
+class analyzer_state_graph : public diagnostics::digraphs::digraph
 {
 public:
-  xml_state (const program_state &state,
-	     const extrinsic_state &ext_state);
-
-  xml::element &
-  get_or_create_element (const region &reg);
+  analyzer_state_graph (const program_state &state,
+			const extrinsic_state &ext_state);
+  diagnostics::state_graphs::state_node_ref
+  get_or_create_state_node (const region &reg);
 
 private:
-  xml::element&
-  create_and_add_element (const region &reg);
-
-  static std::unique_ptr<xml::element>
-  make_memory_space_element (const char *label);
-
-  std::unique_ptr<xml::element>
-  create_element (const region &reg);
+  struct pending_edge
+  {
+    diagnostics::state_graphs::state_node_ref m_src_node;
+    const region &m_dst_reg;
+  };
+  
+  diagnostics::state_graphs::state_node_ref
+  create_and_add_state_node (const region &reg);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  make_state_node (diagnostics::state_graphs::node_kind kind,
+		   std::string id);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  make_memspace_state_node (const region &reg,
+			    enum diagnostics::state_graphs::node_kind kind);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  create_state_node (const region &reg);
 
   /* Spatially sorted concrete bindings.  */
   typedef std::map<bit_range, const svalue *> concrete_bindings_t;
 
   void
-  create_elements_for_binding_cluster (const binding_cluster &cluster,
-				       bool create_all);
+  create_state_nodes_for_binding_cluster (const binding_cluster &cluster,
+					  bool create_all);
 
-  std::unique_ptr<xml::element>
-  create_element_for_conc_bindings (const concrete_bindings_t &conc_bindings);
+  std::unique_ptr<diagnostics::digraphs::node>
+  create_state_node_for_conc_bindings (const concrete_bindings_t &conc_bindings);
 
   // Try to get the bit_range of REG within its base region
   bool
@@ -60,30 +71,36 @@ private:
 				    bit_range &out);
 
   void
-  populate_element_for_typed_region (xml::element &e,
-				     const region &reg,
-				     const concrete_bindings_t &conc_bindings,
-				     bool create_all);
+  populate_state_node_for_typed_region (diagnostics::state_graphs::state_node_ref,
+					const region &reg,
+					const concrete_bindings_t &conc_bindings,
+					bool create_all);
 
   void
-  set_attr_for_dynamic_extents (const region &reg, xml::element &e);
+  set_attr_for_dynamic_extents (const region &reg,
+				diagnostics::state_graphs::state_node_ref);
 
   bool
-  show_child_element_for_child_region_p (const region &reg,
-					 const concrete_bindings_t &conc_bindings,
-					 bool create_all);
+  show_child_state_node_for_child_region_p (const region &reg,
+					    const concrete_bindings_t &conc_bindings,
+					    bool create_all);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  create_state_node_for_svalue (const svalue *sval);
 
-  std::unique_ptr<xml::element>
-  create_element_for_svalue (const svalue *sval);
+  std::string make_node_id (const region &reg);
+  std::string make_node_id (const char *prefix);
 
+  tree_logical_location_manager m_logical_loc_mgr;
   const program_state &m_state;
   const extrinsic_state &m_ext_state;
   region_model_manager &m_mgr;
-  xml::element *m_root;
-  std::map<const region *, xml::element *> m_region_to_element_map;
+  std::map<const region *, diagnostics::digraphs::node *> m_region_to_state_node_map;
   std::map<const region *, tree> m_types_for_untyped_regions;
+  unsigned m_next_id;
+  std::vector<pending_edge> m_pending_edges;
 };
 
 } // namespace ana
 
-#endif /* GCC_ANALYZER_ANA_STATE_TO_XML_STATE_H */
+#endif /* GCC_ANALYZER_ANA_STATE_TO_DIAGNOSTIC_STATE_H */
diff --git a/gcc/analyzer/analyzer.cc b/gcc/analyzer/analyzer.cc
index 938ad6d..9f6f7b4 100644
--- a/gcc/analyzer/analyzer.cc
+++ b/gcc/analyzer/analyzer.cc
@@ -21,7 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/common.h"
 
 #include "tree-pretty-print.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "tree-dfa.h"
 #include "intl.h"
 
@@ -232,7 +232,7 @@ tree_to_json (tree node)
    for unknown).  */
 
 std::unique_ptr<json::value>
-diagnostic_event_id_to_json (const diagnostic_event_id_t &event_id)
+diagnostic_event_id_to_json (const diagnostics::paths::event_id_t &event_id)
 {
   if (event_id.known_p ())
     {
diff --git a/gcc/analyzer/bounds-checking.cc b/gcc/analyzer/bounds-checking.cc
index d2e2b34..921ad16 100644
--- a/gcc/analyzer/bounds-checking.cc
+++ b/gcc/analyzer/bounds-checking.cc
@@ -20,8 +20,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/common.h"
 
 #include "intl.h"
-#include "diagnostic-diagram.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/diagram.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/region-model.h"
@@ -51,7 +51,7 @@ public:
     }
     void prepare_for_emission (checker_path *path,
 			       pending_diagnostic *pd,
-			       diagnostic_event_id_t emission_id) override
+			       diagnostics::paths::event_id_t emission_id) override
     {
       region_creation_event_capacity::prepare_for_emission (path,
 							    pd,
@@ -101,10 +101,11 @@ public:
 							       *this));
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/out_of_bounds/"
     props.set_string (PROPERTY_PREFIX "dir",
 		      get_dir () == access_direction::read ? "read" : "write");
@@ -181,7 +182,7 @@ protected:
 	       a problem.  Give up if that's happened.  */
 	    return;
 	  }
-	diagnostic_diagram diagram
+	diagnostics::diagram diagram
 	  (canvas,
 	   /* Alt text.  */
 	   _("Diagram visualizing the predicted out-of-bounds access"));
@@ -203,7 +204,7 @@ protected:
   const region *m_reg;
   tree m_diag_arg;
   const svalue *m_sval_hint;
-  diagnostic_event_id_t m_region_creation_event_id;
+  diagnostics::paths::event_id_t m_region_creation_event_id;
 };
 
 /* Abstract base class for all out-of-bounds warnings where the
@@ -228,11 +229,11 @@ public:
 	    && m_out_of_bounds_bits == other.m_out_of_bounds_bits);
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const override
   {
     out_of_bounds::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/concrete_out_of_bounds/"
     props.set (PROPERTY_PREFIX "out_of_bounds_bits",
 	       m_out_of_bounds_bits.to_json ());
@@ -294,11 +295,11 @@ public:
 							       *this));
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     concrete_out_of_bounds::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/concrete_past_the_end/"
     props.set (PROPERTY_PREFIX "bit_bound",
 	       tree_to_json (m_bit_bound));
@@ -966,11 +967,12 @@ public:
 	    && pending_diagnostic::same_tree_p (m_capacity, other.m_capacity));
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     out_of_bounds::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/symbolic_past_the_end/"
     props.set (PROPERTY_PREFIX "offset", tree_to_json (m_offset));
     props.set (PROPERTY_PREFIX "num_bytes", tree_to_json (m_num_bytes));
diff --git a/gcc/analyzer/call-details.cc b/gcc/analyzer/call-details.cc
index cb95843..ede1229 100644
--- a/gcc/analyzer/call-details.cc
+++ b/gcc/analyzer/call-details.cc
@@ -25,7 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-pretty-print.h"
 #include "stringpool.h"
 #include "attribs.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/region-model.h"
@@ -528,10 +528,10 @@ public:
     return true;
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/overlapping_buffers/"
     props.set (PROPERTY_PREFIX "bytes_range_a",
 	       m_byte_range_a.to_json ());
diff --git a/gcc/analyzer/call-info.cc b/gcc/analyzer/call-info.cc
index 9a698ef..f431143 100644
--- a/gcc/analyzer/call-info.cc
+++ b/gcc/analyzer/call-info.cc
@@ -24,7 +24,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cfg.h"
 #include "digraph.h"
 #include "sbitmap.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/supergraph.h"
diff --git a/gcc/analyzer/checker-event.cc b/gcc/analyzer/checker-event.cc
index af336df..4eac945 100644
--- a/gcc/analyzer/checker-event.cc
+++ b/gcc/analyzer/checker-event.cc
@@ -1,4 +1,4 @@
-/* Subclasses of diagnostic_event for analyzer diagnostics.
+/* Subclasses of diagnostics::paths::event for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -27,8 +27,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-iterator.h"
 #include "inlining-iterator.h"
 #include "tree-logical-location.h"
-#include "diagnostic-format-sarif.h"
-#include "xml.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/state-graphs.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
@@ -101,7 +101,7 @@ event_kind_to_string (enum event_kind ek)
     }
 }
 
-/* class checker_event : public diagnostic_event.  */
+/* class checker_event : public diagnostics::paths::event.  */
 
 /* checker_event's ctor.  */
 
@@ -131,24 +131,24 @@ checker_event::checker_event (enum event_kind kind,
     }
 }
 
-/* No-op implementation of diagnostic_event::get_meaning vfunc for
+/* No-op implementation of diagnostics::paths::event::get_meaning vfunc for
    checker_event: checker events have no meaning by default.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 checker_event::get_meaning () const
 {
-  return meaning ();
+  return diagnostics::paths::event::meaning ();
 }
 
-/* Implementation of diagnostic_event::maybe_add_sarif_properties
+/* Implementation of diagnostics::paths::event::maybe_add_sarif_properties
    for checker_event.  */
 
 void
 checker_event::
-maybe_add_sarif_properties (sarif_builder &builder,
-			    sarif_object &thread_flow_loc_obj) const
+maybe_add_sarif_properties (diagnostics::sarif_builder &builder,
+			    diagnostics::sarif_object &thread_flow_loc_obj) const
 {
-  sarif_property_bag &props = thread_flow_loc_obj.get_or_create_properties ();
+  auto &props = thread_flow_loc_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/checker_event/"
   props.set (PROPERTY_PREFIX "emission_id",
 	     diagnostic_event_id_to_json  (m_emission_id));
@@ -215,7 +215,7 @@ checker_event::debug () const
 void
 checker_event::prepare_for_emission (checker_path *path,
 				     pending_diagnostic *pd,
-				     diagnostic_event_id_t emission_id)
+				     diagnostics::paths::event_id_t emission_id)
 {
   m_path = path;
   m_pending_diagnostic = pd;
@@ -225,8 +225,8 @@ checker_event::prepare_for_emission (checker_path *path,
   print_desc (*pp.get ());
 }
 
-std::unique_ptr<xml::document>
-checker_event::maybe_make_xml_state (bool debug) const
+std::unique_ptr<diagnostics::digraphs::digraph>
+checker_event::maybe_make_diagnostic_state_graph (bool debug) const
 {
   const program_state *state = get_program_state ();
   if (!state)
@@ -235,14 +235,16 @@ checker_event::maybe_make_xml_state (bool debug) const
   gcc_assert (m_path);
   const extrinsic_state &ext_state = m_path->get_ext_state ();
 
-  auto result = state->make_xml (ext_state);
+  auto result = state->make_diagnostic_state_graph (ext_state);
 
   if (debug)
     {
       pretty_printer pp;
       text_art::theme *theme = global_dc->get_diagram_theme ();
       text_art::dump_to_pp (*state, theme, &pp);
-      result->add_comment (pp_formatted_text (&pp));
+      result->set_attr (STATE_GRAPH_PREFIX,
+			"analyzer/program_state/",
+			pp_formatted_text (&pp));
     }
 
   return result;
@@ -250,7 +252,7 @@ checker_event::maybe_make_xml_state (bool debug) const
 
 /* class debug_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    debug_event.
    Use the saved string as the event's description.  */
 
@@ -262,7 +264,7 @@ debug_event::print_desc (pretty_printer &pp) const
 
 /* class precanned_custom_event : public custom_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    precanned_custom_event.
    Use the saved string as the event's description.  */
 
@@ -285,7 +287,7 @@ statement_event::statement_event (const gimple *stmt, tree fndecl, int depth,
 {
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    statement_event.
    Use the statement's dump form as the event's description.  */
 
@@ -381,7 +383,7 @@ function_entry_event::function_entry_event (const program_point &dst_point,
 {
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    function_entry_event.
 
    Use a string such as "entry to 'foo'" as the event's description.  */
@@ -392,10 +394,10 @@ function_entry_event::print_desc (pretty_printer &pp) const
   pp_printf (&pp, "entry to %qE", m_effective_fndecl);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    function entry.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 function_entry_event::get_meaning () const
 {
   return meaning (verb::enter, noun::function);
@@ -427,7 +429,7 @@ state_change_event::state_change_event (const supernode *node,
 {
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    state_change_event.
 
    Attempt to generate a nicer human-readable description.
@@ -469,7 +471,7 @@ state_change_event::print_desc (pretty_printer &pp) const
 		pp_string (&pp, "NULL origin");
 
 	      /* Get any "meaning" of event.  */
-	      diagnostic_event::meaning meaning = get_meaning ();
+	      diagnostics::paths::event::meaning meaning = get_meaning ();
 	      pp_string (&pp, ", meaning: ");
 	      meaning.dump_to_pp (&pp);
 	      pp_string (&pp, ")");
@@ -506,11 +508,11 @@ state_change_event::print_desc (pretty_printer &pp) const
     }
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    state change events: delegate to the pending_diagnostic to
    get any meaning.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 state_change_event::get_meaning () const
 {
   if (m_pending_diagnostic)
@@ -528,16 +530,17 @@ state_change_event::get_meaning () const
 
 /* class superedge_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::maybe_add_sarif_properties
+/* Implementation of diagnostics::paths::event::maybe_add_sarif_properties
    for superedge_event.  */
 
 void
-superedge_event::maybe_add_sarif_properties (sarif_builder &builder,
-					     sarif_object &thread_flow_loc_obj)
+superedge_event::
+maybe_add_sarif_properties (diagnostics::sarif_builder &builder,
+			    diagnostics::sarif_object &thread_flow_loc_obj)
   const
 {
   checker_event::maybe_add_sarif_properties (builder, thread_flow_loc_obj);
-  sarif_property_bag &props = thread_flow_loc_obj.get_or_create_properties ();
+  auto &props = thread_flow_loc_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/superedge_event/"
   if (m_sedge)
     props.set (PROPERTY_PREFIX "superedge", m_sedge->to_json ());
@@ -624,10 +627,10 @@ cfg_edge_event::cfg_edge_event (enum event_kind kind,
   gcc_assert (eedge.m_sedge->m_kind == SUPEREDGE_CFG_EDGE);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    CFG edge events.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 cfg_edge_event::get_meaning () const
 {
   const cfg_superedge& cfg_sedge = get_cfg_superedge ();
@@ -641,7 +644,7 @@ cfg_edge_event::get_meaning () const
 
 /* class start_cfg_edge_event : public cfg_edge_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    start_cfg_edge_event.
 
    If -fanalyzer-verbose-edges, then generate low-level descriptions, such
@@ -842,7 +845,7 @@ call_event::call_event (const exploded_edge &eedge,
    m_dest_snode = eedge.m_dest->get_supernode ();
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    call_event.
 
    If this call event passes critical state for an sm-based warning,
@@ -874,10 +877,10 @@ call_event::print_desc (pretty_printer &pp) const
 	     get_caller_fndecl ());
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    function call events.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 call_event::get_meaning () const
 {
   return meaning (verb::call, noun::function);
@@ -926,7 +929,7 @@ return_event::return_event (const exploded_edge &eedge,
   m_dest_snode = eedge.m_dest->get_supernode ();
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    return_event.
 
    If this return event returns critical state for an sm-based warning,
@@ -958,10 +961,10 @@ return_event::print_desc (pretty_printer &pp) const
 	     m_src_snode->m_fun->decl);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    function return events.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 return_event::get_meaning () const
 {
   return meaning (verb::return_, noun::function);
@@ -985,10 +988,10 @@ start_consolidated_cfg_edges_event::print_desc (pretty_printer &pp) const
 	     m_edge_sense ? "true" : "false");
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    start_consolidated_cfg_edges_event.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 start_consolidated_cfg_edges_event::get_meaning () const
 {
   return meaning (verb::branch,
@@ -1006,10 +1009,10 @@ inlined_call_event::print_desc (pretty_printer &pp) const
 	     m_apparent_caller_fndecl);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    reconstructed inlined function calls.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 inlined_call_event::get_meaning () const
 {
   return meaning (verb::call, noun::function);
@@ -1017,7 +1020,7 @@ inlined_call_event::get_meaning () const
 
 /* class setjmp_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    setjmp_event.  */
 
 void
@@ -1036,7 +1039,7 @@ setjmp_event::print_desc (pretty_printer &pp) const
 void
 setjmp_event::prepare_for_emission (checker_path *path,
 				    pending_diagnostic *pd,
-				    diagnostic_event_id_t emission_id)
+				    diagnostics::paths::event_id_t emission_id)
 {
   checker_event::prepare_for_emission (path, pd, emission_id);
   path->record_setjmp_event (m_enode, emission_id);
@@ -1075,7 +1078,7 @@ rewind_event::rewind_event (const exploded_edge *eedge,
 
 /* class rewind_from_longjmp_event : public rewind_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    rewind_from_longjmp_event.  */
 
 void
@@ -1099,7 +1102,7 @@ rewind_from_longjmp_event::print_desc (pretty_printer &pp) const
 
 /* class rewind_to_setjmp_event : public rewind_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    rewind_to_setjmp_event.  */
 
 void
@@ -1148,7 +1151,7 @@ rewind_to_setjmp_event::print_desc (pretty_printer &pp) const
 void
 rewind_to_setjmp_event::prepare_for_emission (checker_path *path,
 					      pending_diagnostic *pd,
-					      diagnostic_event_id_t emission_id)
+					      diagnostics::paths::event_id_t emission_id)
 {
   checker_event::prepare_for_emission (path, pd, emission_id);
   path->get_setjmp_event (m_rewind_info->get_enode_origin (),
@@ -1201,7 +1204,7 @@ unwind_event::print_desc (pretty_printer &pp) const
 
 /* class warning_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    warning_event.
 
    If the pending diagnostic implements describe_final_event, use it,
@@ -1246,10 +1249,10 @@ warning_event::print_desc (pretty_printer &pp) const
     pp_string (&pp, "here");
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    warning_event.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 warning_event::get_meaning () const
 {
   return meaning (verb::danger, noun::unknown);
diff --git a/gcc/analyzer/checker-event.h b/gcc/analyzer/checker-event.h
index 7c44f1e..909e388 100644
--- a/gcc/analyzer/checker-event.h
+++ b/gcc/analyzer/checker-event.h
@@ -1,4 +1,4 @@
-/* Subclasses of diagnostic_event for analyzer diagnostics.
+/* Subclasses of diagnostics::paths::event for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -24,6 +24,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-logical-location.h"
 #include "analyzer/program-state.h"
 #include "analyzer/event-loc-info.h"
+#include "diagnostics/digraphs.h"
 
 namespace ana {
 
@@ -61,7 +62,7 @@ extern const char *event_kind_to_string (enum event_kind ek);
    The class hierarchy looks like this (using indentation to show
    inheritance, and with event_kinds shown for the concrete subclasses):
 
-   diagnostic_event
+   diagnostics::paths::event
      checker_event
        debug_event (event_kind::debug)
        custom_event (event_kind::custom)
@@ -90,30 +91,32 @@ extern const char *event_kind_to_string (enum event_kind ek);
        unwind_event (event_kind::unwind)
        warning_event (event_kind::warning).  */
 
-/* Abstract subclass of diagnostic_event; the base class for use in
-   checker_path (the analyzer's diagnostic_path subclass).  */
+/* Abstract subclass of diagnostics::paths::event; the base class for use in
+   checker_path (the analyzer's diagnostics::paths::path subclass).  */
 
-class checker_event : public diagnostic_event
+class checker_event : public diagnostics::paths::event
 {
 public:
-  /* Implementation of diagnostic_event.  */
+  /* Implementation of diagnostics::paths::event.  */
 
   location_t get_location () const final override { return m_loc; }
   int get_stack_depth () const final override { return m_effective_depth; }
-  logical_location get_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_logical_location () const final override
   {
     return m_logical_loc;
   }
   meaning get_meaning () const override;
   bool connect_to_next_event_p () const override { return false; }
-  diagnostic_thread_id_t get_thread_id () const final override
+  diagnostics::paths::thread_id_t get_thread_id () const final override
   {
     return 0;
   }
 
   void
-  maybe_add_sarif_properties (sarif_builder &,
-			      sarif_object &thread_flow_loc_obj) const override;
+  maybe_add_sarif_properties (diagnostics::sarif_builder &,
+			      diagnostics::sarif_object &thread_flow_loc_obj)
+    const override;
 
   /* Additional functionality.  */
   enum event_kind get_kind () const { return m_kind; }
@@ -123,19 +126,19 @@ public:
 
   virtual void prepare_for_emission (checker_path *,
 				     pending_diagnostic *pd,
-				     diagnostic_event_id_t emission_id);
+				     diagnostics::paths::event_id_t emission_id);
   virtual bool is_call_p () const { return false; }
   virtual bool is_function_entry_p () const  { return false; }
   virtual bool is_return_p () const  { return false; }
 
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  maybe_make_diagnostic_state_graph (bool debug) const final override;
+
   virtual const program_state *
   get_program_state () const { return nullptr; }
 
-  std::unique_ptr<xml::document>
-  maybe_make_xml_state (bool debug) const final override;
-
   /* For use with %@.  */
-  const diagnostic_event_id_t *get_id_ptr () const
+  const diagnostics::paths::event_id_t *get_id_ptr () const
   {
     return &m_emission_id;
   }
@@ -159,8 +162,8 @@ protected:
   int m_original_depth;
   int m_effective_depth;
   pending_diagnostic *m_pending_diagnostic;
-  diagnostic_event_id_t m_emission_id; // only set once all pruning has occurred
-  logical_location m_logical_loc;
+  diagnostics::paths::event_id_t m_emission_id; // only set once all pruning has occurred
+  diagnostics::logical_locations::key m_logical_loc;
 };
 
 /* A concrete event subclass for a purely textual event, for use in
@@ -421,8 +424,9 @@ public:
 class superedge_event : public checker_event
 {
 public:
-  void maybe_add_sarif_properties (sarif_builder &,
-				   sarif_object &thread_flow_loc_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_builder &,
+			      diagnostics::sarif_object &thread_flow_loc_obj)
     const override;
 
   /* Mark this edge event as being either an interprocedural call or
@@ -664,7 +668,7 @@ public:
 
   void prepare_for_emission (checker_path *path,
 			     pending_diagnostic *pd,
-			     diagnostic_event_id_t emission_id) final override;
+			     diagnostics::paths::event_id_t emission_id) final override;
 
 private:
   const exploded_node *m_enode;
@@ -730,10 +734,10 @@ public:
 
   void prepare_for_emission (checker_path *path,
 			     pending_diagnostic *pd,
-			     diagnostic_event_id_t emission_id) final override;
+			     diagnostics::paths::event_id_t emission_id) final override;
 
 private:
-  diagnostic_event_id_t m_original_setjmp_event_id;
+  diagnostics::paths::event_id_t m_original_setjmp_event_id;
 };
 
 /* An abstract subclass for throwing/rethrowing an exception.  */
diff --git a/gcc/analyzer/checker-path.cc b/gcc/analyzer/checker-path.cc
index 45593e0..646d72c 100644
--- a/gcc/analyzer/checker-path.cc
+++ b/gcc/analyzer/checker-path.cc
@@ -1,4 +1,4 @@
-/* Subclass of diagnostic_path for analyzer diagnostics.
+/* Subclass of diagnostics::paths::path for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
diff --git a/gcc/analyzer/checker-path.h b/gcc/analyzer/checker-path.h
index 3c174bf..490d21b 100644
--- a/gcc/analyzer/checker-path.h
+++ b/gcc/analyzer/checker-path.h
@@ -1,4 +1,4 @@
-/* Subclass of diagnostic_path for analyzer diagnostics.
+/* Subclass of diagnostics::paths::path for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -26,28 +26,29 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace ana {
 
-/* Subclass of diagnostic_path for analyzer diagnostics.  */
+/* Subclass of diagnostic path for analyzer diagnostics.  */
 
-class checker_path : public diagnostic_path
+class checker_path : public diagnostics::paths::path
 {
 public:
-  checker_path (const logical_location_manager &logical_loc_mgr,
+  checker_path (const diagnostics::logical_locations::manager &logical_loc_mgr,
 		const extrinsic_state &ext_state,
 		logger *logger)
-  : diagnostic_path (logical_loc_mgr),
+  : diagnostics::paths::path (logical_loc_mgr),
     m_ext_state (ext_state),
     m_thread ("main"),
     m_logger (logger)
   {}
 
-  /* Implementation of diagnostic_path vfuncs.  */
+  /* Implementation of diagnostics::paths::path vfuncs.  */
 
   unsigned num_events () const final override
   {
     return m_events.length ();
   }
 
-  const diagnostic_event & get_event (int idx) const final override
+  const diagnostics::paths::event &
+  get_event (int idx) const final override
   {
     return *m_events[idx];
   }
@@ -55,8 +56,8 @@ public:
   {
     return 1;
   }
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override
+  const diagnostics::paths::thread &
+  get_thread (diagnostics::paths::thread_id_t) const final override
   {
     return m_thread;
   }
@@ -115,21 +116,22 @@ public:
     checker_event *e;
     int i;
     FOR_EACH_VEC_ELT (m_events, i, e)
-      e->prepare_for_emission (this, pd, diagnostic_event_id_t (i));
+      e->prepare_for_emission (this, pd, diagnostics::paths::event_id_t (i));
   }
 
   void fixup_locations (pending_diagnostic *pd);
 
   void record_setjmp_event (const exploded_node *enode,
-			    diagnostic_event_id_t setjmp_emission_id)
+			    diagnostics::paths::event_id_t setjmp_emission_id)
   {
     m_setjmp_event_ids.put (enode, setjmp_emission_id);
   }
 
   bool get_setjmp_event (const exploded_node *enode,
-			 diagnostic_event_id_t *out_emission_id)
+			 diagnostics::paths::event_id_t *out_emission_id)
   {
-    if (diagnostic_event_id_t *emission_id = m_setjmp_event_ids.get (enode))
+    if (diagnostics::paths::event_id_t *emission_id
+	  = m_setjmp_event_ids.get (enode))
       {
 	*out_emission_id = *emission_id;
 	return true;
@@ -154,7 +156,7 @@ private:
   /* During prepare_for_emission (and after), the setjmp_event for each
      exploded_node *, so that rewind events can refer to them in their
      descriptions.  */
-  hash_map <const exploded_node *, diagnostic_event_id_t> m_setjmp_event_ids;
+  hash_map <const exploded_node *, diagnostics::paths::event_id_t> m_setjmp_event_ids;
 
   logger *m_logger;
 };
diff --git a/gcc/analyzer/common.h b/gcc/analyzer/common.h
index 148bfdd..ac3048c 100644
--- a/gcc/analyzer/common.h
+++ b/gcc/analyzer/common.h
@@ -34,7 +34,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "options.h"
 #include "bitmap.h"
 #include "diagnostic-core.h"
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 #include "rich-location.h"
 #include "function.h"
 #include "json.h"
@@ -443,7 +443,7 @@ extern std::unique_ptr<json::value>
 tree_to_json (tree node);
 
 extern std::unique_ptr<json::value>
-diagnostic_event_id_to_json (const diagnostic_event_id_t &);
+diagnostic_event_id_to_json (const diagnostics::paths::event_id_t &);
 
 extern std::unique_ptr<json::value>
 bit_offset_to_json (const bit_offset_t &offset);
diff --git a/gcc/analyzer/diagnostic-manager.cc b/gcc/analyzer/diagnostic-manager.cc
index c083b8c..88f72d1 100644
--- a/gcc/analyzer/diagnostic-manager.cc
+++ b/gcc/analyzer/diagnostic-manager.cc
@@ -29,7 +29,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cgraph.h"
 #include "digraph.h"
 #include "gcc-rich-location.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
@@ -1017,9 +1017,10 @@ saved_diagnostic::emit_any_notes () const
    This extra data is intended for use when debugging the analyzer.  */
 
 void
-saved_diagnostic::maybe_add_sarif_properties (sarif_object &result_obj) const
+saved_diagnostic::
+maybe_add_sarif_properties (diagnostics::sarif_object &result_obj) const
 {
-  sarif_property_bag &props = result_obj.get_or_create_properties ();
+  auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/saved_diagnostic/"
   if (m_sm)
     props.set_string (PROPERTY_PREFIX "sm", m_sm->get_name ());
@@ -1044,7 +1045,7 @@ saved_diagnostic::maybe_add_sarif_properties (sarif_object &result_obj) const
       auto duplicates_arr = std::make_unique<json::array> ();
       for (auto iter : m_duplicates)
 	{
-	  auto sd_obj = std::make_unique<sarif_object> ();
+	  auto sd_obj = std::make_unique<diagnostics::sarif_object> ();
 	  iter->maybe_add_sarif_properties (*sd_obj);
 	  duplicates_arr->append (std::move (sd_obj));
 	}
@@ -1541,12 +1542,12 @@ diagnostic_manager::emit_saved_diagnostics (const exploded_graph &eg)
   best_candidates.emit_best (this, eg);
 }
 
-/* Custom subclass of diagnostic_metadata which, for SARIF output,
+/* Custom subclass of diagnostics::metadata which, for SARIF output,
    populates the property bag of the diagnostic's "result" object
    with information from the saved_diagnostic and the
    pending_diagnostic.  */
 
-class pending_diagnostic_metadata : public diagnostic_metadata
+class pending_diagnostic_metadata : public diagnostics::metadata
 {
 public:
   pending_diagnostic_metadata (const saved_diagnostic &sd)
@@ -1555,7 +1556,8 @@ public:
   }
 
   void
-  maybe_add_sarif_properties (sarif_object &result_obj) const override
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
+    const override
   {
     m_sd.maybe_add_sarif_properties (result_obj);
   }
@@ -1583,7 +1585,7 @@ diagnostic_manager::emit_saved_diagnostic (const exploded_graph &eg,
   /* Precompute all enodes from which the diagnostic is reachable.  */
   path_builder pb (eg, *epath, sd.get_feasibility_problem (), sd);
 
-  /* This is the diagnostic_path subclass that will be built for
+  /* This is the diagnostics::paths::path subclass that will be built for
      the diagnostic.  */
   checker_path emission_path (get_logical_location_manager (),
 			      eg.get_ext_state (),
@@ -1663,7 +1665,7 @@ diagnostic_manager::emit_saved_diagnostic (const exploded_graph &eg,
     }
 }
 
-const logical_location_manager &
+const diagnostics::logical_locations::manager &
 diagnostic_manager::get_logical_location_manager () const
 {
   gcc_assert (global_dc);
diff --git a/gcc/analyzer/diagnostic-manager.h b/gcc/analyzer/diagnostic-manager.h
index aa0bd79..d9cf109 100644
--- a/gcc/analyzer/diagnostic-manager.h
+++ b/gcc/analyzer/diagnostic-manager.h
@@ -67,7 +67,8 @@ public:
 
   void emit_any_notes () const;
 
-  void maybe_add_sarif_properties (sarif_object &result_obj) const;
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj) const;
 
   //private:
   const state_machine *m_sm;
@@ -191,7 +192,7 @@ public:
   }
 
 private:
-  const logical_location_manager &
+  const diagnostics::logical_locations::manager &
   get_logical_location_manager () const;
 
   void build_emission_path (const path_builder &pb,
diff --git a/gcc/analyzer/engine.cc b/gcc/analyzer/engine.cc
index 67024e9..745ef7e 100644
--- a/gcc/analyzer/engine.cc
+++ b/gcc/analyzer/engine.cc
@@ -1580,9 +1580,9 @@ exploded_node::on_stmt_pre (exploded_graph &eg,
 	  state->dump (eg.get_ext_state (), true);
 	  return;
 	}
-      else if (is_special_named_call_p (call, "__analyzer_dump_xml", 0))
+      else if (is_special_named_call_p (call, "__analyzer_dump_sarif", 0))
 	{
-	  state->dump_xml (eg.get_ext_state ());
+	  state->dump_sarif (eg.get_ext_state ());
 	  return;
 	}
       else if (is_special_named_call_p (call, "__analyzer_dump_dot", 0))
diff --git a/gcc/analyzer/infinite-loop.cc b/gcc/analyzer/infinite-loop.cc
index ec0b079..a53807c 100644
--- a/gcc/analyzer/infinite-loop.cc
+++ b/gcc/analyzer/infinite-loop.cc
@@ -25,7 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-pretty-print.h"
 #include "cgraph.h"
 #include "digraph.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/call-string.h"
@@ -307,10 +307,11 @@ public:
       }
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/infinite_loop_diagnostic/"
     props.set (PROPERTY_PREFIX "inf_loop", m_inf_loop->to_json ());
 #undef PROPERTY_PREFIX
diff --git a/gcc/analyzer/infinite-recursion.cc b/gcc/analyzer/infinite-recursion.cc
index b80b94a..960b487 100644
--- a/gcc/analyzer/infinite-recursion.cc
+++ b/gcc/analyzer/infinite-recursion.cc
@@ -25,6 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-pretty-print.h"
 #include "cgraph.h"
 #include "digraph.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/call-string.h"
@@ -40,7 +41,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/exploded-graph.h"
 #include "analyzer/checker-path.h"
 #include "analyzer/feasible-graph.h"
-#include "diagnostic-format-sarif.h"
 
 /* A subclass of pending_diagnostic for complaining about suspected
    infinite recursion.  */
@@ -223,10 +223,11 @@ public:
     return false;
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/infinite_recursion_diagnostic/"
     props.set_integer (PROPERTY_PREFIX "prev_entry_enode",
 		       m_prev_entry_enode->m_index);
diff --git a/gcc/analyzer/kf.cc b/gcc/analyzer/kf.cc
index fe25520..2a7c357 100644
--- a/gcc/analyzer/kf.cc
+++ b/gcc/analyzer/kf.cc
@@ -783,7 +783,7 @@ public:
 
     /* SEI CERT C Coding Standard: "POS34-C. Do not call putenv() with a
        pointer to an automatic variable as the argument".  */
-    diagnostic_metadata::precanned_rule
+    diagnostics::metadata::precanned_rule
       rule ("POS34-C", "https://wiki.sei.cmu.edu/confluence/x/6NYxBQ");
     ctxt.add_rule (rule);
 
diff --git a/gcc/analyzer/pending-diagnostic.cc b/gcc/analyzer/pending-diagnostic.cc
index 5e95edd..cc2d795 100644
--- a/gcc/analyzer/pending-diagnostic.cc
+++ b/gcc/analyzer/pending-diagnostic.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "cpplib.h"
 #include "digraph.h"
 #include "ordered-hash-map.h"
@@ -92,7 +92,7 @@ diagnostic_emission_context::warn (const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  const bool result = emit_diagnostic_valist_meta (DK_WARNING,
+  const bool result = emit_diagnostic_valist_meta (diagnostics::kind::warning,
 						   &m_rich_loc, &m_metadata,
 						   pd.get_controlling_option (),
 						   gmsgid, &ap);
@@ -110,7 +110,7 @@ diagnostic_emission_context::inform (const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  emit_diagnostic_valist_meta (DK_NOTE,
+  emit_diagnostic_valist_meta (diagnostics::kind::note,
 			       &m_rich_loc, &m_metadata,
 			       pd.get_controlling_option (),
 			       gmsgid, &ap);
diff --git a/gcc/analyzer/pending-diagnostic.h b/gcc/analyzer/pending-diagnostic.h
index 469513c..b5d90a2 100644
--- a/gcc/analyzer/pending-diagnostic.h
+++ b/gcc/analyzer/pending-diagnostic.h
@@ -21,8 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_ANALYZER_PENDING_DIAGNOSTIC_H
 #define GCC_ANALYZER_PENDING_DIAGNOSTIC_H
 
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
+#include "diagnostics/metadata.h"
 #include "analyzer/sm.h"
 
 namespace ana {
@@ -44,7 +43,7 @@ struct interesting_t
 };
 
 /* Various bundles of information used for generating more precise
-   messages for events within a diagnostic_path, for passing to the
+   messages for events within a diagnostic path, for passing to the
    various "describe_*" vfuncs of pending_diagnostic.  See those
    for more information.  */
 
@@ -58,7 +57,7 @@ struct state_change
 		tree origin,
 		state_machine::state_t old_state,
 		state_machine::state_t new_state,
-		diagnostic_event_id_t event_id,
+		diagnostics::paths::event_id_t event_id,
 		const state_change_event &event)
   : m_expr (expr), m_origin (origin),
     m_old_state (old_state), m_new_state (new_state),
@@ -71,7 +70,7 @@ struct state_change
   tree m_origin;
   state_machine::state_t m_old_state;
   state_machine::state_t m_new_state;
-  diagnostic_event_id_t m_event_id;
+  diagnostics::paths::event_id_t m_event_id;
   const state_change_event &m_event;
 };
 
@@ -131,14 +130,14 @@ struct final_event
     pending_diagnostic::emit vfunc.
 
     The rich_location will have already been populated with a
-    diagnostic_path.  */
+    diagnostics::paths::path.  */
 
 class diagnostic_emission_context
 {
 public:
   diagnostic_emission_context (const saved_diagnostic &sd,
 			       rich_location &rich_loc,
-			       diagnostic_metadata &metadata,
+			       diagnostics::metadata &metadata,
 			       logger *logger)
   : m_sd (sd),
     m_rich_loc (rich_loc),
@@ -156,7 +155,7 @@ public:
   logger *get_logger () const { return m_logger; }
 
   void add_cwe (int cwe) { m_metadata.add_cwe (cwe); }
-  void add_rule (const diagnostic_metadata::rule &r)
+  void add_rule (const diagnostics::metadata::rule &r)
   {
     m_metadata.add_rule (r);
   }
@@ -164,7 +163,7 @@ public:
 private:
   const saved_diagnostic &m_sd;
   rich_location &m_rich_loc;
-  diagnostic_metadata &m_metadata;
+  diagnostics::metadata &m_metadata;
   logger *m_logger;
 };
 
@@ -182,7 +181,7 @@ private:
 
    As well as emitting a diagnostic, the class has various "precision of
    wording" virtual functions, for generating descriptions for events
-   within a diagnostic_path.  These are optional, but implementing these
+   within a diagnostic path.  These are optional, but implementing these
    allows for more precise wordings than the more generic
    implementation.  */
 
@@ -238,7 +237,7 @@ class pending_diagnostic
   virtual location_t fixup_location (location_t loc, bool primary) const;
 
   /* Precision-of-wording vfunc for describing a critical state change
-     within the diagnostic_path.
+     within the diagnostic path.
 
      For example, a double-free diagnostic might use the descriptions:
      - "first 'free' happens here"
@@ -260,13 +259,13 @@ class pending_diagnostic
     return false;
   }
 
-  /* Vfunc for implementing diagnostic_event::get_meaning for
+  /* Vfunc for implementing event::get_meaning for
      state_change_event.  */
-  virtual diagnostic_event::meaning
+  virtual diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &) const
   {
     /* Default no-op implementation.  */
-    return diagnostic_event::meaning ();
+    return diagnostics::paths::event::meaning ();
   }
 
   /* Precision-of-wording vfunc for describing an interprocedural call
@@ -285,7 +284,7 @@ class pending_diagnostic
   }
 
   /* Precision-of-wording vfunc for describing an interprocedural return
-     within the diagnostic_path that carries critial state for the
+     within the diagnostic path that carries critial state for the
      diagnostic, from callee back to caller.
 
      For example, a deref-of-unchecked-malloc diagnostic might use:
@@ -301,7 +300,7 @@ class pending_diagnostic
   }
 
   /* Precision-of-wording vfunc for describing the final event within a
-     diagnostic_path.
+     diagnostic path.
 
      For example a double-free diagnostic might use:
       - "second 'free' here; first 'free' was at (3)"
@@ -402,7 +401,8 @@ class pending_diagnostic
      the opportunity to add diagnostic-specific properties to the SARIF
      "result" object for the diagnostic.
      This is intended for use when debugging a diagnostic.  */
-  virtual void maybe_add_sarif_properties (sarif_object &/*result_obj*/) const
+  virtual void
+  maybe_add_sarif_properties (diagnostics::sarif_object &/*result_obj*/) const
   {
     /* Default no-op implementation.  */
   }
diff --git a/gcc/analyzer/program-point.cc b/gcc/analyzer/program-point.cc
index 1f82559..9baa007 100644
--- a/gcc/analyzer/program-point.cc
+++ b/gcc/analyzer/program-point.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "gcc-rich-location.h"
 #include "gimple-pretty-print.h"
 #include "sbitmap.h"
@@ -234,17 +234,18 @@ function_point::before_supernode (const supernode *supernode,
   return function_point (supernode, from_edge, 0, PK_BEFORE_SUPERNODE);
 }
 
-/* A subclass of diagnostic_context for use by
+/* A subclass of diagnostics::context for use by
    program_point::print_source_line.  */
 
-class debug_diagnostic_context : public diagnostic_context
+class debug_diagnostic_context : public diagnostics::context
 {
 public:
   debug_diagnostic_context ()
   {
     diagnostic_initialize (this, 0);
-    m_source_printing.show_line_numbers_p = true;
-    m_source_printing.enabled = true;
+    auto &source_printing_opts = get_source_printing_options ();
+    source_printing_opts.show_line_numbers_p = true;
+    source_printing_opts.enabled = true;
   }
   ~debug_diagnostic_context ()
   {
@@ -263,9 +264,9 @@ function_point::print_source_line (pretty_printer *pp) const
   // TODO: monospace font
   debug_diagnostic_context tmp_dc;
   gcc_rich_location richloc (stmt->location);
-  diagnostic_source_print_policy source_policy (tmp_dc);
+  diagnostics::source_print_policy source_policy (tmp_dc);
   gcc_assert (pp);
-  source_policy.print (*pp, richloc, DK_ERROR, nullptr);
+  source_policy.print (*pp, richloc, diagnostics::kind::error, nullptr);
   pp_string (pp, pp_formatted_text (tmp_dc.get_reference_printer ()));
 }
 
diff --git a/gcc/analyzer/program-state.cc b/gcc/analyzer/program-state.cc
index c0befac..e16a50f 100644
--- a/gcc/analyzer/program-state.cc
+++ b/gcc/analyzer/program-state.cc
@@ -27,8 +27,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-iterator.h"
 #include "cgraph.h"
 #include "digraph.h"
-#include "diagnostic-event-id.h"
-#include "diagnostic-state.h"
+#include "diagnostics/event-id.h"
+#include "diagnostics/state-graphs.h"
 #include "graphviz.h"
 
 #include "text-art/tree-widget.h"
@@ -1230,8 +1230,14 @@ program_state::make_dump_widget (const text_art::dump_widget_info &dwi) const
 void
 program_state::dump_dot (const extrinsic_state &ext_state) const
 {
-  auto doc = make_xml (ext_state);
-  auto graph = make_dot_graph_from_xml_state (*doc);
+  auto state_graph = make_diagnostic_state_graph (ext_state);
+
+  gcc_assert (global_dc);
+  auto logical_loc_mgr = global_dc->get_logical_location_manager ();
+  gcc_assert (logical_loc_mgr);
+
+  auto graph = diagnostics::state_graphs::make_dot_graph (*state_graph,
+							  *logical_loc_mgr);
 
   pretty_printer pp;
   dot::writer w (pp);
diff --git a/gcc/analyzer/program-state.h b/gcc/analyzer/program-state.h
index e2076c1..4278237 100644
--- a/gcc/analyzer/program-state.h
+++ b/gcc/analyzer/program-state.h
@@ -26,8 +26,6 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/store.h"
 
-namespace xml { class document; }
-
 namespace ana {
 
 /* Data shared by all program_state instances.  */
@@ -248,11 +246,14 @@ public:
   void dump (const extrinsic_state &ext_state, bool simple) const;
   void dump () const;
 
-  std::unique_ptr<xml::document> make_xml (const extrinsic_state &ext_state) const;
-  void dump_xml_to_pp (const extrinsic_state &ext_state, pretty_printer *pp) const;
-  void dump_xml_to_file (const extrinsic_state &ext_state, FILE *outf) const;
-  void dump_xml (const extrinsic_state &ext_state) const;
-  void dump_dot (const extrinsic_state &ext_state) const;
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  make_diagnostic_state_graph (const extrinsic_state &ext_state) const;
+
+  void
+  dump_sarif (const extrinsic_state &ext_state) const;
+
+  void
+  dump_dot (const extrinsic_state &ext_state) const;
 
   std::unique_ptr<json::object>
   to_json (const extrinsic_state &ext_state) const;
diff --git a/gcc/analyzer/region-model-asm.cc b/gcc/analyzer/region-model-asm.cc
index d280ae2..fe70490 100644
--- a/gcc/analyzer/region-model-asm.cc
+++ b/gcc/analyzer/region-model-asm.cc
@@ -171,7 +171,8 @@ region_model::on_asm_stmt (const gasm *stmt, region_model_context *ctxt)
 	 no point in going further.  */
       constraint = constraints[i];
       if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
-				    &allows_mem, &allows_reg, &is_inout))
+				    &allows_mem, &allows_reg, &is_inout,
+				    nullptr))
 	{
 	  if (logger)
 	    logger->log ("error parsing constraint for output %i: %qs",
@@ -204,8 +205,8 @@ region_model::on_asm_stmt (const gasm *stmt, region_model_context *ctxt)
       const char *constraint = constraints[i + noutputs];
       bool allows_reg, allows_mem;
       if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				    constraints.address (),
-				    &allows_mem, &allows_reg))
+				    constraints.address (), &allows_mem,
+				    &allows_reg, nullptr))
 	{
 	  if (logger)
 	    logger->log ("error parsing constraint for input %i: %qs",
diff --git a/gcc/analyzer/region-model.cc b/gcc/analyzer/region-model.cc
index 6df3842..618d96b 100644
--- a/gcc/analyzer/region-model.cc
+++ b/gcc/analyzer/region-model.cc
@@ -26,7 +26,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cgraph.h"
 #include "cfg.h"
 #include "sbitmap.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "stor-layout.h"
 #include "stringpool.h"
 #include "attribs.h"
@@ -40,7 +40,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "is-a.h"
 #include "gcc-rich-location.h"
 #include "gcc-urlifier.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 #include "tree-pretty-print.h"
 #include "fold-const.h"
 #include "selftest-tree.h"
@@ -907,9 +907,10 @@ public:
   }
 
   void
-  maybe_add_sarif_properties (sarif_object &result_obj) const final override
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
+    const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/poisoned_value_diagnostic/"
     props.set (PROPERTY_PREFIX "expr", tree_to_json (m_expr));
     props.set_string (PROPERTY_PREFIX "kind", poison_kind_to_str (m_pkind));
@@ -3835,10 +3836,11 @@ public:
     interest->add_region_creation (m_rhs);
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/dubious_allocation_size/"
     props.set (PROPERTY_PREFIX "lhs", m_lhs->to_json ());
     props.set (PROPERTY_PREFIX "rhs", m_rhs->to_json ());
@@ -7595,9 +7597,10 @@ public:
   }
 
   void
-  maybe_add_sarif_properties (sarif_object &result_obj) const final override
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
+    const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/-Wanalyzer-exposure-through-uninit-copy/"
     props.set (PROPERTY_PREFIX "src_region", m_src_region->to_json ());
     props.set (PROPERTY_PREFIX "dest_region", m_dest_region->to_json ());
diff --git a/gcc/analyzer/sm-fd.cc b/gcc/analyzer/sm-fd.cc
index 370d7a0..b8ae0a1 100644
--- a/gcc/analyzer/sm-fd.cc
+++ b/gcc/analyzer/sm-fd.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "stringpool.h"
 #include "attribs.h"
 
@@ -392,21 +392,22 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (
       const evdesc::state_change &change) const final override
   {
+    using event = diagnostics::paths::event;
     if (change.m_old_state == m_sm.get_start_state ()
 	&& (m_sm.is_unchecked_fd_p (change.m_new_state)
 	    || change.m_new_state == m_sm.m_new_datagram_socket
 	    || change.m_new_state == m_sm.m_new_stream_socket
 	    || change.m_new_state == m_sm.m_new_unknown_socket))
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::resource);
+      return event::meaning (event::verb::acquire,
+			     event::noun::resource);
     if (change.m_new_state == m_sm.m_closed)
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::resource);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::release,
+			     event::noun::resource);
+    return event::meaning ();
   }
 
 protected:
@@ -561,7 +562,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_open_event;
+  diagnostics::paths::event_id_t m_open_event;
   std::unique_ptr<program_state> m_final_state;
 };
 
@@ -705,7 +706,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_close_event;
+  diagnostics::paths::event_id_t m_first_close_event;
 };
 
 class fd_use_after_close : public fd_param_diagnostic
@@ -784,7 +785,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_close_event;
+  diagnostics::paths::event_id_t m_first_close_event;
 };
 
 class fd_use_without_check : public fd_param_diagnostic
@@ -854,7 +855,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_open_event;
+  diagnostics::paths::event_id_t m_first_open_event;
 };
 
 /* Concrete pending_diagnostic subclass for -Wanalyzer-fd-phase-mismatch.  */
diff --git a/gcc/analyzer/sm-file.cc b/gcc/analyzer/sm-file.cc
index 4b1fc77..c852c02 100644
--- a/gcc/analyzer/sm-file.cc
+++ b/gcc/analyzer/sm-file.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "selftest.h"
 
 #include "analyzer/analyzer-logging.h"
@@ -155,18 +155,20 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
+
     if (change.m_old_state == m_sm.get_start_state ()
 	&& change.m_new_state == m_sm.m_unchecked)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::resource);
+      return event::meaning (event::verb::acquire,
+			     event::noun::resource);
     if (change.m_new_state == m_sm.m_closed)
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::resource);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::release,
+			     event::noun::resource);
+    return event::meaning ();
   }
 
 protected:
@@ -225,7 +227,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_fclose_event;
+  diagnostics::paths::event_id_t m_first_fclose_event;
 };
 
 class file_leak : public file_diagnostic
@@ -303,7 +305,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_fopen_event;
+  diagnostics::paths::event_id_t m_fopen_event;
   std::unique_ptr<program_state> m_final_state;
 };
 
diff --git a/gcc/analyzer/sm-malloc.cc b/gcc/analyzer/sm-malloc.cc
index 3581dbb..a6b1421 100644
--- a/gcc/analyzer/sm-malloc.cc
+++ b/gcc/analyzer/sm-malloc.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "stringpool.h"
 #include "attribs.h"
 #include "xml-printer.h"
@@ -436,9 +436,9 @@ public:
       const extrinsic_state &ext_state) const;
 
   void
-  add_state_to_xml (xml_state &out_xml,
-		    const svalue &sval,
-		    state_machine::state_t state) const final override;
+  add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+			    const svalue &sval,
+			    state_machine::state_t state) const final override;
 
   standard_deallocator_set m_free;
   standard_deallocator_set m_scalar_delete;
@@ -814,18 +814,18 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
     if (change.m_old_state == m_sm.get_start_state ()
 	&& unchecked_p (change.m_new_state))
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::memory);
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::acquire,
+					diagnostics::paths::event::noun::memory);
     if (freed_p (change.m_new_state))
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::memory);
-    return diagnostic_event::meaning ();
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::release,
+					diagnostics::paths::event::noun::memory);
+    return diagnostics::paths::event::meaning ();
   }
 
 protected:
@@ -918,7 +918,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_alloc_event;
+  diagnostics::paths::event_id_t m_alloc_event;
   const deallocator_set *m_expected_deallocators;
   const deallocator *m_actual_dealloc;
 };
@@ -988,7 +988,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_free_event;
+  diagnostics::paths::event_id_t m_first_free_event;
   const char *m_funcname;
 };
 
@@ -1031,7 +1031,7 @@ public:
   }
 
 protected:
-  diagnostic_event_id_t m_origin_of_unchecked_event;
+  diagnostics::paths::event_id_t m_origin_of_unchecked_event;
 };
 
 /* Concrete subclass for describing dereference of a possible NULL
@@ -1419,7 +1419,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_free_event;
+  diagnostics::paths::event_id_t m_free_event;
   const deallocator *m_deallocator;
 };
 
@@ -1500,7 +1500,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_alloc_event;
+  diagnostics::paths::event_id_t m_alloc_event;
   std::unique_ptr<program_state> m_final_state;
 };
 
@@ -1768,7 +1768,7 @@ private:
     return result;
   }
 
-  diagnostic_event_id_t m_first_deref_event;
+  diagnostics::paths::event_id_t m_first_deref_event;
   const exploded_node *m_deref_enode;
   tree m_deref_expr;
   const exploded_node *m_check_enode;
@@ -2735,27 +2735,57 @@ malloc_state_machine::transition_ptr_sval_non_null (region_model *model,
   smap->set_state (model, new_ptr_sval, m_free.m_nonnull, nullptr, ext_state);
 }
 
+static enum diagnostics::state_graphs::node_dynalloc_state
+get_dynalloc_state_for_state (enum resource_state rs)
+{
+  switch (rs)
+    {
+    default:
+      gcc_unreachable ();
+    case RS_START:
+    case RS_NULL:
+    case RS_NON_HEAP:
+    case RS_STOP:
+      return diagnostics::state_graphs::node_dynalloc_state::unknown;
+
+    case RS_ASSUMED_NON_NULL:
+      return diagnostics::state_graphs::node_dynalloc_state::nonnull;
+
+    case RS_UNCHECKED:
+      return diagnostics::state_graphs::node_dynalloc_state::unchecked;
+    case RS_NONNULL:
+      return diagnostics::state_graphs::node_dynalloc_state::nonnull;
+    case RS_FREED:
+      return diagnostics::state_graphs::node_dynalloc_state::freed;
+    }
+}
+
 void
-malloc_state_machine::add_state_to_xml (xml_state &out_xml,
-					const svalue &sval,
-					state_machine::state_t state) const
+malloc_state_machine::
+add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+			  const svalue &sval,
+			  state_machine::state_t state) const
 {
   if (const region *reg = sval.maybe_get_region ())
     {
-      auto &reg_element = out_xml.get_or_create_element (*reg);
+      auto reg_node = out_state_graph.get_or_create_state_node (*reg);
       auto alloc_state = as_a_allocation_state (state);
       gcc_assert (alloc_state);
 
-      reg_element.set_attr ("dynamic-alloc-state", state->get_name ());
+      reg_node.set_dynalloc_state
+	(get_dynalloc_state_for_state (alloc_state->m_rs));
       if (alloc_state->m_deallocators)
 	{
 	  pretty_printer pp;
 	  alloc_state->m_deallocators->dump_to_pp (&pp);
-	  reg_element.set_attr ("expected-deallocators", pp_formatted_text (&pp));
+	  reg_node.m_node.set_attr (STATE_NODE_PREFIX,
+				    "expected-deallocators",
+				    pp_formatted_text (&pp));
 	}
       if (alloc_state->m_deallocator)
-	reg_element.set_attr ("deallocator",
-			      alloc_state->m_deallocator->m_name);
+	reg_node.m_node.set_attr (STATE_NODE_PREFIX,
+				  "deallocator",
+				  alloc_state->m_deallocator->m_name);
     }
 }
 
diff --git a/gcc/analyzer/sm-pattern-test.cc b/gcc/analyzer/sm-pattern-test.cc
index 02b32ac..12449a1 100644
--- a/gcc/analyzer/sm-pattern-test.cc
+++ b/gcc/analyzer/sm-pattern-test.cc
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/common.h"
 
 #include "tree-pretty-print.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
diff --git a/gcc/analyzer/sm-sensitive.cc b/gcc/analyzer/sm-sensitive.cc
index f8fcded..4611b10e 100644
--- a/gcc/analyzer/sm-sensitive.cc
+++ b/gcc/analyzer/sm-sensitive.cc
@@ -21,7 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
@@ -106,14 +106,15 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
+
     if (change.m_new_state == m_sm.m_sensitive)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::sensitive);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::acquire, event::noun::sensitive);
+    return event::meaning ();
   }
   bool
   describe_call_with_state (pretty_printer &pp,
@@ -162,7 +163,7 @@ public:
 private:
   const sensitive_state_machine &m_sm;
   tree m_arg;
-  diagnostic_event_id_t m_first_sensitive_event;
+  diagnostics::paths::event_id_t m_first_sensitive_event;
 };
 
 /* sensitive_state_machine's ctor.  */
diff --git a/gcc/analyzer/sm-signal.cc b/gcc/analyzer/sm-signal.cc
index 5a4b384..35ecde1 100644
--- a/gcc/analyzer/sm-signal.cc
+++ b/gcc/analyzer/sm-signal.cc
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "sbitmap.h"
 #include "ordered-hash-map.h"
 #include "selftest.h"
diff --git a/gcc/analyzer/sm-taint.cc b/gcc/analyzer/sm-taint.cc
index 5c8cc7e..f2a94e8 100644
--- a/gcc/analyzer/sm-taint.cc
+++ b/gcc/analyzer/sm-taint.cc
@@ -30,7 +30,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "stringpool.h"
 #include "attribs.h"
 #include "fold-const.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 #include "gcc-urlifier.h"
 
 #include "analyzer/analyzer-logging.h"
@@ -209,20 +209,22 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
     if (change.m_new_state == m_sm.m_tainted)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::taint);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::acquire,
+			     event::noun::taint);
+    return event::meaning ();
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/taint_diagnostic/"
     props.set (PROPERTY_PREFIX "arg", tree_to_json (m_arg));
     props.set_string (PROPERTY_PREFIX "has_bounds",
@@ -495,11 +497,12 @@ public:
 	}
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     taint_diagnostic::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/tainted_offset/"
     props.set (PROPERTY_PREFIX "offset", m_offset->to_json ());
 #undef PROPERTY_PREFIX
@@ -864,11 +867,12 @@ public:
 	}
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     taint_diagnostic::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/tainted_allocation_size/"
     props.set (PROPERTY_PREFIX "size_in_bytes", m_size_in_bytes->to_json ());
 #undef PROPERTY_PREFIX
diff --git a/gcc/analyzer/sm.cc b/gcc/analyzer/sm.cc
index 840806a..c93e9c2 100644
--- a/gcc/analyzer/sm.cc
+++ b/gcc/analyzer/sm.cc
@@ -161,16 +161,16 @@ state_machine::to_json () const
 }
 
 void
-state_machine::add_state_to_xml (xml_state &out_xml,
-				 const svalue &sval,
-				 state_machine::state_t state) const
+state_machine::add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+					 const svalue &sval,
+					 state_machine::state_t state) const
 {
   // no-op
 }
 
 void
-state_machine::add_global_state_to_xml (xml_state &out_xml,
-					state_machine::state_t state) const
+state_machine::add_global_state_to_state_graph (analyzer_state_graph &out_state_graph,
+						state_machine::state_t state) const
 {
   // no-op
 }
diff --git a/gcc/analyzer/sm.h b/gcc/analyzer/sm.h
index 6298fb6..4633fac 100644
--- a/gcc/analyzer/sm.h
+++ b/gcc/analyzer/sm.h
@@ -28,7 +28,7 @@ namespace ana {
 class state_machine;
 class sm_context;
 class pending_diagnostic;
-class xml_state;
+class analyzer_state_graph;
 
 extern bool any_pointer_p (tree expr);
 extern bool any_pointer_p (const svalue *sval);
@@ -188,13 +188,13 @@ public:
   state_t get_start_state () const { return m_start; }
 
   virtual void
-  add_state_to_xml (xml_state &out_xml,
-		    const svalue &sval,
-		    state_machine::state_t state) const;
+  add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+			    const svalue &sval,
+			    state_machine::state_t state) const;
 
   virtual void
-  add_global_state_to_xml (xml_state &out_xml,
-			   state_machine::state_t state) const;
+  add_global_state_to_state_graph (analyzer_state_graph &out_state_graph,
+				   state_machine::state_t state) const;
 
 protected:
   state_t add_state (const char *name);
diff --git a/gcc/analyzer/varargs.cc b/gcc/analyzer/varargs.cc
index 4947271..1fdfd34 100644
--- a/gcc/analyzer/varargs.cc
+++ b/gcc/analyzer/varargs.cc
@@ -334,17 +334,17 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
     if (change.m_new_state == m_sm.m_started)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::resource);
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::acquire,
+					diagnostics::paths::event::noun::resource);
     if (change.m_new_state == m_sm.m_ended)
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::resource);
-    return diagnostic_event::meaning ();
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::release,
+					diagnostics::paths::event::noun::resource);
+    return diagnostics::paths::event::meaning ();
   }
 
 protected:
@@ -452,7 +452,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_va_end_event;
+  diagnostics::paths::event_id_t m_va_end_event;
   const char *m_usage_fnname;
 };
 
@@ -539,7 +539,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_start_event;
+  diagnostics::paths::event_id_t m_start_event;
   const char *m_start_event_fnname;
   std::unique_ptr<program_state> m_final_state;
 };
diff --git a/gcc/attribs.cc b/gcc/attribs.cc
index f666783..3fce9d6 100644
--- a/gcc/attribs.cc
+++ b/gcc/attribs.cc
@@ -1076,21 +1076,15 @@ apply_tm_attr (tree fndecl, tree attr)
    it to CHAIN.  */
 
 tree
-make_attribute (const char *name, const char *arg_name, tree chain)
+make_attribute (string_slice name, string_slice arg_name, tree chain)
 {
-  tree attr_name;
-  tree attr_arg_name;
-  tree attr_args;
-  tree attr;
-
-  attr_name = get_identifier (name);
-  attr_arg_name = build_string (strlen (arg_name), arg_name);
-  attr_args = tree_cons (NULL_TREE, attr_arg_name, NULL_TREE);
-  attr = tree_cons (attr_name, attr_args, chain);
+  tree attr_name = get_identifier_with_length (name.begin (), name.size ());
+  tree attr_arg_name = build_string (arg_name.size (), arg_name.begin ());
+  tree attr_args = tree_cons (NULL_TREE, attr_arg_name, NULL_TREE);
+  tree attr = tree_cons (attr_name, attr_args, chain);
   return attr;
 }
 
-
 /* Common functions used for target clone support.  */
 
 /* Comparator function to be used in qsort routine to sort attribute
diff --git a/gcc/attribs.h b/gcc/attribs.h
index 4b94639..b8b6838 100644
--- a/gcc/attribs.h
+++ b/gcc/attribs.h
@@ -45,7 +45,7 @@ extern bool cxx11_attribute_p (const_tree);
 extern tree get_attribute_name (const_tree);
 extern tree get_attribute_namespace (const_tree);
 extern void apply_tm_attr (tree, tree);
-extern tree make_attribute (const char *, const char *, tree);
+extern tree make_attribute (string_slice, string_slice, tree);
 extern bool attribute_ignored_p (tree);
 extern bool attribute_ignored_p (const attribute_spec *const);
 extern bool any_nonignored_attribute_p (tree);
diff --git a/gcc/auto-obstack.h b/gcc/auto-obstack.h
new file mode 100644
index 0000000..fe16dbd
--- /dev/null
+++ b/gcc/auto-obstack.h
@@ -0,0 +1,58 @@
+/* RAII wrapper around obstack.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_AUTO_OBSTACK_H
+#define GCC_AUTO_OBSTACK_H
+
+/* RAII wrapper around obstack.  */
+
+struct auto_obstack
+{
+  auto_obstack ()
+  {
+    obstack_init (&m_obstack);
+  }
+
+  ~auto_obstack ()
+  {
+    obstack_free (&m_obstack, NULL);
+  }
+
+  operator obstack & () { return m_obstack; }
+
+  void grow (const void *src, size_t length)
+  {
+    obstack_grow (&m_obstack, src, length);
+  }
+
+  void *object_base () const
+  {
+    return m_obstack.object_base;
+  }
+
+  size_t object_size () const
+  {
+    return obstack_object_size (&m_obstack);
+  }
+
+  obstack m_obstack;
+};
+
+#endif /* GCC_AUTO_OBSTACK_H */
diff --git a/gcc/auto-profile.cc b/gcc/auto-profile.cc
index 64f4cda..7ff9526 100644
--- a/gcc/auto-profile.cc
+++ b/gcc/auto-profile.cc
@@ -53,6 +53,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "auto-profile.h"
 #include "tree-pretty-print.h"
 #include "gimple-pretty-print.h"
+#include "output.h"
 
 /* The following routines implements AutoFDO optimization.
 
@@ -239,6 +240,8 @@ public:
   /* Add new name and return its index.  */
   int add_name (char *);
 
+  /* Return cgraph node corresponding to given name index.  */
+  cgraph_node *get_cgraph_node (int);
 private:
   typedef std::map<const char *, unsigned, string_compare> string_index_map;
   string_vector vector_;
@@ -430,7 +433,8 @@ public:
   void
   set_call_location (location_t l)
   {
-    gcc_checking_assert (call_location_ == UNKNOWN_LOCATION);
+    gcc_checking_assert (call_location_ == UNKNOWN_LOCATION
+			 && l != UNKNOWN_LOCATION);
     call_location_= l;
   }
 
@@ -443,7 +447,6 @@ public:
   /* Lookup count and warn about duplicates.  */
   count_info *lookup_count (location_t loc, inline_stack &stack,
 			    cgraph_node *node);
-
 private:
   /* Callsite, represented as (decl_lineno, callee_function_name_index).  */
   typedef std::pair<unsigned, unsigned> callsite;
@@ -622,9 +625,11 @@ get_original_name (const char *name, bool alloc = true)
     }
   /* Suffixes of clones that compiler generates after auto-profile.  */
   const char *suffixes[] = {"isra", "constprop", "lto_priv", "part", "cold"};
-  for (unsigned i = 0; i < sizeof (suffixes); ++i)
+  for (unsigned i = 0; i < sizeof (suffixes) / sizeof (const char *); ++i)
     {
-      if (strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
+      int len = strlen (suffixes[i]);
+      if (len == last_dot - next_dot - 1
+	  && strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
 	{
 	  *next_dot = 0;
 	  return get_original_name (ret, false);
@@ -683,6 +688,26 @@ dump_afdo_loc (FILE *f, unsigned loc)
     fprintf (f, "%i", loc >> 16);
 }
 
+/* Return assembler name as in symbol table and DW_AT_linkage_name.  */
+
+static const char *
+raw_symbol_name (const char *asmname)
+{
+  /* If we start supporting user_label_prefixes, add_linkage_attr will also
+     need to be fixed.  */
+  if (strlen (user_label_prefix))
+    sorry ("auto-profile is not supported for targets with user label prefix");
+  return asmname + (asmname[0] == '*');
+}
+
+/* Convenience wrapper that looks up assembler name.  */
+
+static const char *
+raw_symbol_name (tree decl)
+{
+  return raw_symbol_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
+}
+
 /* Dump STACK to F.  */
 
 static void
@@ -693,7 +718,7 @@ dump_inline_stack (FILE *f, inline_stack *stack)
     {
       fprintf (f, "%s%s:",
 	       first ? "" : "; ",
-	       IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (p.decl)));
+	       raw_symbol_name (p.decl));
       dump_afdo_loc (f, p.afdo_loc);
       first = false;
     }
@@ -815,7 +840,7 @@ string_table::get_index (const char *name) const
 int
 string_table::get_index_by_decl (tree decl) const
 {
-  const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
+  const char *name = raw_symbol_name (decl);
   int ret = get_index (name);
   if (ret != -1)
     return ret;
@@ -860,10 +885,35 @@ string_table::read ()
     {
       vector_.quick_push (xstrdup (gcov_read_string ()));
       map_[vector_.last ()] = i;
+      if (gcov_is_error ())
+	return false;
     }
   return true;
 }
 
+/* Return cgraph node corresponding to given NAME_INDEX,
+   NULL if unavailable.  */
+cgraph_node *
+string_table::get_cgraph_node (int name_index)
+{
+  const char *sname = get_name (name_index);
+
+  symtab_node *n = cgraph_node::get_for_asmname (get_identifier (sname));
+  for (;n; n = n->next_sharing_asm_name)
+    if (cgraph_node *cn = dyn_cast <cgraph_node *> (n))
+      if (cn->definition && cn->has_gimple_body_p ())
+	return cn;
+  return NULL;
+}
+
+/* Return corresponding cgraph node.  */
+
+cgraph_node *
+function_instance::get_cgraph_node ()
+{
+  return afdo_string_table->get_cgraph_node (name ());
+}
+
 /* Member functions for function_instance.  */
 
 function_instance::~function_instance ()
@@ -874,20 +924,6 @@ function_instance::~function_instance ()
     delete iter->second;
 }
 
-/* Return corresponding cgraph node, NULL if unavailable.  */
-cgraph_node *
-function_instance::get_cgraph_node ()
-{
-  for (symtab_node *n = cgraph_node::get_for_asmname
-			    (get_identifier
-			       (afdo_string_table->get_name (name ())));
-       n; n = n->next_sharing_asm_name)
-    if (cgraph_node *cn = dyn_cast <cgraph_node *> (n))
-      if (cn->definition && cn->has_gimple_body_p ())
-	return cn;
-  return NULL;
-}
-
 /* Traverse callsites of the current function_instance to find one at the
    location of LINENO and callee name represented in DECL.  */
 
@@ -919,10 +955,10 @@ function_instance::get_function_instance_by_decl (unsigned lineno,
 	  dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
 			   dump_user_location_t::from_location_t (location),
 			   "auto-profile has mismatched function name %s"
-			   " instaed of %s at loc %i:%i",
+			   " insteed of %s at loc %i:%i",
 			   afdo_string_table->get_name (iter.first.second),
-			   IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
-			   lineno << 16,
+			   raw_symbol_name (decl),
+			   lineno >> 16,
 			   lineno & 65535);
     }
 
@@ -1125,10 +1161,13 @@ function_instance::offline_if_in_set (name_index_set &seen,
    Return non-zero if it correspons and 2 if renaming was done.  */
 
 static int
-match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
+match_with_target (cgraph_node *n,
+		   gimple *stmt,
+		   function_instance *inlined_fn,
+		   cgraph_node *orig_callee)
 {
-  const char *symbol_name = IDENTIFIER_POINTER
-			       (DECL_ASSEMBLER_NAME (n->decl));
+  cgraph_node *callee = orig_callee->ultimate_alias_target ();
+  const char *symbol_name = raw_symbol_name (callee->decl);
   const char *name = afdo_string_table->get_name (inlined_fn->name ());
   if (strcmp (name, symbol_name))
     {
@@ -1142,7 +1181,7 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
 	    in_suffix = true;
 	}
       /* Accept dwarf names and stripped suffixes.  */
-      if (!strcmp (lang_hooks.dwarf_name (n->decl, 0),
+      if (!strcmp (lang_hooks.dwarf_name (callee->decl, 0),
 		   afdo_string_table->get_name (inlined_fn->name ()))
 	  || (!name[i] && symbol_name[i] == '.')
 	  || in_suffix)
@@ -1157,10 +1196,14 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
 	  inlined_fn->set_name (index);
 	  return 2;
 	}
-      warning_at (gimple_location (stmt), OPT_Wauto_profile,
-		  "auto-profile contains inlined "
-		  "function with symbol name %s instead of symbol name %s",
-		  name, symbol_name);
+      /* Only warn about declarations.  It is possible that the function
+	 is declared as alias in other module and we inlined cross-module.  */
+      if (callee->definition
+	  && warning (OPT_Wauto_profile,
+		      "auto-profile of %q+F contains inlined "
+		      "function with symbol name %s instead of symbol name %s",
+		      n->decl, name, symbol_name))
+	inform (gimple_location (stmt), "corresponding call");
       return 0;
     }
   return 1;
@@ -1203,13 +1246,14 @@ function_instance::lookup_count (location_t loc, inline_stack &stack,
   if (stack.length ())
     {
       int c = pos_counts.count (stack[0].afdo_loc);
-      if (c > 1)
-	warning_at (loc, OPT_Wauto_profile,
-		    "duplicated count information"
-		    " in auto-profile of %q+F"
-		    " with relative location %i discriminator %i",
-		    node->decl, stack[0].afdo_loc >> 16,
-		    stack[0].afdo_loc & 65535);
+      if (c > 1
+	  && warning (OPT_Wauto_profile,
+		      "duplicated count information"
+		      " in auto-profile of %q+F"
+		      " with relative location %i discriminator %i",
+		      node->decl, stack[0].afdo_loc >> 16,
+		      stack[0].afdo_loc & 65535))
+	  inform (loc, "corresponding source location");
       if (c)
 	return &pos_counts[stack[0].afdo_loc];
     }
@@ -1271,6 +1315,7 @@ function_instance::match (cgraph_node *node,
   hash_set<const count_info *> counts;
   hash_set<const count_info *> targets;
   hash_set<const function_instance *> functions;
+  hash_set<const function_instance *> functions_to_offline;
 
   /* We try to fill in lost disciminator if there is unique call
      with given line number.  This map is used to record them.  */
@@ -1374,20 +1419,23 @@ function_instance::match (cgraph_node *node,
 			  inlined_fn_nodisc = iter.second;
 			cnodis++;
 		      }
-		  if (c > 1 || cnodis > 1)
-		    warning_at (gimple_location (stmt), OPT_Wauto_profile,
-				"duplicated callsite in auto-profile of %q+F"
-				" with relative location %i, discriminator %i",
-				node->decl, stack[0].afdo_loc >> 16,
-				stack[0].afdo_loc & 65535);
-		  if (inlined_fn && info && info->targets.size ())
-		    warning_at (gimple_location (stmt), OPT_Wauto_profile,
-				"both call targets and inline callsite"
-				" information is present in auto-profile"
-				" of function %q+F with relative location"
-				" %i, discriminator %i",
-				node->decl, stack[0].afdo_loc >> 16,
-				stack[0].afdo_loc & 65535);
+		  if ((c > 1 || (!c && cnodis > 1))
+		      && warning (OPT_Wauto_profile,
+				  "duplicated callsite in auto-profile of %q+F"
+				  " with relative location %i,"
+				  " discriminator %i",
+				  node->decl, stack[0].afdo_loc >> 16,
+				  stack[0].afdo_loc & 65535))
+		    inform (gimple_location (stmt), "corresponding call");
+		  if (inlined_fn && info && info->targets.size ()
+		      && warning (OPT_Wauto_profile,
+				  "both call targets and inline callsite"
+				  " information is present in auto-profile"
+				  " of function %q+F with relative location"
+				  " %i, discriminator %i",
+				  node->decl, stack[0].afdo_loc >> 16,
+				  stack[0].afdo_loc & 65535))
+		    inform (gimple_location (stmt), "corresponding call");
 		  tree callee = gimple_call_fndecl (stmt);
 		  cgraph_node *callee_node;
 		  unsigned int loc = stack[0].afdo_loc;
@@ -1420,11 +1468,17 @@ function_instance::match (cgraph_node *node,
 		      if (lineno_to_call.get
 			      (stack[0].afdo_loc >> 16)->length () == 1)
 			{
-			  warning_at (gimple_location (stmt), OPT_Wauto_profile,
-				      "auto-profile of %q+F seem to contain"
-				      " lost discriminator %i for call at"
-				      " relative location %i",
-				      node->decl, loc & 65535, loc >> 16);
+			  if (warning (OPT_Wauto_profile,
+				       "auto-profile of %q+F seem to contain"
+				       " lost discriminator %i for"
+				       " call of %s at relative location %i",
+				       node->decl,
+				       loc & 65535,
+				       afdo_string_table->get_name
+					 (inlined_fn_nodisc->name ()),
+				       loc >> 16))
+			    inform (gimple_location (stmt),
+				    "corresponding call");
 			  inlined_fn = inlined_fn_nodisc;
 			  if (dump_file)
 			    fprintf (dump_file, "   Lost discriminator %i\n",
@@ -1435,11 +1489,10 @@ function_instance::match (cgraph_node *node,
 		    }
 		  if (callee && (callee_node = cgraph_node::get (callee)))
 		    {
-		      callee_node = callee_node->ultimate_alias_target ();
 		      if (inlined_fn)
 			{
 			  int old_name = inlined_fn->name ();
-			  int r = match_with_target (stmt, inlined_fn,
+			  int r = match_with_target (node, stmt, inlined_fn,
 						     callee_node);
 			  if (r == 2)
 			    {
@@ -1451,11 +1504,13 @@ function_instance::match (cgraph_node *node,
 						      == inlined_fn);
 			      callsite key2 = {stack[0].afdo_loc,
 						inlined_fn->name ()};
-			      callsites[key2] = inlined_fn;
 			      callsites.erase (iter);
+			      callsites[key2] = inlined_fn;
 			    }
 			  if (r)
 			    functions.add (inlined_fn);
+			  else
+			    functions_to_offline.add (inlined_fn);
 			}
 
 		      if (info && info->targets.size () > 1)
@@ -1473,19 +1528,24 @@ function_instance::match (cgraph_node *node,
 		    {
 		      if (inlined_fn
 			  && inlined_fn->get_call_location ()
-				  != UNKNOWN_LOCATION
-			  && warning_at (gimple_location (stmt),
-					 OPT_Wauto_profile,
-					 "%q+F contains two calls of the same"
-					 " relative location +%i,"
-					 " discrimnator %i,"
-					 " that leads to lost auto-profile",
-					 node->decl,
-					 loc << 16,
-					 loc & 65535))
+				  != UNKNOWN_LOCATION)
 			{
-			  inform (inlined_fn->get_call_location (),
-				  "location of the earlier call");
+			  if (warning (OPT_Wauto_profile,
+				       "function contains two calls of the same"
+				       " relative location +%i,"
+				       " discrimnator %i,"
+				       " that leads to lost auto-profile",
+				       loc >> 16,
+				       loc & 65535))
+			    {
+			      inform (gimple_location (stmt),
+				      "location of the first call");
+			      inform (inlined_fn->get_call_location (),
+				      "location of the second call");
+			    }
+			  if (dump_file)
+			    fprintf (dump_file,
+				     "   Duplicated call location\n");
 			  inlined_fn = NULL;
 			}
 		      if (inlined_fn)
@@ -1507,10 +1567,10 @@ function_instance::match (cgraph_node *node,
 			      callsite key2 = {stack[0].afdo_loc,
 					       newn ? *newn
 					       : inlined_fn->name ()};
+			      callsites.erase (iter);
 			      callsites[key2] = inlined_fn;
 			      inlined_fn->set_name (newn ? *newn
 						    : inlined_fn->name ());
-			      callsites.erase (iter);
 			    }
 			  functions.add (inlined_fn);
 			}
@@ -1559,12 +1619,25 @@ function_instance::match (cgraph_node *node,
     (DECL_STRUCT_FUNCTION (node->decl)->function_end_locus, node->decl);
   unsigned int start_location = get_combined_location
     (DECL_STRUCT_FUNCTION (node->decl)->function_start_locus, node->decl);
+  /* When outputting code to builtins location we use line number 0.
+     craeate_gcov is stupid and hapilly computes offsets across files.
+     Silently ignore it.  */
+  unsigned int zero_location
+	  = ((unsigned)(1-DECL_SOURCE_LINE (node->decl))) << 16;
   for (position_count_map::const_iterator iter = pos_counts.begin ();
        iter != pos_counts.end ();)
     if (!counts.contains (&iter->second))
       {
-	if (iter->first != end_location && iter->first != start_location
-	    && iter->first)
+	if (iter->first != end_location
+	    && iter->first != start_location
+	    && (iter->first & 65535) != zero_location
+	    && iter->first
+	    /* FIXME: dwarf5 does not represent inline stack of debug
+	       statements and consequently create_gcov is sometimes
+	       mixing up statements from other functions.  Do not warn
+	       user about this until this problem is solved.
+	       We still write info into dump file.  */
+	    && 0)
 	  {
 	    if (!warned)
 	      warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
@@ -1601,25 +1674,38 @@ function_instance::match (cgraph_node *node,
        iter != callsites.end ();)
     if (!functions.contains (iter->second))
       {
-	if (!warned)
-	  warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
-			       OPT_Wauto_profile,
-			       "auto-profile of %q+F contains extra callsites",
-			       node->decl);
-	if (warned)
-	  inform (DECL_SOURCE_LOCATION (node->decl),
-		  "call of %s with relative location +%i, discriminator %i",
-		  afdo_string_table->get_name (iter->first.second),
-		  iter->first.first >> 16, iter->first.first & 65535);
-	if ((iter->first.first >> 16) > (end_location >> 16) && warned)
-	  inform (DECL_SOURCE_LOCATION (node->decl),
-		  "location is after end of function");
-	warned = true;
 	function_instance *f = iter->second;
-	if (dump_file)
+	/* If we did not see the corresponding statement, warn.  */
+	if (!functions_to_offline.contains (iter->second))
+	  {
+	    if (!warned)
+	      warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
+				   OPT_Wauto_profile,
+				   "auto-profile of %q+F contains"
+				   " extra callsites",
+				   node->decl);
+	    if (warned)
+	      inform (DECL_SOURCE_LOCATION (node->decl),
+		      "call of %s with total count %" PRId64
+		      ", relative location +%i, discriminator %i",
+		      afdo_string_table->get_name (iter->first.second),
+		      iter->second->total_count (),
+		      iter->first.first >> 16, iter->first.first & 65535);
+	    if ((iter->first.first >> 16) > (end_location >> 16) && warned)
+	      inform (DECL_SOURCE_LOCATION (node->decl),
+		      "location is after end of function");
+	    if (dump_file)
+	      {
+		fprintf (dump_file,
+			 "Offlining inline with no corresponding gimple stmt ");
+		f->dump_inline_stack (dump_file);
+		fprintf (dump_file, "\n");
+	      }
+	  }
+	else if (dump_file)
 	  {
 	    fprintf (dump_file,
-		     "Offlining inline with no corresponding gimple stmt ");
+		     "Offlining mismatched inline ");
 	    f->dump_inline_stack (dump_file);
 	    fprintf (dump_file, "\n");
 	  }
@@ -1684,9 +1770,9 @@ function_instance::remove_external_functions
       auto iter = callsites.find (key);
       callsite key2 = key;
       key2.second = *to_symbol_name.get (key.second);
-      callsites[key2] = iter->second;
       iter->second->set_name (key2.second);
       callsites.erase (iter);
+      callsites[key2] = iter->second;
     }
   auto_vec <int, 20> target_to_rename;
   for (auto &iter : pos_counts)
@@ -1883,6 +1969,7 @@ autofdo_source_profile::offline_external_functions ()
   cgraph_node *node;
   name_index_set seen;
   name_index_map to_symbol_name;
+  size_t last_name;
 
   /* Add renames erasing suffixes produced by late clones, such as
      .isra, .ipcp.  */
@@ -1918,10 +2005,10 @@ autofdo_source_profile::offline_external_functions ()
 	index = afdo_string_table->add_name (n2);
       to_symbol_name.put (i, index);
     }
+  last_name = afdo_string_table->num_entries ();
   FOR_EACH_DEFINED_FUNCTION (node)
     {
-      const char *name
-	  = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
+      const char *name = raw_symbol_name (node->decl);
       const char *dwarf_name = lang_hooks.dwarf_name (node->decl, 0);
       int index = afdo_string_table->get_index (name);
 
@@ -1958,11 +2045,17 @@ autofdo_source_profile::offline_external_functions ()
 	{
 	  if (dump_file)
 	    {
-	      fprintf (dump_file,
-		       "Node %s not in auto profile (%s neither %s)\n",
-		       node->dump_name (),
-		       name,
-		       dwarf_name);
+	      if (dwarf_name && strcmp (dwarf_name, name))
+		fprintf (dump_file,
+			 "Node %s not in auto profile (%s neither %s)\n",
+			 node->dump_name (),
+			 name,
+			 dwarf_name);
+	      else
+		fprintf (dump_file,
+			 "Node %s (symbol %s) not in auto profile\n",
+			 node->dump_name (),
+			 name);
 	    }
 	}
     }
@@ -2006,72 +2099,87 @@ autofdo_source_profile::offline_external_functions ()
      should be done on unmodified profile and merging works better if
      mismatches are already resolved both in source and destination.  */
   while (fns.length () || fns2.length ())
-    if (fns.length ())
-      {
-	function_instance *f = fns.pop ();
-	if (f->get_location () == UNKNOWN_LOCATION)
-	  {
-	    int index = f->name ();
-	    int *newn = to_symbol_name.get (index);
-	    if (newn)
-	      {
-		f->set_name (*newn);
-		if (map_.count (index)
-		    && map_[index] == f)
-		  map_.erase (index);
-		if (!map_.count (*newn))
-		  map_[*newn] = f;
-	      }
-	    if (cgraph_node *n = f->get_cgraph_node ())
-	      {
-		gcc_checking_assert (seen.contains (f->name ()));
-		f->match (n, fns, to_symbol_name);
-	      }
-	  }
-	fns2.safe_push (f);
-      }
-    else
-      {
-	function_instance *f = fns2.pop ();
-	int index = f->name ();
-	gcc_checking_assert (f->in_worklist_p ());
+    {
+      /* In case renaming introduced new name, keep seen up to date.  */
+      for (; last_name < afdo_string_table->num_entries (); last_name++)
+	{
+	  const char *name = afdo_string_table->get_name (last_name);
+	  symtab_node *n
+	    = afdo_string_table->get_cgraph_node (last_name);
+	  if (dump_file)
+	    fprintf (dump_file, "New name %s %s\n", name,
+		     n ? "wth corresponding definition"
+		     : "with no corresponding definition");
+	  if (n)
+	    seen.add (last_name);
+	}
+      if (fns.length ())
+	{
+	  function_instance *f = fns.pop ();
+	  if (f->get_location () == UNKNOWN_LOCATION)
+	    {
+	      int index = f->name ();
+	      int *newn = to_symbol_name.get (index);
+	      if (newn)
+		{
+		  f->set_name (*newn);
+		  if (map_.count (index)
+		      && map_[index] == f)
+		    map_.erase (index);
+		  if (!map_.count (*newn))
+		    map_[*newn] = f;
+		}
+	      if (cgraph_node *n = f->get_cgraph_node ())
+		{
+		  gcc_checking_assert (seen.contains (f->name ()));
+		  f->match (n, fns, to_symbol_name);
+		}
+	    }
+	  fns2.safe_push (f);
+	}
+      else
+	{
+	  function_instance *f = fns2.pop ();
+	  int index = f->name ();
+	  gcc_checking_assert (f->in_worklist_p ());
 
-	/* If map has different function_instance of same name, then
-	   this is a duplicated entry which needs to be merged.  */
-	if (map_.count (index) && map_[index] != f)
-	  {
-	    if (dump_file)
-	      {
-		fprintf (dump_file, "Merging duplicate instance: ");
-		f->dump_inline_stack (dump_file);
-		fprintf (dump_file, "\n");
-	      }
-	    map_[index]->merge (f, fns);
-	    gcc_checking_assert (!f->inlined_to ());
-	    f->clear_in_worklist ();
-	    delete f;
-	  }
-	/* If name was not seen in the symbol table, remove it.  */
-	else if (!seen.contains (index))
-	  {
-	    f->offline_if_in_set (seen, fns);
-	    f->clear_in_worklist ();
-	    if (dump_file)
-	      fprintf (dump_file, "Removing external %s\n",
-		       afdo_string_table->get_name (f->name ()));
-	    if (map_.count (index) && map_[index] == f)
-	      map_.erase (f->name ());
-	    delete f;
-	  }
-	/* If this is offline function instance seen in this
-	   translation unit offline external inlines and possibly
-	   rename from dwarf name.  */
-	else
-	  {
-	    f->remove_external_functions (seen, to_symbol_name, fns);
-	    f->clear_in_worklist ();
-	  }
-      }
+	  /* If map has different function_instance of same name, then
+	     this is a duplicated entry which needs to be merged.  */
+	  if (map_.count (index) && map_[index] != f)
+	    {
+	      if (dump_file)
+		{
+		  fprintf (dump_file, "Merging duplicate instance: ");
+		  f->dump_inline_stack (dump_file);
+		  fprintf (dump_file, "\n");
+		}
+	      map_[index]->merge (f, fns);
+	      gcc_checking_assert (!f->inlined_to ());
+	      f->clear_in_worklist ();
+	      delete f;
+	    }
+	  /* If name was not seen in the symbol table, remove it.  */
+	  else if (!seen.contains (index))
+	    {
+	      f->offline_if_in_set (seen, fns);
+	      f->clear_in_worklist ();
+	      if (dump_file)
+		fprintf (dump_file, "Removing external %s\n",
+			 afdo_string_table->get_name (f->name ()));
+	      if (map_.count (index) && map_[index] == f)
+		map_.erase (f->name ());
+	      delete f;
+	    }
+	  /* If this is offline function instance seen in this
+	     translation unit offline external inlines and possibly
+	     rename from dwarf name.  */
+	  else
+	    {
+	      f->remove_external_functions (seen, to_symbol_name, fns);
+	      f->clear_in_worklist ();
+	    }
+	}
+    }
   if (dump_file)
     for (auto const &iter : map_)
       {
@@ -2103,8 +2211,7 @@ walk_block (tree fn, function_instance *s, tree block)
 	      fprintf (dump_file, ":");
 	      dump_afdo_loc (dump_file, loc);
 	      fprintf (dump_file, " %s\n",
-		       IDENTIFIER_POINTER
-			 (DECL_ASSEMBLER_NAME (BLOCK_ABSTRACT_ORIGIN (block))));
+		       raw_symbol_name (BLOCK_ABSTRACT_ORIGIN (block)));
 	    }
 	  return;
 	}
@@ -2221,7 +2328,7 @@ autofdo_source_profile::offline_unrealized_inlines ()
 
 function_instance *
 function_instance::read_function_instance (function_instance_stack *stack,
-                                           gcov_type head_count)
+					   gcov_type head_count)
 {
   unsigned name = gcov_read_unsigned ();
   unsigned num_pos_counts = gcov_read_unsigned ();
@@ -2244,10 +2351,10 @@ function_instance::read_function_instance (function_instance_stack *stack,
         (*stack)[j]->total_count_ += count;
       for (unsigned j = 0; j < num_targets; j++)
         {
-          /* Only indirect call target histogram is supported now.  */
-          gcov_read_unsigned ();
-          gcov_type target_idx = gcov_read_counter ();
-          s->pos_counts[offset].targets[target_idx] = gcov_read_counter ();
+	  /* Only indirect call target histogram is supported now.  */
+	  gcov_read_unsigned ();
+	  gcov_type target_idx = gcov_read_counter ();
+	  s->pos_counts[offset].targets[target_idx] = gcov_read_counter ();
         }
     }
   for (unsigned i = 0; i < num_callsites; i++)
@@ -2462,7 +2569,7 @@ autofdo_source_profile::get_callsite_total_count (
     {
       if (dump_file)
 	fprintf (dump_file, "Mismatched name of callee %s and profile %s\n",
-		 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (edge->callee->decl)),
+		 raw_symbol_name (edge->callee->decl),
 		 afdo_string_table->get_name (s->name ()));
       return 0;
     }
@@ -2522,6 +2629,7 @@ autofdo_source_profile::read ()
     afdo_count_scale
       = MAX (((gcov_type)1 << (profile_count::n_bits / 2))
 	     / afdo_profile_info->sum_max, 1);
+  afdo_profile_info->cutoff *= afdo_count_scale;
   afdo_hot_bb_threshod
     = hot_frac
       ? afdo_profile_info->sum_max * afdo_count_scale / hot_frac
@@ -2531,10 +2639,12 @@ autofdo_source_profile::read ()
     fprintf (dump_file, "Max count in profile %" PRIu64 "\n"
 			"Setting scale %" PRIu64 "\n"
 			"Scaled max count %" PRIu64 "\n"
+			"Cutoff %" PRIu64 "\n"
 			"Hot count threshold %" PRIu64 "\n\n",
 	     (int64_t)afdo_profile_info->sum_max,
 	     (int64_t)afdo_count_scale,
 	     (int64_t)(afdo_profile_info->sum_max * afdo_count_scale),
+	     (int64_t)afdo_profile_info->cutoff,
 	     (int64_t)afdo_hot_bb_threshod);
   afdo_profile_info->sum_max *= afdo_count_scale;
   return true;
@@ -2553,8 +2663,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack (
     {
       if (dump_file)
 	fprintf (dump_file, "No offline instance for %s\n",
-		 IDENTIFIER_POINTER
-		   (DECL_ASSEMBLER_NAME (stack[stack.length () - 1].decl)));
+		 raw_symbol_name (stack[stack.length () - 1].decl));
       return NULL;
     }
   function_instance *s = iter->second;
@@ -2576,8 +2685,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack (
 			      "auto-profile has no inlined function instance "
 			      "for inlined call of %s at relative "
 			      " locaction +%i, discriminator %i\n",
-			     IDENTIFIER_POINTER
-			      (DECL_ASSEMBLER_NAME (stack[i - 1].decl)),
+			     raw_symbol_name (stack[i - 1].decl),
 			     stack[i].afdo_loc >> 16,
 			     stack[i].afdo_loc & 65535);
 	  return NULL;
@@ -2641,14 +2749,22 @@ read_profile (void)
 
   /* autofdo_source_profile.  */
   afdo_source_profile = autofdo_source_profile::create ();
-  if (afdo_source_profile == NULL)
+  if (afdo_source_profile == NULL
+      || gcov_is_error ())
     {
       error ("cannot read function profile from %s", auto_profile_file);
+      delete afdo_source_profile;
+      afdo_source_profile = NULL;
       return;
     }
 
   /* autofdo_module_profile.  */
   fake_read_autofdo_module_profile ();
+  if (gcov_is_error ())
+    {
+      error ("cannot read module profile from %s", auto_profile_file);
+      return;
+    }
 }
 
 /* From AutoFDO profiles, find values inside STMT for that we want to measure
@@ -3301,10 +3417,22 @@ cmp (const void *a, const void *b)
   return 0;
 }
 
+/* To scalle a connected component of graph we collect desired scales of
+   basic blocks on the boundary and then compute a robust average.  */
+
+struct scale
+{
+  /* Scale descired.  */
+  sreal scale;
+  /* Weight for averaging computed from execution count of the edge
+     scale originates from.  */
+  uint64_t weight;
+};
+
 /* Add scale ORIG/ANNOTATED to SCALES.  */
 
 static void
-add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig)
+add_scale (vec <scale> *scales, profile_count annotated, profile_count orig)
 {
   if (dump_file)
     {
@@ -3319,9 +3447,9 @@ add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig)
 	= annotated.guessed_local ()
 		.to_sreal_scale (orig);
       if (dump_file)
-	fprintf (dump_file, "    adding scale %.16f\n",
-		 scale.to_double ());
-      scales->safe_push (scale);
+	fprintf (dump_file, "    adding scale %.16f, weight %" PRId64 "\n",
+		 scale.to_double (), annotated.value () + 1);
+      scales->safe_push ({scale, annotated.value () + 1});
     }
 }
 
@@ -3367,7 +3495,7 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
   /* Basic blocks of connected component currently processed.  */
   auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun));
   /* Scale factors found.  */
-  auto_vec <sreal, 20> scales;
+  auto_vec <scale, 20> scales;
   auto_vec <basic_block, 20> stack (n_basic_blocks_for_fn (cfun));
 
   basic_block seed_bb;
@@ -3379,9 +3507,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
      >=2 is an id of the component BB belongs to.  */
   auto_vec <unsigned int, 20> component;
   component.safe_grow (last_basic_block_for_fn (cfun));
+  profile_count max_count_in_fn = profile_count::zero ();
   FOR_ALL_BB_FN (seed_bb, cfun)
-    component[seed_bb->index]
-	= is_bb_annotated (seed_bb, *annotated_bb) ? 1 : 0;
+    if (is_bb_annotated (seed_bb, *annotated_bb))
+      {
+	component[seed_bb->index] = 1;
+	max_count_in_fn = max_count_in_fn.max (seed_bb->count);
+      }
+    else
+      component[seed_bb->index] = 0;
   FOR_ALL_BB_FN (seed_bb, cfun)
    if (!component[seed_bb->index])
      {
@@ -3504,12 +3638,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
 		 profile_count annotated_count = e->dest->count;
 		 profile_count out_count = profile_count::zero ();
 		 bool ok = true;
+
 		 for (edge e2: e->dest->preds)
 		   if (AFDO_EINFO (e2)->is_annotated ())
 		     annotated_count -= AFDO_EINFO (e2)->get_count ();
-		   else if (component[e->src->index] == component_id)
-		     out_count += e->count ();
-		   else if (e->probability.nonzero_p ())
+		   else if (component[e2->src->index] == component_id)
+		     out_count += e2->count ();
+		   else if (is_bb_annotated (e2->src, *annotated_bb))
+		     annotated_count -= e2->count ();
+		   else if (e2->probability.nonzero_p ())
 		     {
 		       ok = false;
 		       break;
@@ -3551,12 +3688,52 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
 	 {
 	   if (dump_file)
 	     fprintf (dump_file,
-		      "  Can not determine count from the boundary; giving up");
+		      "  Can not determine count from the boundary; giving up\n");
 	   continue;
 	 }
        gcc_checking_assert (scales.length ());
        scales.qsort (cmp);
-       scale_bbs (bbs, scales[scales.length () / 2]);
+
+       uint64_t overall_weight = 0;
+       for (scale &e : scales)
+	 overall_weight += e.weight;
+
+       uint64_t cummulated = 0, weight_sum = 0;
+       sreal scale_sum = 0;
+       for (scale &e : scales)
+	 {
+	   uint64_t prev = cummulated;
+	   cummulated += e.weight;
+	   if (cummulated >= overall_weight / 4
+	       && prev <= 3 * overall_weight / 4)
+	     {
+	       scale_sum += e.scale * e.weight;
+	       weight_sum += e.weight;
+	       if (dump_file)
+		 fprintf (dump_file, "    accounting scale %.16f, weight %" PRId64 "\n",
+			  e.scale.to_double (), e.weight);
+	     }
+	   else if (dump_file)
+	     fprintf (dump_file, "    ignoring scale %.16f, weight %" PRId64 "\n",
+		      e.scale.to_double (), e.weight);
+	  }
+       sreal scale = scale_sum / (sreal)weight_sum;
+
+       /* Avoid scaled regions to have very large counts.
+	  Otherwise they may dominate ipa-profile's histogram computing cutoff
+	  of hot basic blocks.  */
+       if (max_count * scale > max_count_in_fn.guessed_local ())
+	 {
+	   if (dump_file)
+	     {
+	       fprintf (dump_file, "Scaling by %.16f produces max count ",
+			scale.to_double ());
+	       (max_count * scale).dump (dump_file);
+	       fprintf (dump_file, " that exceeds max count in fn; capping\n");
+	     }
+	   scale = max_count_in_fn.guessed_local ().to_sreal_scale (max_count);
+	 }
+       scale_bbs (bbs, scale);
      }
 }
 
@@ -3865,6 +4042,7 @@ read_autofdo_file (void)
   autofdo::afdo_profile_info = XNEW (gcov_summary);
   autofdo::afdo_profile_info->runs = 1;
   autofdo::afdo_profile_info->sum_max = 0;
+  autofdo::afdo_profile_info->cutoff = 1;
 
   /* Read the profile from the profile file.  */
   autofdo::read_profile ();
diff --git a/gcc/avoid-store-forwarding.cc b/gcc/avoid-store-forwarding.cc
index 37e0953..78ed736 100644
--- a/gcc/avoid-store-forwarding.cc
+++ b/gcc/avoid-store-forwarding.cc
@@ -119,17 +119,6 @@ generate_bit_insert_sequence (store_fwd_info *store_info, rtx dest)
   unsigned HOST_WIDE_INT bitsize = store_size * BITS_PER_UNIT;
   unsigned HOST_WIDE_INT start = store_info->offset * BITS_PER_UNIT;
 
-  /* Adjust START for machines with BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN.
-     Given that the bytes will be reversed in this case, we need to
-     calculate the starting position from the end of the destination
-     register.  */
-  if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
-    {
-      unsigned HOST_WIDE_INT load_mode_bitsize
-	= (GET_MODE_BITSIZE (GET_MODE (dest))).to_constant ();
-      start = load_mode_bitsize - bitsize - start;
-    }
-
   rtx mov_reg = store_info->mov_reg;
   store_bit_field (dest, bitsize, start, 0, 0, GET_MODE (mov_reg), mov_reg,
 		   false, false);
@@ -156,11 +145,18 @@ is_store_forwarding (rtx store_mem, rtx load_mem, HOST_WIDE_INT *off_val)
   poly_int64 load_offset, store_offset;
   rtx load_base = strip_offset (XEXP (load_mem, 0), &load_offset);
   rtx store_base = strip_offset (XEXP (store_mem, 0), &store_offset);
+  poly_int64 off_diff = store_offset - load_offset;
+
+  HOST_WIDE_INT off_val_tmp = 0;
+  bool is_off_diff_constant = off_diff.is_constant (&off_val_tmp);
+  if (off_val)
+    *off_val = off_val_tmp;
+
   return (MEM_SIZE (load_mem).is_constant ()
 	  && rtx_equal_p (load_base, store_base)
 	  && known_subrange_p (store_offset, MEM_SIZE (store_mem),
 			       load_offset, MEM_SIZE (load_mem))
-	  && (store_offset - load_offset).is_constant (off_val));
+	  && is_off_diff_constant);
 }
 
 /* Given a list of small stores that are forwarded to LOAD_INSN, try to
@@ -242,17 +238,39 @@ process_store_forwarding (vec<store_fwd_info> &stores, rtx_insn *load_insn,
 
   int move_to_front = -1;
   int total_cost = 0;
+  int base_offset_index = -1;
+
+  /* Find the last store that has the same offset the load, in the case that
+     we're eliminating the load.  We will try to use it as a base register
+     to avoid bit inserts (see second loop below).  We want the last one, as
+     it will be wider and we don't want to overwrite the base register if
+     there are many of them.  */
+  if (load_elim)
+    {
+      FOR_EACH_VEC_ELT_REVERSE (stores, i, it)
+	{
+	  const bool has_base_offset
+	    = known_eq (poly_uint64 (it->offset),
+			subreg_size_lowpart_offset (MEM_SIZE (it->store_mem),
+						    load_size));
+	  if (has_base_offset)
+	    {
+	      base_offset_index = i;
+	      break;
+	    }
+	}
+    }
 
   /* Check if we can emit bit insert instructions for all forwarded stores.  */
   FOR_EACH_VEC_ELT (stores, i, it)
     {
       it->mov_reg = gen_reg_rtx (GET_MODE (it->store_mem));
       rtx_insn *insns = NULL;
-      const bool has_zero_offset = it->offset == 0;
 
-      /* If we're eliminating the load then find the store with zero offset
-	 and use it as the base register to avoid a bit insert if possible.  */
-      if (load_elim && has_zero_offset)
+      /* Check if this is a store with base offset, if we're eliminating the
+	 load, and use it as the base register to avoid a bit insert if
+	 possible.  Load elimination is implied by base_offset_index != -1.  */
+      if (i == (unsigned) base_offset_index)
 	{
 	  start_sequence ();
 
@@ -445,9 +463,22 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
     return;
 
   auto_vec<store_fwd_info, 8> store_exprs;
+  auto_vec<rtx> store_exprs_del;
   rtx_insn *insn;
   unsigned int insn_cnt = 0;
 
+  /* We are iterating over the basic block's instructions detecting store
+     instructions.  Upon reaching a load instruction, we check if any of the
+     previously detected stores could result in store forwarding.  In that
+     case, we try to reorder the load and store instructions.
+     We skip this transformation when we encounter complex memory operations,
+     instructions that might throw an exception, instruction dependencies,
+     etc.  This is done by clearing the vector of detected stores, while
+     keeping the removed stores in another vector.  By doing so, we can check
+     if any of the removed stores operated on the load's address range, when
+     reaching a subsequent store that operates on the same address range,
+     as this would lead to incorrect values on the register that keeps the
+     loaded value.  */
   FOR_BB_INSNS (bb, insn)
     {
       if (!NONDEBUG_INSN_P (insn))
@@ -460,6 +491,10 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 
       if (!set || insn_could_throw_p (insn))
 	{
+	  unsigned int i;
+	  store_fwd_info *it;
+	  FOR_EACH_VEC_ELT (store_exprs, i, it)
+	    store_exprs_del.safe_push (it->store_mem);
 	  store_exprs.truncate (0);
 	  continue;
 	}
@@ -483,6 +518,10 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 	  || (load_mem && (!MEM_SIZE_KNOWN_P (load_mem)
 			   || !MEM_SIZE (load_mem).is_constant ())))
 	{
+	  unsigned int i;
+	  store_fwd_info *it;
+	  FOR_EACH_VEC_ELT (store_exprs, i, it)
+	    store_exprs_del.safe_push (it->store_mem);
 	  store_exprs.truncate (0);
 	  continue;
 	}
@@ -534,6 +573,7 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 		    it->remove = true;
 		    removed_count++;
 		    remove_rest = true;
+		    store_exprs_del.safe_push (it->store_mem);
 		  }
 	      }
 	  }
@@ -573,23 +613,46 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 		  it->remove = true;
 		  removed_count++;
 		  remove_rest = true;
+		  forwardings.truncate (0);
 		}
 	      else if (is_store_forwarding (store_mem, load_mem, &off_val))
 		{
+		  unsigned int j;
+		  rtx *del_it;
+		  bool same_range_as_removed = false;
+
+		  /* Check if another store in the load's address range has
+		     been deleted due to a constraint violation.  In this case
+		     we can't forward any other stores that operate in this
+		     range, as it would lead to partial update of the register
+		     that holds the loaded value.  */
+		  FOR_EACH_VEC_ELT (store_exprs_del, j, del_it)
+		    {
+		      rtx del_store_mem = *del_it;
+		      same_range_as_removed
+			= is_store_forwarding (del_store_mem, load_mem, NULL);
+		      if (same_range_as_removed)
+			break;
+		    }
+
 		  /* Check if moving this store after the load is legal.  */
 		  bool write_dep = false;
-		  for (unsigned int j = store_exprs.length () - 1; j != i; j--)
+		  if (!same_range_as_removed)
 		    {
-		      if (!store_exprs[j].forwarded
-			  && output_dependence (store_mem,
-						store_exprs[j].store_mem))
+		      unsigned int j = store_exprs.length () - 1;
+		      for (; j != i; j--)
 			{
-			  write_dep = true;
-			  break;
+			  if (!store_exprs[j].forwarded
+			      && output_dependence (store_mem,
+						    store_exprs[j].store_mem))
+			    {
+			      write_dep = true;
+			      break;
+			    }
 			}
 		    }
 
-		  if (!write_dep)
+		  if (!same_range_as_removed && !write_dep)
 		    {
 		      it->forwarded = true;
 		      it->offset = off_val;
@@ -609,6 +672,7 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 		  it->remove = true;
 		  removed_count++;
 		  remove_rest = true;
+		  forwardings.truncate (0);
 		}
 	    }
 
@@ -616,9 +680,12 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 	    process_store_forwarding (forwardings, insn, load_mem);
 	}
 
+	/* Abort in case that we encounter a memory read/write that is not a
+	   simple store/load, as we can't make safe assumptions about the
+	   side-effects of this.  */
 	if ((writes_mem && !is_simple_store)
 	     || (reads_mem && !is_simple_load))
-	   store_exprs.truncate (0);
+	  return;
 
 	if (removed_count)
 	{
diff --git a/gcc/builtins.cc b/gcc/builtins.cc
index a2ce372..7f580a3 100644
--- a/gcc/builtins.cc
+++ b/gcc/builtins.cc
@@ -7799,11 +7799,17 @@ expand_builtin_crc_table_based (internal_fn fn, scalar_mode crc_mode,
 
   rtx op1 = expand_normal (rhs1);
   rtx op2 = expand_normal (rhs2);
-  gcc_assert (TREE_CODE (rhs3) == INTEGER_CST);
-  rtx op3 = gen_int_mode (TREE_INT_CST_LOW (rhs3), crc_mode);
+  rtx op3;
+  if (TREE_CODE (rhs3) != INTEGER_CST)
+    {
+      error ("third argument to %<crc%> builtins must be a constant");
+      op3 = const0_rtx;
+    }
+  else
+    op3 = convert_to_mode (crc_mode, expand_normal (rhs3), 0);
 
   if (CONST_INT_P (op2))
-    op2 = gen_int_mode (INTVAL (op2), crc_mode);
+    op2 = convert_to_mode (crc_mode, op2, 0);
 
   if (fn == IFN_CRC)
     expand_crc_table_based (target, op1, op2, op3, data_mode);
diff --git a/gcc/c-family/ChangeLog b/gcc/c-family/ChangeLog
index dd2ae5c..c9ab153 100644
--- a/gcc/c-family/ChangeLog
+++ b/gcc/c-family/ChangeLog
@@ -1,3 +1,181 @@
+2025-08-02  Martin Uecker  <uecker@tugraz.at>
+
+	* c-attribs.cc (handle_argspec_attribute): Update.
+	(build_arg_spec): New function.
+	(build_attr_access_from_parms): Rewrite `arg spec' handling.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Make diagnostics::context::m_source_printing
+	private.
+	* c-format.cc: Likewise.
+	* c-opts.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update usage of "diagnostic_info" to explicitly
+	refer to "diagnostics::diagnostic_info".
+	* c-opts.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-format.cc: Update for file_cache and char_span moving from
+	input.h to diagnostics/file-cache.h and into the "diagnostics::"
+	namespace.
+	* c-indentation.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc (c_family_tests): Add include of
+	"diagnostics/diagnostics-selftests.h".  Replace
+	c_diagnostic_cc_tests with
+	diagnostics::selftest::context_cc_tests.
+	* c-common.h: Drop c_diagnostic_cc_tests decl.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* c-format.cc: Likewise.
+	* c-lex.cc: Likewise.
+	* c-opts.cc: Likewise.
+	* c-pragma.cc: Likewise.
+	* c-warn.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+	* c-common.h: Likewise.
+	* c-cppbuiltin.cc: Likewise.
+	* known-headers.cc: Likewise.
+	* known-headers.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update comment for renaming of edit_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-format.cc (test_type_mismatch_range_labels): Update for
+	move of selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-format.cc: Update for move of selftest-diagnostic.h to
+	diagnostics/selftest-context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.h: Update for diagnostic_context becoming
+	diagnostics::context.
+	* c-opts.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-opts.cc (c_diagnostic_text_finalizer): Add "m_" prefix to
+	fields of diagnostic_info.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-opts.cc: Update for move of "diagnostic-macro-unwinding.h"
+	to "diagnostics/macro-unwinding.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-opts.cc: Update for move of diagnostics output formats into
+	namespace "diagnostics" as "sinks".
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* c-omp.cc (c_finish_omp_depobj): Use OMP_ITERATOR_DECL_P.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* c-attribs.cc (handle_target_clones_attribute): Change to use
+	get_clone_versions.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* c-format.cc (local_string_slice_node): New node type.
+	(asm_fprintf_char_table): New entry.
+	(init_dynamic_diag_info): Add support for string_slice.
+	* c-format.h (T_STRING_SLICE): New node type.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* c-opts.cc (c_common_post_options): Change
+	warn_unused_but_set_parameter and warn_unused_but_set_variable
+	from 1 to 3 if they were set only implicitly.
+	* c-attribs.cc (build_attr_access_from_parms): Remove unused
+	but set variable nelts.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* c.opt (Wc++26-compat): New option.
+	* c.opt.urls: Regenerate.
+	* c-opts.cc (c_common_post_options): Clear warn_cxx26_compat for
+	C++26 or later.
+	* c-cppbuiltin.cc (c_cpp_builtins): For C++26 predefine
+	__cpp_trivial_relocatability=202502L.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117785
+	* c-cppbuiltin.cc (c_cpp_builtins): Predefine
+	__cpp_constexpr_exceptions=202411L for C++26.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-ubsan.cc (get_bound_from_access_with_size): Adjust the position
+	of the arguments per the new design.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-attribs.cc (handle_counted_by_attribute): Accept counted_by
+	attribute for pointer fields.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-gimplify.cc (is_address_with_access_with_size): New function.
+	(ubsan_walk_array_refs_r): Instrument an INDIRECT_REF whose base
+	address is .ACCESS_WITH_SIZE or an address computation whose base
+	address is .ACCESS_WITH_SIZE.
+	* c-ubsan.cc (ubsan_instrument_bounds_pointer_address): New function.
+	(struct factor_t): New structure.
+	(get_factors_from_mul_expr): New function.
+	(get_index_from_offset): New function.
+	(get_index_from_pointer_addr_expr): New function.
+	(is_instrumentable_pointer_array_address): New function.
+	(ubsan_array_ref_instrumented_p): Change prototype.
+	Handle MEM_REF in addtional to ARRAY_REF.
+	(ubsan_maybe_instrument_array_ref): Handle MEM_REF in addtional
+	to ARRAY_REF.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* c.opt: Add -Wno-abbreviated-auto-in-template-arg.
+	* c.opt.urls: Regenerate.
+
+2025-07-04  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c/120837
+	* c-common.cc (pointer_int_sum): Rewrite the intop PLUS_EXPR or
+	MINUS_EXPR optimization into extension of both intop operands,
+	their separate multiplication and then addition/subtraction followed
+	by rest of pointer_int_sum handling after the multiplication.
+
 2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
 
 	* c-gimplify.cc (is_address_with_access_with_size): New function.
diff --git a/gcc/c-family/c-attribs.cc b/gcc/c-family/c-attribs.cc
index ea04ed7..a0d832b 100644
--- a/gcc/c-family/c-attribs.cc
+++ b/gcc/c-family/c-attribs.cc
@@ -2906,53 +2906,22 @@ handle_counted_by_attribute (tree *node, tree name,
 		" declaration %q+D", name, decl);
       *no_add_attrs = true;
     }
-  /* This attribute only applies to a field with array type or pointer type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) != ARRAY_TYPE
-	   && TREE_CODE (TREE_TYPE (decl)) != POINTER_TYPE)
+  /* This attribute only applies to field with array type.  */
+  else if (TREE_CODE (TREE_TYPE (decl)) != ARRAY_TYPE)
     {
       error_at (DECL_SOURCE_LOCATION (decl),
-		"%qE attribute is not allowed for a non-array"
-		" or non-pointer field", name);
+		"%qE attribute is not allowed for a non-array field",
+		name);
       *no_add_attrs = true;
     }
   /* This attribute only applies to a C99 flexible array member type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
-	   && !c_flexible_array_member_type_p (TREE_TYPE (decl)))
+  else if (! c_flexible_array_member_type_p (TREE_TYPE (decl)))
     {
       error_at (DECL_SOURCE_LOCATION (decl),
 		"%qE attribute is not allowed for a non-flexible"
 		" array member field", name);
       *no_add_attrs = true;
     }
-  /* This attribute cannot be applied to a pointer to void type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
-	   && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == VOID_TYPE)
-    {
-      error_at (DECL_SOURCE_LOCATION (decl),
-		"%qE attribute is not allowed for a pointer to void",
-		name);
-      *no_add_attrs = true;
-    }
-  /* This attribute cannot be applied to a pointer to function type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
-	   && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == FUNCTION_TYPE)
-    {
-      error_at (DECL_SOURCE_LOCATION (decl),
-		"%qE attribute is not allowed for a pointer to"
-		" function", name);
-      *no_add_attrs = true;
-    }
-  /* This attribute cannot be applied to a pointer to structure or union
-     with flexible array member.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
-	   && RECORD_OR_UNION_TYPE_P (TREE_TYPE (TREE_TYPE (decl)))
-	   && TYPE_INCLUDES_FLEXARRAY (TREE_TYPE (TREE_TYPE (decl))))
-    {
-	error_at (DECL_SOURCE_LOCATION (decl),
-		  "%qE attribute is not allowed for a pointer to"
-		  " structure or union with flexible array member", name);
-	*no_add_attrs = true;
-    }
   /* The argument should be an identifier.  */
   else if (TREE_CODE (argval) != IDENTIFIER_NODE)
     {
@@ -2961,8 +2930,7 @@ handle_counted_by_attribute (tree *node, tree name,
       *no_add_attrs = true;
     }
   /* Issue error when there is a counted_by attribute with a different
-     field as the argument for the same flexible array member or
-     pointer field.  */
+     field as the argument for the same flexible array member field.  */
   else if (old_counted_by != NULL_TREE)
     {
       tree old_fieldname = TREE_VALUE (TREE_VALUE (old_counted_by));
@@ -4152,10 +4120,11 @@ handle_argspec_attribute (tree *, tree, tree args, int, bool *)
 {
   /* Verify the attribute has one or two arguments and their kind.  */
   gcc_assert (args && TREE_CODE (TREE_VALUE (args)) == STRING_CST);
-  for (tree next = TREE_CHAIN (args); next; next = TREE_CHAIN (next))
+  if (TREE_CHAIN (args))
     {
-      tree val = TREE_VALUE (next);
-      gcc_assert (DECL_P (val) || EXPR_P (val));
+      tree val = TREE_VALUE (TREE_CHAIN (args));
+      gcc_assert (!TREE_CHAIN (TREE_CHAIN (args)));
+      gcc_assert (TYPE_P (val));
     }
   return NULL_TREE;
 }
@@ -5768,6 +5737,71 @@ handle_access_attribute (tree node[3], tree name, tree args, int flags,
   return NULL_TREE;
 }
 
+
+/* This function builds a string which is concatenated to SPEC and returns
+   list of variably bounds corresponding to an array/VLA parameter with
+   type TYPE.  The string consists of one dollar symbol for each specified
+   variable bound, one asterisk for each unspecified variable bound,
+   a space for an array of unknown size (only possibly for the outermost),
+   and a zero for a zero-sized array.
+
+   The chainof variable bounds starts with the most significant bound.
+   For example, the TYPE T[2][m][3][n] will produce "$$" and (m, (n, nil)).  */
+
+static tree
+build_arg_spec (tree type, std::string *spec)
+{
+  while (POINTER_TYPE_P (type))
+    type = TREE_TYPE (type);
+
+  if (TREE_CODE (type) != ARRAY_TYPE)
+    return NULL_TREE;
+
+  tree list = build_arg_spec (TREE_TYPE (type), spec);
+
+  if (!COMPLETE_TYPE_P (type))
+    {
+      (*spec) += ' ';
+      return list;
+    }
+
+  tree mval = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
+
+  if (!mval)
+    {
+     (*spec) += '0';
+     return list;
+    }
+
+  if (TREE_CODE (mval) == COMPOUND_EXPR
+      && integer_zerop (TREE_OPERAND (mval, 0))
+      && integer_zerop (TREE_OPERAND (mval, 1)))
+    {
+      (*spec) += '*';
+      return list;
+    }
+
+  if (TREE_CODE (mval) == INTEGER_CST)
+    return list;
+
+  /* A variable bound.  */
+  (*spec) += '$';
+
+  mval = array_type_nelts_top (type);
+
+  /* Remove NOP_EXPR and SAVE_EXPR to uncover possible PARM_DECLS.  */
+  if (TREE_CODE (mval) == NOP_EXPR)
+    mval = TREE_OPERAND (mval, 0);
+   if (TREE_CODE (mval) == SAVE_EXPR)
+    {
+      mval = TREE_OPERAND (mval, 0);
+      if (TREE_CODE (mval) == NOP_EXPR)
+	mval = TREE_OPERAND (mval, 0);
+    }
+
+  return tree_cons (NULL_TREE, mval, list);
+}
+
 /* Extract attribute "arg spec" from each FNDECL argument that has it,
    build a single attribute access corresponding to all the arguments,
    and return the result.  SKIP_VOIDPTR set to ignore void* parameters
@@ -5844,15 +5878,16 @@ build_attr_access_from_parms (tree parms, bool skip_voidptr)
       argspec = TREE_VALUE (argspec);
 
       /* The attribute arg spec string.  */
-      tree str = TREE_VALUE (argspec);
-      const char *s = TREE_STRING_POINTER (str);
+      const char *s = TREE_STRING_POINTER (TREE_VALUE (argspec));
+      bool static_p = s && (0 == strcmp("static", s));
 
       /* Collect the list of nonnull arguments which use "[static ..]".  */
-      if (s != NULL && s[0] == '[' && s[1] == 's')
+      if (static_p)
 	nnlist = tree_cons (NULL_TREE, build_int_cst (integer_type_node,
 						      argpos + 1), nnlist);
 
-      /* Create the attribute access string from the arg spec string,
+      tree argvbs;
+      /* Create the attribute access string from the arg spec data,
 	 optionally followed by position of the VLA bound argument if
 	 it is one.  */
       {
@@ -5863,21 +5898,52 @@ build_attr_access_from_parms (tree parms, bool skip_voidptr)
 	    specend = 1;
 	  }
 
-	/* Format the access string in place.  */
-	int len = snprintf (NULL, 0, "%c%u%s",
-			    attr_access::mode_chars[access_deferred],
-			    argpos, s);
-	spec.resize (specend + len + 1);
-	sprintf (&spec[specend], "%c%u%s",
-		 attr_access::mode_chars[access_deferred],
-		 argpos, s);
+	spec += attr_access::mode_chars[access_deferred];
+	spec += std::to_string (argpos);
+	spec += '[';
+	tree type = TREE_VALUE (TREE_CHAIN (argspec));
+	argvbs = build_arg_spec (type, &spec);
+
+	/* Postprocess the string to bring it in the format expected
+	   by the code handling the access attribute.  First, we
+	   add 's' if the array was declared as [static ...].  */
+	if (static_p)
+	  {
+	    size_t send = spec.length();
+
+	    if (spec[send - 1] == '[')
+	      {
+		spec += 's';
+	      }
+	    else
+	      {
+		/* If there is a symbol, we need to swap the order.  */
+		spec += spec[send - 1];
+		spec[send - 1] = 's';
+	      }
+	  }
+
+	/* If the outermost bound is an integer constant, we need to write
+	   the size  if it is constant.  */
+	if (type && TYPE_DOMAIN (type))
+	  {
+	    tree mval = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
+	    if (mval && TREE_CODE (mval) == INTEGER_CST)
+	      {
+		char buf[40];
+		unsigned HOST_WIDE_INT n = tree_to_uhwi (mval) + 1;
+		sprintf (buf, HOST_WIDE_INT_PRINT_UNSIGNED, n);
+		spec += buf;
+	      }
+	  }
+	spec += ']';
+
 	/* Trim the trailing NUL.  */
-	spec.resize (specend + len);
+	spec.resize (spec.length ());
       }
 
       /* The (optional) list of expressions denoting the VLA bounds
 	 N in ARGTYPE <arg>[Ni]...[Nj]...[Nk].  */
-      tree argvbs = TREE_CHAIN (argspec);
       if (argvbs)
 	{
 	  spec += ',';
@@ -5890,8 +5956,7 @@ build_attr_access_from_parms (tree parms, bool skip_voidptr)
 	     order.  */
 	  vblist = tree_cons (NULL_TREE, argvbs, vblist);
 
-	  unsigned nelts = 0;
-	  for (tree vb = argvbs; vb; vb = TREE_CHAIN (vb), ++nelts)
+	  for (tree vb = argvbs; vb; vb = TREE_CHAIN (vb))
 	    {
 	      tree bound = TREE_VALUE (vb);
 	      if (const unsigned *psizpos = arg2pos.get (bound))
@@ -6173,7 +6238,9 @@ handle_target_clones_attribute (tree *node, tree name, tree ARG_UNUSED (args),
 	    }
 	}
 
-      if (get_target_clone_attr_len (args) == -1)
+      auto_vec<string_slice> versions = get_clone_attr_versions (args, NULL);
+
+      if (versions.length () == 1)
 	{
 	  warning (OPT_Wattributes,
 		   "single %<target_clones%> attribute is ignored");
diff --git a/gcc/c-family/c-common.cc b/gcc/c-family/c-common.cc
index 9d69298..e7dd460 100644
--- a/gcc/c-family/c-common.cc
+++ b/gcc/c-family/c-common.cc
@@ -56,6 +56,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-pretty-print-markup.h"
 #include "gcc-rich-location.h"
 #include "gcc-urlifier.h"
+#include "diagnostics/diagnostics-selftests.h"
 
 cpp_reader *parse_in;		/* Declared in c-pragma.h.  */
 
@@ -3438,20 +3439,41 @@ pointer_int_sum (location_t loc, enum tree_code resultcode,
 	 an overflow error if the constant is negative but INTOP is not.  */
       && (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (intop))
 	  || (TYPE_PRECISION (TREE_TYPE (intop))
-	      == TYPE_PRECISION (TREE_TYPE (ptrop)))))
-    {
-      enum tree_code subcode = resultcode;
-      tree int_type = TREE_TYPE (intop);
-      if (TREE_CODE (intop) == MINUS_EXPR)
-	subcode = (subcode == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR);
-      /* Convert both subexpression types to the type of intop,
-	 because weird cases involving pointer arithmetic
-	 can result in a sum or difference with different type args.  */
-      ptrop = build_binary_op (EXPR_LOCATION (TREE_OPERAND (intop, 1)),
-			       subcode, ptrop,
-			       convert (int_type, TREE_OPERAND (intop, 1)),
-			       true);
-      intop = convert (int_type, TREE_OPERAND (intop, 0));
+	      == TYPE_PRECISION (TREE_TYPE (ptrop))))
+      && TYPE_PRECISION (TREE_TYPE (intop)) <= TYPE_PRECISION (sizetype))
+    {
+      tree intop0 = TREE_OPERAND (intop, 0);
+      tree intop1 = TREE_OPERAND (intop, 1);
+      if (TYPE_PRECISION (TREE_TYPE (intop)) != TYPE_PRECISION (sizetype)
+	  || TYPE_UNSIGNED (TREE_TYPE (intop)) != TYPE_UNSIGNED (sizetype))
+	{
+	  tree optype = c_common_type_for_size (TYPE_PRECISION (sizetype),
+						TYPE_UNSIGNED (sizetype));
+	  intop0 = convert (optype, intop0);
+	  intop1 = convert (optype, intop1);
+	}
+      tree t = fold_build2_loc (loc, MULT_EXPR, TREE_TYPE (intop0), intop0,
+				convert (TREE_TYPE (intop0), size_exp));
+      intop0 = convert (sizetype, t);
+      if (TREE_OVERFLOW_P (intop0) && !TREE_OVERFLOW (t))
+	intop0 = wide_int_to_tree (TREE_TYPE (intop0), wi::to_wide (intop0));
+      t = fold_build2_loc (loc, MULT_EXPR, TREE_TYPE (intop1), intop1,
+			   convert (TREE_TYPE (intop1), size_exp));
+      intop1 = convert (sizetype, t);
+      if (TREE_OVERFLOW_P (intop1) && !TREE_OVERFLOW (t))
+	intop1 = wide_int_to_tree (TREE_TYPE (intop1), wi::to_wide (intop1));
+      intop = build_binary_op (EXPR_LOCATION (intop), TREE_CODE (intop),
+			       intop0, intop1, true);
+
+      /* Create the sum or difference.  */
+      if (resultcode == MINUS_EXPR)
+	intop = fold_build1_loc (loc, NEGATE_EXPR, sizetype, intop);
+
+      ret = fold_build_pointer_plus_loc (loc, ptrop, intop);
+
+      fold_undefer_and_ignore_overflow_warnings ();
+
+      return ret;
     }
 
   /* Convert the integer argument to a type the same size as sizetype
@@ -7004,7 +7026,7 @@ c_parse_error (const char *gmsgid, enum cpp_ttype token_type,
 /* Return the gcc option code associated with the reason for a cpp
    message, or 0 if none.  */
 
-static diagnostic_option_id
+static diagnostics::option_id
 c_option_controlling_cpp_diagnostic (enum cpp_warning_reason reason)
 {
   const struct cpp_reason_option_codes_t *entry;
@@ -7046,8 +7068,8 @@ c_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
 		  rich_location *richloc,
 		  const char *msg, va_list *ap)
 {
-  diagnostic_info diagnostic;
-  diagnostic_t dlevel;
+  diagnostics::diagnostic_info diagnostic;
+  enum diagnostics::kind dlevel;
   bool save_warn_system_headers = global_dc->m_warn_system_headers;
   bool ret;
 
@@ -7061,24 +7083,24 @@ c_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
     case CPP_DL_WARNING:
       if (flag_no_output)
 	return false;
-      dlevel = DK_WARNING;
+      dlevel = diagnostics::kind::warning;
       break;
     case CPP_DL_PEDWARN:
       if (flag_no_output && !flag_pedantic_errors)
 	return false;
-      dlevel = DK_PEDWARN;
+      dlevel = diagnostics::kind::pedwarn;
       break;
     case CPP_DL_ERROR:
-      dlevel = DK_ERROR;
+      dlevel = diagnostics::kind::error;
       break;
     case CPP_DL_ICE:
-      dlevel = DK_ICE;
+      dlevel = diagnostics::kind::ice;
       break;
     case CPP_DL_NOTE:
-      dlevel = DK_NOTE;
+      dlevel = diagnostics::kind::note;
       break;
     case CPP_DL_FATAL:
-      dlevel = DK_FATAL;
+      dlevel = diagnostics::kind::fatal;
       break;
     default:
       gcc_unreachable ();
@@ -9942,8 +9964,11 @@ c_family_tests (void)
   c_indentation_cc_tests ();
   c_pretty_print_cc_tests ();
   c_spellcheck_cc_tests ();
-  c_diagnostic_cc_tests ();
   c_opt_problem_cc_tests ();
+
+  /* According to https://gcc.gnu.org/pipermail/gcc/2021-November/237703.html
+     this has some language-specific assumptions, so we run it here.  */
+  diagnostics::selftest::context_cc_tests ();
 }
 
 } // namespace selftest
@@ -10025,7 +10050,7 @@ try_to_locate_new_include_insertion_point (const char *file, location_t loc)
     return UNKNOWN_LOCATION;
 
   /* The "start_location" is column 0, meaning "the whole line".
-     rich_location and edit_context can't cope with this, so use
+     rich_location and diagnostics::changes can't cope with this, so use
      column 1 instead.  */
   location_t col_0 = ord_map_for_insertion->start_location;
   return linemap_position_for_loc_and_offset (line_table, col_0, 1);
@@ -10089,7 +10114,7 @@ maybe_add_include_fixit (rich_location *richloc, const char *header,
   richloc->add_fixit_insert_before (include_insert_loc, text);
   free (text);
 
-  if (override_location && global_dc->m_source_printing.enabled)
+  if (override_location && global_dc->get_source_printing_options ().enabled)
     {
       /* Replace the primary location with that of the insertion point for the
 	 fix-it hint.
diff --git a/gcc/c-family/c-common.h b/gcc/c-family/c-common.h
index 675708d..7c7e21d 100644
--- a/gcc/c-family/c-common.h
+++ b/gcc/c-family/c-common.h
@@ -979,7 +979,7 @@ extern tree build_va_arg (location_t, tree, tree);
 
 extern const unsigned int c_family_lang_mask;
 extern unsigned int c_common_option_lang_mask (void);
-extern void c_common_diagnostics_set_defaults (diagnostic_context *);
+extern void c_common_diagnostics_set_defaults (diagnostics::context *);
 extern bool c_common_complain_wrong_lang_p (const struct cl_option *);
 extern void c_common_init_options_struct (struct gcc_options *);
 extern void c_common_init_options (unsigned int, struct cl_decoded_option *);
@@ -1265,7 +1265,7 @@ extern void c_stddef_cpp_builtins (void);
 extern void fe_file_change (const line_map_ordinary *);
 extern void c_parse_error (const char *, enum cpp_ttype, tree, unsigned char,
 			   rich_location *richloc);
-extern diagnostic_option_id get_option_for_builtin_define (const char *macro_name);
+extern diagnostics::option_id get_option_for_builtin_define (const char *macro_name);
 
 /* In c-ppoutput.cc  */
 extern void init_pp_output (FILE *);
@@ -1729,7 +1729,6 @@ extern tree braced_lists_to_strings (tree, tree);
 namespace selftest {
   /* Declarations for specific families of tests within c-family,
      by source file, in alphabetical order.  */
-  extern void c_diagnostic_cc_tests (void);
   extern void c_format_cc_tests (void);
   extern void c_indentation_cc_tests (void);
   extern void c_opt_problem_cc_tests (void);
diff --git a/gcc/c-family/c-cppbuiltin.cc b/gcc/c-family/c-cppbuiltin.cc
index 459fd86..4aea902 100644
--- a/gcc/c-family/c-cppbuiltin.cc
+++ b/gcc/c-family/c-cppbuiltin.cc
@@ -1087,6 +1087,7 @@ c_cpp_builtins (cpp_reader *pfile)
 	{
 	  /* Set feature test macros for C++26.  */
 	  cpp_define (pfile, "__cpp_constexpr=202406L");
+	  cpp_define (pfile, "__cpp_constexpr_exceptions=202411L");
 	  cpp_define (pfile, "__cpp_static_assert=202306L");
 	  cpp_define (pfile, "__cpp_placeholder_variables=202306L");
 	  cpp_define (pfile, "__cpp_structured_bindings=202403L");
@@ -1095,6 +1096,7 @@ c_cpp_builtins (cpp_reader *pfile)
 	  cpp_define (pfile, "__cpp_pack_indexing=202311L");
 	  cpp_define (pfile, "__cpp_pp_embed=202502L");
 	  cpp_define (pfile, "__cpp_constexpr_virtual_inheritance=202506L");
+	  cpp_define (pfile, "__cpp_trivial_relocatability=202502L");
 	}
       if (flag_concepts && cxx_dialect > cxx14)
 	cpp_define (pfile, "__cpp_concepts=202002L");
@@ -1679,7 +1681,7 @@ c_cpp_builtins (cpp_reader *pfile)
 /* Given NAME, return the command-line option that would make it be
    a builtin define, or 0 if unrecognized.  */
 
-diagnostic_option_id
+diagnostics::option_id
 get_option_for_builtin_define (const char *name)
 {
   if (!strcmp (name, "_OPENACC"))
diff --git a/gcc/c-family/c-format.cc b/gcc/c-family/c-format.cc
index a44249a..bf144d0 100644
--- a/gcc/c-family/c-format.cc
+++ b/gcc/c-family/c-format.cc
@@ -32,7 +32,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "diagnostic.h"
 #include "substring-locations.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/file-cache.h"
 #include "builtins.h"
 #include "attribs.h"
 #include "c-family/c-type-mismatch.h"
@@ -70,6 +71,7 @@ static GTY(()) tree local_event_ptr_node;
 static GTY(()) tree local_pp_element_ptr_node;
 static GTY(()) tree local_gimple_ptr_node;
 static GTY(()) tree local_cgraph_node_ptr_node;
+static GTY(()) tree local_string_slice_node;
 static GTY(()) tree locus;
 
 static bool decode_format_attr (const_tree, tree, tree, function_format_info *,
@@ -770,6 +772,7 @@ static const format_char_info asm_fprintf_char_table[] =
   { "p",   1, STD_C89, { T89_V,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "q",  "c",  NULL }, \
   { "r",   1, STD_C89, { T89_C,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "",    "//cR",   NULL }, \
   { "@",   1, STD_C89, { T_EVENT_PTR,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "", "\"",   NULL }, \
+  { "B",   1, STD_C89, { T_STRING_SLICE,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "q", "",   NULL }, \
   { "e",   1, STD_C89, { T_PP_ELEMENT_PTR,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "", "\"", NULL }, \
   { "<",   0, STD_C89, NOARGUMENTS, "",      "<",   NULL }, \
   { ">",   0, STD_C89, NOARGUMENTS, "",      ">",   NULL }, \
@@ -4632,7 +4635,7 @@ get_corrected_substring (const substring_loc &fmt_loc,
   if (caret.column > finish.column)
     return NULL;
 
-  char_span line
+  diagnostics::char_span line
     = global_dc->get_file_cache ().get_source_line (start.file, start.line);
   if (!line)
     return NULL;
@@ -4644,7 +4647,8 @@ get_corrected_substring (const substring_loc &fmt_loc,
      specification, up to the (but not including) the length modifier.
      In the above example, this would be "%-+*.*".  */
   int length_up_to_type = caret.column - start.column;
-  char_span prefix_span = line.subspan (start.column - 1, length_up_to_type);
+  diagnostics::char_span prefix_span
+    = line.subspan (start.column - 1, length_up_to_type);
   char *prefix = prefix_span.xstrdup ();
 
   /* Now attempt to generate a suggestion for the rest of the specification
@@ -5211,6 +5215,11 @@ init_dynamic_diag_info (void)
       || local_cgraph_node_ptr_node == void_type_node)
     local_cgraph_node_ptr_node = get_named_type ("cgraph_node");
 
+  /* Similar to the above but for string_slice*.  */
+  if (!local_string_slice_node
+      || local_string_slice_node == void_type_node)
+    local_string_slice_node = get_named_type ("string_slice");
+
   /* Similar to the above but for diagnostic_event_id_t*.  */
   if (!local_event_ptr_node
       || local_event_ptr_node == void_type_node)
@@ -5577,10 +5586,12 @@ test_type_mismatch_range_labels ()
   gcc_rich_location richloc (fmt, &fmt_label, nullptr);
   richloc.add_range (param, SHOW_RANGE_WITHOUT_CARET, &param_label);
 
-  test_diagnostic_context dc;
+  diagnostics::selftest::test_context dc;
   diagnostic_show_locus (&dc,
-			 dc.m_source_printing,
-			 &richloc, DK_ERROR, dc.get_reference_printer ());
+			 dc.get_source_printing_options (),
+			 &richloc,
+			 diagnostics::kind::error,
+			 dc.get_reference_printer ());
   if (c_dialect_cxx ())
     /* "char*", without a space.  */
     ASSERT_STREQ ("   printf (\"msg: %i\\n\", msg);\n"
diff --git a/gcc/c-family/c-format.h b/gcc/c-family/c-format.h
index 323338c..d44d386 100644
--- a/gcc/c-family/c-format.h
+++ b/gcc/c-family/c-format.h
@@ -317,6 +317,7 @@ struct format_kind_info
 #define T89_G   { STD_C89, NULL, &local_gimple_ptr_node }
 #define T_CGRAPH_NODE   { STD_C89, NULL, &local_cgraph_node_ptr_node }
 #define T_EVENT_PTR    { STD_C89, NULL, &local_event_ptr_node }
+#define T_STRING_SLICE    { STD_C89, NULL, &local_string_slice_node }
 #define T_PP_ELEMENT_PTR    { STD_C89, NULL, &local_pp_element_ptr_node }
 #define T89_T   { STD_C89, NULL, &local_tree_type_node }
 #define T89_V	{ STD_C89, NULL, T_V }
diff --git a/gcc/c-family/c-gimplify.cc b/gcc/c-family/c-gimplify.cc
index e905059..c6fb764 100644
--- a/gcc/c-family/c-gimplify.cc
+++ b/gcc/c-family/c-gimplify.cc
@@ -66,20 +66,6 @@ along with GCC; see the file COPYING3.  If not see
     walk back up, we check that they fit our constraints, and copy them
     into temporaries if not.  */
 
-
-/* Check whether TP is an address computation whose base is a call to
-   .ACCESS_WITH_SIZE.  */
-
-static bool
-is_address_with_access_with_size (tree tp)
-{
-  if (TREE_CODE (tp) == POINTER_PLUS_EXPR
-      && (TREE_CODE (TREE_OPERAND (tp, 0)) == INDIRECT_REF)
-      && (is_access_with_size_p (TREE_OPERAND (TREE_OPERAND (tp, 0), 0))))
-	return true;
-  return false;
-}
-
 /* Callback for c_genericize.  */
 
 static tree
@@ -135,20 +121,6 @@ ubsan_walk_array_refs_r (tree *tp, int *walk_subtrees, void *data)
       walk_tree (&TREE_OPERAND (*tp, 1), ubsan_walk_array_refs_r, pset, pset);
       walk_tree (&TREE_OPERAND (*tp, 0), ubsan_walk_array_refs_r, pset, pset);
     }
-  else if (TREE_CODE (*tp) == INDIRECT_REF
-	   && is_address_with_access_with_size (TREE_OPERAND (*tp, 0)))
-    {
-      ubsan_maybe_instrument_array_ref (&TREE_OPERAND (*tp, 0), false);
-      /* Make sure ubsan_maybe_instrument_array_ref is not called again on
-	 the POINTER_PLUS_EXPR, so ensure it is not walked again and walk
-	 its subtrees manually.  */
-      tree aref = TREE_OPERAND (*tp, 0);
-      pset->add (aref);
-      *walk_subtrees = 0;
-      walk_tree (&TREE_OPERAND (aref, 0), ubsan_walk_array_refs_r, pset, pset);
-    }
-  else if (is_address_with_access_with_size (*tp))
-    ubsan_maybe_instrument_array_ref (tp, true);
   return NULL_TREE;
 }
 
diff --git a/gcc/c-family/c-indentation.cc b/gcc/c-family/c-indentation.cc
index 2e8261d..bb214fc 100644
--- a/gcc/c-family/c-indentation.cc
+++ b/gcc/c-family/c-indentation.cc
@@ -25,6 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "c-indentation.h"
 #include "selftest.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 
 /* Round up VIS_COLUMN to nearest tab stop. */
 
@@ -45,13 +46,13 @@ next_tab_stop (unsigned int vis_column, unsigned int tab_width)
    on the line (up to or before EXPLOC).  */
 
 static bool
-get_visual_column (file_cache &fc,
+get_visual_column (diagnostics::file_cache &fc,
 		   expanded_location exploc,
 		   unsigned int *out,
 		   unsigned int *first_nws,
 		   unsigned int tab_width)
 {
-  char_span line = fc.get_source_line (exploc.file, exploc.line);
+  diagnostics::char_span line = fc.get_source_line (exploc.file, exploc.line);
   if (!line)
     return false;
   if ((size_t)exploc.column > line.length ())
@@ -88,14 +89,14 @@ get_visual_column (file_cache &fc,
    Otherwise, return false, leaving *FIRST_NWS untouched.  */
 
 static bool
-get_first_nws_vis_column (file_cache &fc,
+get_first_nws_vis_column (diagnostics::file_cache &fc,
 			  const char *file, int line_num,
 			  unsigned int *first_nws,
 			  unsigned int tab_width)
 {
   gcc_assert (first_nws);
 
-  char_span line = fc.get_source_line (file, line_num);
+  diagnostics::char_span line = fc.get_source_line (file, line_num);
   if (!line)
     return false;
   unsigned int vis_column = 0;
@@ -160,7 +161,7 @@ get_first_nws_vis_column (file_cache &fc,
    Return true if such an unindent/outdent is detected.  */
 
 static bool
-detect_intervening_unindent (file_cache &fc,
+detect_intervening_unindent (diagnostics::file_cache &fc,
 			     const char *file,
 			     int body_line,
 			     int next_stmt_line,
@@ -335,7 +336,7 @@ should_warn_for_misleading_indentation (const token_indent_info &guard_tinfo,
   if (next_stmt_exploc.file != body_exploc.file)
     return false;
 
-  file_cache &fc = global_dc->get_file_cache ();
+  diagnostics::file_cache &fc = global_dc->get_file_cache ();
 
   /* If NEXT_STMT_LOC and BODY_LOC are on the same line, consider
      the location of the guard.
@@ -691,7 +692,7 @@ test_next_tab_stop ()
 
 static void
 assert_get_visual_column_succeeds (const location &loc,
-				   file_cache &fc,
+				   diagnostics::file_cache &fc,
 				   const char *file, int line, int column,
 				   const unsigned int tab_width,
 				   unsigned int expected_visual_column,
@@ -735,7 +736,7 @@ assert_get_visual_column_succeeds (const location &loc,
 
 static void
 assert_get_visual_column_fails (const location &loc,
-				file_cache &fc,
+				diagnostics::file_cache &fc,
 				const char *file, int line, int column,
 				const unsigned int tab_width)
 {
@@ -783,7 +784,7 @@ test_get_visual_column ()
 			 "\t line 2\n");
   line_table_test ltt;
   temp_source_file tmp (SELFTEST_LOCATION, ".txt", content);
-  file_cache fc;
+  diagnostics::file_cache fc;
 
   const unsigned int tab_width = 8;
   const char *file = tmp.get_filename ();
diff --git a/gcc/c-family/c-lex.cc b/gcc/c-family/c-lex.cc
index fef6ae6..b45d722 100644
--- a/gcc/c-family/c-lex.cc
+++ b/gcc/c-family/c-lex.cc
@@ -1176,7 +1176,7 @@ interpret_integer (const cpp_token *token, unsigned int flags,
 	  && (flags & CPP_N_WIDTH) != CPP_N_LARGE)
 	emit_diagnostic
 	  ((c_dialect_cxx () ? cxx_dialect == cxx98 : !flag_isoc99)
-	   ? DK_PEDWARN : DK_WARNING,
+	   ? diagnostics::kind::pedwarn : diagnostics::kind::warning,
 	   input_location, OPT_Wlong_long,
 	   (flags & CPP_N_UNSIGNED)
 	   ? "integer constant is too large for %<unsigned long%> type"
diff --git a/gcc/c-family/c-omp.cc b/gcc/c-family/c-omp.cc
index 4352214..fe272888 100644
--- a/gcc/c-family/c-omp.cc
+++ b/gcc/c-family/c-omp.cc
@@ -769,9 +769,7 @@ c_finish_omp_depobj (location_t loc, tree depobj,
 	  kind = OMP_CLAUSE_DEPEND_KIND (clause);
 	  t = OMP_CLAUSE_DECL (clause);
 	  gcc_assert (t);
-	  if (TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (OMP_ITERATOR_DECL_P (t))
 	    {
 	      error_at (OMP_CLAUSE_LOCATION (clause),
 			"%<iterator%> modifier may not be specified on "
diff --git a/gcc/c-family/c-opts.cc b/gcc/c-family/c-opts.cc
index 3ee9444..c652e82 100644
--- a/gcc/c-family/c-opts.cc
+++ b/gcc/c-family/c-opts.cc
@@ -32,7 +32,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "flags.h"
 #include "toplev.h"
 #include "langhooks.h"
-#include "diagnostic-macro-unwinding.h" /* for virt_loc_aware_diagnostic_finalizer */
+#include "diagnostics/macro-unwinding.h" /* for virt_loc_aware_diagnostic_finalizer */
 #include "intl.h"
 #include "cppdefault.h"
 #include "incpath.h"
@@ -43,7 +43,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "dumpfile.h"
 #include "file-prefix-map.h"    /* add_*_prefix_map()  */
 #include "context.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 #ifndef DOLLARS_IN_IDENTIFIERS
 # define DOLLARS_IN_IDENTIFIERS true
@@ -169,9 +169,9 @@ c_common_option_lang_mask (void)
 
 /* Diagnostic finalizer for C/C++/Objective-C/Objective-C++.  */
 static void
-c_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-			     const diagnostic_info *diagnostic,
-			     diagnostic_t)
+c_diagnostic_text_finalizer (diagnostics::text_sink &text_output,
+			     const diagnostics::diagnostic_info *diagnostic,
+			     enum diagnostics::kind)
 {
   pretty_printer *const pp = text_output.get_printer ();
   char *saved_prefix = pp_take_prefix (pp);
@@ -179,19 +179,19 @@ c_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
   pp_newline (pp);
   diagnostic_show_locus (&text_output.get_context (),
 			 text_output.get_source_printing_options (),
-			 diagnostic->richloc, diagnostic->kind, pp);
+			 diagnostic->m_richloc, diagnostic->m_kind, pp);
   /* By default print macro expansion contexts in the diagnostic
      finalizer -- for tokens resulting from macro expansion.  */
-  virt_loc_aware_diagnostic_finalizer (text_output, diagnostic);
+  diagnostics::virt_loc_aware_text_finalizer (text_output, diagnostic);
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
 }
 
 /* Common default settings for diagnostics.  */
 void
-c_common_diagnostics_set_defaults (diagnostic_context *context)
+c_common_diagnostics_set_defaults (diagnostics::context *context)
 {
-  diagnostic_text_finalizer (context) = c_diagnostic_text_finalizer;
+  diagnostics::text_finalizer (context) = c_diagnostic_text_finalizer;
   context->set_permissive_option (OPT_fpermissive);
 }
 
@@ -278,7 +278,7 @@ c_common_init_options (unsigned int decoded_options_count,
   if (c_dialect_cxx ())
     set_std_cxx17 (/*ISO*/false);
 
-  global_dc->m_source_printing.colorize_source_p = true;
+  global_dc->get_source_printing_options ().colorize_source_p = true;
 }
 
 /* Handle switch SCODE with argument ARG.  VALUE is true, unless no-
@@ -1165,6 +1165,9 @@ c_common_post_options (const char **pfilename)
       warn_cxx20_compat = 0;
       cpp_opts->cpp_warn_cxx20_compat = 0;
     }
+  if (cxx_dialect >= cxx26)
+    /* Don't warn about C++26 compatibility changes in C++26 or later.  */
+    warn_cxx26_compat = 0;
 
   /* C++17 has stricter evaluation order requirements; let's use some of them
      for earlier C++ as well, so chaining works as expected.  */
@@ -1223,6 +1226,17 @@ c_common_post_options (const char **pfilename)
   SET_OPTION_IF_UNSET (&global_options, &global_options_set,
 		       flag_range_for_ext_temps, cxx_dialect >= cxx23);
 
+  /* EnabledBy unfortunately can't specify value to use if set and
+     LangEnabledBy can't specify multiple options with &&.  For -Wunused
+     or -Wunused -Wextra we want these to default to 3 unless user specified
+     some other level explicitly.  */
+  if (warn_unused_but_set_parameter == 1)
+    SET_OPTION_IF_UNSET (&global_options, &global_options_set,
+			 warn_unused_but_set_parameter, 3);
+  if (warn_unused_but_set_variable == 1)
+    SET_OPTION_IF_UNSET (&global_options, &global_options_set,
+			 warn_unused_but_set_variable, 3);
+
   /* -fimmediate-escalation has no effect when immediate functions are not
      supported.  */
   if (flag_immediate_escalation && cxx_dialect < cxx20)
diff --git a/gcc/c-family/c-pragma.cc b/gcc/c-family/c-pragma.cc
index 137b83bf..8a1218b 100644
--- a/gcc/c-family/c-pragma.cc
+++ b/gcc/c-family/c-pragma.cc
@@ -781,7 +781,7 @@ public:
       PK_IGNORED_ATTRIBUTES,
       PK_DIAGNOSTIC,
     } pd_kind;
-  diagnostic_t diagnostic_kind;
+  enum diagnostics::kind diagnostic_kind;
   const char *kind_str;
   const char *option_str;
   bool own_option_str;
@@ -792,7 +792,7 @@ public:
     valid = false;
     loc_kind = loc_option = UNKNOWN_LOCATION;
     pd_kind = PK_INVALID;
-    diagnostic_kind = DK_UNSPECIFIED;
+    diagnostic_kind = diagnostics::kind::unspecified;
     kind_str = option_str = nullptr;
     own_option_str = false;
   }
@@ -808,7 +808,7 @@ public:
     kind_str = kind_string;
 
     pd_kind = PK_INVALID;
-    diagnostic_kind = DK_UNSPECIFIED;
+    diagnostic_kind = diagnostics::kind::unspecified;
     if (strcmp (kind_str, "push") == 0)
       pd_kind = PK_PUSH;
     else if (strcmp (kind_str, "pop") == 0)
@@ -818,17 +818,17 @@ public:
     else if (strcmp (kind_str, "error") == 0)
       {
 	pd_kind = PK_DIAGNOSTIC;
-	diagnostic_kind = DK_ERROR;
+	diagnostic_kind = diagnostics::kind::error;
       }
     else if (strcmp (kind_str, "warning") == 0)
       {
 	pd_kind = PK_DIAGNOSTIC;
-	diagnostic_kind = DK_WARNING;
+	diagnostic_kind = diagnostics::kind::warning;
       }
     else if (strcmp (kind_str, "ignored") == 0)
       {
 	pd_kind = PK_DIAGNOSTIC;
-	diagnostic_kind = DK_IGNORED;
+	diagnostic_kind = diagnostics::kind::ignored;
       }
   }
 
@@ -1016,7 +1016,8 @@ handle_pragma_diagnostic_impl ()
      what we used to do here before and changing it breaks e.g.
      PR69543 and PR69558.  */
   control_warning_option (option_index, (int) data.diagnostic_kind,
-			  arg, data.diagnostic_kind != DK_IGNORED,
+			  arg,
+			  data.diagnostic_kind != diagnostics::kind::ignored,
 			  input_location, lang_mask, &handlers,
 			  &global_options, &global_options_set,
 			  global_dc);
diff --git a/gcc/c-family/c-ubsan.cc b/gcc/c-family/c-ubsan.cc
index 38514a4..a4dc310 100644
--- a/gcc/c-family/c-ubsan.cc
+++ b/gcc/c-family/c-ubsan.cc
@@ -397,8 +397,7 @@ get_bound_from_access_with_size (tree call)
     return NULL_TREE;
 
   tree ref_to_size = CALL_EXPR_ARG (call, 1);
-  unsigned int class_of_size = TREE_INT_CST_LOW (CALL_EXPR_ARG (call, 2));
-  tree type = TREE_TYPE (CALL_EXPR_ARG (call, 3));
+  tree type = TREE_TYPE (TREE_TYPE (CALL_EXPR_ARG (call, 2)));
   tree size = fold_build2 (MEM_REF, type, unshare_expr (ref_to_size),
 			   build_int_cst (ptr_type_node, 0));
   /* If size is negative value, treat it as zero.  */
@@ -410,12 +409,7 @@ get_bound_from_access_with_size (tree call)
 			build_zero_cst (type), size);
   }
 
-  /* Only when class_of_size is 1, i.e, the number of the elements of
-     the object type, return the size.  */
-  if (class_of_size != 1)
-    return NULL_TREE;
-  else
-    size = fold_convert (sizetype, size);
+  size = fold_convert (sizetype, size);
 
   return size;
 }
@@ -554,322 +548,38 @@ ubsan_instrument_bounds (location_t loc, tree array, tree *index,
 				       *index, bound);
 }
 
-
-/* Instrument array bounds for the pointer array address which is
-   an INDIRECT_REF to the call to .ACCESS_WITH_SIZE.  We create special
-   builtin, that gets expanded in the sanopt pass, and make an array
-   dimention of it.  POINTER_ADDR is the pointer array's base address.
-   *INDEX is an index to the array.
-   IGNORE_OFF_BY_ONE is true if the POINTER_ADDR is not inside an
-   INDIRECT_REF.
-   Return NULL_TREE if no instrumentation is emitted.  */
-
-tree
-ubsan_instrument_bounds_pointer_address (location_t loc, tree pointer_addr,
-					 tree *index,
-					 bool ignore_off_by_one)
-{
-  gcc_assert (TREE_CODE (pointer_addr) == INDIRECT_REF);
-  tree call = TREE_OPERAND (pointer_addr, 0);
-  if (!is_access_with_size_p (call))
-    return NULL_TREE;
-  tree bound = get_bound_from_access_with_size (call);
-
-  if (ignore_off_by_one)
-    bound = fold_build2 (PLUS_EXPR, TREE_TYPE (bound), bound,
-			 build_int_cst (TREE_TYPE (bound),
-			 1));
-
-  /* Don't emit instrumentation in the most common cases.  */
-  tree idx = NULL_TREE;
-  if (TREE_CODE (*index) == INTEGER_CST)
-    idx = *index;
-  else if (TREE_CODE (*index) == BIT_AND_EXPR
-	   && TREE_CODE (TREE_OPERAND (*index, 1)) == INTEGER_CST)
-    idx = TREE_OPERAND (*index, 1);
-  if (idx
-      && TREE_CODE (bound) == INTEGER_CST
-      && tree_int_cst_sgn (idx) >= 0
-      && tree_int_cst_lt (idx, bound))
-    return NULL_TREE;
-
-  *index = save_expr (*index);
-
-  /* Create an array_type for the corresponding pointer array.  */
-  tree itype = build_range_type (sizetype, size_zero_node, NULL_TREE);
-  /* The array's element type can be get from the return type of the call to
-     .ACCESS_WITH_SIZE.  */
-  tree element_type = TREE_TYPE (TREE_TYPE (TREE_TYPE (call)));
-  tree array_type = build_array_type (element_type, itype);
-  /* Create a "(T *) 0" tree node to describe the array type.  */
-  tree zero_with_type = build_int_cst (build_pointer_type (array_type), 0);
-  return build_call_expr_internal_loc (loc, IFN_UBSAN_BOUNDS,
-				       void_type_node, 3, zero_with_type,
-				       *index, bound);
-}
-
-/* This structure is to combine a factor with its parent and its position
- * in its parent tree.  */
-struct factor_t
-{
-  tree factor;
-  tree parent; /* the parent tree of this factor.  */
-  int pos;     /* the position of this factor in its parent tree.  */
-};
-
-/* for a multiply expression like:
-    ((long unsigned int) m * (long unsigned int) SAVE_EXPR <n>) * 4
-
-   locate all the factors, the parents of the factor and the position of
-   the factor in its parent, and put them to VEC_FACTORS.  */
-
-static void
-get_factors_from_mul_expr (tree mult_expr, tree parent,
-			   int pos, auto_vec<factor_t> *vec_factors)
-{
-  struct factor_t mult_factor = {0, 0, -1};
-  mult_factor.factor = mult_expr;
-  mult_factor.parent = parent;
-  mult_factor.pos = pos;
-
-  while (CONVERT_EXPR_CODE_P (TREE_CODE (mult_expr)))
-    {
-      mult_factor.parent = mult_expr;
-      mult_factor.pos = 0;
-      mult_expr = TREE_OPERAND (mult_expr, 0);
-      mult_factor.factor = mult_expr;
-    }
-  if (TREE_CODE (mult_expr) != MULT_EXPR)
-    vec_factors->safe_push (mult_factor);
-  else
-    {
-      get_factors_from_mul_expr (TREE_OPERAND (mult_expr, 0), mult_expr,
-				 0, vec_factors);
-      get_factors_from_mul_expr (TREE_OPERAND (mult_expr, 1), mult_expr,
-				 1, vec_factors);
-    }
-}
-
-/* Given an OFFSET expression, and the ELEMENT_SIZE,
-   get the index expression from OFFSET and return it.
-   For example:
-   OFFSET:
-    ((long unsigned int) m * (long unsigned int) SAVE_EXPR <n>) * 4
-   ELEMENT_SIZE:
-    (sizetype) SAVE_EXPR <n> * 4
-   get the index as (long unsigned int) m, and return it.
-   The INDEX_P holds the pointer to the parent tree of the index,
-   INDEX_N holds the position of the index in its parent.  */
-
-static tree
-get_index_from_offset (tree offset, tree *index_p,
-		       int *index_n, tree element_size)
-{
-  if (TREE_CODE (offset) != MULT_EXPR)
-    return NULL_TREE;
-
-  auto_vec<factor_t> e_factors, o_factors;
-  get_factors_from_mul_expr (element_size, NULL, -1, &e_factors);
-  get_factors_from_mul_expr (offset, *index_p, *index_n, &o_factors);
-
-  if (e_factors.is_empty () || o_factors.is_empty ())
-    return NULL_TREE;
-
-  bool all_found = true;
-  for (unsigned i = 0; i < e_factors.length (); i++)
-    {
-      factor_t e_size_factor = e_factors[i];
-      bool found = false;
-      for (unsigned j = 0; j < o_factors.length ();)
-	{
-	  factor_t o_exp_factor = o_factors[j];
-	  if (operand_equal_p (e_size_factor.factor, o_exp_factor.factor))
-	    {
-	      o_factors.unordered_remove (j);
-	      found = true;
-	      break;
-	    }
-	  else
-	    j++;
-	}
-      if (!found)
-	all_found = false;
-    }
-
-  if (!all_found)
-    return NULL_TREE;
-
-  if (o_factors.length () != 1)
-    return NULL_TREE;
-
-  *index_p = o_factors[0].parent;
-  *index_n = o_factors[0].pos;
-  return o_factors[0].factor;
-}
-
-/* For an pointer + offset computation expression, such as,
-   *.ACCESS_WITH_SIZE (p->c, &p->b, 1, 0, -1, 0B)
-    + (sizetype) ((long unsigned int) index * 4
-   Return the index of this pointer array reference,
-   set the parent tree of INDEX to *INDEX_P.
-   set the operand position of the INDEX in the parent tree to *INDEX_N.
-   If failed, return NULL_TREE.  */
-
-static tree
-get_index_from_pointer_addr_expr (tree pointer, tree *index_p, int *index_n)
-{
-  *index_p = NULL_TREE;
-  *index_n = -1;
-  if (TREE_CODE (TREE_OPERAND (pointer, 0)) != INDIRECT_REF)
-    return NULL_TREE;
-  tree call = TREE_OPERAND (TREE_OPERAND (pointer, 0), 0);
-  if (!is_access_with_size_p (call))
-    return NULL_TREE;
-
-  /* Get the pointee type of the call to .ACCESS_WITH_SIZE.
-     This should be the element type of the pointer array.  */
-  tree pointee_type = TREE_TYPE (TREE_TYPE (TREE_TYPE (call)));
-  tree pointee_size = TYPE_SIZE_UNIT (pointee_type);
-
-  tree index_exp = TREE_OPERAND (pointer, 1);
-  *index_p = pointer;
-  *index_n = 1;
-
-  if (!(TREE_CODE (index_exp) != MULT_EXPR
-	&& tree_int_cst_equal (pointee_size, integer_one_node)))
-    {
-      while (CONVERT_EXPR_CODE_P (TREE_CODE (index_exp)))
-	{
-	  *index_p = index_exp;
-	  *index_n = 0;
-	  index_exp = TREE_OPERAND (index_exp, 0);
-	}
-      index_exp = get_index_from_offset (index_exp, index_p,
-					 index_n, pointee_size);
-
-      if (!index_exp)
-      return NULL_TREE;
-    }
-
-  while (CONVERT_EXPR_CODE_P (TREE_CODE (index_exp)))
-    {
-      *index_p = index_exp;
-      *index_n = 0;
-      index_exp = TREE_OPERAND (index_exp, 0);
-    }
-
-  return index_exp;
-}
-
-/* Return TRUE when the EXPR is a pointer array address that could be
-   instrumented.
-   We only instrument an address computation similar as the following:
-      *.ACCESS_WITH_SIZE (p->c, &p->b, 1, 0, -1, 0B)
-	+ (sizetype) ((long unsigned int) index * 4)
-   if the EXPR is instrumentable, return TRUE and
-   set the index to *INDEX.
-   set the *.ACCESS_WITH_SIZE to *BASE.
-   set the parent tree of INDEX to *INDEX_P.
-   set the operand position of the INDEX in the parent tree to INDEX_N.  */
-
-static bool
-is_instrumentable_pointer_array_address (tree expr, tree *base,
-					 tree *index, tree *index_p,
-					 int *index_n)
-{
-  /* For a poiner array address as:
-	(*.ACCESS_WITH_SIZE (p->c, &p->b, 1, 0, -1, 0B)
-	  + (sizetype) ((long unsigned int) index * 4)
-     op0 is the call to *.ACCESS_WITH_SIZE;
-     op1 is the index.  */
-  if (TREE_CODE (expr) != POINTER_PLUS_EXPR)
-    return false;
-
-  tree op0 = TREE_OPERAND (expr, 0);
-  if (TREE_CODE (op0) != INDIRECT_REF)
-    return false;
-  if (!is_access_with_size_p (TREE_OPERAND (op0, 0)))
-    return false;
-  tree op1 = get_index_from_pointer_addr_expr (expr, index_p, index_n);
-  if (op1 != NULL_TREE)
-    {
-      *base = op0;
-      *index = op1;
-      return true;
-    }
-  return false;
-}
-
-/* Return true iff T is an array or an indirect reference that was
-   instrumented by SANITIZE_BOUNDS.  */
+/* Return true iff T is an array that was instrumented by SANITIZE_BOUNDS.  */
 
 bool
-ubsan_array_ref_instrumented_p (tree t)
+ubsan_array_ref_instrumented_p (const_tree t)
 {
-  if (TREE_CODE (t) != ARRAY_REF
-      && TREE_CODE (t) != MEM_REF)
+  if (TREE_CODE (t) != ARRAY_REF)
     return false;
 
-  bool is_array = (TREE_CODE (t) == ARRAY_REF);
-  tree op0 = NULL_TREE;
-  tree op1 = NULL_TREE;
-  tree index_p = NULL_TREE;
-  int index_n = 0;
-  if (is_array)
-    {
-      op1 = TREE_OPERAND (t, 1);
-      return TREE_CODE (op1) == COMPOUND_EXPR
-	     && TREE_CODE (TREE_OPERAND (op1, 0)) == CALL_EXPR
-	     && CALL_EXPR_FN (TREE_OPERAND (op1, 0)) == NULL_TREE
-	     && CALL_EXPR_IFN (TREE_OPERAND (op1, 0)) == IFN_UBSAN_BOUNDS;
-    }
-  else if (is_instrumentable_pointer_array_address (t, &op0, &op1,
-						    &index_p, &index_n))
-    return TREE_CODE (op1) == COMPOUND_EXPR
-	   && TREE_CODE (TREE_OPERAND (op1, 0)) == CALL_EXPR
-	   && CALL_EXPR_FN (TREE_OPERAND (op1, 0)) == NULL_TREE
-	   && CALL_EXPR_IFN (TREE_OPERAND (op1, 0)) == IFN_UBSAN_BOUNDS;
-
-  return false;
+  tree op1 = TREE_OPERAND (t, 1);
+  return TREE_CODE (op1) == COMPOUND_EXPR
+	 && TREE_CODE (TREE_OPERAND (op1, 0)) == CALL_EXPR
+	 && CALL_EXPR_FN (TREE_OPERAND (op1, 0)) == NULL_TREE
+	 && CALL_EXPR_IFN (TREE_OPERAND (op1, 0)) == IFN_UBSAN_BOUNDS;
 }
 
-/* Instrument an ARRAY_REF or an address computation whose base address is
-   a call to .ACCESS_WITH_SIZE, if it hasn't already been instrumented.
-   IGNORE_OFF_BY_ONE is true if the ARRAY_REF is inside a ADDR_EXPR, or the
-   address computation is not inside a INDIRECT_REF.  */
+/* Instrument an ARRAY_REF, if it hasn't already been instrumented.
+   IGNORE_OFF_BY_ONE is true if the ARRAY_REF is inside a ADDR_EXPR.  */
 
 void
 ubsan_maybe_instrument_array_ref (tree *expr_p, bool ignore_off_by_one)
 {
-  tree e = NULL_TREE;
-  tree op0 = NULL_TREE;
-  tree op1 = NULL_TREE;
-  tree index_p = NULL_TREE;  /* the parent tree of INDEX.  */
-  int index_n = 0;  /* the operand position of INDEX in the parent tree.  */
-
   if (!ubsan_array_ref_instrumented_p (*expr_p)
       && sanitize_flags_p (SANITIZE_BOUNDS | SANITIZE_BOUNDS_STRICT)
       && current_function_decl != NULL_TREE)
     {
-      if (TREE_CODE (*expr_p) == ARRAY_REF)
-	{
-	  op0 = TREE_OPERAND (*expr_p, 0);
-	  op1 = TREE_OPERAND (*expr_p, 1);
-	  index_p = *expr_p;
-	  index_n = 1;
-	  e = ubsan_instrument_bounds (EXPR_LOCATION (*expr_p), op0,
-				       &op1, ignore_off_by_one);
-	}
-      else if (is_instrumentable_pointer_array_address (*expr_p, &op0, &op1,
-							&index_p, &index_n))
-	e = ubsan_instrument_bounds_pointer_address (EXPR_LOCATION (*expr_p),
-						     op0, &op1,
-						     ignore_off_by_one);
-
-      /* Replace the original INDEX with the instrumented INDEX.  */
+      tree op0 = TREE_OPERAND (*expr_p, 0);
+      tree op1 = TREE_OPERAND (*expr_p, 1);
+      tree e = ubsan_instrument_bounds (EXPR_LOCATION (*expr_p), op0, &op1,
+					ignore_off_by_one);
       if (e != NULL_TREE)
-	TREE_OPERAND (index_p, index_n)
-	  = build2 (COMPOUND_EXPR, TREE_TYPE (op1), e, op1);
+	TREE_OPERAND (*expr_p, 1) = build2 (COMPOUND_EXPR, TREE_TYPE (op1),
+					    e, op1);
     }
 }
 
diff --git a/gcc/c-family/c-warn.cc b/gcc/c-family/c-warn.cc
index d547b08..09517d2 100644
--- a/gcc/c-family/c-warn.cc
+++ b/gcc/c-family/c-warn.cc
@@ -3835,7 +3835,7 @@ do_warn_array_compare (location_t location, tree_code code, tree op0, tree op1)
     op1 = TREE_OPERAND (op1, 0);
 
   auto_diagnostic_group d;
-  diagnostic_t kind = DK_WARNING;
+  enum diagnostics::kind kind = diagnostics::kind::warning;
   const char *msg;
   if (c_dialect_cxx () && cxx_dialect >= cxx20)
     {
@@ -3843,7 +3843,7 @@ do_warn_array_compare (location_t location, tree_code code, tree op0, tree op1)
       if (cxx_dialect >= cxx26)
 	{
 	  msg = G_("comparison between two arrays is not allowed in C++26");
-	  kind = DK_PERMERROR;
+	  kind = diagnostics::kind::permerror;
 	}
       else
 	msg = G_("comparison between two arrays is deprecated in C++20");
diff --git a/gcc/c-family/c.opt b/gcc/c-family/c.opt
index 8af466d..12877eb 100644
--- a/gcc/c-family/c.opt
+++ b/gcc/c-family/c.opt
@@ -397,6 +397,10 @@ Wassign-intercept
 ObjC ObjC++ Var(warn_assign_intercept) Warning
 Warn whenever an Objective-C assignment is being intercepted by the garbage collector.
 
+Wabbreviated-auto-in-template-arg
+C++ ObjC++ Warning Var(warn_abbev_auto_targ) Init(1)
+Diagnose a placeholder type in a template argument in a function parameter type.
+
 Wbad-function-cast
 C ObjC Var(warn_bad_function_cast) Warning
 Warn about casting functions to incompatible types.
@@ -493,6 +497,10 @@ Wc++20-compat
 C++ ObjC++ Var(warn_cxx20_compat) Warning LangEnabledBy(C++ ObjC++,Wall) Init(0) CPP(cpp_warn_cxx20_compat) CppReason(CPP_W_CXX20_COMPAT)
 Warn about C++ constructs whose meaning differs between ISO C++ 2017 and ISO C++ 2020.
 
+Wc++26-compat
+C++ ObjC++ Var(warn_cxx26_compat) Warning LangEnabledBy(C++ ObjC++,Wall) Init(0)
+Warn about C++ constructs whose meaning differs between ISO C++ 2023 and ISO C++ 2026.
+
 Wc++11-extensions
 C++ ObjC++ Var(warn_cxx11_extensions) Warning Init(1)
 Warn about C++11 constructs in code compiled with an older standard.
diff --git a/gcc/c-family/c.opt.urls b/gcc/c-family/c.opt.urls
index 65d1221..5c97593 100644
--- a/gcc/c-family/c.opt.urls
+++ b/gcc/c-family/c.opt.urls
@@ -139,6 +139,9 @@ UrlSuffix(gcc/Warning-Options.html#index-Warray-parameter)
 Wassign-intercept
 UrlSuffix(gcc/Objective-C-and-Objective-C_002b_002b-Dialect-Options.html#index-Wassign-intercept)
 
+Wabbreviated-auto-in-template-arg
+UrlSuffix(gcc/C_002b_002b-Dialect-Options.html#index-Wabbreviated-auto-in-template-arg)
+
 Wbad-function-cast
 UrlSuffix(gcc/Warning-Options.html#index-Wbad-function-cast)
 
@@ -187,6 +190,9 @@ UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b17-compat)
 Wc++20-compat
 UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b20-compat)
 
+Wc++26-compat
+UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b26-compat)
+
 Wc++11-extensions
 UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b11-extensions)
 
diff --git a/gcc/c-family/known-headers.cc b/gcc/c-family/known-headers.cc
index 7e0fa95..e3c0113 100644
--- a/gcc/c-family/known-headers.cc
+++ b/gcc/c-family/known-headers.cc
@@ -329,9 +329,10 @@ suggest_missing_header::~suggest_missing_header ()
 
 /* suggest_missing_option's ctor.  */
 
-suggest_missing_option::suggest_missing_option (location_t loc,
-						const char *macro_name,
-						diagnostic_option_id option_id)
+suggest_missing_option::
+suggest_missing_option (location_t loc,
+			const char *macro_name,
+			diagnostics::option_id option_id)
 : deferred_diagnostic (loc), m_name_str (macro_name), m_option_id (option_id)
 {
   gcc_assert (macro_name);
diff --git a/gcc/c-family/known-headers.h b/gcc/c-family/known-headers.h
index b1da757f..3ffe5f3 100644
--- a/gcc/c-family/known-headers.h
+++ b/gcc/c-family/known-headers.h
@@ -48,12 +48,12 @@ class suggest_missing_option : public deferred_diagnostic
 {
  public:
   suggest_missing_option (location_t loc, const char *name,
-			  diagnostic_option_id option_id);
+			  diagnostics::option_id option_id);
   ~suggest_missing_option ();
 
  private:
   const char *m_name_str;
-  diagnostic_option_id m_option_id;
+  diagnostics::option_id m_option_id;
 };
 
 #endif /* GCC_KNOWN_HEADERS_H */
diff --git a/gcc/c/ChangeLog b/gcc/c/ChangeLog
index cb69b8c..464e5a1 100644
--- a/gcc/c/ChangeLog
+++ b/gcc/c/ChangeLog
@@ -1,3 +1,97 @@
+2025-08-02  Martin Uecker  <uecker@tugraz.at>
+
+	* c-decl.cc (get_parm_array_spec): Remove.
+	(push_parm_decl): Do not add `arg spec` attribute.
+	(build_arg_spec_attribute): New function.
+	(grokdeklarator): Add `arg spec` attribute.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-errors.cc: Update usage of "diagnostic_info" to explicitly
+	refer to "diagnostics::diagnostic_info".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-errors.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* c-parser.cc: Likewise.
+	* c-typeck.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-decl.cc: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+	* c-errors.cc: Likewise.
+	* c-tree.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-objc-common.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+	* c-tree.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-errors.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* c-typeck.cc (build_asm_expr): Pass null pointer to
+	parse_{input,output}_constraint().
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* c-typeck.cc (handle_omp_array_sections): Use OMP_ITERATOR_DECL_P.
+	(c_finish_omp_clauses): Likewise.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* c-parser.cc (c_parser_unary_expression): Clear DECL_READ_P
+	after default_function_array_read_conversion for
+	-Wunused-but-set-{parameter,variable}={2,3} on
+	PRE{IN,DE}CREMENT_EXPR argument.
+	(c_parser_postfix_expression_after_primary): Similarly for
+	POST{IN,DE}CREMENT_EXPR.
+	* c-decl.cc (pop_scope): Use OPT_Wunused_but_set_variable_
+	instead of OPT_Wunused_but_set_variable.
+	(finish_function): Use OPT_Wunused_but_set_parameter_
+	instead of OPT_Wunused_but_set_parameter.
+	* c-typeck.cc (mark_exp_read): Handle {PRE,POST}{IN,DE}CREMENT_EXPR
+	and don't handle it when cast to void.
+	(build_modify_expr): Clear DECL_READ_P after build_binary_op
+	for -Wunused-but-set-{parameter,variable}=3.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-typeck.cc (build_access_with_size_for_counted_by): Update comments.
+	Adjust the arguments per the new design.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	PR middle-end/121000
+	* c-typeck.cc (build_access_with_size_for_counted_by): Update comments.
+	Pass TYPE_SIZE_UNIT of the element as the 6th argument.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-decl.cc (verify_counted_by_attribute): Change the 2nd argument
+	to a vector of fields with counted_by attribute. Verify all fields
+	in this vector.
+	(finish_struct): Collect all the fields with counted_by attribute
+	to a vector and pass this vector to verify_counted_by_attribute.
+	* c-typeck.cc (build_counted_by_ref): Handle pointers with counted_by.
+	Add one more argument, issue error when the pointee type is a structure
+	or union including a flexible array member.
+	(build_access_with_size_for_counted_by): Handle pointers with counted_by.
+	(handle_counted_by_for_component_ref): Call build_counted_by_ref
+	with the new prototype.
+
 2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
 
 	* c-decl.cc (verify_counted_by_attribute): Change the 2nd argument
diff --git a/gcc/c/c-decl.cc b/gcc/c/c-decl.cc
index 7e1c197..7850365 100644
--- a/gcc/c/c-decl.cc
+++ b/gcc/c/c-decl.cc
@@ -1363,7 +1363,7 @@ pop_scope (void)
 	case VAR_DECL:
 	  /* Warnings for unused variables.  */
 	  if ((!TREE_USED (p) || !DECL_READ_P (p))
-	      && !warning_suppressed_p (p, OPT_Wunused_but_set_variable)
+	      && !warning_suppressed_p (p, OPT_Wunused_but_set_variable_)
 	      && !DECL_IN_SYSTEM_HEADER (p)
 	      && DECL_NAME (p)
 	      && !DECL_ARTIFICIAL (p)
@@ -1377,7 +1377,7 @@ pop_scope (void)
 		}
 	      else if (DECL_CONTEXT (p) == current_function_decl)
 		warning_at (DECL_SOURCE_LOCATION (p),
-			    OPT_Wunused_but_set_variable,
+			    OPT_Wunused_but_set_variable_,
 			    "variable %qD set but not used", p);
 	    }
 
@@ -4556,7 +4556,7 @@ lookup_name_fuzzy (tree name, enum lookup_name_fuzzy_kind kind, location_t loc)
   /* Next, look for exact matches for builtin defines that would have been
      defined if the user had passed a command-line option (e.g. -fopenmp
      for "_OPENMP").  */
-  diagnostic_option_id option_id
+  diagnostics::option_id option_id
     = get_option_for_builtin_define (IDENTIFIER_POINTER (name));
   if (option_id.m_idx > 0)
     return name_hint
@@ -6208,184 +6208,7 @@ grokparm (const struct c_parm *parm, tree *expr)
   return decl;
 }
 
-/* Return attribute "arg spec" corresponding to an array/VLA parameter
-   described by PARM, concatenated onto attributes ATTRS.
-   The spec consists of one dollar symbol for each specified variable
-   bound, one asterisk for each unspecified variable bound, followed
-   by at most one specification of the most significant bound of
-   an ordinary array parameter.  For ordinary arrays the specification
-   is either the constant bound itself, or the space character for
-   an array with an unspecified bound (the [] form).  Finally, a chain
-   of specified variable bounds is appended to the spec, starting with
-   the most significant bound.  For example, the PARM T a[2][m][3][n]
-   will produce __attribute__((arg spec ("[$$2]", m, n)).
-   For T a typedef for an array with variable bounds, the bounds are
-   included in the specification in the expected order.
-   No "arg spec"  is created for parameters of pointer types, making
-   a distinction between T(*)[N] (or, equivalently, T[][N]) and
-   the T[M][N] form, all of which have the same type and are represented
-   the same, but only the last of which gets an "arg spec" describing
-   the most significant bound M.  */
 
-static tree
-get_parm_array_spec (const struct c_parm *parm, tree attrs)
-{
-  /* The attribute specification string, minor bound first.  */
-  std::string spec;
-
-  /* A list of VLA variable bounds, major first, or null if unspecified
-     or not a VLA.  */
-  tree vbchain = NULL_TREE;
-  /* True for a pointer parameter.  */
-  bool pointer = false;
-  /* True for an ordinary array with an unpecified bound.  */
-  bool nobound = false;
-
-  /* Create a string representation for the bounds of the array/VLA.  */
-  for (c_declarator *pd = parm->declarator, *next; pd; pd = next)
-    {
-      next = pd->declarator;
-      while (next && next->kind == cdk_attrs)
-	next = next->declarator;
-
-      /* Remember if a pointer has been seen to avoid storing the constant
-	 bound.  */
-      if (pd->kind == cdk_pointer)
-	pointer = true;
-
-      if ((pd->kind == cdk_pointer || pd->kind == cdk_function)
-	  && (!next || next->kind == cdk_id))
-	{
-	  /* Do nothing for the common case of a pointer.  The fact that
-	     the parameter is one can be deduced from the absence of
-	     an arg spec for it.  */
-	  return attrs;
-	}
-
-      if (pd->kind == cdk_id)
-	{
-	  if (pointer
-	      || !parm->specs->type
-	      || TREE_CODE (parm->specs->type) != ARRAY_TYPE
-	      || !TYPE_DOMAIN (parm->specs->type)
-	      || !TYPE_MAX_VALUE (TYPE_DOMAIN (parm->specs->type)))
-	    continue;
-
-	  tree max = TYPE_MAX_VALUE (TYPE_DOMAIN (parm->specs->type));
-	  if (!vbchain
-	      && TREE_CODE (max) == INTEGER_CST)
-	    {
-	      /* Extract the upper bound from a parameter of an array type
-		 unless the parameter is an ordinary array of unspecified
-		 bound in which case a next iteration of the loop will
-		 exit.  */
-	      if (spec.empty () || spec.end ()[-1] != ' ')
-		{
-		  if (!tree_fits_shwi_p (max))
-		    continue;
-
-		  /* The upper bound is the value of the largest valid
-		     index.  */
-		  HOST_WIDE_INT n = tree_to_shwi (max) + 1;
-		  char buf[40];
-		  sprintf (buf, HOST_WIDE_INT_PRINT_UNSIGNED, n);
-		  spec += buf;
-		}
-	      continue;
-	    }
-
-	  /* For a VLA typedef, create a list of its variable bounds and
-	     append it in the expected order to VBCHAIN.  */
-	  tree tpbnds = NULL_TREE;
-	  for (tree type = parm->specs->type; TREE_CODE (type) == ARRAY_TYPE;
-	       type = TREE_TYPE (type))
-	    {
-	      tree nelts_minus_one = array_type_nelts_minus_one (type);
-	      if (error_operand_p (nelts_minus_one))
-		return attrs;
-	      if (TREE_CODE (nelts_minus_one) != INTEGER_CST)
-		{
-		  /* Each variable VLA bound is represented by the dollar
-		     sign.  */
-		  spec += "$";
-		  tpbnds = tree_cons (NULL_TREE, nelts_minus_one, tpbnds);
-		}
-	    }
-	  tpbnds = nreverse (tpbnds);
-	  vbchain = chainon (vbchain, tpbnds);
-	  continue;
-	}
-
-      if (pd->kind != cdk_array)
-	continue;
-
-      if (pd->u.array.vla_unspec_p)
-	{
-	  /* Each unspecified bound is represented by a star.  There
-	     can be any number of these in a declaration (but none in
-	     a definition).  */
-	  spec += '*';
-	  continue;
-	}
-
-      tree nelts = pd->u.array.dimen;
-      if (!nelts)
-	{
-	  /* Ordinary array of unspecified size.  There can be at most
-	     one for the most significant bound.  Exit on the next
-	     iteration which determines whether or not PARM is declared
-	     as a pointer or an array.  */
-	  nobound = true;
-	  continue;
-	}
-
-      if (pd->u.array.static_p)
-	spec += 's';
-
-      if (!INTEGRAL_TYPE_P (TREE_TYPE (nelts)))
-	/* Avoid invalid NELTS.  */
-	return attrs;
-
-      STRIP_NOPS (nelts);
-      nelts = c_fully_fold (nelts, false, nullptr);
-      if (TREE_CODE (nelts) == INTEGER_CST)
-	{
-	  /* Skip all constant bounds except the most significant one.
-	     The interior ones are included in the array type.  */
-	  if (next && (next->kind == cdk_array || next->kind == cdk_pointer))
-	    continue;
-
-	  if (!tree_fits_uhwi_p (nelts))
-	    /* Bail completely on invalid bounds.  */
-	    return attrs;
-
-	  char buf[40];
-	  unsigned HOST_WIDE_INT n = tree_to_uhwi (nelts);
-	  sprintf (buf, HOST_WIDE_INT_PRINT_UNSIGNED, n);
-	  spec += buf;
-	  break;
-	}
-
-      /* Each variable VLA bound is represented by a dollar sign.  */
-      spec += "$";
-      vbchain = tree_cons (NULL_TREE, nelts, vbchain);
-    }
-
-  if (spec.empty () && !nobound)
-    return attrs;
-
-  spec.insert (0, "[");
-  if (nobound)
-    /* Ordinary array of unspecified bound is represented by a space.
-       It must be last in the spec.  */
-    spec += ' ';
-  spec += ']';
-
-  tree acsstr = build_string (spec.length () + 1, spec.c_str ());
-  tree args = tree_cons (NULL_TREE, acsstr, vbchain);
-  tree name = get_identifier ("arg spec");
-  return tree_cons (name, args, attrs);
-}
 
 /* Given a parsed parameter declaration, decode it into a PARM_DECL
    and push that on the current scope.  EXPR is a pointer to an
@@ -6401,7 +6224,6 @@ push_parm_decl (const struct c_parm *parm, tree *expr)
   if (decl && DECL_P (decl))
     DECL_SOURCE_LOCATION (decl) = parm->loc;
 
-  attrs = get_parm_array_spec (parm, attrs);
   decl_attributes (&decl, attrs, 0);
 
   decl = pushdecl (decl);
@@ -6775,6 +6597,25 @@ add_decl_expr (location_t loc, tree type, tree *expr, bool set_name_p)
     }
 }
 
+
+/* Add attribute "arg spec" to ATTRS corresponding to an array/VLA parameter
+   declared with type TYPE.  The attribute has two arguments.  The first is
+   a string that encodes the presence of the static keyword.  The second is
+   the declared type of the array before adjustment, i.e. as an array type
+   including the outermost bound.  */
+
+static tree
+build_arg_spec_attribute (tree type, bool static_p, tree attrs)
+{
+  tree vbchain = tree_cons (NULL_TREE, type, NULL_TREE);
+  tree acsstr = static_p ? build_string (7, "static") :
+			   build_string (1, "");
+  tree args = tree_cons (NULL_TREE, acsstr, vbchain);
+  tree name = get_identifier ("arg spec");
+  return tree_cons (name, args, attrs);
+}
+
+
 /* Given declspecs and a declarator,
    determine the name and type of the object declared
    and construct a ..._DECL node for it.
@@ -6834,6 +6675,7 @@ grokdeclarator (const struct c_declarator *declarator,
   bool funcdef_flag = false;
   bool funcdef_syntax = false;
   bool size_varies = false;
+  bool size_error = false;
   tree decl_attr = declspecs->decl_attr;
   int array_ptr_quals = TYPE_UNQUALIFIED;
   tree array_ptr_attrs = NULL_TREE;
@@ -7326,6 +7168,7 @@ grokdeclarator (const struct c_declarator *declarator,
 				"size of unnamed array has non-integer type");
 		    size = integer_one_node;
 		    size_int_const = true;
+		    size_error = true;
 		  }
 		/* This can happen with enum forward declaration.  */
 		else if (!COMPLETE_TYPE_P (TREE_TYPE (size)))
@@ -7338,6 +7181,7 @@ grokdeclarator (const struct c_declarator *declarator,
 				"type");
 		    size = integer_one_node;
 		    size_int_const = true;
+		    size_error = true;
 		  }
 
 		size = c_fully_fold (size, false, &size_maybe_const);
@@ -7363,6 +7207,7 @@ grokdeclarator (const struct c_declarator *declarator,
 			  error_at (loc, "size of unnamed array is negative");
 			size = integer_one_node;
 			size_int_const = true;
+			size_error = true;
 		      }
 		    /* Handle a size folded to an integer constant but
 		       not an integer constant expression.  */
@@ -7978,6 +7823,10 @@ grokdeclarator (const struct c_declarator *declarator,
 
 	if (TREE_CODE (type) == ARRAY_TYPE)
 	  {
+	    if (!size_error)
+	      *decl_attrs = build_arg_spec_attribute (type, array_parm_static,
+						      *decl_attrs);
+
 	    /* Transfer const-ness of array into that of type pointed to.  */
 	    type = TREE_TYPE (type);
 	    if (orig_qual_type != NULL_TREE)
@@ -9432,62 +9281,56 @@ c_update_type_canonical (tree t)
     }
 }
 
-/* Verify the argument of the counted_by attribute of each of the
-   FIELDS_WITH_COUNTED_BY is a valid field of the containing structure,
-   STRUCT_TYPE, Report error and remove the corresponding attribute
-   when it's not.  */
+/* Verify the argument of the counted_by attribute of the flexible array
+   member FIELD_DECL is a valid field of the containing structure,
+   STRUCT_TYPE, Report error and remove this attribute when it's not.  */
 
 static void
-verify_counted_by_attribute (tree struct_type,
-			     auto_vec<tree> *fields_with_counted_by)
+verify_counted_by_attribute (tree struct_type, tree field_decl)
 {
-  for (tree field_decl : *fields_with_counted_by)
-    {
-      tree attr_counted_by = lookup_attribute ("counted_by",
-						DECL_ATTRIBUTES (field_decl));
+  tree attr_counted_by = lookup_attribute ("counted_by",
+					   DECL_ATTRIBUTES (field_decl));
 
-      if (!attr_counted_by)
-	continue;
+  if (!attr_counted_by)
+    return;
 
-      /* If there is an counted_by attribute attached to the field,
-	 verify it.  */
+  /* If there is an counted_by attribute attached to the field,
+     verify it.  */
 
-      tree fieldname = TREE_VALUE (TREE_VALUE (attr_counted_by));
+  tree fieldname = TREE_VALUE (TREE_VALUE (attr_counted_by));
 
-      /* Verify the argument of the attrbute is a valid field of the
-	 containing structure.  */
+  /* Verify the argument of the attrbute is a valid field of the
+     containing structure.  */
 
-      tree counted_by_field = lookup_field (struct_type, fieldname);
+  tree counted_by_field = lookup_field (struct_type, fieldname);
 
-      /* Error when the field is not found in the containing structure and
-	 remove the corresponding counted_by attribute from the field_decl.  */
-      if (!counted_by_field)
+  /* Error when the field is not found in the containing structure and
+     remove the corresponding counted_by attribute from the field_decl.  */
+  if (!counted_by_field)
+    {
+      error_at (DECL_SOURCE_LOCATION (field_decl),
+		"argument %qE to the %<counted_by%> attribute"
+		" is not a field declaration in the same structure"
+		" as %qD", fieldname, field_decl);
+      DECL_ATTRIBUTES (field_decl)
+	= remove_attribute ("counted_by", DECL_ATTRIBUTES (field_decl));
+    }
+  else
+  /* Error when the field is not with an integer type.  */
+    {
+      while (TREE_CHAIN (counted_by_field))
+	counted_by_field = TREE_CHAIN (counted_by_field);
+      tree real_field = TREE_VALUE (counted_by_field);
+
+      if (!INTEGRAL_TYPE_P (TREE_TYPE (real_field)))
 	{
 	  error_at (DECL_SOURCE_LOCATION (field_decl),
 		    "argument %qE to the %<counted_by%> attribute"
-		    " is not a field declaration in the same structure"
-		    " as %qD", fieldname, field_decl);
+		    " is not a field declaration with an integer type",
+		    fieldname);
 	  DECL_ATTRIBUTES (field_decl)
 	    = remove_attribute ("counted_by", DECL_ATTRIBUTES (field_decl));
 	}
-      else
-      /* Error when the field is not with an integer type.  */
-	{
-	  while (TREE_CHAIN (counted_by_field))
-	    counted_by_field = TREE_CHAIN (counted_by_field);
-	  tree real_field = TREE_VALUE (counted_by_field);
-
-	  if (!INTEGRAL_TYPE_P (TREE_TYPE (real_field)))
-	    {
-	      error_at (DECL_SOURCE_LOCATION (field_decl),
-			"argument %qE to the %<counted_by%> attribute"
-			" is not a field declaration with an integer type",
-			fieldname);
-	      DECL_ATTRIBUTES (field_decl)
-		= remove_attribute ("counted_by",
-				    DECL_ATTRIBUTES (field_decl));
-	    }
-	}
     }
 }
 
@@ -9562,7 +9405,7 @@ finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
      until now.)  */
 
   bool saw_named_field = false;
-  auto_vec<tree> fields_with_counted_by;
+  tree counted_by_fam_field = NULL_TREE;
   for (x = fieldlist; x; x = DECL_CHAIN (x))
     {
       /* Whether this field is the last field of the structure or union.
@@ -9643,16 +9486,9 @@ finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
 	     record it here and do more verification later after the
 	     whole structure is complete.  */
 	  if (lookup_attribute ("counted_by", DECL_ATTRIBUTES (x)))
-	    fields_with_counted_by.safe_push (x);
+	    counted_by_fam_field = x;
 	}
 
-      if (TREE_CODE (TREE_TYPE (x)) == POINTER_TYPE)
-	/* If there is a counted_by attribute attached to this field,
-	   record it here and do more verification later after the
-	   whole structure is complete.  */
-	if (lookup_attribute ("counted_by", DECL_ATTRIBUTES (x)))
-	  fields_with_counted_by.safe_push (x);
-
       if (pedantic && TREE_CODE (t) == RECORD_TYPE
 	  && flexible_array_type_p (TREE_TYPE (x)))
 	pedwarn (DECL_SOURCE_LOCATION (x), OPT_Wpedantic,
@@ -9951,8 +9787,8 @@ finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
 	struct_parse_info->struct_types.safe_push (t);
      }
 
-  if (fields_with_counted_by.length () > 0)
-    verify_counted_by_attribute (t, &fields_with_counted_by);
+  if (counted_by_fam_field)
+    verify_counted_by_attribute (t, counted_by_fam_field);
 
   return t;
 }
@@ -11478,9 +11314,9 @@ finish_function (location_t end_loc)
 	    && !DECL_READ_P (decl)
 	    && DECL_NAME (decl)
 	    && !DECL_ARTIFICIAL (decl)
-	    && !warning_suppressed_p (decl, OPT_Wunused_but_set_parameter))
+	    && !warning_suppressed_p (decl, OPT_Wunused_but_set_parameter_))
 	  warning_at (DECL_SOURCE_LOCATION (decl),
-		      OPT_Wunused_but_set_parameter,
+		      OPT_Wunused_but_set_parameter_,
 		      "parameter %qD set but not used", decl);
     }
 
diff --git a/gcc/c/c-errors.cc b/gcc/c/c-errors.cc
index 4682dca..e2c74fe 100644
--- a/gcc/c/c-errors.cc
+++ b/gcc/c/c-errors.cc
@@ -33,10 +33,10 @@ along with GCC; see the file COPYING3.  If not see
 
 bool
 pedwarn_c23 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -48,8 +48,9 @@ pedwarn_c23 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc2y)
-			   ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Wc23_c2y_compat;
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
+      diagnostic.m_option_id = OPT_Wc23_c2y_compat;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c23-c2y-compat suppresses even the pedwarns.  */
@@ -58,8 +59,9 @@ pedwarn_c23 (location_t location,
   /* For -pedantic outside C2Y, issue a pedwarn.  */
   else if (pedantic && !flag_isoc2y)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   va_end (ap);
@@ -77,10 +79,10 @@ pedwarn_c23 (location_t location,
 
 bool
 pedwarn_c11 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -93,11 +95,12 @@ pedwarn_c11 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc23)
-			   ? DK_PEDWARN : DK_WARNING);
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
       if (option_id == OPT_Wpedantic)
-	diagnostic.option_id = OPT_Wc11_c23_compat;
+	diagnostic.m_option_id = OPT_Wc11_c23_compat;
       else
-	diagnostic.option_id = option_id;
+	diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c11-c23-compat suppresses even the pedwarns.  */
@@ -106,8 +109,9 @@ pedwarn_c11 (location_t location,
   /* For -pedantic outside C23, issue a pedwarn.  */
   else if (pedantic && !flag_isoc23)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   va_end (ap);
@@ -122,10 +126,10 @@ pedwarn_c11 (location_t location,
 
 bool
 pedwarn_c99 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -137,8 +141,9 @@ pedwarn_c99 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc11)
-			   ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Wc99_c11_compat;
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
+      diagnostic.m_option_id = OPT_Wc99_c11_compat;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c99-c11-compat suppresses even the pedwarns.  */
@@ -147,8 +152,9 @@ pedwarn_c99 (location_t location,
   /* For -pedantic outside C11, issue a pedwarn.  */
   else if (pedantic && !flag_isoc11)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   va_end (ap);
@@ -164,10 +170,10 @@ pedwarn_c99 (location_t location,
 
 bool
 pedwarn_c90 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -183,8 +189,9 @@ pedwarn_c90 (location_t location,
 	{
 	  diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			       (pedantic && !flag_isoc99)
-			       ? DK_PEDWARN : DK_WARNING);
-	  diagnostic.option_id = option_id;
+			       ? diagnostics::kind::pedwarn
+			       : diagnostics::kind::warning);
+	  diagnostic.m_option_id = option_id;
 	  diagnostic_report_diagnostic (global_dc, &diagnostic);
 	  warned = true;
 	  goto out;
@@ -196,8 +203,9 @@ pedwarn_c90 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc99)
-			   ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Wc90_c99_compat;
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
+      diagnostic.m_option_id = OPT_Wc90_c99_compat;
       diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c90-c99-compat suppresses the pedwarns.  */
@@ -206,8 +214,9 @@ pedwarn_c90 (location_t location,
   /* For -pedantic outside C99, issue a pedwarn.  */
   else if (pedantic && !flag_isoc99)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       diagnostic_report_diagnostic (global_dc, &diagnostic);
       warned = true;
     }
diff --git a/gcc/c/c-objc-common.cc b/gcc/c/c-objc-common.cc
index d574bc7..5c50983 100644
--- a/gcc/c/c-objc-common.cc
+++ b/gcc/c/c-objc-common.cc
@@ -414,7 +414,7 @@ has_c_linkage (const_tree decl ATTRIBUTE_UNUSED)
 }
 
 void
-c_initialize_diagnostics (diagnostic_context *context)
+c_initialize_diagnostics (diagnostics::context *context)
 {
   context->set_pretty_printer (std::make_unique<c_pretty_printer> ());
   c_common_diagnostics_set_defaults (context);
diff --git a/gcc/c/c-parser.cc b/gcc/c/c-parser.cc
index 0c3e3e2..4a13fc0 100644
--- a/gcc/c/c-parser.cc
+++ b/gcc/c/c-parser.cc
@@ -10547,15 +10547,31 @@ c_parser_unary_expression (c_parser *parser)
       c_parser_consume_token (parser);
       exp_loc = c_parser_peek_token (parser)->location;
       op = c_parser_cast_expression (parser, NULL);
-
-      op = default_function_array_read_conversion (exp_loc, op);
+      if ((VAR_P (op.value) || TREE_CODE (op.value) == PARM_DECL)
+	  && !DECL_READ_P (op.value)
+	  && (VAR_P (op.value) ? warn_unused_but_set_variable
+			       : warn_unused_but_set_parameter) > 1)
+	{
+	  op = default_function_array_read_conversion (exp_loc, op);
+	  DECL_READ_P (op.value) = 0;
+	}
+      else
+	op = default_function_array_read_conversion (exp_loc, op);
       return parser_build_unary_op (op_loc, PREINCREMENT_EXPR, op);
     case CPP_MINUS_MINUS:
       c_parser_consume_token (parser);
       exp_loc = c_parser_peek_token (parser)->location;
       op = c_parser_cast_expression (parser, NULL);
-
-      op = default_function_array_read_conversion (exp_loc, op);
+      if ((VAR_P (op.value) || TREE_CODE (op.value) == PARM_DECL)
+	  && !DECL_READ_P (op.value)
+	  && (VAR_P (op.value) ? warn_unused_but_set_variable
+			       : warn_unused_but_set_parameter) > 1)
+	{
+	  op = default_function_array_read_conversion (exp_loc, op);
+	  DECL_READ_P (op.value) = 0;
+	}
+      else
+	op = default_function_array_read_conversion (exp_loc, op);
       return parser_build_unary_op (op_loc, PREDECREMENT_EXPR, op);
     case CPP_AND:
       c_parser_consume_token (parser);
@@ -13933,7 +13949,17 @@ c_parser_postfix_expression_after_primary (c_parser *parser,
 	  start = expr.get_start ();
 	  finish = c_parser_peek_token (parser)->get_finish ();
 	  c_parser_consume_token (parser);
-	  expr = default_function_array_read_conversion (expr_loc, expr);
+	  if ((VAR_P (expr.value) || TREE_CODE (expr.value) == PARM_DECL)
+	      && !DECL_READ_P (expr.value)
+	      && (VAR_P (expr.value) ? warn_unused_but_set_variable
+				     : warn_unused_but_set_parameter) > 1
+	      && TREE_CODE (TREE_TYPE (expr.value)) != ARRAY_TYPE)
+	    {
+	      expr = default_function_array_read_conversion (expr_loc, expr);
+	      DECL_READ_P (expr.value) = 0;
+	    }
+	  else
+	    expr = default_function_array_read_conversion (expr_loc, expr);
 	  expr.value = build_unary_op (op_loc, POSTINCREMENT_EXPR,
 				       expr.value, false);
 	  set_c_expr_source_range (&expr, start, finish);
@@ -13945,7 +13971,17 @@ c_parser_postfix_expression_after_primary (c_parser *parser,
 	  start = expr.get_start ();
 	  finish = c_parser_peek_token (parser)->get_finish ();
 	  c_parser_consume_token (parser);
-	  expr = default_function_array_read_conversion (expr_loc, expr);
+	  if ((VAR_P (expr.value) || TREE_CODE (expr.value) == PARM_DECL)
+	      && !DECL_READ_P (expr.value)
+	      && (VAR_P (expr.value) ? warn_unused_but_set_variable
+				     : warn_unused_but_set_parameter) > 1
+	      && TREE_CODE (TREE_TYPE (expr.value)) != ARRAY_TYPE)
+	    {
+	      expr = default_function_array_read_conversion (expr_loc, expr);
+	      DECL_READ_P (expr.value) = 0;
+	    }
+	  else
+	    expr = default_function_array_read_conversion (expr_loc, expr);
 	  expr.value = build_unary_op (op_loc, POSTDECREMENT_EXPR,
 				       expr.value, false);
 	  set_c_expr_source_range (&expr, start, finish);
@@ -29126,11 +29162,14 @@ c_parser_omp_error (c_parser *parser, enum pragma_context context)
       if (msg == NULL)
 	msg = _("<message unknown at compile time>");
     }
+  const enum diagnostics::kind diag_kind = (severity_fatal
+					    ? diagnostics::kind::error
+					    : diagnostics::kind::warning);
   if (msg)
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered: %s", msg);
   else
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered");
   return false;
 }
diff --git a/gcc/c/c-tree.h b/gcc/c/c-tree.h
index 364f51d..bb0b113 100644
--- a/gcc/c/c-tree.h
+++ b/gcc/c/c-tree.h
@@ -757,7 +757,7 @@ extern tree c_finish_bc_name (location_t, tree, bool);
 extern bool c_objc_common_init (void);
 extern bool c_missing_noreturn_ok_p (tree);
 extern bool c_warn_unused_global_decl (const_tree);
-extern void c_initialize_diagnostics (diagnostic_context *);
+extern void c_initialize_diagnostics (diagnostics::context *);
 extern bool c_var_mod_p (tree x, tree fn);
 extern alias_set_type c_get_alias_set (tree);
 extern int c_type_dwarf_attribute (const_tree, int);
@@ -960,13 +960,13 @@ extern void c_bind (location_t, tree, bool);
 extern bool tag_exists_p (enum tree_code, tree);
 
 /* In c-errors.cc */
-extern bool pedwarn_c90 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c90 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool pedwarn_c99 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c99 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool pedwarn_c11 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c11 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool pedwarn_c23 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c23 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
 extern void add_note_about_new_keyword (location_t loc,
 					tree keyword_id);
diff --git a/gcc/c/c-typeck.cc b/gcc/c/c-typeck.cc
index 7948106..ed6e56e 100644
--- a/gcc/c/c-typeck.cc
+++ b/gcc/c/c-typeck.cc
@@ -2310,14 +2310,30 @@ mark_exp_read (tree exp)
     case PARM_DECL:
       DECL_READ_P (exp) = 1;
       break;
+    CASE_CONVERT:
+      if (VOID_TYPE_P (TREE_TYPE (exp)))
+	switch (TREE_CODE (TREE_OPERAND (exp, 0)))
+	  {
+	  case PREINCREMENT_EXPR:
+	  case PREDECREMENT_EXPR:
+	  case POSTINCREMENT_EXPR:
+	  case POSTDECREMENT_EXPR:
+	    return;
+	  default:
+	    break;
+	  }
+      /* FALLTHRU */
     case ARRAY_REF:
     case COMPONENT_REF:
     case MODIFY_EXPR:
     case REALPART_EXPR:
     case IMAGPART_EXPR:
-    CASE_CONVERT:
     case ADDR_EXPR:
     case VIEW_CONVERT_EXPR:
+    case PREINCREMENT_EXPR:
+    case PREDECREMENT_EXPR:
+    case POSTINCREMENT_EXPR:
+    case POSTDECREMENT_EXPR:
       mark_exp_read (TREE_OPERAND (exp, 0));
       break;
     case COMPOUND_EXPR:
@@ -2922,8 +2938,8 @@ should_suggest_deref_p (tree datum_type)
 
 /* For a SUBDATUM field of a structure or union DATUM, generate a REF to
    the object that represents its counted_by per the attribute counted_by
-   attached to this field if it's a flexible array member or a pointer
-   field, otherwise return NULL_TREE.
+   attached to this field if it's a flexible array member field, otherwise
+   return NULL_TREE.
    Set COUNTED_BY_TYPE to the TYPE of the counted_by field.
    For example, if:
 
@@ -2941,34 +2957,18 @@ should_suggest_deref_p (tree datum_type)
 
 */
 static tree
-build_counted_by_ref (location_t loc, tree datum, tree subdatum,
-		      tree *counted_by_type)
+build_counted_by_ref (tree datum, tree subdatum, tree *counted_by_type)
 {
   tree type = TREE_TYPE (datum);
-  tree sub_type = TREE_TYPE (subdatum);
-  if (!c_flexible_array_member_type_p (sub_type)
-      && TREE_CODE (sub_type) != POINTER_TYPE)
+  if (!c_flexible_array_member_type_p (TREE_TYPE (subdatum)))
     return NULL_TREE;
 
-  tree element_type = TREE_TYPE (sub_type);
-
   tree attr_counted_by = lookup_attribute ("counted_by",
 					   DECL_ATTRIBUTES (subdatum));
   tree counted_by_ref = NULL_TREE;
   *counted_by_type = NULL_TREE;
   if (attr_counted_by)
     {
-      /* Issue error when the element_type is a structure or
-	 union including a flexible array member.  */
-      if (RECORD_OR_UNION_TYPE_P (element_type)
-	  && TYPE_INCLUDES_FLEXARRAY (element_type))
-	{
-	  error_at (loc,
-		    "%<counted_by%> attribute is not allowed for a pointer to"
-		    " structure or union with flexible array member");
-	  return error_mark_node;
-	}
-
       tree field_id = TREE_VALUE (TREE_VALUE (attr_counted_by));
       counted_by_ref
 	= build_component_ref (UNKNOWN_LOCATION,
@@ -2991,29 +2991,28 @@ build_counted_by_ref (location_t loc, tree datum, tree subdatum,
 }
 
 /* Given a COMPONENT_REF REF with the location LOC, the corresponding
-   COUNTED_BY_REF, and the COUNTED_BY_TYPE, generate the corresponding
-   call to the internal function .ACCESS_WITH_SIZE.
-
-   Generate an INDIRECT_REF to a call to the internal function
-   .ACCESS_WITH_SIZE.
+   COUNTED_BY_REF, and the COUNTED_BY_TYPE, generate an INDIRECT_REF
+   to a call to the internal function .ACCESS_WITH_SIZE.
 
    REF
 
    to:
 
-   (*.ACCESS_WITH_SIZE (REF, COUNTED_BY_REF, 1, (TYPE_OF_SIZE)0, -1,
-			(TYPE_OF_ARRAY *)0))
+   (*.ACCESS_WITH_SIZE (REF, COUNTED_BY_REF, (* TYPE_OF_SIZE)0,
+			TYPE_SIZE_UNIT for element)
 
    NOTE: The return type of this function is the POINTER type pointing
-   to the original flexible array type or the original pointer type.
-   Then the type of the INDIRECT_REF is the original flexible array type
-   or the original pointer type.
+   to the original flexible array type.
+   Then the type of the INDIRECT_REF is the original flexible array type.
+
+   The type of the first argument of this function is a POINTER type
+   to the original flexible array type.
 
-   The 4th argument of the call is a constant 0 with the TYPE of the
-   object pointed by COUNTED_BY_REF.
+   The 3rd argument of the call is a constant 0 with the pointer TYPE whose
+   pointee type is the TYPE of the object pointed by COUNTED_BY_REF.
 
-   The 6th argument of the call is a constant 0 of the same TYPE as
-   the return type of the call.
+   The 4th argument of the call is the TYPE_SIZE_UNIT of the element TYPE
+   of the array.
 
   */
 static tree
@@ -3021,29 +3020,26 @@ build_access_with_size_for_counted_by (location_t loc, tree ref,
 				       tree counted_by_ref,
 				       tree counted_by_type)
 {
-  gcc_assert (c_flexible_array_member_type_p (TREE_TYPE (ref))
-	      || TREE_CODE (TREE_TYPE (ref)) == POINTER_TYPE);
-  bool is_fam = c_flexible_array_member_type_p (TREE_TYPE (ref));
-  tree first_param = is_fam ? array_to_pointer_conversion (loc, ref)
-		     : build_unary_op (loc, ADDR_EXPR, ref, false);
-
-  /* The result type of the call is a pointer to the original type
-     of the ref.  */
+  gcc_assert (c_flexible_array_member_type_p (TREE_TYPE (ref)));
+  /* The result type of the call is a pointer to the flexible array type.  */
   tree result_type = c_build_pointer_type (TREE_TYPE (ref));
-  first_param = c_fully_fold (first_param, false, NULL);
+  tree element_size = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ref)));
+
+  tree first_param
+    = c_fully_fold (array_to_pointer_conversion (loc, ref), false, NULL);
   tree second_param
     = c_fully_fold (counted_by_ref, false, NULL);
+  tree third_param = build_int_cst (build_pointer_type (counted_by_type), 0);
 
   tree call
     = build_call_expr_internal_loc (loc, IFN_ACCESS_WITH_SIZE,
-				    result_type, 6,
+				    result_type, 4,
 				    first_param,
 				    second_param,
-				    build_int_cst (integer_type_node, 1),
-				    build_int_cst (counted_by_type, 0),
-				    build_int_cst (integer_type_node, -1),
-				    build_int_cst (result_type, 0));
-  /* Wrap the call with an INDIRECT_REF with the original type of the ref.  */
+				    third_param,
+				    element_size);
+
+  /* Wrap the call with an INDIRECT_REF with the flexible array type.  */
   call = build1 (INDIRECT_REF, TREE_TYPE (ref), call);
   SET_EXPR_LOCATION (call, loc);
   return call;
@@ -3061,7 +3057,7 @@ handle_counted_by_for_component_ref (location_t loc, tree ref)
   tree datum = TREE_OPERAND (ref, 0);
   tree subdatum = TREE_OPERAND (ref, 1);
   tree counted_by_type = NULL_TREE;
-  tree counted_by_ref = build_counted_by_ref (loc, datum, subdatum,
+  tree counted_by_ref = build_counted_by_ref (datum, subdatum,
 					      &counted_by_type);
   if (counted_by_ref)
     ref = build_access_with_size_for_counted_by (loc, ref,
@@ -6437,14 +6433,16 @@ build_conditional_expr (location_t colon_loc, tree ifexp, bool ifexp_bcp,
       else
 	{
 	  int qual = ENCODE_QUAL_ADDR_SPACE (as_common);
-	  diagnostic_t kind = DK_PERMERROR;
+	  enum diagnostics::kind kind = diagnostics::kind::permerror;
 	  if (!flag_isoc99)
 	    /* This downgrade to a warning ensures that -std=gnu89
 	       -pedantic-errors does not flag these mismatches between
-	       builtins as errors (as DK_PERMERROR would).  ISO C99
-	       and later do not have implicit function declarations,
+	       builtins as errors (as diagnostics::kind::permerror would)
+	       ISO C99 and later do not have implicit function declarations,
 	       so the mismatch cannot occur naturally there.  */
-	    kind = bltin1 && bltin2 ? DK_WARNING : DK_PEDWARN;
+	    kind = (bltin1 && bltin2
+		    ? diagnostics::kind::warning
+		    : diagnostics::kind::pedwarn);
 	  if (emit_diagnostic (kind, colon_loc, OPT_Wincompatible_pointer_types,
 			       "pointer type mismatch "
 			       "in conditional expression"))
@@ -7328,8 +7326,21 @@ build_modify_expr (location_t location, tree lhs, tree lhs_origtype,
 		newrhs = build1 (EXCESS_PRECISION_EXPR, TREE_TYPE (rhs),
 				 newrhs);
 	    }
+	  bool clear_decl_read = false;
+	  if ((VAR_P (lhs) || TREE_CODE (lhs) == PARM_DECL)
+	      && !DECL_READ_P (lhs)
+	      && (VAR_P (lhs) ? warn_unused_but_set_variable
+			      : warn_unused_but_set_parameter) > 2)
+	    {
+	      mark_exp_read (newrhs);
+	      if (!DECL_READ_P (lhs))
+		clear_decl_read = true;
+	    }
+
 	  newrhs = build_binary_op (location,
 				    modifycode, lhs, newrhs, true);
+	  if (clear_decl_read)
+	    DECL_READ_P (lhs) = 0;
 
 	  /* The original type of the right hand side is no longer
 	     meaningful.  */
@@ -7589,7 +7600,8 @@ error_init (location_t loc, const char *gmsgid, ...)
   /* The gmsgid may be a format string with %< and %>. */
   va_list ap;
   va_start (ap, gmsgid);
-  bool warned = emit_diagnostic_valist (DK_ERROR, loc, -1, gmsgid, &ap);
+  bool warned = emit_diagnostic_valist (diagnostics::kind::error,
+					loc, -1, gmsgid, &ap);
   va_end (ap);
 
   ofwhat = print_spelling ((char *) alloca (spelling_length () + 1));
@@ -7601,7 +7613,7 @@ error_init (location_t loc, const char *gmsgid, ...)
 
 static bool ATTRIBUTE_GCC_DIAG (3,0)
 pedwarn_permerror_init (location_t loc, int opt, const char *gmsgid,
-			va_list *ap, diagnostic_t kind)
+			va_list *ap, enum diagnostics::kind kind)
 {
   /* Use the location where a macro was expanded rather than where
      it was defined to make sure macros defined in system headers
@@ -7628,7 +7640,8 @@ pedwarn_init (location_t loc, int opt, const char *gmsgid, ...)
 {
   va_list ap;
   va_start (ap, gmsgid);
-  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap, DK_PEDWARN);
+  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap,
+					diagnostics::kind::pedwarn);
   va_end (ap);
   return warned;
 }
@@ -7640,7 +7653,8 @@ permerror_init (location_t loc, int opt, const char *gmsgid, ...)
 {
   va_list ap;
   va_start (ap, gmsgid);
-  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap, DK_PERMERROR);
+  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap,
+					diagnostics::kind::permerror);
   va_end (ap);
   return warned;
 }
@@ -12637,7 +12651,8 @@ build_asm_expr (location_t loc, tree string, tree outputs, tree inputs,
       oconstraints[i] = constraint;
 
       if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-				   &allows_mem, &allows_reg, &is_inout))
+				   &allows_mem, &allows_reg, &is_inout,
+				   nullptr))
 	{
 	  /* If the operand is going to end up in memory,
 	     mark it addressable.  */
@@ -12698,7 +12713,8 @@ build_asm_expr (location_t loc, tree string, tree outputs, tree inputs,
       input = TREE_VALUE (tail);
 
       if (parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				  oconstraints, &allows_mem, &allows_reg))
+				  oconstraints, &allows_mem, &allows_reg,
+				  nullptr))
 	{
 	  /* If the operand is going to end up in memory,
 	     mark it addressable.  */
@@ -12880,7 +12896,9 @@ c_finish_return (location_t loc, tree retval, tree origtype, bool musttail_p)
 	  && valtype != NULL_TREE && TREE_CODE (valtype) != VOID_TYPE)
 	{
 	  no_warning = true;
-	  if (emit_diagnostic (flag_isoc99 ? DK_PERMERROR : DK_WARNING,
+	  if (emit_diagnostic (flag_isoc99
+			       ? diagnostics::kind::permerror
+			       : diagnostics::kind::warning,
 			       loc, OPT_Wreturn_mismatch,
 			       "%<return%> with no value,"
 			       " in function returning non-void"))
@@ -15628,9 +15646,7 @@ handle_omp_array_sections (tree &c, enum c_omp_region_type ort)
   tree *tp = &OMP_CLAUSE_DECL (c);
   if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
        || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_AFFINITY)
-      && TREE_CODE (*tp) == TREE_LIST
-      && TREE_PURPOSE (*tp)
-      && TREE_CODE (TREE_PURPOSE (*tp)) == TREE_VEC)
+      && OMP_ITERATOR_DECL_P (*tp))
     tp = &TREE_VALUE (*tp);
   tree first = handle_omp_array_sections_1 (c, *tp, types,
 					    maybe_zero_len, first_non_one,
@@ -16827,9 +16843,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	  /* FALLTHRU */
 	case OMP_CLAUSE_AFFINITY:
 	  t = OMP_CLAUSE_DECL (c);
-	  if (TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (OMP_ITERATOR_DECL_P (t))
 	    {
 	      if (TREE_PURPOSE (t) != last_iterators)
 		last_iterators_remove
@@ -16929,10 +16943,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 		      break;
 		    }
 		}
-	      if (TREE_CODE (OMP_CLAUSE_DECL (c)) == TREE_LIST
-		  && TREE_PURPOSE (OMP_CLAUSE_DECL (c))
-		  && (TREE_CODE (TREE_PURPOSE (OMP_CLAUSE_DECL (c)))
-		      == TREE_VEC))
+	      if (OMP_ITERATOR_DECL_P (OMP_CLAUSE_DECL (c)))
 		TREE_VALUE (OMP_CLAUSE_DECL (c)) = t;
 	      else
 		OMP_CLAUSE_DECL (c) = t;
diff --git a/gcc/calls.cc b/gcc/calls.cc
index e16190c..2711c4e 100644
--- a/gcc/calls.cc
+++ b/gcc/calls.cc
@@ -2589,7 +2589,8 @@ can_implement_as_sibling_call_p (tree exp,
       return false;
     }
 
-  if (TYPE_VOLATILE (TREE_TYPE (TREE_TYPE (addr))))
+  if (TYPE_VOLATILE (TREE_TYPE (TREE_TYPE (addr)))
+      && !CALL_EXPR_MUST_TAIL_CALL (exp))
     {
       maybe_complain_about_tail_call (exp, _("volatile function type"));
       return false;
diff --git a/gcc/cfgexpand.cc b/gcc/cfgexpand.cc
index 33649d4..8950294 100644
--- a/gcc/cfgexpand.cc
+++ b/gcc/cfgexpand.cc
@@ -3275,44 +3275,6 @@ expand_asm_loc (tree string, int vol, location_t locus)
   emit_insn (body);
 }
 
-/* Return the number of times character C occurs in string S.  */
-static int
-n_occurrences (int c, const char *s)
-{
-  int n = 0;
-  while (*s)
-    n += (*s++ == c);
-  return n;
-}
-
-/* A subroutine of expand_asm_operands.  Check that all operands have
-   the same number of alternatives.  Return true if so.  */
-
-static bool
-check_operand_nalternatives (const vec<const char *> &constraints)
-{
-  unsigned len = constraints.length();
-  if (len > 0)
-    {
-      int nalternatives = n_occurrences (',', constraints[0]);
-
-      if (nalternatives + 1 > MAX_RECOG_ALTERNATIVES)
-	{
-	  error ("too many alternatives in %<asm%>");
-	  return false;
-	}
-
-      for (unsigned i = 1; i < len; ++i)
-	if (n_occurrences (',', constraints[i]) != nalternatives)
-	  {
-	    error ("operand constraints for %<asm%> differ "
-		   "in number of alternatives");
-	    return false;
-	  }
-    }
-  return true;
-}
-
 /* Check for overlap between registers marked in CLOBBERED_REGS and
    anything inappropriate in T.  Emit error and return the register
    variable definition for error, NULL_TREE for ok.  */
@@ -3478,10 +3440,6 @@ expand_asm_stmt (gasm *stmt)
 	= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
     }
 
-  /* ??? Diagnose during gimplification?  */
-  if (! check_operand_nalternatives (constraints))
-    return;
-
   /* Count the number of meaningful clobbered registers, ignoring what
      we would ignore later.  */
   auto_vec<rtx> clobber_rvec;
@@ -3555,7 +3513,8 @@ expand_asm_stmt (gasm *stmt)
 	 no point in going further.  */
       constraint = constraints[i];
       if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
-				    &allows_mem, &allows_reg, &is_inout))
+				    &allows_mem, &allows_reg, &is_inout,
+				    nullptr))
 	return;
 
       /* If the output is a hard register, verify it doesn't conflict with
@@ -3633,8 +3592,8 @@ expand_asm_stmt (gasm *stmt)
 
       constraint = constraints[i + noutputs];
       if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				    constraints.address (),
-				    &allows_mem, &allows_reg))
+				    constraints.address (), &allows_mem,
+				    &allows_reg, nullptr))
 	return;
 
       if (! allows_reg && allows_mem)
@@ -3664,7 +3623,7 @@ expand_asm_stmt (gasm *stmt)
 
       ok = parse_output_constraint (&constraints[i], i, ninputs,
 				    noutputs, &allows_mem, &allows_reg,
-				    &is_inout);
+				    &is_inout, nullptr);
       gcc_assert (ok);
 
       /* If an output operand is not a decl or indirect ref and our constraint
@@ -3769,7 +3728,7 @@ expand_asm_stmt (gasm *stmt)
       constraint = constraints[i + noutputs];
       ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
 				   constraints.address (),
-				   &allows_mem, &allows_reg);
+				   &allows_mem, &allows_reg, nullptr);
       gcc_assert (ok);
 
       /* EXPAND_INITIALIZER will not generate code for valid initializer
@@ -5358,6 +5317,9 @@ expand_debug_expr (tree exp)
       return simplify_gen_binary (MULT, mode, op0, op1);
 
     case RDIV_EXPR:
+      gcc_assert (FLOAT_MODE_P (mode)
+		  || ALL_FIXED_POINT_MODE_P (mode));
+      /* Fall through.  */
     case TRUNC_DIV_EXPR:
     case EXACT_DIV_EXPR:
       if (unsignedp)
diff --git a/gcc/cgraph.cc b/gcc/cgraph.cc
index 94a2e6e..32071a8 100644
--- a/gcc/cgraph.cc
+++ b/gcc/cgraph.cc
@@ -1790,6 +1790,19 @@ cgraph_update_edges_for_call_stmt_node (cgraph_node *node,
 
       if (e)
 	{
+	  /* If call was devirtualized during cloning, mark edge
+	     as resolved.  */
+	  if (e->speculative)
+	    {
+	      if (new_stmt && is_gimple_call (new_stmt))
+		{
+		  tree decl = gimple_call_fndecl (new_stmt);
+		  if (decl)
+		    e = cgraph_edge::resolve_speculation (e, decl);
+		}
+	      else
+		e = cgraph_edge::resolve_speculation (e, NULL);
+	    }
 	  /* Keep calls marked as dead dead.  */
 	  if (new_stmt && is_gimple_call (new_stmt) && e->callee
 	      && fndecl_built_in_p (e->callee->decl, BUILT_IN_UNREACHABLE,
diff --git a/gcc/cgraphunit.cc b/gcc/cgraphunit.cc
index fa54a59..9f4af63 100644
--- a/gcc/cgraphunit.cc
+++ b/gcc/cgraphunit.cc
@@ -63,7 +63,7 @@ along with GCC; see the file COPYING3.  If not see
       final assembler is generated.  This is done in the following way. Note
       that with link time optimization the process is split into three
       stages (compile time, linktime analysis and parallel linktime as
-      indicated bellow).
+      indicated below).
 
       Compile time:
 
@@ -2400,9 +2400,10 @@ symbol_table::compile (void)
     if (node->alias
 	&& lookup_attribute ("weakref", DECL_ATTRIBUTES (node->decl)))
       {
-	IDENTIFIER_TRANSPARENT_ALIAS
-	   (DECL_ASSEMBLER_NAME (node->decl)) = 1;
-	TREE_CHAIN (DECL_ASSEMBLER_NAME (node->decl))
+	tree id = DECL_ASSEMBLER_NAME (node->decl);
+	gcc_assert (!IDENTIFIER_INTERNAL_P (id));
+	IDENTIFIER_TRANSPARENT_ALIAS (id) = 1;
+	TREE_CHAIN (id)
 	   = (node->alias_target ? node->alias_target
 	      : DECL_ASSEMBLER_NAME (node->get_alias_target ()->decl));
       }
diff --git a/gcc/cobol/ChangeLog b/gcc/cobol/ChangeLog
index 4b05399..35d645c 100644
--- a/gcc/cobol/ChangeLog
+++ b/gcc/cobol/ChangeLog
@@ -1,3 +1,483 @@
+2025-08-02  Jakub Jelinek  <jakub@redhat.com>
+
+	* parse.y (intrinsic): Use %td format specifier with no cast on
+	argument instead of %ld with cast to long.
+	* scan_ante.h (numstr_of): Likewise.
+	* util.cc (cbl_field_t::report_invalid_initial_value): Likewise.
+
+2025-08-01  Robert Dubner  <rdubner@symas.com>
+
+	PR cobol/119324
+	* cbldiag.h (location_dump): Inline suppression of knownConditionTrueFalse.
+	* genapi.cc (parser_statement_begin): Combine two if() statements.
+	* genutil.cc (get_binary_value): File-level suppression of duplicateBreak.
+	* symbols.cc (symbol_elem_cmp): File-level suppression of duplicateBreak.
+
+2025-07-31  Robert Dubner  <rdubner@symas.com>
+
+	PR cobol/120244
+	* genapi.cc (get_level_88_domain): Increase array size for final byte.
+	(psa_FldLiteralA): Use correct length in build_string_literal call.
+	* scan.l: Use a loop instead of std:transform to avoid EOF overrun.
+	* scan_ante.h (binary_integer_usage): Use a variable-length buffer.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* util.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* util.cc: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* util.cc: Remove redundant #includes
+
+2025-07-24  Robert Dubner  <rdubner@symas.com>
+
+	PR cobol/119231
+	* gcobolspec.cc: (lang_specific_driver): Pass OPT_static through.
+	Handle -static and -static-libgcobol properly.
+
+2025-07-23  Robert Dubner  <rdubner@symas.com>
+
+	* genapi.cc (leave_procedure): Adjust location_t for PERFORM.
+	(parser_perform_times): Likewise.
+	(internal_perform_through_times): Likewise.
+	(perform_outofline_before_until): Likewise.
+	(perform_outofline_after_until): Likewise.
+	(perform_outofline_testafter_varying): Likewise.
+	(perform_outofline_before_varying): Likewise.
+
+2025-07-21  Robert Dubner  <rdubner@symas.com>
+	    James K. Lowden  <jklowden@cobolworx.com>
+
+	PR cobol/120402
+	* Make-lang.in: Elminate commented-out scripting.
+	* cbldiag.h (_CBLDIAG_H): Change #if 0 to #if GCOBOL_GETENV
+	(warn_msg): Add printf attributes.
+	(location_dump): Add debugging message.
+	* cdf.y: Improved linemap tracking.
+	* genapi.cc (treeplet_fill_source): const attribute for formal parameter.
+	(insert_nop): Created to consolidate var_decl_nop writes.
+	(build_main_that_calls_something): Move generation to the end of executable.
+	(level_88_helper): Formatting.
+	(parser_call_targets_dump): Formatting.
+	(function_pointer_from_name): const attribute for formal parameter.
+	(parser_initialize_programs): const attribute for formal parameter.
+	(parser_statement_begin): Improved linemap handling.
+	(section_label):  Improved linemap handling.
+	(paragraph_label): Improved linemap handling.
+	(pseudo_return_pop): Improved linemap handling.
+	(leave_procedure): Formatting.
+	(parser_enter_section):  Improved linemap handling.
+	(parser_enter_paragraph): Improved linemap handling.
+	(parser_perform): Formatting.
+	(parser_leave_file): Move creation of main() to this routine.
+	(parser_enter_program): Move creation of main from here to leave_file.
+	(parser_accept): Formatting. const attribute for formal parameter.
+	(parser_accept_command_line): const attribute for formal parameter.
+	(parser_accept_command_line_count): const attribute for formal parameter.
+	(parser_accept_envar): Likewise.
+	(parser_set_envar): Likewise.
+	(parser_display): Likewise.
+	(get_exhibit_name): Implement EXHIBIT verb.
+	(parser_exhibit): Likewise.
+	(parser_sleep): const attribute for formal parameter.
+	(parser_division): Improved linemap handling.
+	(parser_classify): const attribute for formal parameter.
+	(create_iline_address_pairs): Improved linemap handling.
+	(parser_perform_start): Likewise.
+	(perform_inline_until): Likewise.
+	(perform_inline_testbefore_varying): Likewise.
+	(parser_perform_until): Likewise.
+	(parser_perform_inline_times): Likewise.
+	(parser_intrinsic_subst): const attribute for formal parameter.
+	(parser_file_merge): Formatting.
+	(create_and_call): Improved linemap handling.
+	(mh_identical): const attribute for formal parameter.
+	(mh_numeric_display): const attribute for formal parameter.
+	(mh_little_endian): Likewise.
+	(mh_source_is_group): Likewise.
+	(psa_FldLiteralA): Formatting.
+	* genapi.h (parser_accept): const attribute for formal parameter.
+	(parser_accept_envar): Likewise.
+	(parser_set_envar): Likewise.
+	(parser_accept_command_line): Likewise.
+	(parser_accept_command_line_count): Likewise.
+	(parser_add): Likewise.
+	(parser_classify): Likewise.
+	(parser_sleep): Likewise.
+	(parser_exhibit): Likewise.
+	(parser_display): Likewise.
+	(parser_initialize_programs): Likewise.
+	(parser_intrinsic_subst): Likewise.
+	* gengen.cc (gg_assign): Improved linemap handling.
+	(gg_add_field_to_structure): Likewise.
+	(gg_define_from_declaration): Likewise.
+	(gg_build_relational_expression): Likewise.
+	(gg_goto_label_decl): Likewise.
+	(gg_goto): Likewise.
+	(gg_printf): Likewise.
+	(gg_fprintf): Likewise.
+	(gg_memset): Likewise.
+	(gg_memchr): Likewise.
+	(gg_memcpy): Likewise.
+	(gg_memmove): Likewise.
+	(gg_strcpy): Likewise.
+	(gg_strcmp): Likewise.
+	(gg_strncmp): Likewise.
+	(gg_return): Likewise.
+	(chain_parameter_to_function): Likewise.
+	(gg_define_function): Likewise.
+	(gg_get_function_decl): Likewise.
+	(gg_call_expr): Likewise.
+	(gg_call): Likewise.
+	(gg_call_expr_list): Likewise.
+	(gg_exit): Likewise.
+	(gg_abort): Likewise.
+	(gg_strlen): Likewise.
+	(gg_strdup): Likewise.
+	(gg_malloc): Likewise.
+	(gg_realloc): Likewise.
+	(gg_free): Likewise.
+	(gg_set_current_line_number): Likewise.
+	(gg_get_current_line_number): Likewise.
+	(gg_insert_into_assembler): Likewise.
+	(token_location_override): Likewise.
+	(gg_token_location): Likewise.
+	* gengen.h (location_from_lineno): Likewise.
+	(gg_set_current_line_number): Likewise.
+	(gg_get_current_line_number): Likewise.
+	(gg_token_location): Likewise.
+	(current_token_location): Likewise.
+	(current_location_minus_one): Likewise.
+	(current_location_minus_one_clear): Likewise.
+	(token_location_override): Likewise.
+	* genmath.cc (fast_divide):  const attribute for formal parameter.
+	* genutil.cc (get_and_check_refstart_and_reflen): Likewise.
+	(get_data_offset): Likewise.
+	(refer_refmod_length): Likewise.
+	(refer_offset): Likewise.
+	(refer_size): Likewise.
+	(refer_size_dest): Likewise.
+	(refer_size_source): Likewise.
+	(qualified_data_location): Likewise.
+	* genutil.h (refer_offset): Likewise.
+	(refer_size_source): Likewise.
+	(refer_size_dest): Likewise.
+	(qualified_data_location): Likewise.
+	* parse.y: EVALUATE token; Implement EXHIBIT verb;
+	Improved linemap handling.
+	* parse_ante.h (input_file_status_notify): Improved linemap handling.
+	(location_set): Likewise.
+	* scan.l: PICTURE string validation.
+	* scan_ante.h (class picture_t): PICTURE string validation.
+	(validate_picture): Likewise.
+	* symbols.cc (symbol_currency): Revised default currency handling.
+	* symbols.h (symbol_currency): Likewise.
+	* util.cc (location_from_lineno): Improved linemap handling.
+	(current_token_location): Improved linemap handling.
+	(current_location_minus_one): Improved linemap handling.
+	(current_location_minus_one_clear): Improved linemap handling.
+	(gcc_location_set_impl): Improved linemap handling.
+	(warn_msg): Improved linemap handling.
+	* util.h (cobol_lineno): Improved linemap handling.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* gcobolspec.cc (lang_specific_driver): Remove unused but set variable
+	n_cobol_files.
+
+2025-07-14  Robert Dubner  <rdubner@symas.com>
+
+	* cobol1.cc (cobol_langhook_handle_option): Eliminate cppcheck warnings.
+	* dts.h: Likewise.
+	* except.cc (cbl_enabled_exceptions_t::dump): Likewise.
+	* gcobolspec.cc (lang_specific_driver): Likewise.
+	* genapi.cc (parser_file_merge): Likewise.
+	* gengen.cc (gg_unique_in_function): Likewise.
+	(gg_declare_variable): Likewise.
+	(gg_peek_fn_decl): Likewise.
+	(gg_define_function): Likewise.
+	* genmath.cc (set_up_on_exception_label): Likewise.
+	(set_up_compute_error_label): Likewise.
+	(arithmetic_operation): Likewise.
+	(fast_divide): Likewise.
+	* genutil.cc (get_and_check_refstart_and_reflen): Likewise.
+	(get_depending_on_value_from_odo): Likewise.
+	(get_data_offset): Likewise.
+	(get_binary_value): Likewise.
+	(process_this_exception): Likewise.
+	(copy_little_endian_into_place): Likewise.
+	(refer_is_clean): Likewise.
+	(refer_fill_depends): Likewise.
+	* genutil.h (process_this_exception): Likewise.
+	(copy_little_endian_into_place): Likewise.
+	(refer_is_clean): Likewise.
+	* lexio.cc (check_push_pop_directive): Likewise.
+	(check_source_format_directive): Likewise.
+	(location_in): Likewise.
+	(lexer_input): Likewise.
+	(cdftext::lex_open): Likewise.
+	(lexio_dialect_mf): Likewise.
+	(valid_sequence_area): Likewise.
+	(cdftext::free_form_reference_format): Likewise.
+	(cdftext::segment_line): Likewise.
+	* lexio.h (struct span_t): Likewise.
+	* scan_ante.h (trim_location): Likewise.
+	* symbols.cc (symbol_elem_cmp): Likewise.
+	(symbol_alphabet): Likewise.
+	(end_of_group): Likewise.
+	(cbl_field_t::attr_str): Likewise.
+	(symbols_update): Likewise.
+	(symbol_typedef_add): Likewise.
+	(symbol_field_add): Likewise.
+	(new_temporary_impl): Likewise.
+	(symbol_label_section_exists): Likewise.
+	(symbol_program_callables): Likewise.
+	(file_status_status_of): Likewise.
+	* symfind.cc (is_data_field): Likewise.
+	(finalize_symbol_map2): Likewise.
+	(class in_scope): Likewise.
+	(symbol_match2): Likewise.
+	* util.cc (get_current_dir_name): Likewise.
+	(gb4): Likewise.
+	(class cdf_directives_t): Likewise.
+	(cbl_field_t::report_invalid_initial_value): Likewise.
+	(literal_subscript_oob): Likewise.
+	(cbl_refer_t::str): Likewise.
+	(date_time_fmt): Likewise.
+	(class unique_stack): Likewise.
+	(cobol_set_pp_option): Likewise.
+	(cobol_filename): Likewise.
+	(cobol_filename_restore): Likewise.
+	(gcc_location_set_impl): Likewise.
+	(ydferror): Likewise.
+	(error_msg_direct): Likewise.
+	(yyerror): Likewise.
+	(cbl_unimplemented_at): Likewise.
+
+2025-07-13  Robert Dubner  <rdubner@symas.com>
+
+	* Make-lang.in: Eliminate the .cc.o override.
+	* genapi.cc (level_88_helper): Eliminate cppcheck warning.
+	(get_level_88_domain): Likewise.
+	(get_class_condition_string): Likewise.
+	(parser_call_targets_dump): Likewise.
+	(parser_compile_ecs): Likewise.
+	(initialize_variable_internal): Likewise.
+	(move_tree): Likewise.
+	(combined_name): Likewise.
+	(assembler_label): Likewise.
+	(find_procedure): Likewise.
+	(parser_perform): Likewise.
+	(parser_perform_times): Likewise.
+	(internal_perform_through): Likewise.
+	(internal_perform_through_times): Likewise.
+	(psa_FldLiteralN): Likewise.
+	(psa_FldBlob): Likewise.
+	(parser_accept): Likewise.
+	(parser_accept_exception): Likewise.
+	(parser_accept_exception_end): Likewise.
+	(parser_accept_command_line): Likewise.
+	(parser_accept_envar): Likewise.
+	(parser_display_internal): Likewise.
+	(parser_display): Likewise.
+	(parser_assign): Likewise.
+	(parser_initialize_table): Likewise.
+	(parser_arith_error): Likewise.
+	(parser_arith_error_end): Likewise.
+	(parser_division): Likewise.
+	(label_fetch): Likewise.
+	(parser_label_label): Likewise.
+	(parser_label_goto): Likewise.
+	(parser_perform_start): Likewise.
+	(parser_perform_conditional): Likewise.
+	(parser_perform_conditional_end): Likewise.
+	(parser_perform_until): Likewise.
+	(parser_file_delete): Likewise.
+	(parser_intrinsic_subst): Likewise.
+	(create_lsearch_address_pairs): Likewise.
+	(parser_bsearch_start): Likewise.
+	(is_ascending_key): Likewise.
+	(parser_sort): Likewise.
+	(parser_file_sort): Likewise.
+	(parser_return_start): Likewise.
+	(parser_file_merge): Likewise.
+	(parser_string_overflow): Likewise.
+	(parser_unstring): Likewise.
+	(parser_string): Likewise.
+	(parser_call_exception): Likewise.
+	(create_and_call): Likewise.
+	(mh_identical): Likewise.
+	(move_helper): Likewise.
+	(binary_initial_from_float128): Likewise.
+	(initial_from_initial): Likewise.
+	(psa_FldLiteralA): Likewise.
+	(parser_local_add): Likewise.
+	(parser_symbol_add): Likewise.
+	* genapi.h (parser_display): Likewise.
+	* gengen.cc (gg_call_expr): Explict check for NULL_TREE.
+	(gg_call): Likewise.
+	* show_parse.h (SHOW_PARSE_LABEL_OK): Likewise.
+	(TRACE1_FIELD_VALUE): Likewise.
+	(CHECK_FIELD): Likewise.
+	(CHECK_FIELD2): Likewise.
+	(CHECK_LABEL): Likewise.
+	* util.cc (cbl_internal_error): Apply [[noreturn]] attribute.
+	* util.h (cbl_internal_error): Likewise.
+
+2025-07-11  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR cobol/120621
+	* lexio.cc (parse_replace_pairs): Cast mfile.lineno() to fmt_size_t.
+	* parse.y (intrinsic): Print ptrdiff_t using %ld, cast arg to long.
+	* scan_ante.h (numstr_of): Print nx using %ld, cast arg to long.
+	* util.cc (cbl_field_t::report_invalid_initial_value): Print
+	ptrdiff_t using %ld, cast arg to long.
+
+2025-07-10  James K. Lowden  <jklowden@cobolworx.com>
+
+	PR cobol/120765
+	* cdf.y: Extend grammar for new CDF syntax, relocate dictionary.
+	* cdfval.h (cdf_dictionary): Use new CDF dictionary.
+	* dts.h: Remove useless assignment, note incorrect behavior.
+	* except.cc: Remove obsolete EC state.
+	* gcobol.1: Document CDF in its own section.
+	* genapi.cc (parser_statement_begin): Use new EC state function.
+	(parser_file_merge): Same.
+	(parser_check_fatal_exception): Same.
+	* genutil.cc (get_and_check_refstart_and_reflen): Same.
+	(get_depending_on_value_from_odo): Same.
+	(get_data_offset): Same.
+	(process_this_exception): Same.
+	* lexio.cc (check_push_pop_directive): New function.
+	(check_source_format_directive): Restrict regex search to 1 line.
+	(cdftext::free_form_reference_format): Use new function.
+	* parse.y: Define new CDF tokens, use new CDF state.
+	* parse_ante.h (cdf_tokens): Use new CDF state.
+	(redefined_token): Same.
+	(class prog_descr_t): Remove obsolete CDF state.
+	(class program_stack_t): Same.
+	(current_call_convention): Same.
+	* scan.l: Recognize new CDF tokens.
+	* scan_post.h (is_cdf_token): Same.
+	* symbols.h (cdf_current_tokens): Change current_call_convention to return void.
+	* token_names.h: Regenerate.
+	* udf/stored-char-length.cbl: Use new PUSH/POP CDF functionality.
+	* util.cc (class cdf_directives_t): Define cdf_directives_t.
+	(current_call_convention): Same.
+	(cdf_current_tokens): Same.
+	(cdf_dictionary): Same.
+	(cdf_enabled_exceptions): Same.
+	(cdf_push): Same.
+	(cdf_push_call_convention): Same.
+	(cdf_push_current_tokens): Same.
+	(cdf_push_dictionary): Same.
+	(cdf_push_enabled_exceptions): Same.
+	(cdf_push_source_format): Same.
+	(cdf_pop): Same.
+	(cdf_pop_call_convention): Same.
+	(cdf_pop_current_tokens): Same.
+	(cdf_pop_dictionary): Same.
+	(cdf_pop_enabled_exceptions): Same.
+	(cdf_pop_source_format): Same.
+	* util.h (cdf_push): Declare cdf_directives_t.
+	(cdf_push_call_convention): Same.
+	(cdf_push_current_tokens): Same.
+	(cdf_push_dictionary): Same.
+	(cdf_push_enabled_exceptions): Same.
+	(cdf_push_source_format): Same.
+	(cdf_pop): Same.
+	(cdf_pop_call_convention): Same.
+	(cdf_pop_current_tokens): Same.
+	(cdf_pop_dictionary): Same.
+	(cdf_pop_source_format): Same.
+	(cdf_pop_enabled_exceptions): Same.
+
+2025-07-09  Robert Dubner  <rdubner@symas.com>
+	    James K. Lowden  <jklowden@cobolworx.com>
+
+	PR cobol/120765
+	PR cobol/119337
+	PR cobol/120794
+	* Make-lang.in: Take control of the .cc.o rule.
+	* cbldiag.h (error_msg_direct): New declaration.
+	(gcc_location_dump): Forward declaration.
+	(location_dump): Use gcc_location_dump.
+	* cdf.y: Change some tokens.
+	* gcobc: Change dialect handling.
+	* genapi.cc (parser_call_targets_dump): Temporarily remove from service.
+	(parser_compile_dcls): Combine temporary arrays.
+	(get_binary_value_from_float): Apply const to one parameter.
+	(depending_on_value): Localize a boolean variable.
+	(normal_normal_compare): Likewise.
+	(cobol_compare): Eliminate cppcheck warning.
+	(combined_name): Apply const to an input parameter.
+	(parser_perform): Apply const to a variable.
+	(parser_accept): Improve handling of special_name_t parameter and
+	the exception conditions.
+	(parser_display): Improve handling of speciat_name_t parameter; use the
+	os_filename[] string when appropriate.
+	(program_end_stuff): Rename shadowing variable.
+	(parser_division): Consolidate temporary char[] arrays.
+	(parser_file_start): Apply const to a parameter.
+	(inspect_replacing): Likewise.
+	(parser_program_hierarchy): Rename shadowing variable.
+	(mh_identical): Apply const to parameters.
+	(float_type_of): Likewise.
+	(picky_memcpy): Likewise.
+	(mh_numeric_display): Likewise.
+	(mh_little_endian): Likewise.
+	(mh_source_is_group): Apply static to a variable it.
+	(move_helper): Quiet a cppcheck warning.
+	* genapi.h (parser_accept): Add exceptions to declaration.
+	(parser_accept_under_discussion): Add declaration.
+	(parser_display): Change to std::vector; add exceptions to declaration.
+	* lexio.cc (cdf_source_format): Improve source code location handling.
+	(source_format_t::infer): Likewise.
+	(is_fixed_format): Likewise.
+	(is_reference_format): Likewise.
+	(left_margin): Likewise.
+	(right_margin): Likewise.
+	(cobol_set_indicator_column): Likewise.
+	(include_debug): Likewise.
+	(continues_at): Likewise.
+	(indicated): Likewise.
+	(check_source_format_directive): Likewise.
+	(cdftext::free_form_reference_format): Likewise.
+	* parse.y: Tokens; program and function names; DISPLAY and ACCEPT
+	handling.
+	* parse_ante.h (class tokenset_t): Removed.
+	(class current_tokens_t): Removed.
+	(field_of): Removed.
+	* scan.l: Token handling.
+	* scan_ante.h (level_found): Comment.
+	* scan_post.h (start_condition_str): Remove cast author_state:.
+	* symbols.cc (symbols_update): Change error message.
+	(symbol_table_init): Correct and reorder entries.
+	(symbol_unresolved_file_key): New function definition.
+	(cbl_file_key_t::deforward): Change error message.
+	* symbols.h (symbol_unresolved_file_key): New declaration.
+	(keyword_tok): New function.
+	(redefined_token): New function.
+	(class current_tokens_t): New class.
+	* symfind.cc (symbol_match): Revise error message.
+	* token_names.h: Reorder and change numbers in comments.
+	* util.cc (class cdf_directives_t): New class.
+	(cobol_set_indicator_column): New function.
+	(cdf_source_format): New function.
+	(gcc_location_set_impl): Improve column handling in token_location.
+	(gcc_location_dump): New function.
+	(class temp_loc_t): Modify constructor.
+	(error_msg_direct): New function.
+	* util.h (class source_format_t): New class.
+
 2025-07-01  James K. Lowden  <jklowden@cobolworx.com>
 
 	* Make-lang.in: Use && instead of semicolon between commands.
diff --git a/gcc/cobol/Make-lang.in b/gcc/cobol/Make-lang.in
index e884212..0e2a773 100644
--- a/gcc/cobol/Make-lang.in
+++ b/gcc/cobol/Make-lang.in
@@ -384,3 +384,4 @@ cobol.stagefeedback: stagefeedback-start
 selftest-cobol:
 
 lang_checks += check-cobol
+
diff --git a/gcc/cobol/cbldiag.h b/gcc/cobol/cbldiag.h
index 548b0f2..dd16190 100644
--- a/gcc/cobol/cbldiag.h
+++ b/gcc/cobol/cbldiag.h
@@ -33,7 +33,7 @@
 #else
 #define _CBLDIAG_H
 
-#if 0
+#if GCOBOL_GETENV
 #define gcobol_getenv(x) getenv(x)
 #else
 #define gcobol_getenv(x) ((char *)nullptr)
@@ -78,10 +78,19 @@ struct YDFLTYPE
 
 #endif
 
+// Diagnostic format specifiers are documented in gcc/pretty-print.cc
 // an error at a location, called from the parser for semantic errors
 void error_msg( const YYLTYPE& loc, const char gmsgid[], ... )
   ATTRIBUTE_GCOBOL_DIAG(2, 3);
 
+bool
+warn_msg( const YYLTYPE& loc, const char gmsgid[], ... )
+  ATTRIBUTE_GCOBOL_DIAG(2, 3);
+
+// an error that uses token_location, not yylloc
+void error_msg_direct( const char gmsgid[], ... )
+  ATTRIBUTE_GCOBOL_DIAG(1, 2);
+
 void dialect_error( const YYLTYPE& loc, const char term[], const char dialect[] );
 
 
@@ -104,16 +113,23 @@ void dbgmsg( const char fmt[], ... ) ATTRIBUTE_PRINTF_1;
 
 void gcc_location_set( const YYLTYPE& loc );
 
+void gcc_location_dump();
+
 // tree.h defines yy_flex_debug as a macro because options.h
 #if ! defined(yy_flex_debug)
 template <typename LOC>
 static void
 location_dump( const char func[], int line, const char tag[], const LOC& loc) {
   extern int yy_flex_debug; // cppcheck-suppress shadowVariable
-  if( yy_flex_debug && gcobol_getenv("update_location") )
-    fprintf(stderr, "%s:%d: %s location (%d,%d) to (%d,%d)\n",
-            func, line, tag,
-            loc.first_line, loc.first_column, loc.last_line, loc.last_column);
+  if( yy_flex_debug ) {
+    const char *detail = gcobol_getenv("update_location"); // cppcheck-suppress knownConditionTrueFalse
+    if( detail ) {
+      fprintf(stderr, "%s:%d: %s location (%d,%d) to (%d,%d)\n",
+              func, line, tag,
+              loc.first_line, loc.first_column, loc.last_line, loc.last_column);
+      if( *detail == '2' ) gcc_location_dump();
+    }
+  }
 }
 #endif // defined(yy_flex_debug)
 
diff --git a/gcc/cobol/cdf.y b/gcc/cobol/cdf.y
index 3344271..53fea5d 100644
--- a/gcc/cobol/cdf.y
+++ b/gcc/cobol/cdf.y
@@ -95,7 +95,7 @@ void input_file_status_notify();
         }                                                               \
       location_dump("cdf.c", __LINE__, "current", (Current));		\
       input_file_status_notify();					\
-      gcc_location_set( location_set(Current) );			\
+      location_set(Current);                                            \
   } while (0)
 
 %}
@@ -105,14 +105,14 @@ void input_file_status_notify();
 
   using std::map;
 
-  static map<std::string, cdfval_t> dictionary;
-
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-function"
   static bool
   cdfval_add( const char name[],
 	       const cdfval_t& value, bool override = false )
   {
+    cdf_values_t& dictionary( cdf_dictionary() );
+    
     if( scanner_parsing() ) {
       if( ! override ) {
 	if( dictionary.find(name) != dictionary.end() ) return false;
@@ -123,6 +123,8 @@ void input_file_status_notify();
   }
   static void
   cdfval_off( const char name[] ) {
+    cdf_values_t& dictionary( cdf_dictionary() );
+    
     if( scanner_parsing() ) {
       auto p = dictionary.find(name);
       if( p == dictionary.end() ) {
@@ -159,6 +161,8 @@ exception_turn_t exception_turn;
 			
 bool
 apply_cdf_turn( const exception_turn_t& turn ) {
+  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
+  
   for( auto elem : turn.exception_files() ) {
     std::set<size_t> files(elem.second.begin(), elem.second.end());
     enabled_exceptions.turn_on_off(turn.enabled,
@@ -204,15 +208,17 @@ apply_cdf_turn( const exception_turn_t& turn ) {
 %type   <file>		filename
 %type   <files>         filenames
 
-%token BY 478
+%type   <number>        cdf_stackable
+
+%token BY 486
 %token COPY 362
 %token CDF_DISPLAY 384    ">>DISPLAY"
-%token IN 597
+%token IN 605
 %token NAME 286
 %token NUMSTR 305    "numeric literal"
-%token OF 678
-%token PSEUDOTEXT 713
-%token REPLACING 735
+%token OF 686
+%token PSEUDOTEXT 721
+%token REPLACING 743
 %token LITERAL 298
 %token SUPPRESS 376
 
@@ -227,25 +233,32 @@ apply_cdf_turn( const exception_turn_t& turn ) {
 %token CDF_WHEN 389    ">>WHEN"
 %token CDF_END_EVALUATE 390    ">>END-EVALUATE"
 
-%token AS 460  CONSTANT 361  DEFINED 363
+%token ALL 450
+%token CALL_CONVENTION 391    ">>CALL-CONVENTION"
+%token COBOL_WORDS 380    ">>COBOL-WORDS"
+%token CDF_PUSH 394    ">>PUSH"
+%token CDF_POP 395    ">>POP"
+%token SOURCE_FORMAT 396    ">>SOURCE FORMAT"
+
+%token AS 468  CONSTANT 361  DEFINED 363
 %type	<boolean>	     DEFINED
-%token OTHER 690  PARAMETER_kw 368    "PARAMETER"
-%token OFF 679  OVERRIDE 369
-%token THRU 931
-%token TRUE_kw 805    "True"
+%token OTHER 698  PARAMETER_kw 368    "PARAMETER"
+%token OFF 687  OVERRIDE 369
+%token THRU 939
+%token TRUE_kw 813    "True"
 
-%token CALL_COBOL 391    "CALL"
-%token CALL_VERBATIM 392    "CALL (as C)"
+%token CALL_COBOL 392    "CALL"
+%token CALL_VERBATIM 393    "CALL (as C)"
 
-%token TURN 807  CHECKING 488  LOCATION 641  ON 681  WITH 833
+%token TURN 815  CHECKING 496  LOCATION 649  ON 689  WITH 841
 
-%left OR 932
-%left AND 933
-%right NOT 934
-%left '<'  '>'  '='  NE 935  LE 936  GE 937
+%left OR 940
+%left AND 941
+%right NOT 942
+%left '<'  '>'  '='  NE 943  LE 944  GE 945
 %left '-'  '+'
 %left '*'  '/'
-%right NEG 939
+%right NEG 947
 
 %define api.prefix {ydf}
 %define api.token.prefix{YDF_}
@@ -277,6 +290,8 @@ complete:	cdf_define
 	|	cdf_display
 	|	cdf_turn
         |       cdf_call_convention
+        |       cdf_push
+        |       cdf_pop
 		;
 
 		/*
@@ -328,6 +343,7 @@ cdf_define:	CDF_DEFINE cdf_constant NAME as cdf_expr[value] override
 		  }
 		  if( !cdfval_add( $NAME, cdfval_t($value), $override) ) {
 		    error_msg(@NAME, "name already in dictionary: %s", $NAME);
+                    cdf_values_t& dictionary( cdf_dictionary() );
 		    const cdfval_t& entry = dictionary[$NAME];
 		    if( entry.filename ) {
 		      error_msg(@NAME, "%s previously defined in %s:%d",
@@ -357,7 +373,7 @@ cdf_define:	CDF_DEFINE cdf_constant NAME as cdf_expr[value] override
 		 * available regardless.
 		 */
 		{
-		  if( 0 == dictionary.count($NAME) ) {
+		  if( 0 == cdf_dictionary().count($NAME) ) {
 		    yywarn("CDF: '%s' is defined AS PARAMETER "
 			    "but was not defined", $NAME);
 		  }
@@ -400,6 +416,35 @@ cdf_call_convention:
                 }
                 ;
 
+cdf_push:       CDF_PUSH cdf_stackable {
+		  switch( $cdf_stackable ) {
+                  case YDF_ALL: 		cdf_push(); break;
+                  case YDF_CALL_CONVENTION: cdf_push_call_convention(); break;
+                  case YDF_CDF_DEFINE: 	cdf_push_dictionary(); break;
+                  case YDF_COBOL_WORDS: 	cdf_push_current_tokens(); break;
+                  case YDF_SOURCE_FORMAT:	cdf_push_source_format(); break;
+                  default: gcc_unreachable(); 
+                  }
+                }
+                ;
+cdf_pop:        CDF_POP cdf_stackable {
+		  switch( $cdf_stackable ) {
+                  case YDF_ALL: 		cdf_pop(); break;
+                  case YDF_CALL_CONVENTION: cdf_pop_call_convention(); break;
+                  case YDF_CDF_DEFINE: 	cdf_pop_dictionary(); break;
+                  case YDF_COBOL_WORDS: 	cdf_pop_current_tokens(); break;
+                  case YDF_SOURCE_FORMAT:	cdf_pop_source_format(); break; 
+                  default: gcc_unreachable(); 
+                  }
+                }
+                ;
+
+cdf_stackable:  ALL		{ $$ = YDF_ALL; }
+        |       CALL_CONVENTION	{ $$ = YDF_CALL_CONVENTION; }
+        |       COBOL_WORDS	{ $$ = YDF_COBOL_WORDS; }
+        |       CDF_DEFINE	{ $$ = YDF_CDF_DEFINE; }
+        |       SOURCE_FORMAT	{ $$ = YDF_SOURCE_FORMAT; }
+                ;
 
 except_names: 	except_name
 	|	except_names except_name
@@ -471,6 +516,7 @@ cdf_eval_obj:	cdf_cond_expr
 cdf_cond_expr:	BOOL
 	|	NAME DEFINED
 		{
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  auto p = dictionary.find($1);
 		  bool found = p != dictionary.end();
 		  if( !$DEFINED ) found = ! found;
@@ -552,6 +598,7 @@ cdf_expr:	cdf_expr '+' cdf_expr { $$ = $1(@1) + $3(@3); }
         ;
 
 cdf_factor:     NAME {
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  auto that = dictionary.find($1);
 		  if( that != dictionary.end() ) {
 		    $$ = that->second;
@@ -651,6 +698,7 @@ name_any:	namelit
 
 name_one:	NAME
 		{
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  cdf_arg_t arg = { YDF_NAME, $1 };
 		  auto p = dictionary.find($1);
 
@@ -665,6 +713,7 @@ name_one:	NAME
 
 namelit:	name
 		{
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  cdf_arg_t arg = { YDF_NAME, $1 };
 		  auto p = dictionary.find($1);
 
@@ -745,6 +794,7 @@ location_set( const YYLTYPE& loc ) {
 bool // used by cobol1.cc
 defined_cmd( const char arg[] )
 {
+  cdf_values_t& dictionary( cdf_dictionary() );
   cdfval_t value(1);
 
   char *name = xstrdup(arg);
@@ -868,6 +918,7 @@ static int ydflex(void) {
 
 bool
 cdf_value( const char name[], const cdfval_t& value ) {
+  cdf_values_t& dictionary( cdf_dictionary() );
   auto p = dictionary.find(name);
 
   if( p != dictionary.end() ) return false;
@@ -878,6 +929,7 @@ cdf_value( const char name[], const cdfval_t& value ) {
 
 const cdfval_t *
 cdf_value( const char name[] ) {
+  cdf_values_t& dictionary( cdf_dictionary() );
   auto p = dictionary.find(name);
 
   if( p == dictionary.end() ) return NULL;
diff --git a/gcc/cobol/cdfval.h b/gcc/cobol/cdfval.h
index 465bdbb..cc474a2 100644
--- a/gcc/cobol/cdfval.h
+++ b/gcc/cobol/cdfval.h
@@ -126,4 +126,8 @@ cdf_value( const char name[] );
 bool
 cdf_value( const char name[], const cdfval_t& value );
 
+typedef std::map<std::string, cdfval_t> cdf_values_t;
+
+cdf_values_t& cdf_dictionary();
+
 #endif
diff --git a/gcc/cobol/cobol1.cc b/gcc/cobol/cobol1.cc
index 4bd79f1..3146da5 100644
--- a/gcc/cobol/cobol1.cc
+++ b/gcc/cobol/cobol1.cc
@@ -357,7 +357,7 @@ cobol_langhook_handle_option (size_t scode,
             return true;
 
         case OPT_M:
-	    cobol_set_pp_option('M');
+            cobol_set_pp_option('M');
             return true;
 
         case OPT_fstatic_call:
@@ -368,16 +368,18 @@ cobol_langhook_handle_option (size_t scode,
             wsclear(cobol_default_byte);
             return true;
 
-        case OPT_fflex_debug:
+        case OPT_fflex_debug: // cppcheck-suppress syntaxError // The need for this is a mystery
             yy_flex_debug = 1;
             cobol_set_debugging( true, yy_debug == 1, cobol_trace_debug == 1 );
             return true;
+
         case OPT_fyacc_debug:
             yy_debug = 1;
             cobol_set_debugging(yy_flex_debug == 1,
                                 true,
                                 cobol_trace_debug == 1 );
             return true;
+
         case OPT_ftrace_debug:
             cobol_set_debugging( yy_flex_debug == 1, yy_debug == 1, true );
             return true;
@@ -406,11 +408,13 @@ cobol_langhook_handle_option (size_t scode,
         case OPT_fsyntax_only:
           mode_syntax_only(identification_div_e);
           break;
+
         case OPT_preprocess:
           if( ! preprocess_filter_add(arg) ) {
             cbl_errx( "could not execute preprocessor %s", arg);
           }
           return true;
+
         case OPT_include:
           if( ! include_file_add(arg) ) {
             cbl_errx( "could not include %s", arg);
diff --git a/gcc/cobol/dts.h b/gcc/cobol/dts.h
index aa3fa58..c900c45 100644
--- a/gcc/cobol/dts.h
+++ b/gcc/cobol/dts.h
@@ -86,24 +86,30 @@ namespace dts {
 #if __cpp_exceptions
       static const char msg[] = "input not NUL-terminated";
       throw std::domain_error( msg );
-#else
-      // eoinput terminates input
-      eoinput = strchr(input, '\0'); // cppcheck-suppress uselessAssignmentPtrArg
 #endif
     }
     auto ncm = re.size();
     cm.resize(ncm);
     std::vector <regmatch_t> cms(ncm);
 
-
     int erc = regexec( &re, input, ncm, cms.data(), 0 );
     if( erc != 0 ) return false;
+#if  __cpp_exceptions
+    // This is not correct at all, but current use depends on current behavior.
+    // The following line is excluded from the GCC build, which is compiled
+    // without __cpp_exceptions.  parse_copy_directive (for one) depends on
+    // regex_search returning true even if the match is beyond eoinput.
+    if( eoinput < cm[0].second ) return false;
+    // Correct behavior would return match only between input and eoinput.
+    // Because regex(3) uses a NUL terminator, it may match text between
+    // eoinput and the NUL.
+#endif
     std::transform( cms.begin(), cms.end(), cm.begin(),
                     [input]( const regmatch_t& m ) {
                       return csub_match( input, m );
                     } );
     return true;
   }
-};
+}
 
 
diff --git a/gcc/cobol/except.cc b/gcc/cobol/except.cc
index 60b8416..df1c7df 100644
--- a/gcc/cobol/except.cc
+++ b/gcc/cobol/except.cc
@@ -84,8 +84,6 @@ cbl_enabled_exception_t::dump( int i ) const {
               file );
 }
 
-cbl_enabled_exceptions_t enabled_exceptions;
-
 void
 cbl_enabled_exceptions_t::dump() const {
   extern int yydebug;
@@ -98,7 +96,7 @@ cbl_enabled_exceptions_t::dump() const {
     return;
   }
   int i = 1;
-  for( auto& elem : *this ) {
+  for( auto& elem : *this ) { // cppcheck-suppress constVariableReference
     dbgmsg("cbl_enabled_exceptions_t: %2d  {%s, %s, %lu}",
            i++,
            elem.location? "with location" : "  no location", 
diff --git a/gcc/cobol/gcobc b/gcc/cobol/gcobc
index 01c75dd..fa9f609 100755
--- a/gcc/cobol/gcobc
+++ b/gcc/cobol/gcobc
@@ -125,25 +125,24 @@ $0 recognizes the following GnuCOBOL cobc compilation options:
         -std=mvs -std=mvs-strict
         -std=mf  -std=mf-strict
         -std=cobol85 -std=cobol2002 -std=cobol2014
- Options that are the same in gcobol and cobc are passed through verbatim.
- Options that have no analog in gcobol produce a warning message.
- To produce this message, use -HELP.
+Options that are the same in gcobol and cobc are passed through verbatim.
+Options that have no analog in gcobol produce a warning message.
+To produce this message, use -HELP.
 To see the constructed cobc command-line, use -echo.
 To override the default cobc, set the "cobc" environment variable.
 By default, gcobc invokes the gcobol the same directory the gcobc resides.
 To override, set the gcobol environment variable.
- EOF
- }
+EOF
+}
 
-dialect="gnu"
+dialect="mf gnu"
 out_set=""
 first=""
 
- #
- # Simply iterate over the command-line tokens.  We can't use getopts
- # here because it's not designed for single-dash words (e.g. -shared).
 #
-
+# Iterate over the command-line tokens.  We can't use getopts here
+# because it's not designed for single-dash words (e.g. -shared).
+#
 for opt in "$@"
 do
     if [ "$skip_arg" ]
@@ -441,11 +440,13 @@ do
         -std=mvs | -std=mvs-strict | -std=ibm | -std=ibm-strict) dialect=ibm
                   ;;
         -std=mf | -std=mf-strict)  dialect=mf
-                  ;;
-        -std=default)  dialect=gnu  # that's GnuCOBOL's default and GCC's dialect for GnuCOBOL
-                   ;;
-        -std=cobol*)    dialect=""   # GCC COBOL targets COBOL2024 "mostly backward to COBOL85"
-                   ;;
+                                   ;;
+                      # GnuCOBOL's default and GCC's dialect for GnuCOBOL
+        -std=default) dialect=gnu  
+                      ;;
+                      # GCC COBOL targets COBOL2024 "mostly backward to COBOL85"
+        -std=cobol*)  dialect="" 
+                      ;;
         -std=*)
             dialect=""
             warn "$opt (unkown dialect)"
@@ -480,7 +481,8 @@ do
            opts="$opts /dev/stdin"
            ;;
 
-        *) if [ -z "$output_name" ]  # first non-option argument is source file name
+           # First file name argument is default output filename.
+        *) if [ -z "$output_name" -a -e "$opt" ]  
            then
                output_name=$(basename "${opt%.*}")
                case $mode in
@@ -512,6 +514,11 @@ fi
 # To override the default gcobol, set the "gcobol" environment variable.
 gcobol="${gcobol:-${0%/*}/gcobol}"
 
+if [ "$dialect" ]
+then
+    dialect=$(echo $dialect | sed -E 's/[[:alnum:]]+/-dialect &/g')
+fi
+
 if [ "$echo" ]
 then
     echo $gcobol $mode $opts
@@ -523,4 +530,4 @@ then
     set -x
 fi
 
-exec $gcobol $mode $opts
+exec $gcobol $mode $dialect $opts
diff --git a/gcc/cobol/gcobol.1 b/gcc/cobol/gcobol.1
index 6db5400..ebb833c 100644
--- a/gcc/cobol/gcobol.1
+++ b/gcc/cobol/gcobol.1
@@ -587,6 +587,21 @@ statement, regardless of compile-time constants.
 ..
 .
 .Ss Implemented Exception Conditions
+By default, per ISO, no EC is enabled.  Implemented ECs may be enabled
+on the command line or via the
+.Sy TURN
+directive. Any attempt to enable an EC that is not implemented is
+treated as an error.
+.Pp
+An enabled EC not handled by a
+.Sy DECLARATIVE
+is written to the system log and to standard error.  (The authors
+intend to make that an option.)  A fatal EC not handled with
+.Sy RESUME
+ends with a call to
+.Xr abort 3
+and process termination. 
+.Pp
 Not all Exception Conditions are implemented.  Any attempt to enable
 an EC that that is not implemented produces a warning message.
 The following are implemented:
@@ -913,11 +928,90 @@ or
 has no effect; the two are interchangeable.
 ..
 .
-.Ss Compiler-Directing Facility (CDF)
+.Sh COMPILER-DIRECTING FACILITY
 The CDF should be used with caution because no comprehensive test
-suite has been identified.
+suite has been identified.  
 .
-.Ss Conditional Compilation
+.Ss CDF Text Manipulation
+.Bl -tag -width >>DEFINE
+.It Sy COPY Ar copybook Li Oo OF|BY Ar library Oc Oo Sy REPLACING ... Oc
+If
+.Ar copybook
+is a literal, it treated a literal filename, which either does or does not exist.  If
+.Ar copybook
+is a \*[lang] word,
+.Nm
+looks first for an environment variable named
+.Va copybook
+and, if found, uses the contents of that variable as the name of the
+copybook file.  If that file does not exist, it continues looking for
+a file named one of:
+.sp
+.Bl -bullet -compact -offset 5n
+.It
+.Pa copybook
+(literally)
+.It
+.Pa copybook.cpy
+.It
+.Pa copybook.CPY
+.It
+.Pa copybook.cbl
+.It
+.Pa copybook.CBL
+.It
+.Pa copybook.cob
+.It
+.Pa copybook.COB
+.El
+.sp
+in that order.  It looks first in the same directory as the source
+code file, and then in any
+.Ar copybook-path
+named with the
+.Fl I
+option.
+.
+.\" FIXME: need escape mechanism for directories with ':' in the name.
+.Ar copybook-path
+may (like the shell's
+.Ev PATH
+variable) be a colon-separated list.
+The
+.Fl I
+option may occur multiple times on the command line.  Each successive
+.Ar copybook-path
+is concatenated to previous ones.
+Relative paths (having no leading
+.Ql / Ns
+\&)
+are searched relative to the compiler's current working directory.
+.Pp
+For example,
+.D1 \&
+.D1 Fl I Li /usr/local/include:include
+.D1 \&
+searches first the directory where the \*[lang] program is found, next in
+.Pa /usr/local/include ,
+and finally in an
+.Pa include
+subdirectory of the directory from which
+.Nm
+was invoked.
+.Pp
+For the
+.Sy REPLACING
+phrase, both the modern pseudo-text and the \*[lang]/85 forms are
+recognized. (The older forms are used in the NIST CCVS/85 test suite.) 
+.It Sy REPLACE ...
+.Nm
+supports the full ISO
+.Sy REPLACE
+syntax.  
+.El
+.
+.Ss CDF Directives
+.\"Bl -tag -width >>PROPAGATE
 .Bl -tag -width >>DEFINE
 .It >> Ns Sy DEFINE Ar name Sy AS Bro Ar expression Li | Sy PARAMETER Brc Op Sy OVERRIDE
 Define
@@ -959,10 +1053,6 @@ is supported. Boolean literals are not supported.
 .
 .It >> Ns Sy EVALUATE
 Not implemented.
-.El
-.
-.Ss Other CDF Directives
-.Bl -tag -width >>PROPAGATE
 .It >> Ns Sy CALL-CONVENTION Ar convention
 .Ar convention
 may be one of:
@@ -1036,6 +1126,24 @@ Not implemented.
 Not implemented.
 .It >> Ns Sy PROPAGATE
 Not implemented.
+.It >> Ns Sy PUSH Ar directive
+.It >> Ns Sy POP  Ar directive
+With
+.Sy PUSH ,
+push CDF state onto a stack.
+With
+.Sy POP ,
+return to the prior pushed state.
+.Ar directive
+may be one of
+.Bl -tag -compact
+.It Sy CALL-CONVENTION
+.It Sy COBOL-WORDS
+.It Sy DEFINE
+.It Sy SOURCE FORMAT
+.It Sy TURN
+.El
+.
 .It >> Ns Sy TURN Oo
 .Ar ec Oo Ar file Li ... Oc ...
 .Oc Sy CHECKING Bro Oo Sy ON Oc Oo Oo Sy WITH Oc Sy LOCATION Oc | Sy OFF Brc
@@ -1087,76 +1195,6 @@ the directive must appear before
 To test a feature-set variable, use
 .Dl >>IF Ar feature Li DEFINED
 .
-.Ss Copybooks
-.Nm
-supports the CDF
-.Sy COPY
-statement, with or without its
-.Sy REPLACING
-component.  For any statement
-.sp
-.D1 COPY Ar copybook
-.sp
-.Nm
-looks first for an environment variable named
-.Va copybook
-and, if found, uses the contents of that variable as the name of the
-copybook file.  If that file does not exist, it continues looking for
-a file named one of:
-.sp
-.Bl -bullet -compact -offset 5n
-.It
-.Pa copybook
-(literally)
-.It
-.Pa copybook.cpy
-.It
-.Pa copybook.CPY
-.It
-.Pa copybook.cbl
-.It
-.Pa copybook.CBL
-.It
-.Pa copybook.cob
-.It
-.Pa copybook.COB
-.El
-.sp
-in that order.  It looks first in the same directory as the source
-code file, and then in any
-.Ar copybook-path
-named with the
-.Fl I
-option.
-.
-.\" FIXME: need escape mechanism for directories with ':' in the name.
-.Ar copybook-path
-may (like the shell's
-.Ev PATH
-variable) be a colon-separated list.
-.
-The
-.Fl I
-option may occur multiple times on the command line.  Each successive
-.Ar copybook-path
-is concatenated to previous ones.
-Relative paths (having no leading
-.Ql / Ns
-\&)
-are searched relative to the compiler's current working directory.
-.Pp
-For example,
-.D1 \&
-.D1 Fl I Li /usr/local/include:include
-.D1 \&
-searches first the directory where the \*[lang] program is found, next in
-.Pa /usr/local/include ,
-and finally in an
-.Pa include
-subdirectory of the directory from which
-.Nm
-was invoked.
-.
 .Ss Intrinsic functions
 .Nm
 implements all intrinsic functions defined by \*[isostd], plus a few
diff --git a/gcc/cobol/gcobolspec.cc b/gcc/cobol/gcobolspec.cc
index 70784d7..1f1b463 100644
--- a/gcc/cobol/gcobolspec.cc
+++ b/gcc/cobol/gcobolspec.cc
@@ -142,9 +142,6 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
   int n_infiles = 0;
   int n_outfiles = 0;
 
-  // The number of input files when the language is "none" or "cobol"
-  int n_cobol_files = 0;
-
   // saw_OPT_no_main means "don't expect -main"
   bool saw_OPT_no_main = false;
 
@@ -234,11 +231,6 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
       case OPT_SPECIAL_input_file:
         no_files_error = false;
         n_infiles += 1;
-        if(    strcmp(language, "none")  == 0
-            || strcmp(language, "cobol") == 0 )
-          {
-          n_cobol_files += 1;
-          }
         if( strstr(decoded_options[i].orig_option_with_args_text, "libgcobol.a") )
           {
           // We have been given an explicit libgcobol.a.  We need to note that.
@@ -478,7 +470,10 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
 ////        break;
 ////#endif
       case OPT_static:
+#if defined (HAVE_LD_STATIC_DYNAMIC)
+        append_arg(decoded_options[i]);
         static_in_general = true;
+#endif        
         break;
 
       default:
@@ -506,17 +501,23 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
     need_libgcobol = false;
     }
 
+  if( static_in_general )
+    {
+    // These two options interfere with each other.
+    static_libgcobol = false;
+    }
+
   if( need_libgcobol )
     {
     add_arg_lib(COBOL_LIBRARY, static_libgcobol);
     }
   if( need_libdl )
     {
-    add_arg_lib(DL_LIBRARY, static_in_general);
+    add_arg_lib(DL_LIBRARY, false);
     }
   if( need_libstdc )
     {
-    add_arg_lib(STDCPP_LIBRARY, static_in_general);
+    add_arg_lib(STDCPP_LIBRARY, false);
     }
 
   if( prior_main )
@@ -529,7 +530,8 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
   // cl_decoded_option
 
   size_t new_option_count = new_opt.size();
-  struct cl_decoded_option *new_options = XNEWVEC (struct cl_decoded_option, new_option_count);
+  struct cl_decoded_option *new_options = XNEWVEC (struct cl_decoded_option,
+                                                    new_option_count);
 
   for(size_t i=0; i<new_option_count; i++)
     {
@@ -539,7 +541,7 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
 #ifdef NOISY
   verbose = true;
 #endif
-  if( verbose && new_options != original_options )
+  if( verbose && new_options != original_options ) // cppcheck-suppress knownConditionTrueFalse
     {
     fprintf(stderr, _("Driving: (" HOST_SIZE_T_PRINT_DEC ")\n"),
             (fmt_size_t)new_option_count);
diff --git a/gcc/cobol/genapi.cc b/gcc/cobol/genapi.cc
index 8017788..c9d2da4 100644
--- a/gcc/cobol/genapi.cc
+++ b/gcc/cobol/genapi.cc
@@ -27,6 +27,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+
 #include "cobol-system.h"
 
 #include "coretypes.h"
@@ -117,7 +118,7 @@ typedef struct TREEPLET
 
 static
 void
-treeplet_fill_source(TREEPLET &treeplet, cbl_refer_t &refer)
+treeplet_fill_source(TREEPLET &treeplet, const cbl_refer_t &refer)
   {
   treeplet.pfield = gg_get_address_of(refer.field->var_decl_node);
   treeplet.offset = refer_offset(refer);
@@ -190,6 +191,9 @@ const char *gv_trace_switch = NULL;
 char const *bTRACE1 = NULL;
 tree trace_handle;
 tree trace_indent;
+
+// This variable is set to true when the output cursor is known to be at the
+// start-of-line.
 bool cursor_at_sol = true;
 
 static void
@@ -229,6 +233,13 @@ trace1_init()
     }
   }
 
+static
+void
+insert_nop(int n)
+  {
+  gg_assign(var_decl_nop, build_int_cst_type(INT, n));
+  }
+
 static void
 create_cblc_string_variable(const char *var_name, const char *var_contents)
   {
@@ -266,8 +277,6 @@ build_main_that_calls_something(const char *something)
     SHOW_PARSE_END
     }
 
-  gg_set_current_line_number(DEFAULT_LINE_NUMBER);
-
   tree function_decl = gg_define_function( INT,
                                            "main",
                                            "main",
@@ -321,7 +330,6 @@ build_main_that_calls_something(const char *something)
                                        argc,
                                        argv,
                                        NULL_TREE)));
-  strncpy(ach_cobol_entry_point, psz, sizeof(ach_cobol_entry_point)-1);
   free(psz);
   gg_finalize_function();
   }
@@ -361,8 +369,11 @@ level_88_helper(size_t parent_capacity,
                 size_t &returned_size)
   {
   // We return a MALLOCed return value, which the caller must free.
-  char *retval  = (char *)xmalloc(parent_capacity + 64);
-  char *builder = (char *)xmalloc(parent_capacity + 64);
+  char *retval  = static_cast<char *>(xmalloc(parent_capacity + 64));
+  gcc_assert(retval);
+  char *builder = static_cast<char *>(xmalloc(parent_capacity + 64));
+  gcc_assert(builder);
+
   size_t nbuild = 0;
 
   cbl_figconst_t figconst = cbl_figconst_of( elem.name());
@@ -403,7 +414,8 @@ level_88_helper(size_t parent_capacity,
 
     // Pick up the string
     size_t first_name_length  = elem.size();
-    char *first_name = (char *)xmalloc(first_name_length + 1);
+    char *first_name = static_cast<char *>(xmalloc(first_name_length + 1));
+    gcc_assert(first_name);
     memcpy(first_name, elem.name(), first_name_length);
     first_name[first_name_length] = '\0';
 
@@ -480,7 +492,7 @@ get_level_88_domain(size_t parent_capacity, cbl_field_t *var, size_t &returned_s
   // Numerics are converted to strings, and handled as above
 
   size_t retval_capacity = 64;
-  char *retval = (char *)xmalloc(retval_capacity);
+  char *retval = static_cast<char *>(xmalloc(retval_capacity));
   size_t output_index = 0;
 
   // Loop through the provided domains:
@@ -497,8 +509,9 @@ get_level_88_domain(size_t parent_capacity, cbl_field_t *var, size_t &returned_s
     if( output_index + stream_len > retval_capacity )
       {
       retval_capacity *= 2;
-      retval = (char *)xrealloc(retval, retval_capacity);
+      retval = static_cast<char *>(xrealloc(retval, retval_capacity));
       }
+    gcc_assert(retval);
     memcpy(retval + output_index, stream, stream_len);
     output_index += stream_len;
     returned_size += stream_len;
@@ -509,14 +522,23 @@ get_level_88_domain(size_t parent_capacity, cbl_field_t *var, size_t &returned_s
     if( output_index + stream_len > retval_capacity )
       {
       retval_capacity *= 2;
-      retval = (char *)xrealloc(retval, retval_capacity);
+      retval = static_cast<char *>(xrealloc(retval, retval_capacity));
       }
+    gcc_assert(retval);
     memcpy(retval + output_index, stream, stream_len);
     output_index += stream_len;
     returned_size += stream_len;
     free(stream);
     domain += 1;
     }
+
+  if( returned_size >= retval_capacity)
+    {
+    retval_capacity *= 2;
+    retval = static_cast<char *>(xrealloc(retval, retval_capacity));
+    }
+
+  gcc_assert(returned_size < retval_capacity);
   retval[returned_size++] = '\0';
   return retval;
   }
@@ -608,13 +630,8 @@ get_class_condition_string(cbl_field_t *var)
       // Since the first.name is a single character, we can do this as
       // a single-character pair.
 
-      // Keep in mind that the single character might be a two-byte UTF-8
-      // codepoint
-        uint8_t ch1 = domain->first.name()[0];
-        uint8_t ch2 = domain->last.name()[0];
-
-        gcc_assert(first_name_length <= 2);
-        gcc_assert(last_name_length <= 2);
+      uint8_t ch1;
+      uint8_t ch2;
 
       char *p2;
       size_t one;
@@ -766,8 +783,10 @@ parser_call_target_convention( tree func )
 void
 parser_call_targets_dump()
   {
-    dbgmsg( "call targets for #" HOST_SIZE_T_PRINT_UNSIGNED,
+    dbgmsg( "call targets for #" HOST_SIZE_T_PRINT_UNSIGNED " NOT dumping",
             (fmt_size_t)current_program_index() );
+#if 0 // A change to call_targets rendered this routine useless.  Until we get
+      // around to repairing it, this code is left for reference.
     for( const auto& elem : call_targets ) {
       const auto& k = elem.first;
       const auto& v = elem.second;
@@ -781,6 +800,7 @@ parser_call_targets_dump()
       }
       fprintf(stderr, " ]\n");
     }
+#endif
   }
 
 size_t
@@ -808,8 +828,8 @@ parser_call_target_update( size_t caller,
   }
 
 static tree
-function_pointer_from_name(cbl_refer_t &name,
-                          tree function_return_type)
+function_pointer_from_name(const cbl_refer_t &name,
+                           tree function_return_type)
   {
   Analyze();
 
@@ -885,7 +905,8 @@ function_pointer_from_name(cbl_refer_t &name,
   }
 
 void
-parser_initialize_programs(size_t nprogs, struct cbl_refer_t *progs)
+parser_initialize_programs( size_t nprogs,
+                            const struct cbl_refer_t *progs)
   {
   Analyze();
   SHOW_PARSE
@@ -985,14 +1006,13 @@ parser_compile_ecs( const std::vector<uint64_t>& ecs )
     return NULL_TREE;
     }
 
-  char ach[32];
+  char ach[64];
   static int counter = 1;
   sprintf(ach, "_ecs_table_%d", counter++);
   tree retval =  array_of_long_long(ach, ecs);
   SHOW_IF_PARSE(nullptr)
     {
     SHOW_PARSE_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(ecs.size()), as_voidp(retval));
     SHOW_PARSE_TEXT(ach)
@@ -1001,7 +1021,6 @@ parser_compile_ecs( const std::vector<uint64_t>& ecs )
   TRACE1
     {
     TRACE1_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(ecs.size()), as_voidp(retval));
     TRACE1_TEXT_ABC("", ach, "");
@@ -1034,14 +1053,13 @@ parser_compile_dcls( const std::vector<uint64_t>& dcls )
     return NULL_TREE;
     }
 
-  char ach[32];
+  char ach[64];
   static int counter = 1;
   sprintf(ach, "_dcls_table_%d", counter++);
   tree retval =  array_of_long_long(ach, dcls);
   SHOW_IF_PARSE(nullptr)
     {
     SHOW_PARSE_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(dcls.size()), as_voidp(retval));
     SHOW_PARSE_TEXT(ach);
@@ -1050,7 +1068,6 @@ parser_compile_dcls( const std::vector<uint64_t>& dcls )
   TRACE1
     {
     TRACE1_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(dcls.size()), as_voidp(retval));
     TRACE1_TEXT_ABC("", ach, "");
@@ -1167,21 +1184,13 @@ parser_statement_begin( const cbl_name_t statement_name,
   //  the execution time of a program doing two-billion simple adds in an inner
   //  loop dropped from 3.8 seconds to 0.175 seconds.
 
-  bool exception_processing = enabled_exceptions.size() ;
+  bool exception_processing = cdf_enabled_exceptions().size() ;
 
   if( !exception_processing )
     {
     exception_processing = file_ops.find(statement_name) != file_ops.end();
     }
 
-  if( gg_get_current_line_number() == DEFAULT_LINE_NUMBER )
-    {
-    // This code is intended to prevert GDB anomalies when the first line of a
-    // program is a PERFORM <proc> ... TEST AFTER ... UNTIL ...
-    gg_set_current_line_number(CURRENT_LINE_NUMBER-1);
-    gg_assign(var_decl_nop, build_int_cst_type(INT, 106));
-    }
-
   // At this point, if any exception is enabled, we store the location stuff.
   // Each file I-O routine calls store_location_stuff explicitly, because
   // those exceptions can't be defeated.
@@ -1189,14 +1198,9 @@ parser_statement_begin( const cbl_name_t statement_name,
   if( exception_processing )
     {
     store_location_stuff(statement_name);
-    }
-
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
-  if( exception_processing )
-    {
     set_exception_environment(ecs, dcls);
     }
+
   sv_is_i_o = false;
   }
 
@@ -1210,10 +1214,9 @@ initialize_variable_internal( cbl_refer_t refer,
             // gg_string_literal(refer.field->name),
             // NULL_TREE);
   cbl_field_t *parsed_var = refer.field;
-
-  if( parsed_var->type == FldLiteralA )
+  if( !parsed_var )
     {
-    return;
+    cbl_internal_error("%s should not be null", "parsed_var");
     }
 
   if( parsed_var->is_key_name() )
@@ -1229,7 +1232,7 @@ initialize_variable_internal( cbl_refer_t refer,
     return;
     }
 
-  if( parsed_var && parsed_var->type == FldBlob )
+  if( parsed_var->type == FldBlob )
     {
     return;
     }
@@ -1347,8 +1350,6 @@ initialize_variable_internal( cbl_refer_t refer,
     SHOW_PARSE_END
     }
 
-  CHECK_FIELD(parsed_var);
-
   // When initializing a variable, we have to ignore any DEPENDING ON clause
   // that might otherwise apply
   suppress_dest_depends = true;
@@ -1588,7 +1589,7 @@ parser_initialize(const cbl_refer_t& refer, bool like_parser_symbol_add)
 
 static void
 get_binary_value_from_float(tree         value,
-                            cbl_refer_t &dest,
+                      const cbl_refer_t &dest,
                             cbl_field_t *source,
                             tree         source_offset
                             )
@@ -1682,6 +1683,7 @@ depending_on_value(tree depending_on, cbl_field_t *current_sizer)
 //  gg_assign(occurs_lower, build_int_cst_type(LONG, current_sizer->occurs.bounds.lower));
 //  gg_assign(occurs_upper, build_int_cst_type(LONG, current_sizer->occurs.bounds.upper));
 
+  gcc_assert(current_sizer);
   if( current_sizer->occurs.depending_on )
     {
     get_depending_on_value_from_odo(depending_on, current_sizer);
@@ -1825,16 +1827,12 @@ normal_normal_compare(bool debugging,
               NULL_TREE);
     }
 
-  bool needs_adjusting;
   if( !left_intermediate && !right_intermediate )
     {
     // Yay!  Both sides have fixed rdigit values.
 
-    // Flag needs_adjusting as false, because we are going to do it here:
-    needs_adjusting = false;
     int adjust =   get_scaled_rdigits(left_side_ref->field)
                  - get_scaled_rdigits(right_side_ref->field);
-
     if( adjust > 0 )
       {
       // We need to make right_side bigger to match the scale of left_side
@@ -1849,6 +1847,7 @@ normal_normal_compare(bool debugging,
   else
     {
     // At least one side is right_intermediate
+    bool needs_adjusting;
 
     tree adjust;
     if( !left_intermediate && right_intermediate )
@@ -2357,7 +2356,7 @@ cobol_compare(  tree return_int,
                 build_int_cst_type(INT, rightflags),
                 integer_zero_node,
                 NULL_TREE));
-    compared = true;
+    // compared = true;  // Commented out to quiet cppcheck
     }
 
 //  gg_printf("   result is %d\n", return_int, NULL_TREE);
@@ -2377,6 +2376,8 @@ move_tree(  cbl_field_t  *dest,
     SHOW_PARSE_END
     }
 
+  CHECK_FIELD(dest);
+
   bool moved = true;
 
   tree source_length = gg_define_size_t();
@@ -2460,7 +2461,7 @@ move_tree(  cbl_field_t  *dest,
               psz_source,
               min_length,
               member(dest->var_decl_node, "picture"),
-              NULL);
+              NULL_TREE);
       break;
       }
 
@@ -2563,7 +2564,7 @@ get_string_from(cbl_field_t *field)
   }
 
 static char *
-combined_name(cbl_label_t *label)
+combined_name(const cbl_label_t *label)
   {
   // This routine returns a pointer to a static, so make sure you use the result
   // before calling the routine again
@@ -2578,7 +2579,7 @@ combined_name(cbl_label_t *label)
     if( label->parent )
       {
       // It's possible for implicit
-      cbl_label_t *section_label = cbl_label_of(symbol_at(label->parent));
+      const cbl_label_t *section_label = cbl_label_of(symbol_at(label->parent));
       sect_name = section_label->name;
       }
     }
@@ -2588,7 +2589,7 @@ combined_name(cbl_label_t *label)
     }
 
   static size_t retval_size = 256;
-  static char *retval= (char *)xmalloc(retval_size);
+  static char *retval= static_cast<char *>(xmalloc(retval_size));
 
   char *paragraph             = cobol_name_mangler(para_name);
   char *section         = cobol_name_mangler(sect_name);
@@ -2600,8 +2601,9 @@ combined_name(cbl_label_t *label)
                   + 24 )
     {
     retval_size *= 2;
-    retval = (char *)xrealloc(retval, retval_size);
+    retval = static_cast<char *>(xrealloc(retval, retval_size));
     }
+  gcc_assert(retval);
 
   *retval = '\0';
   char ach[24];
@@ -2648,8 +2650,9 @@ assembler_label(const char *label)
     {
     length = strlen(label) + strlen(local_text) + 1;
     free(build);
-    build = (char *)xmalloc(length);
+    build = static_cast<char *>(xmalloc(length));
     }
+  gcc_assert(build);
 
   strcpy(build, label);
   strcat(build, local_text);
@@ -2663,8 +2666,6 @@ section_label(struct cbl_proc_t *procedure)
   // With nested programs, you can have multiple program/section pairs with the
   // the same names; we use a deconflictor to avoid collisions
 
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   size_t deconflictor = symbol_label_id(procedure->label);
 
   cbl_label_t *label = procedure->label;
@@ -2689,7 +2690,7 @@ section_label(struct cbl_proc_t *procedure)
     }
   assembler_label(psz2);
   free(psz2);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 108));
+  insert_nop(108);
   }
 
 static void
@@ -2704,8 +2705,6 @@ paragraph_label(struct cbl_proc_t *procedure)
   //      are not referenced by the program.  We provide a deconflictor to
   //      separate such labels.
 
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   cbl_label_t *paragraph  = procedure->label;
   cbl_label_t *section    = nullptr;
 
@@ -2727,6 +2726,9 @@ paragraph_label(struct cbl_proc_t *procedure)
           section_name ? section_name: "(null)" ,
           current_function->our_unmangled_name ? current_function->our_unmangled_name: "" ,
           (fmt_size_t)deconflictor );
+
+  // (0) is wrong, so back up one
+
   gg_insert_into_assembler(psz1);
 
   SHOW_PARSE
@@ -2743,7 +2745,25 @@ paragraph_label(struct cbl_proc_t *procedure)
                           combined_name(procedure->label));
   assembler_label(psz2);
   free(psz2);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 109));
+
+  // We are inserting a NOP after having created a label for the procedure.
+  // This means that when using GDC_COBOL to step into a procedure, the
+  // execution will stop there and show "123 para-name." at the stopped point.
+  //
+  // Note that because there is no user-specified executable code at that point
+  // the user can't set a working breakpoint with "break 123".  But because
+  // GDB will pick up the psz2 text and set a breakpoint there (which is the
+  // location of the NOP) "break para-name" will actually stop and show line
+  // 123.
+  //
+  // This really only makes sense when you look at the assembly language. Keep
+  // in mind as you read it that issuing a "break 123" causes GDB to set a
+  // breakpoint at the first executable machine language code following the
+  // first ".loc 123" directive.
+  //
+  // Yes, trying to understand this causes headaches for many people who read
+  // this.  Take an aspirin.
+  insert_nop(109);
   }
 
 static void
@@ -2787,6 +2807,7 @@ pseudo_return_pop(cbl_proc_t *procedure)
               NULL_TREE);
     }
 
+  token_location_override(current_location_minus_one());
   IF( var_decl_exit_address, eq_op, procedure->exit.addr )
     {
     TRACE1
@@ -2796,11 +2817,13 @@ pseudo_return_pop(cbl_proc_t *procedure)
     // The top of the stack is us!
 
     // Pick up the return address from the pseudo_return stack:
+    token_location_override(current_location_minus_one());
     gg_assign(current_function->void_star_temp,
               gg_call_expr( VOID_P,
                             "__gg__pseudo_return_pop",
                             NULL_TREE));
     // And do the return:
+    token_location_override(current_location_minus_one());
     gg_goto(current_function->void_star_temp);
     }
   ELSE
@@ -2834,11 +2857,13 @@ leave_procedure(struct cbl_proc_t *procedure, bool /*section*/)
     // procedure->bottom.label);
     // Procedure can be null, for example at the beginning of a
     // new program, or after somebody else has cleared it out.
+
     gg_append_statement(procedure->exit.label);
 
     char *psz;
     psz = xasprintf("_procret." HOST_SIZE_T_PRINT_DEC ":",
                     (fmt_size_t)symbol_label_id(procedure->label));
+    token_location_override(current_location_minus_one());
     gg_insert_into_assembler(psz);
     free(psz);
     pseudo_return_pop(procedure);
@@ -2951,7 +2976,9 @@ find_procedure(cbl_label_t *label)
     static int counter=1;
 
     // This is a new section or paragraph; we need to create its values:
-    retval = (struct cbl_proc_t *)xmalloc(sizeof(struct cbl_proc_t));
+    retval = static_cast<struct cbl_proc_t *>
+                                          (xmalloc(sizeof(struct cbl_proc_t)));
+    gcc_assert(retval);
     retval->label = label;
 
     gg_create_goto_pair(&retval->top.go_to,
@@ -3007,6 +3034,8 @@ parser_enter_section(cbl_label_t *label)
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
+    SHOW_PARSE_INDENT
+    linemap_dump_location( line_table, current_token_location(), stderr );
     SHOW_PARSE_END
     }
 
@@ -3014,8 +3043,7 @@ parser_enter_section(cbl_label_t *label)
 
   // This NOP is needed to give GDB a line number for the entry point of
   // paragraphs
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 101));
+  insert_nop(101);
 
   struct cbl_proc_t *procedure = find_procedure(label);
   gg_append_statement(procedure->top.label);
@@ -3042,6 +3070,8 @@ parser_enter_paragraph(cbl_label_t *label)
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
+    SHOW_PARSE_INDENT
+    linemap_dump_location( line_table, current_token_location(), stderr );
     SHOW_PARSE_END
     }
 
@@ -3257,16 +3287,20 @@ parser_goto( cbl_refer_t value_ref, size_t narg, cbl_label_t * const labels[] )
 void
 parser_perform(cbl_label_t *label, bool suppress_nexting)
   {
-  label->used = yylineno;
   Analyze();
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
     char ach[32];
-    sprintf(ach, " label is at %p", (void*)label);
+    sprintf(ach, " label is at %p", static_cast<void*>(label));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " label->proc is %p", (void*)label->structs.proc);
+    if( label )
+      {
+      sprintf(ach,
+              " label->proc is %p",
+              static_cast<void*>(label->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
@@ -3279,6 +3313,7 @@ parser_perform(cbl_label_t *label, bool suppress_nexting)
     }
 
   CHECK_LABEL(label);
+  label->used = yylineno;
 
   struct cbl_proc_t *procedure = find_procedure(label);
 
@@ -3315,9 +3350,9 @@ parser_perform(cbl_label_t *label, bool suppress_nexting)
   char ach[256];
   if( label->type == LblParagraph )
     {
-    cbl_label_t *section_label = cbl_label_of(symbol_at(label->parent));
+    const cbl_label_t *sec_label = cbl_label_of(symbol_at(label->parent));
     para_name = label->name;
-    sect_name = section_label->name;
+    sect_name = sec_label->name;
     sprintf(ach,
             "%s PERFORM %s of %s of %s (" HOST_SIZE_T_PRINT_DEC ")",
             ASM_COMMENT_START,
@@ -3377,9 +3412,9 @@ parser_perform_times( cbl_label_t *proc_1, cbl_refer_t count )
     SHOW_PARSE_REF(" ", count)
     SHOW_PARSE_TEXT(" TIMES")
     char ach[32];
-    sprintf(ach, " proc_1 is at %p", (void*)proc_1);
+    sprintf(ach, " proc_1 is at %p", static_cast<void*>(proc_1));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " proc_1->proc is %p", (void*)proc_1->structs.proc);
+    sprintf(ach, " proc_1->proc is %p", static_cast<void*>(proc_1->structs.proc));
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
@@ -3416,6 +3451,7 @@ parser_perform_times( cbl_label_t *proc_1, cbl_refer_t count )
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler(ach);
   }
 
@@ -3430,17 +3466,22 @@ internal_perform_through( cbl_label_t *proc_1,
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", proc_1);
     char ach[32];
-    sprintf(ach, " proc_1 is at %p", (void*)proc_1);
+    sprintf(ach, " proc_1 is at %p", static_cast<void*>(proc_1));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " proc_1->proc is %p", (void*)proc_1->structs.proc);
+    if( proc_1 )
+      {
+      sprintf(ach,
+              " proc_1->proc is %p",
+              static_cast<void*>(proc_1->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     if( proc_2 )
       {
       SHOW_PARSE_INDENT
-      SHOW_PARSE_LABEL("", proc_2);
-      sprintf(ach, " proc_2 is at %p", (void*)proc_2);
+      SHOW_PARSE_LABEL_OK("", proc_2);
+      sprintf(ach, " proc_2 is at %p", static_cast<void*>(proc_2));
       SHOW_PARSE_TEXT(ach)
-      sprintf(ach, " proc_2->proc is %p", (void*)proc_2->structs.proc);
+      sprintf(ach, " proc_2->proc is %p", static_cast<void*>(proc_2->structs.proc));
       SHOW_PARSE_TEXT(ach)
       }
     SHOW_PARSE_END
@@ -3453,14 +3494,12 @@ internal_perform_through( cbl_label_t *proc_1,
 
   CHECK_LABEL(proc_1);
 
-  if(!proc_2)
+  if( !proc_2 )
     {
     parser_perform(proc_1, suppress_nexting);
     return;
     }
 
-  CHECK_LABEL(proc_2);
-
   struct cbl_proc_t *proc1 = find_procedure(proc_1);
   struct cbl_proc_t *proc2 = find_procedure(proc_2);
 
@@ -3515,17 +3554,22 @@ internal_perform_through_times(   cbl_label_t *proc_1,
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", proc_1);
     char ach[32];
-    sprintf(ach, " proc_1 is at %p", (void*)proc_1);
+    sprintf(ach, " proc_1 is at %p", static_cast<void*>(proc_1));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " proc_1->proc is %p", (void*)proc_1->structs.proc);
+    if( proc_1 )
+      {
+      sprintf(ach,
+              " proc_1->proc is %p",
+              static_cast<void*>(proc_1->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     if( proc_2 )
       {
       SHOW_PARSE_INDENT
-      SHOW_PARSE_LABEL("", proc_2);
-      sprintf(ach, " proc_2 is at %p", (void*)proc_2);
+      SHOW_PARSE_LABEL_OK("", proc_2);
+      sprintf(ach, " proc_2 is at %p", static_cast<void*>(proc_2));
       SHOW_PARSE_TEXT(ach)
-      sprintf(ach, " proc_2->proc is %p", (void*)proc_2->structs.proc);
+      sprintf(ach, " proc_2->proc is %p", static_cast<void*>(proc_2->structs.proc));
       SHOW_PARSE_TEXT(ach)
       }
     SHOW_PARSE_REF(" ", count);
@@ -3561,6 +3605,7 @@ internal_perform_through_times(   cbl_label_t *proc_1,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -3752,6 +3797,22 @@ parser_leave_file()
     {
     // We are leaving the top-level file, which means this compilation is
     // done, done, done.
+
+    // There is, however, one thing left to do.  If the command line says
+    // that this module needs a main entry point, then this is where
+    // we create a main() function.  We build it at the end, so that all of
+    // the .loc directives associated with it appear at the end of the
+    // source code.  We used to create the main() entry point at the beginning,
+    // but that created confusion for GDB when trying to debug the generated
+    // executable.
+    if( main_entry_point )
+      {
+      next_program_is_main = false;
+      build_main_that_calls_something(main_entry_point);
+      free(main_entry_point);
+      main_entry_point = NULL;
+      }
+
     gg_leaving_the_source_code_file();
     }
   }
@@ -3861,17 +3922,8 @@ parser_enter_program( const char *funcname_,
   // The first thing we have to do is mangle this name.  This is safe even
   // though the end result will be mangled again, because the mangler doesn't
   // change a mangled name.
-   
-  char *mangled_name;
- 
-  if( current_call_convention() == cbl_call_cobol_e )
-    {
-    mangled_name = cobol_name_mangler(funcname_);
-    }
-  else
-    {
-    mangled_name = xstrdup(funcname_);
-    }
+
+  char *mangled_name = cobol_name_mangler(funcname_);
 
   size_t parent_index = current_program_index();
   char *funcname;
@@ -3899,28 +3951,25 @@ parser_enter_program( const char *funcname_,
 
   if( !is_function && !parent_index )
     {
-    // This is a top_level program, and not a function
+    // This is a top_level program-id, and not a function
     if( next_program_is_main )
       {
+      // This is the first top-level program-id.
       next_program_is_main = false;
-      if(main_entry_point)
+      if( !main_entry_point )
         {
-        build_main_that_calls_something(main_entry_point);
-        free(main_entry_point);
-        main_entry_point = NULL;
-        }
-      else
-        {
-        build_main_that_calls_something(funcname);
+        // Because no explicit main_entry_point was specified, this program-id,
+        // the first in the file, becomes the target of the main() function
+        // that will be created at parser_leave_file time.
+        main_entry_point = xstrdup(funcname);
+
+        char *psz = cobol_name_mangler(main_entry_point);
+        strncpy(ach_cobol_entry_point, psz, sizeof(ach_cobol_entry_point)-1);
+        free(psz);
         }
       }
     }
 
-  // Call this after build_main_that_calls_something, because it manipulates
-  // the current line number to DEFAULT_LINE_NUMBER.  We have to manipulate it
-  // back afterward.
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   if( strcmp(funcname_, "main") == 0 && this_module_has_main )
     {
     // setting 'retval' to 1 let's the caller know that we are being told
@@ -4110,6 +4159,8 @@ psa_FldLiteralN(struct cbl_field_t *field )
   // We are constructing a completely static constant structure, based on the
   // text string in .initial
 
+  CHECK_FIELD(field);
+
   FIXED_WIDE_INT(128) value = 0;
 
   do
@@ -4302,6 +4353,8 @@ psa_FldBlob(struct cbl_field_t *var )
     SHOW_PARSE_END
     }
 
+  CHECK_FIELD(var);
+
   // We are constructing a completely static constant structure.  We know the
   // capacity.  We'll create it from the data.initial.  The var_decl_node will
   // be a pointer to the data
@@ -4339,67 +4392,182 @@ psa_FldBlob(struct cbl_field_t *var )
   }
 
 void
-parser_accept(  struct cbl_refer_t refer,
-                enum special_name_t special_e )
+parser_accept(const struct cbl_refer_t &tgt,
+              special_name_t special_e,
+              cbl_label_t *error,
+              cbl_label_t *not_error )
   {
-  Analyze();
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
-    SHOW_PARSE_REF(" ", refer);
+    if( error )
+      {
+      SHOW_PARSE_LABEL(" error ", error)
+      }
+    if( not_error )
+      {
+      SHOW_PARSE_LABEL(" not_error ", not_error)
+      }
     SHOW_PARSE_END
     }
-  TRACE1
-    {
-    TRACE1_HEADER
-    TRACE1_END
-    }
-
-  /*
-  enum special_name_t
-      {
-        SYSIN_e,
-        SYSIPT_e,
-        SYSOUT_e,
-        SYSLIST_e,
-        SYSLST_e,
-        SYSPUNCH_e,
-        SYSPCH_e,
-        CONSOLE_e,
-        C01_e, C02_e, C03_e, C04_e, C05_e, C06_e,
-        C07_e, C08_e, C09_e, C10_e, C11_e, C12_e,
-        CSP_e,
-        S01_e, S02_e, S03_e, S04_e, S05_e,
-        AFP_5A_e,
-  };
-  */
 
   // The ISO spec describes the valid special names for ACCEPT as implementation
   // dependent.  We are following IBM's lead.
 
   tree environment = build_int_cst_type(INT, special_e);
 
-  switch( special_e )
+  const char *function_to_call = NULL;
+
+  switch(special_e)
     {
+    case STDIN_e:
     case CONSOLE_e:
     case SYSIPT_e:
     case SYSIN_e:
-      break;
-    default:
-      dbgmsg("%s(): We don't know what to do with special_name_t %d,", __func__, special_e);
-      dbgmsg("%s(): so we are ignoring it.", __func__);
-      yywarn("unrecognized SPECIAL NAME ignored");
+      // This is ordinary input from from the stdin:
+      gg_call(VOID,
+              "__gg__accept",
+              environment,
+              gg_get_address_of(tgt.field->var_decl_node),
+              refer_offset(tgt),
+              refer_size_dest(tgt),
+              NULL_TREE);
       return;
       break;
-    }
 
-  gg_call(VOID,
-          "__gg__accept",
-          environment,
-          gg_get_address_of(refer.field->var_decl_node),
-          refer_offset(refer),
-          refer_size_dest(refer),
-          NULL_TREE);
+    case C01_e:
+    case C02_e:
+    case C03_e:
+    case C04_e:
+    case C05_e:
+    case C06_e:
+    case C07_e:
+    case C08_e:
+    case C09_e:
+    case C10_e:
+    case C11_e:
+    case C12_e:
+    case CSP_e:
+    case S01_e:
+    case S02_e:
+    case S03_e:
+    case S04_e:
+    case S05_e:
+    case AFP_5A_e:
+    case STDOUT_e:
+    case SYSOUT_e:
+    case SYSLIST_e:
+    case SYSLST_e:
+    case STDERR_e:
+    case SYSPUNCH_e:
+    case SYSPCH_e:
+    case SYSERR_e:
+      cbl_internal_error("Not valid for ACCEPT statement.");
+      break;
+
+    case ARG_NUM_e:
+      // This ACCEPT statement wants the number of argv values:
+      gg_call(VOID,
+              "__gg__get_argc",
+              gg_get_address_of(tgt.field->var_decl_node),
+              refer_offset(tgt),
+              refer_size_source(tgt),
+              NULL_TREE);
+       return;
+       break;
+
+    case ENV_NAME_e:
+      // This fetches the environment name set by DISPLAY... UPON ENV_NAME_e
+      gg_call(VOID,
+              "__gg__get_env_name",
+              gg_get_address_of(tgt.field->var_decl_node),
+              refer_offset(tgt),
+              refer_size_source(tgt),
+              NULL_TREE);
+       return;
+       break;
+
+    case ENV_VALUE_e:
+      // This fetches the environment value associated with the previously
+      // esablished name
+      function_to_call = "__gg__get_env_value";
+      break;
+
+    case ARG_VALUE_e:
+      // We are fetching the variable whose index was established by a prior
+      // DISPLAY UPON ARGUMENT-NUMBER.  After the fetch, the value will be
+      // incremented by one.
+      function_to_call = "__gg__accept_arg_value";
+      break;
+    }
+  if( function_to_call )
+    {
+    tree erf = gg_define_int();
+    gg_assign(erf,
+              gg_call_expr(  INT,
+                            function_to_call,
+                            gg_get_address_of(tgt.field->var_decl_node),
+                            refer_offset(tgt),
+                            refer_size_dest(tgt),
+                            NULL_TREE));
+    if( error )
+      {
+      // There is an ON EXCEPTION phrase:
+      IF( erf, ne_op, integer_zero_node )
+        {
+        SHOW_PARSE
+          {
+          SHOW_PARSE_INDENT
+          SHOW_PARSE_TEXT("Laying down GOTO     error->INTO for_argv")
+          SHOW_PARSE_LABEL_OK(" ", error)
+          }
+        gg_append_statement( error->structs.arith_error->into.go_to );
+        }
+      ELSE
+        {
+        }
+        ENDIF
+      }
+    if( not_error )
+      {
+      // There is an NOT ON EXCEPTION phrase:
+      IF( erf, eq_op, integer_zero_node )
+        {
+        SHOW_PARSE
+          {
+          SHOW_PARSE_INDENT
+          SHOW_PARSE_TEXT("Laying down GOTO not_error->INTO for_argv")
+          SHOW_PARSE_LABEL_OK(" ", not_error)
+          }
+        gg_append_statement( not_error->structs.arith_error->into.go_to );
+        }
+      ELSE
+        {
+        }
+        ENDIF
+      }
+    if( error )
+      {
+      SHOW_PARSE
+        {
+        SHOW_PARSE_INDENT
+        SHOW_PARSE_TEXT("Laying down LABEL     error->bottom")
+        SHOW_PARSE_LABEL_OK(" ", error)
+        }
+      gg_append_statement( error->structs.arith_error->bottom.label );
+      }
+    if( not_error )
+      {
+      SHOW_PARSE
+        {
+        SHOW_PARSE_INDENT
+        SHOW_PARSE_TEXT("Laying down LABEL not_error->bottom")
+        SHOW_PARSE_LABEL_OK(" ", not_error)
+        SHOW_PARSE_END
+        }
+      gg_append_statement( not_error->structs.arith_error->bottom.label );
+      }
+    }
   }
 
 // TODO: update documentation.
@@ -4411,7 +4579,6 @@ parser_accept_exception( cbl_label_t *accept_label )
 
   // We are entering either SIZE ERROR or NOT SIZE ERROR code
   RETURN_IF_PARSE_ONLY;
-  set_up_on_exception_label(accept_label);
 
   SHOW_PARSE
     {
@@ -4424,6 +4591,9 @@ parser_accept_exception( cbl_label_t *accept_label )
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(accept_label);
+  set_up_on_exception_label(accept_label);
+
   // Jump over the [NOT] ON EXCEPTION code that is about to be laid down
   gg_append_statement( accept_label->structs.arith_error->over.go_to );
   // Create the label that allows the following code to be executed at
@@ -4450,6 +4620,8 @@ parser_accept_exception_end( cbl_label_t *accept_label )
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(accept_label);
+
   // Jump to the end of the arithmetic code:
   gg_append_statement( accept_label->structs.arith_error->bottom.go_to );
   // Lay down the label that allows the ERROR/NOT ERROR instructions
@@ -4459,8 +4631,8 @@ parser_accept_exception_end( cbl_label_t *accept_label )
   }
 
 void
-parser_accept_command_line( cbl_refer_t tgt,
-                            cbl_refer_t source,
+parser_accept_command_line( const cbl_refer_t &tgt,
+                            const cbl_refer_t &source,
                             cbl_label_t *error,
                             cbl_label_t *not_error )
   {
@@ -4500,7 +4672,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO     error->INTO for_command_line")
-          SHOW_PARSE_LABEL(" ", error)
+          SHOW_PARSE_LABEL_OK(" ", error)
           }
         gg_append_statement( error->structs.arith_error->into.go_to );
         }
@@ -4518,7 +4690,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO not_error->INTO for command_line")
-          SHOW_PARSE_LABEL(" ", not_error)
+          SHOW_PARSE_LABEL_OK(" ", not_error)
           }
         gg_append_statement( not_error->structs.arith_error->into.go_to );
         }
@@ -4550,7 +4722,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO     error->INTO for_argv")
-          SHOW_PARSE_LABEL(" ", error)
+          SHOW_PARSE_LABEL_OK(" ", error)
           }
         gg_append_statement( error->structs.arith_error->into.go_to );
         }
@@ -4568,7 +4740,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO not_error->INTO for_argv")
-          SHOW_PARSE_LABEL(" ", not_error)
+          SHOW_PARSE_LABEL_OK(" ", not_error)
           }
         gg_append_statement( not_error->structs.arith_error->into.go_to );
         }
@@ -4584,7 +4756,7 @@ parser_accept_command_line( cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL     error->bottom")
-      SHOW_PARSE_LABEL(" ", error)
+      SHOW_PARSE_LABEL_OK(" ", error)
       }
     gg_append_statement( error->structs.arith_error->bottom.label );
     }
@@ -4594,7 +4766,7 @@ parser_accept_command_line( cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL not_error->bottom")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       SHOW_PARSE_END
       }
     gg_append_statement( not_error->structs.arith_error->bottom.label );
@@ -4602,7 +4774,7 @@ parser_accept_command_line( cbl_refer_t tgt,
   }
 
 void
-parser_accept_command_line_count( cbl_refer_t tgt )
+parser_accept_command_line_count( const cbl_refer_t &tgt )
   {
   Analyze();
   SHOW_PARSE
@@ -4624,10 +4796,10 @@ parser_accept_command_line_count( cbl_refer_t tgt )
   }
 
 void
-parser_accept_envar(struct cbl_refer_t tgt,
-                    struct cbl_refer_t envar,
-                    cbl_label_t *error,
-                    cbl_label_t *not_error )
+parser_accept_envar(const struct cbl_refer_t &tgt,
+                    const struct cbl_refer_t &envar,
+                          cbl_label_t *error,
+                          cbl_label_t *not_error )
   {
   Analyze();
 
@@ -4692,7 +4864,7 @@ parser_accept_envar(struct cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL     error->bottom")
-      SHOW_PARSE_LABEL(" ", error)
+      SHOW_PARSE_LABEL_OK(" ", error)
       }
     gg_append_statement( error->structs.arith_error->bottom.label );
     }
@@ -4702,7 +4874,7 @@ parser_accept_envar(struct cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL not_error->bottom")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       SHOW_PARSE_END
       }
     gg_append_statement( not_error->structs.arith_error->bottom.label );
@@ -4710,7 +4882,8 @@ parser_accept_envar(struct cbl_refer_t tgt,
   }
 
 void
-parser_set_envar( struct cbl_refer_t name, struct cbl_refer_t value )
+parser_set_envar( const struct cbl_refer_t &name,
+                  const struct cbl_refer_t &value )
   {
   Analyze();
   SHOW_PARSE
@@ -5112,7 +5285,6 @@ parser_display_internal(tree file_descriptor,
             build_int_cst_type(SIZE_T, refer.field->data.capacity),
             advance ? integer_one_node : integer_zero_node,
             NULL_TREE );
-    cursor_at_sol = advance;
     }
   else if( refer.field->type == FldLiteralN )
     {
@@ -5150,50 +5322,50 @@ parser_display_internal(tree file_descriptor,
         *p = 'E';
         if( exp < 0 && exp >= -9 )
           {
-            p[1] = '-';
-            p[2] = '0';
-            p[3] = '0' - exp;
-            p[4] = '\0';
+          p[1] = '-';
+          p[2] = '0';
+          p[3] = '0' - exp;
+          p[4] = '\0';
           }
         else if( exp >= 0 && exp <= 9 )
           {
-            p[1] = '+';
-            p[2] = '0';
-            p[3] = '0' + exp;
-            p[4] = '\0';
+          p[1] = '+';
+          p[2] = '0';
+          p[3] = '0' + exp;
+          p[4] = '\0';
           }
         }
       else if (exp == 0)
         {
-          p[-1] = '\0';
+        p[-1] = '\0';
         }
       else if (exp < 0)
         {
-          p[-1] = '\0';
-          char *q = strchr (ach, '.');
-          char dig = q[-1];
-          q[-1] = '\0';
-          char tem[132];
-          snprintf (tem, 132, "%s0.%0*u%c%s", ach, -exp - 1, 0, dig, q + 1);
-          strcpy (ach, tem);
+        p[-1] = '\0';
+        char *q = strchr (ach, '.');
+        char dig = q[-1];
+        q[-1] = '\0';
+        char tem[132];
+        snprintf (tem, 132, "%s0.%0*d%c%s", ach, -exp - 1, 0, dig, q + 1);
+        strcpy (ach, tem);
         }
-      else if (exp > 0)
+      else // if (exp > 0)
         {
-          p[-1] = '\0';
-          char *q = strchr (ach, '.');
-          for (int i = 0; i != exp; ++i)
-            q[i] = q[i + 1];
-          q[exp] = '.';
+        p[-1] = '\0';
+        char *q = strchr (ach, '.');
+        for (int i = 0; i != exp; ++i)
+          q[i] = q[i + 1];
+        q[exp] = '.';
         }
       __gg__remove_trailing_zeroes(ach);
       }
 
     if( symbol_decimal_point() == ',' )
       {
-      char *p = strchr(ach, '.' );
-      if( p )
+      char *pdot = strchr(ach, '.' );
+      if( pdot )
         {
-        *p = symbol_decimal_point();
+        *pdot = symbol_decimal_point();
         }
       }
 
@@ -5250,22 +5422,29 @@ parser_display_field(cbl_field_t *field)
                                 DISPLAY_NO_ADVANCE);
   }
 
-/*
- * The first parameter to parser_display is the "device" upon which to display
- * the data. Besides normal devices, these may include elements that define the
- * Unix command line and environment:
- *  1.  ARG_NUM_e, the ARGUMENT-NUMBER
- *  2.  ARG_VALUE_e, the ARGUMENT-VALUE
- *  3.  ENV_NAME_e, the ENVIRONMENT-NAME
- *  4.  ENV_VALUE_e, the ENVIRONMENT-VALUE
- * that need special care and feeding.
- */
 void
 parser_display( const struct cbl_special_name_t *upon,
-                struct cbl_refer_t refs[],
-                size_t n,
-                bool advance )
+          const std::vector<cbl_refer_t> &refs,
+                bool advance,
+          const cbl_label_t *not_error,
+          const cbl_label_t *error )
   {
+  const size_t n = refs.size();
+  /*
+   * The first parameter to parser_display is the "device" upon which to display
+   * the data. Besides normal devices, these may include elements that define the
+   * Unix command line and environment:
+   *  1.  ARG_NUM_e, the ARGUMENT-NUMBER
+   *  2.  ARG_VALUE_e, the ARGUMENT-VALUE
+   *  3.  ENV_NAME_e, the ENVIRONMENT-NAME
+   *  4.  ENV_VALUE_e, the ENVIRONMENT-VALUE
+   * that need special care and feeding.
+   */
+
+  // At the present time, I am not sure what not_error and error are for
+  gcc_assert(!not_error);
+  gcc_assert(!error);
+
   Analyze();
   SHOW_PARSE
     {
@@ -5274,7 +5453,7 @@ parser_display( const struct cbl_special_name_t *upon,
     for(size_t i=0; i<n; i++)
       {
       SHOW_PARSE_INDENT
-      SHOW_PARSE_REF("", refs[i]);
+      SHOW_PARSE_REF("", refs.at(i));
       }
     if( advance )
       {
@@ -5306,23 +5485,81 @@ parser_display( const struct cbl_special_name_t *upon,
     {
     switch(upon->id)
       {
+      // See table 5 in the IBM Cobol For Linux x86 1.2 document.
+
+      case STDIN_e:
+      case SYSIN_e:
+      case SYSIPT_e:
+        cbl_internal_error("Attempting to send to an input device.");
+        break;
+
+      case C01_e:
+      case C02_e:
+      case C03_e:
+      case C04_e:
+      case C05_e:
+      case C06_e:
+      case C07_e:
+      case C08_e:
+      case C09_e:
+      case C10_e:
+      case C11_e:
+      case C12_e:
+      case CSP_e:
+      case S01_e:
+      case S02_e:
+      case S03_e:
+      case S04_e:
+      case S05_e:
+      case AFP_5A_e:
+      case ARG_VALUE_e:
+        cbl_internal_error("Not valid for DISPLAY statement.");
+        break;
+
       case STDOUT_e:
-      case SYSOUT_e:
-      case SYSLIST_e:
-      case SYSLST_e:
       case CONSOLE_e:
+        // These are inarguably stdout
         gg_assign(file_descriptor, integer_one_node);
         break;
 
       case STDERR_e:
+      case SYSERR_e:
+        // These are inarguably stderr
+        gg_assign(file_descriptor, integer_two_node);
+        break;
+
+      case SYSOUT_e:
+      case SYSLIST_e:
+      case SYSLST_e:
       case SYSPUNCH_e:
       case SYSPCH_e:
-        gg_assign(file_descriptor, integer_two_node);
+        // In the 21st century, when there are no longer valid assumptions to
+        // be made about the existence of line printers, and where things
+        // formerly-ubiquitous card punches no longer exist, there is a need
+        // for the possibility of assigning these "devices" to externally-
+        // determined Unix gadgetry in /dev:
+        gg_assign(file_descriptor,
+                  gg_call_expr( INT,
+                                "__gg__get_file_descriptor",
+                                gg_string_literal(upon->os_filename),
+                                NULL_TREE));
+        needs_closing = true;
         break;
 
+      case ARG_NUM_e:
+        // Set the index number for a subsequent ACCEPT FROM ARG_VALUE_e
+        gg_call(VOID,
+                "__gg__set_arg_num",
+                gg_get_address_of(refs[0].field->var_decl_node),
+                refer_offset(refs[0]),
+                refer_size_source(refs[0]),
+                NULL_TREE);
+         return;
+         break;
+
       case ENV_NAME_e:
-        // This Part I of the slightly absurd method of using DISPLAY...UPON
-        // to fetch, or set, environment variables.
+        // Establish the name of an environment variable for later use with
+        // in either DISPLAY UPON or ACCEPT FROM
         gg_call(VOID,
                 "__gg__set_env_name",
                 gg_get_address_of(refs[0].field->var_decl_node),
@@ -5332,19 +5569,16 @@ parser_display( const struct cbl_special_name_t *upon,
          return;
          break;
 
-      default:
-        if( upon->os_filename[0] )
-          {
-          tree topen = gg_open( gg_string_literal(upon->os_filename),
-                                build_int_cst_type(INT, O_APPEND|O_WRONLY));
-          gg_assign(file_descriptor, topen);
-          needs_closing = true;
-          }
-        else
-          {
-          fprintf(stderr, "We don't know what to do in parser_display\n");
-          gcc_unreachable();
-          }
+      case ENV_VALUE_e:
+        // Set the contents of the environment variable named with ENV_NAME_e
+        gg_call(VOID,
+                "__gg__set_env_value",
+                gg_get_address_of(refs[0].field->var_decl_node),
+                refer_offset(refs[0]),
+                refer_size_source(refs[0]),
+                NULL_TREE);
+         return;
+         break;
       }
     }
   else
@@ -5359,17 +5593,114 @@ parser_display( const struct cbl_special_name_t *upon,
     }
   CHECK_FIELD(refs[n-1].field);
   parser_display_internal(file_descriptor, refs[n-1], advance ? DISPLAY_ADVANCE : DISPLAY_NO_ADVANCE);
-
   if( needs_closing )
     {
-    tree tclose = gg_close(file_descriptor);
-    // We are ignoring the close() return value
-    gg_append_statement(tclose);
+    gg_close(file_descriptor);
     }
 
   cursor_at_sol = advance;
   }
 
+static
+bool  // Returns false for literals; true for named variables
+get_exhibit_name(tree file_descriptor, const cbl_refer_t &arg)
+  {
+  bool retval;
+  if( is_literal(arg.field) )
+    {
+    // If something is a literal, we just display the literal value
+    parser_display_internal(file_descriptor,
+                            arg,
+                            DISPLAY_NO_ADVANCE);
+    retval = false;
+    }
+  else
+    {
+    // It's not a literal, so we show its name, and the names or literal
+    // values) of any qualifier subscripts or refmods
+    gg_write( file_descriptor,
+              gg_string_literal(arg.field->name),
+              build_int_cst_type(SIZE_T, strlen(arg.field->name)) );
+
+    if( arg.subscripts.size() )
+      {
+      // This refer has subscripts:
+      gg_write( file_descriptor,
+                gg_string_literal("("),
+                integer_one_node );
+      for(size_t i=0; i<arg.subscripts.size(); i++)
+        {
+        if( i > 0 )
+          {
+          gg_write( file_descriptor,
+                    gg_string_literal(","),
+                    integer_one_node );
+          }
+        get_exhibit_name(file_descriptor, arg.subscripts[i]);
+        }
+      gg_write( file_descriptor,
+                gg_string_literal(")"),
+                integer_one_node );
+      }
+    if( arg.refmod.from || arg.refmod.len )
+      {
+      gg_write( file_descriptor,
+                gg_string_literal("("),
+                integer_one_node );
+      if( arg.refmod.from )
+        {
+        get_exhibit_name(file_descriptor, *(arg.refmod.from));
+        }
+      gg_write( file_descriptor,
+                gg_string_literal(":"),
+                integer_one_node );
+      if( arg.refmod.len )
+        {
+        get_exhibit_name(file_descriptor, *(arg.refmod.len));
+        }
+      gg_write( file_descriptor,
+                gg_string_literal(")"),
+                integer_one_node );
+      }
+    retval = true;
+    }
+  return retval;
+  }
+
+void
+parser_exhibit( bool /*changed*/, bool /*named*/,
+                const std::vector<cbl_refer_t> &args )
+  {
+  tree file_descriptor = gg_define_int();
+  gg_assign(file_descriptor, integer_one_node);   // stdout is file descriptor 1.
+
+  for(size_t i=0; i<args.size(); i++)
+    {
+    CHECK_FIELD(args[i].field);
+    if(i > 0)
+      {
+      // When there more than one argument, the second through Nth get a space
+      // in front of them.
+      gg_write( file_descriptor,
+                gg_string_literal(" "),
+                integer_one_node);
+      }
+    if( get_exhibit_name(file_descriptor, args[i]) )
+      {
+      gg_write( file_descriptor,
+                gg_string_literal("="),
+                integer_one_node);
+      parser_display_internal(file_descriptor,
+                              args[i],
+                              DISPLAY_NO_ADVANCE);
+      }
+    }
+  gg_write( file_descriptor,
+            gg_string_literal("\n"),
+            integer_one_node);
+  cursor_at_sol = true;
+  }
+
 static tree
 get_literalN_value(cbl_field_t *var)
   {
@@ -5642,12 +5973,12 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down on_error GOTO into")
-      SHOW_PARSE_LABEL(" ", on_error)
+      SHOW_PARSE_LABEL_OK(" ", on_error)
       }
     IF( gg_bitwise_or(error_flag,
                       compute_error->structs.compute_error->compute_error_code),
-        ne_op,
-        integer_zero_node )
+                      ne_op,
+                      integer_zero_node )
       {
       gg_append_statement( on_error->structs.arith_error->into.go_to );
       }
@@ -5673,7 +6004,7 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down not_error GOTO into")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       }
     IF( compute_error->structs.compute_error->compute_error_code, eq_op, integer_zero_node )
       {
@@ -5689,7 +6020,7 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down on_error LABEL BOTTOM:")
-      SHOW_PARSE_LABEL(" ", on_error)
+      SHOW_PARSE_LABEL_OK(" ", on_error)
       }
     gg_append_statement( on_error->structs.arith_error->bottom.label );
     }
@@ -5700,7 +6031,7 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down not_error LABEL BOTTOM:")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       }
     gg_append_statement( not_error->structs.arith_error->bottom.label );
     }
@@ -5975,10 +6306,18 @@ parser_initialize_table(size_t nelem,
     }
   typedef size_t span_t[2];
   static_assert(sizeof(spans[0]) == sizeof(span_t), "pair size wrong");
-  static tree tspans = gg_define_variable(SIZE_T_P, "..pit_v1", vs_file_static);
-  static tree ttbls  = gg_define_variable(SIZE_T_P, "..pit_v2", vs_file_static);
-  gg_assign(tspans, build_array_of_size_t(2*nspan, (const size_t *)spans));
-  gg_assign(ttbls,  build_array_of_size_t(2*ntbl,  (const size_t *)tbls));
+  static tree tspans = gg_define_variable(SIZE_T_P,
+                                          "..pit_v1",
+                                          vs_file_static);
+  static tree ttbls  = gg_define_variable(SIZE_T_P,
+                                          "..pit_v2",
+vs_file_static);
+  gg_assign(tspans,
+            build_array_of_size_t(2*nspan,
+                                  reinterpret_cast<const size_t *>(spans)));
+  gg_assign(ttbls,
+            build_array_of_size_t(2*ntbl,
+                                  reinterpret_cast<const size_t *>(tbls)));
 
   gg_call(VOID,
           "__gg__mirror_range",
@@ -6137,7 +6476,7 @@ is_valuable( cbl_field_type_t type ) {
   return false;
 }
 
-void parser_sleep(cbl_refer_t seconds)
+void parser_sleep(const cbl_refer_t &seconds)
   {
   if( seconds.field )
     {
@@ -6157,7 +6496,7 @@ void parser_sleep(cbl_refer_t seconds)
     // This is a naked place-holding CONTINUE.  Generate some do-nothing
     // code that will stick some .LOC information into the assembly language,
     // so that GDB-COBOL can display the CONTINUE statement.
-    gg_assign(var_decl_nop, build_int_cst_type(INT, 103));
+    insert_nop(103);
     }
   }
 
@@ -6240,12 +6579,12 @@ program_end_stuff(cbl_refer_t refer, ec_type_t ec)
 
       tree array_type = build_array_type_nelts(UCHAR,
                                     returner->data.capacity);
-      tree retval     =  gg_define_variable(array_type, vs_static);
-      gg_memcpy(gg_get_address_of(retval),
+      tree array     =  gg_define_variable(array_type, vs_static);
+      gg_memcpy(gg_get_address_of(array),
                 member(returner->var_decl_node, "data"),
                 member(returner->var_decl_node, "capacity"));
 
-      tree actual = gg_cast(COBOL_FUNCTION_RETURN_TYPE, gg_get_address_of(retval));
+      tree actual = gg_cast(COBOL_FUNCTION_RETURN_TYPE, gg_get_address_of(array));
 
       restore_local_variables();
       gg_return(actual);
@@ -6501,7 +6840,6 @@ parser_arith_error(cbl_label_t *arithmetic_label)
 
   // We are entering either SIZE ERROR or NOT SIZE ERROR code
   RETURN_IF_PARSE_ONLY;
-  set_up_on_exception_label(arithmetic_label);
 
   SHOW_PARSE
     {
@@ -6514,6 +6852,10 @@ parser_arith_error(cbl_label_t *arithmetic_label)
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(arithmetic_label);
+
+  set_up_on_exception_label(arithmetic_label);
+
   // Jump over the [NOT] ON EXCEPTION code that is about to be laid down
   gg_append_statement( arithmetic_label->structs.arith_error->over.go_to );
   // Create the label that allows the following code to be executed at
@@ -6540,6 +6882,8 @@ parser_arith_error_end(cbl_label_t *arithmetic_label)
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(arithmetic_label);
+
   // Jump to the end of the arithmetic code:
   gg_append_statement( arithmetic_label->structs.arith_error->bottom.go_to );
   // Lay down the label that allows the ERROR/NOT ERROR instructions
@@ -6723,8 +7067,6 @@ parser_division(cbl_division_t division,
     SHOW_PARSE_END
     }
 
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   if( division == data_div_e )
     {
     Analyze();
@@ -6876,7 +7218,6 @@ parser_division(cbl_division_t division,
       // expected formal parameter and tacks it onto the end of the
       // function's arguments chain.
 
-      char ach[2*sizeof(cbl_name_t)];
       sprintf(ach, "_p_%s", args[i].refer.field->name);
 
       size_t nbytes = 0;
@@ -6896,14 +7237,13 @@ parser_division(cbl_division_t division,
       chain_parameter_to_function(current_function->function_decl, par_type, ach);
       }
 
-    bool check_for_parameter_count = false;
-
     if( nusing )
       {
       // During the call, we saved the parameter_count and an array of variable
       // lengths.  We need to look at those values if, and only if, one or more
       // of our USING arguments has an OPTIONAL flag or if one of our targets is
       // marked as VARYING.
+      bool check_for_parameter_count = false;
       for(size_t i=0; i<nusing; i++)
         {
         if( args[i].optional )
@@ -7099,7 +7439,6 @@ parser_division(cbl_division_t division,
           // If so, we have to give var2::data_pointer the same value as
           // var1::data_pointer
           //
-          cbl_field_t *next_var;
           size_t our_index = symbol_index(symbol_elem_of(new_var));
           size_t next_index  = our_index + 1;
           // Look ahead in the symbol table for the next LEVEL01/77
@@ -7110,7 +7449,7 @@ parser_division(cbl_division_t division,
               {
               break;
               }
-            next_var = cbl_field_of(e);
+            cbl_field_t *next_var = cbl_field_of(e);
             if( !next_var )
               {
               break;
@@ -7185,6 +7524,11 @@ parser_division(cbl_division_t division,
         ENDIF
       }
       ENDIF
+    // The first token_location that the parser establishes is caused by the
+    // parser scanning all of the lines in the source code.  This messes up the
+    // logic for backing up one line, which is needed to correctly step through
+    // COBOL code with GDB-COBOL.  So, we clear it here.
+    current_location_minus_one_clear();
     }
   }
 
@@ -7629,7 +7973,9 @@ label_fetch(struct cbl_label_t *label)
   if( !label->structs.goto_trees )
     {
     label->structs.goto_trees
-      = (cbl_label_addresses_t *)xmalloc(sizeof(struct cbl_label_addresses_t) );
+      = static_cast<cbl_label_addresses_t *>
+        (xmalloc(sizeof(struct cbl_label_addresses_t)));
+    gcc_assert(label->structs.goto_trees);
 
     gg_create_goto_pair(&label->structs.goto_trees->go_to,
                         &label->structs.goto_trees->label);
@@ -7647,15 +7993,18 @@ parser_label_label(struct cbl_label_t *label)
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL("", label)
     char ach[32];
-    sprintf(ach, " label is at %p", (void*)label);
+    sprintf(ach, " label is at %p", static_cast<void*>(label));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " label->proc is %p", (void*)label->structs.proc);
+    if( label )
+      {
+      sprintf(ach,
+              " label->proc is %p",
+              static_cast<void*>(label->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
 
-  CHECK_LABEL(label);
-
   TRACE1
     {
     TRACE1_HEADER
@@ -7663,6 +8012,8 @@ parser_label_label(struct cbl_label_t *label)
     TRACE1_END
     }
 
+  CHECK_LABEL(label);
+
   if(strcmp(label->name, "_end_declaratives") == 0 )
     {
     suppress_cobol_entry_point = false;
@@ -7674,21 +8025,25 @@ void
 parser_label_goto(struct cbl_label_t *label)
   {
   label->used = yylineno;
+
   Analyze();
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
     char ach[32];
-    sprintf(ach, " label is at %p", (void*)label);
+    sprintf(ach, " label is at %p", static_cast<void*>(label));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " label->proc is %p", (void*)label->structs.proc);
+    if( label )
+      {
+      sprintf(ach,
+              " label->proc is %p",
+              static_cast<void*>(label->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
 
-  CHECK_LABEL(label);
-
   TRACE1
     {
     TRACE1_HEADER
@@ -7696,7 +8051,9 @@ parser_label_goto(struct cbl_label_t *label)
     TRACE1_END
     }
 
-  if(strcmp(label->name, "_end_declaratives") == 0 )
+  CHECK_LABEL(label);
+
+  if( strcmp(label->name, "_end_declaratives") == 0 )
     {
     suppress_cobol_entry_point = true;
     }
@@ -7780,7 +8137,7 @@ parser_setop( struct cbl_field_t *tgt,
 
 void
 parser_classify(    cbl_field_t *tgt,
-                    cbl_refer_t  candidate,
+               const cbl_refer_t  &candidate,
                     enum classify_t type )
   {
   Analyze();
@@ -7877,14 +8234,6 @@ create_iline_address_pairs(struct cbl_perform_tgt_t *tgt)
 
   gg_create_goto_pair(&tgt->addresses.setup.go_to,
                       &tgt->addresses.setup.label);
-
-  // Even in -O0 compilations, the compiler does some elementary optimizations
-  // around JMP instructions.  We have the SETUP code for in-line performats
-  // in an island at the end of the loop code.  With this intervention, NEXTing
-  // through the code shows you the final statement of the loop before the
-  // loop actually starts.
-
-  tgt->addresses.line_number_of_setup_code = gg_get_current_line_number();
   }
 
 void
@@ -7898,7 +8247,7 @@ parser_perform_start( struct cbl_perform_tgt_t *tgt )
       {
       SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
       char ach[32];
-      sprintf(ach, " %p", (void*)tgt);
+      sprintf(ach, " %p", static_cast<void*>(tgt));
       SHOW_PARSE_TEXT(ach);
       SHOW_PARSE_LABEL(" ", tgt->from())
       if( tgt->to() )
@@ -7947,7 +8296,7 @@ parser_perform_start( struct cbl_perform_tgt_t *tgt )
   // Give GDB-COBOL something to chew on when NEXTing.  This instruction will
   // get the line number of the PERFORM N TIMES code.
   gg_append_statement(tgt->addresses.top.label);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 104));
+  insert_nop(104);
   }
 
 void
@@ -7959,7 +8308,7 @@ parser_perform_conditional( struct cbl_perform_tgt_t *tgt )
     SHOW_PARSE_HEADER
     SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
     char ach[32];
-    sprintf(ach, " %p", (void*)tgt);
+    sprintf(ach, " %p", static_cast<void*>(tgt));
     SHOW_PARSE_TEXT(ach);
     SHOW_PARSE_END
     }
@@ -8009,7 +8358,7 @@ parser_perform_conditional_end( struct cbl_perform_tgt_t *tgt )
     SHOW_PARSE_HEADER
     SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
     char ach[32];
-    sprintf(ach, " %p", (void*)tgt);
+    sprintf(ach, " %p", static_cast<void*>(tgt));
     SHOW_PARSE_TEXT(ach);
     SHOW_PARSE_END
     }
@@ -8099,6 +8448,7 @@ perform_outofline_before_until(struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8162,6 +8512,7 @@ perform_outofline_after_until(struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8282,6 +8633,7 @@ perform_outofline_testafter_varying(struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8425,6 +8777,7 @@ perform_outofline_before_varying(   struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8504,8 +8857,6 @@ perform_inline_until(   struct cbl_perform_tgt_t *tgt,
                   GOTO TOP
       EXIT:
   */
-  gg_set_current_line_number(cobol_location().last_line);
-
   gg_append_statement(tgt->addresses.test.label);
 
   // Go to where the conditional is recalculated....
@@ -8620,8 +8971,6 @@ perform_inline_testbefore_varying(  struct cbl_perform_tgt_t *tgt,
     parser_move(varys[i].varying, varys[i].from);
     }
 
-  gg_set_current_line_number(cobol_location().last_line);
-
   // Lay down the testing cycle:
   for(size_t i=0; i<N; i++)
     {
@@ -8933,7 +9282,7 @@ parser_perform_until(   struct cbl_perform_tgt_t *tgt,
     SHOW_PARSE_HEADER
     SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
     char ach[32];
-    sprintf(ach, " %p", (void*)tgt);
+    sprintf(ach, " %p", static_cast<void*>(tgt));
     SHOW_PARSE_TEXT(ach);
     SHOW_PARSE_LABEL(" ", tgt->from())
     if( tgt->to() )
@@ -8943,9 +9292,6 @@ parser_perform_until(   struct cbl_perform_tgt_t *tgt,
     SHOW_PARSE_END
     }
 
-  gg_set_current_line_number(cobol_location().last_line);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 105));
-
   if( tgt->from()->type != LblLoop )
     {
     perform_outofline( tgt, test_before, N, varys);
@@ -9012,10 +9358,6 @@ parser_perform_inline_times(struct cbl_perform_tgt_t *tgt,
   gg_append_statement( tgt->addresses.testA.label );
   gg_append_statement( tgt->addresses.test.label );
 
-  // AT this point, we want to set the line_number to the location of the
-  // END-PERFORM statement.
-  gg_set_current_line_number(cobol_location().last_line);
-
   gg_decrement(counter);
   // Do the test:
   IF( counter, gt_op, gg_cast(LONG, integer_zero_node) )
@@ -9046,8 +9388,6 @@ parser_perform_inline_times(struct cbl_perform_tgt_t *tgt,
     SHOW_PARSE_END
     }
 
-  int stash = gg_get_current_line_number();
-  gg_set_current_line_number(tgt->addresses.line_number_of_setup_code);
   gg_append_statement( tgt->addresses.setup.label );
 
   // Get the count:
@@ -9078,8 +9418,6 @@ parser_perform_inline_times(struct cbl_perform_tgt_t *tgt,
     gg_append_statement( tgt->addresses.exit.go_to );
     ENDIF
 
-  gg_set_current_line_number(stash);
-
   SHOW_PARSE
     {
     SHOW_PARSE_INDENT
@@ -9768,13 +10106,19 @@ void
 parser_file_delete( struct cbl_file_t *file, bool /*sequentially*/ )
   {
   Analyze();
+
+  if( !file )
+    {
+    cbl_internal_error("The file pointer should not be null");
+    abort();  // Because cppcheck doesn't recognize [[noerror]]
+    }
+
   bool sequentially =    file->access == file_access_seq_e
                       || file->org    == file_sequential_e
                       || file->org    == file_line_sequential_e;
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
-    if(file)
       {
       SHOW_PARSE_TEXT(" ");
       SHOW_PARSE_TEXT(file->name);
@@ -9787,10 +10131,6 @@ parser_file_delete( struct cbl_file_t *file, bool /*sequentially*/ )
         SHOW_PARSE_TEXT(" sequentially")
         }
       }
-    else
-      {
-      SHOW_PARSE_TEXT(" *file is NULL")
-      }
     SHOW_PARSE_END
     }
 
@@ -9947,8 +10287,8 @@ parser_file_start(struct cbl_file_t *file,
     // A key has a number of fields
     for(size_t ifield=0; ifield<file->keys[key_number].nfield; ifield++)
       {
-      size_t field_index = file->keys[key_number].fields[ifield];
-      cbl_field_t *field = cbl_field_of(symbol_at(field_index));
+      size_t nfield = file->keys[key_number].fields[ifield];
+      cbl_field_t *field = cbl_field_of(symbol_at(nfield));
       combined_length += field->data.capacity;
       }
     gg_assign(length, build_int_cst_type(SIZE_T, combined_length));
@@ -9975,7 +10315,7 @@ parser_file_start(struct cbl_file_t *file,
 
 static void
 inspect_tally(bool backward,
-              cbl_refer_t identifier_1,
+        const cbl_refer_t &identifier_1,
               cbl_inspect_opers_t& identifier_2)
   {
   Analyze();
@@ -10175,8 +10515,8 @@ inspect_tally(bool backward,
 
 static void
 inspect_replacing(int backward,
-                  cbl_refer_t identifier_1,
-                  cbl_inspect_opers_t& operations)
+            const cbl_refer_t &identifier_1,
+                  cbl_inspect_opers_t &operations)
   {
   Analyze();
   // This is an INSPECT FORMAT 2
@@ -10516,7 +10856,7 @@ parser_intrinsic_numval_c( cbl_field_t *f,
 
 void
 parser_intrinsic_subst( cbl_field_t *f,
-                        cbl_refer_t& ref1,
+                  const cbl_refer_t& ref1,
                         size_t argc,
                         cbl_substitute_t * argv )
   {
@@ -10541,7 +10881,9 @@ parser_intrinsic_subst( cbl_field_t *f,
 
   sv_is_i_o = true;
   store_location_stuff("SUBSTITUTE");
-  unsigned char *control_bytes = (unsigned char *)xmalloc(argc * sizeof(unsigned char));
+  unsigned char *control_bytes =
+           static_cast<unsigned char *>(xmalloc(argc * sizeof(unsigned char)));
+  gcc_assert(control_bytes);
   std::vector<cbl_refer_t> arg1(argc);
   std::vector<cbl_refer_t> arg2(argc);
 
@@ -10978,7 +11320,9 @@ static void
 create_lsearch_address_pairs(struct cbl_label_t *name)
   {
   // Create the lsearch structure
-  name->structs.lsearch = (cbl_lsearch_t *)xmalloc(sizeof(cbl_lsearch_t));
+  name->structs.lsearch =
+                  static_cast<cbl_lsearch_t *>(xmalloc(sizeof(cbl_lsearch_t)));
+  gcc_assert(name->structs.lsearch);
   cbl_lsearch_t *lsearch = name->structs.lsearch;
 
   gg_create_goto_pair(&lsearch->addresses.at_exit.go_to,
@@ -11228,7 +11572,9 @@ parser_bsearch_start(   cbl_label_t* name,
     }
 
   // We need a cbl_bsearch_t structure:
-  name->structs.bsearch = (cbl_bsearch_t *)xmalloc(sizeof(cbl_bsearch_t));
+  name->structs.bsearch =
+                  static_cast<cbl_bsearch_t *>(xmalloc(sizeof(cbl_bsearch_t)));
+  gcc_assert(name->structs.bsearch);
   cbl_bsearch_t *bsearch = name->structs.bsearch;
 
   // Create the address/label pairs we need
@@ -11261,6 +11607,8 @@ parser_bsearch_start(   cbl_label_t* name,
     current = parent_of(current);
     }
 
+  CHECK_FIELD(current);
+
   // There are a number of things we learn from the field "current"
 
   // We get the index:
@@ -11373,7 +11721,6 @@ is_ascending_key(const cbl_refer_t& key)
   bool retval = true;
 
   cbl_field_t *family_tree = key.field;
-  gcc_assert(family_tree);
   while( family_tree )
     {
     if( family_tree->occurs.nkey )
@@ -11382,7 +11729,10 @@ is_ascending_key(const cbl_refer_t& key)
       }
     family_tree = parent_of(family_tree);
     }
+
+  CHECK_FIELD(family_tree);
   gcc_assert(family_tree->occurs.nkey);
+
   for(size_t i=0; i<family_tree->occurs.nkey; i++)
     {
     for(size_t j=0; j<family_tree->occurs.keys[i].field_list.nfield; j++)
@@ -11542,8 +11892,12 @@ parser_sort(cbl_refer_t tableref,
                                          return n + key.fields.size();
                                        } );
   typedef const cbl_field_t * const_field_t;
-  const_field_t *flattened_fields = (const_field_t *)xmalloc(total_keys * sizeof(cbl_field_t *));
-  size_t *flattened_ascending       = (size_t *)xmalloc(total_keys * sizeof(size_t));
+  const_field_t *flattened_fields =
+     static_cast<const_field_t *>(xmalloc(total_keys * sizeof(cbl_field_t *)));
+  gcc_assert(flattened_fields);
+  size_t *flattened_ascending =
+                   static_cast<size_t *>(xmalloc(total_keys * sizeof(size_t)));
+  gcc_assert(flattened_ascending);
 
   size_t key_index = 0;
   for( size_t i=0; i<keys.size(); i++ )
@@ -11679,8 +12033,12 @@ parser_file_sort(   cbl_file_t *workfile,
                                          return n + key.fields.size();
                                        } );
   typedef const cbl_field_t * const_field_t;
-  auto flattened_fields = (const_field_t *)xmalloc(total_keys * sizeof(cbl_field_t *));
-  size_t *flattened_ascending    = (size_t *)      xmalloc(total_keys * sizeof(size_t));
+  auto flattened_fields
+   = static_cast<const_field_t *>(xmalloc(total_keys * sizeof(cbl_field_t *)));
+  gcc_assert(flattened_fields);
+  size_t *flattened_ascending =
+                   static_cast<size_t *>(xmalloc(total_keys * sizeof(size_t)));
+  gcc_assert(flattened_ascending);
 
   size_t key_index = 0;
   for( size_t i=0; i<keys.size(); i++ )
@@ -11839,7 +12197,9 @@ parser_return_start( cbl_file_t *workfile, cbl_refer_t into )
 
   // We assume that workfile is open.
 
-  workfile->addresses = (cbl_sortreturn_t *)xmalloc(sizeof(cbl_sortreturn_t));
+  workfile->addresses = static_cast<cbl_sortreturn_t *>
+                                          (xmalloc(sizeof(cbl_sortreturn_t)));
+  gcc_assert(workfile->addresses);
   gg_create_goto_pair(&workfile->addresses->at_end.go_to,
                       &workfile->addresses->at_end.label);
   gg_create_goto_pair(&workfile->addresses->not_at_end.go_to,
@@ -12025,9 +12385,13 @@ parser_file_merge(  cbl_file_t *workfile,
                                          return i + key.fields.size();
                                        } );
   typedef const cbl_field_t * const_field_t;
-  const_field_t *flattened_fields = (const_field_t *)xmalloc(total_keys * sizeof(cbl_field_t *));
+  const_field_t *flattened_fields
+                 = static_cast<const_field_t *>
+                   (xmalloc(total_keys * sizeof(cbl_field_t *)));
+  gcc_assert(flattened_fields);
   size_t *flattened_ascending
-            = (size_t *)xmalloc(total_keys * sizeof(size_t));
+            = static_cast<size_t *>(xmalloc(total_keys * sizeof(size_t)));
+  gcc_assert(flattened_ascending);
 
   size_t key_index = 0;
   for( size_t i=0; i<keys.size(); i++ )
@@ -12041,8 +12405,9 @@ parser_file_merge(  cbl_file_t *workfile,
     }
 
   // Create the array of cbl_field_t pointers for the keys
-  tree all_keys = gg_array_of_field_pointers(total_keys,
-                                             const_cast<cbl_field_t**>(flattened_fields));
+  tree all_keys = gg_array_of_field_pointers(
+                                 total_keys,
+                                 const_cast<cbl_field_t**>(flattened_fields));
 
   // Create the array of integers that are the flags for ASCENDING:
   tree ascending = gg_array_of_size_t(total_keys, flattened_ascending);
@@ -12066,6 +12431,9 @@ parser_file_merge(  cbl_file_t *workfile,
   ELSE
     ENDIF
 
+  const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
+
   for(size_t i=0; i<ninputs; i++)
     {
     if( process_this_exception(ec_sort_merge_file_open_e) )
@@ -12223,7 +12591,8 @@ parser_string_overflow( cbl_label_t *name )
    */
 
   name->structs.unstring
-    = (cbl_unstring_t *)xmalloc(sizeof(struct cbl_unstring_t) );
+    = static_cast<cbl_unstring_t *>(xmalloc(sizeof(struct cbl_unstring_t)));
+  gcc_assert(name->structs.unstring);
 
   // Set up the address pairs for this clause
   gg_create_goto_pair(&name->structs.unstring->over.go_to,
@@ -12282,8 +12651,8 @@ parser_unstring(cbl_refer_t src,
     }
 
   std::vector<cbl_refer_t> delims(ndelimited);
-  char *alls = (char *)xmalloc(ndelimited+1);
-
+  char *alls = static_cast<char *>(xmalloc(ndelimited+1));
+  gcc_assert(alls);
   for(size_t i=0; i<ndelimited; i++)
     {
     delims[i] = delimiteds[i];
@@ -12374,7 +12743,8 @@ parser_string(const cbl_refer_t& tgt,
     }
 
   // We need an array of nsource+1 integers:
-  size_t *integers = (size_t *)xmalloc((nsource+1)*sizeof(size_t));
+  size_t *integers = static_cast<size_t *>(xmalloc((nsource+1)*sizeof(size_t)));
+  gcc_assert(integers);
 
   // Count up how many treeplets we are going to need:
   size_t cblc_count = 2;  // tgt and pointer
@@ -12465,8 +12835,9 @@ parser_call_exception( cbl_label_t *name )
     }
 
   name->structs.call_exception
-    = (cbl_call_exception_t *)xmalloc(sizeof(struct cbl_call_exception_t) );
-
+    = static_cast<cbl_call_exception_t *>
+                                (xmalloc(sizeof(struct cbl_call_exception_t)));
+  gcc_assert(name->structs.call_exception);
   // Set up the address pairs for this clause
   gg_create_goto_pair(&name->structs.call_exception->over.go_to,
                       &name->structs.call_exception->over.label);
@@ -12526,8 +12897,10 @@ create_and_call(size_t narg,
 
   if(narg)
     {
-    arguments = (tree *)xmalloc(2*narg * sizeof(tree));
-    allocated = (int * )xmalloc(narg * sizeof(int));
+    arguments = static_cast<tree *>(xmalloc(2*narg * sizeof(tree)));
+    gcc_assert(arguments);
+    allocated = static_cast<int *>(xmalloc(narg * sizeof(int)));
+    gcc_assert(allocated);
     }
 
   // Put the arguments onto the "stack" of calling parameters:
@@ -12759,7 +13132,7 @@ create_and_call(size_t narg,
     // Fetch the FUNCTION_DECL for that FUNCTION_TYPE
     tree function_decl = gg_build_fn_decl(funcname, fndecl_type);
     set_call_convention(function_decl, current_call_convention());
-    
+
     // Take the address of the function decl:
     tree address_of_function = gg_get_address_of(function_decl);
 
@@ -12771,7 +13144,7 @@ create_and_call(size_t narg,
     parser_call_target( funcname, assigment );
 
     // Create the call_expr from that address
-    call_expr = build_call_array_loc( location_from_lineno(),
+    call_expr = build_call_array_loc( gg_token_location(),
                                       returned_value_type,
                                       address_of_function,
                                       narg,
@@ -13510,9 +13883,9 @@ parser_program_hierarchy( const cbl_prog_hier_t& hier )
     // are also accessible by us.  Go find them.
     std::vector<const hier_node *>uncles;
     find_uncles(nodes[i], uncles);
-    for( size_t i=0; i<uncles.size(); i++ )
+    for( size_t j=0; j<uncles.size(); j++ )
       {
-      const hier_node *uncle = uncles[i];
+      const hier_node *uncle = uncles[j];
       if( map_of_sets[caller].find(uncle->name) == map_of_sets[caller].end() )
         {
         // We have a COMMON uncle or sibling we haven't seen before.
@@ -13550,7 +13923,6 @@ parser_program_hierarchy( const cbl_prog_hier_t& hier )
       if( callers.find(caller) == callers.end() )
         {
         // We haven't seen this caller before
-        callers.insert(caller);
 
         char ach[3*sizeof(cbl_name_t)];
         tree names_table_type = build_array_type_nelts(CHAR_P, mol->second.size()+1);
@@ -13617,6 +13989,8 @@ parser_program_hierarchy( const cbl_prog_hier_t& hier )
                 (fmt_size_t)caller);
         tree accessible_programs_decl = gg_trans_unit_var_decl(ach);
         gg_assign( accessible_programs_decl, gg_get_address_of(the_constructed_table) );
+
+        callers.insert(caller);
         }
       }
     }
@@ -13734,7 +14108,7 @@ parser_check_fatal_exception()
   // in its innermost loop had an execution time of 19.5 seconds.  By putting in
   // the if() statement, that was reduced to 3.8 seconds.
 
-  if( enabled_exceptions.size() || sv_is_i_o )
+  if( cdf_enabled_exceptions().size() || sv_is_i_o )
     {
     gg_call(VOID,
             "__gg__check_fatal_exception",
@@ -13882,9 +14256,9 @@ conditional_abs(tree source, const cbl_field_t *field)
   }
 
 static bool
-mh_identical(cbl_refer_t &destref,
-             cbl_refer_t &sourceref,
-             TREEPLET    &tsource)
+mh_identical(const cbl_refer_t &destref,
+             const cbl_refer_t &sourceref,
+             const TREEPLET    &tsource)
   {
   // Check to see if the two variables are identical types, thus allowing
   // for a simple byte-for-byte copy of the data areas:
@@ -13906,7 +14280,7 @@ mh_identical(cbl_refer_t &destref,
       )
     {
     // The source and destination are identical in type
-    if( (sourceref.field->attr & intermediate_e) || !symbol_find_odo(sourceref.field) )
+    if( !symbol_find_odo(sourceref.field) )
       {
       Analyze();
       // Source doesn't have a depending_on clause
@@ -14224,7 +14598,7 @@ float_type_of(const cbl_field_t *field)
   }
 
 static tree
-float_type_of(cbl_refer_t *refer)
+float_type_of(const cbl_refer_t *refer)
   {
   return float_type_of(refer->field);
   }
@@ -14456,7 +14830,7 @@ picky_memset(tree &dest_p, unsigned char value, size_t length)
   }
 
 static void
-picky_memcpy(tree &dest_p, tree &source_p, size_t length)
+picky_memcpy(tree &dest_p, const tree &source_p, size_t length)
   {
   if( length )
     {
@@ -14475,10 +14849,10 @@ picky_memcpy(tree &dest_p, tree &source_p, size_t length)
   }
 
 static bool
-mh_numeric_display( cbl_refer_t &destref,
-                    cbl_refer_t &sourceref,
-                    TREEPLET    &tsource,
-                    tree size_error)
+mh_numeric_display( const cbl_refer_t &destref,
+                    const cbl_refer_t &sourceref,
+                    const TREEPLET    &tsource,
+                          tree size_error)
   {
   bool moved = false;
 
@@ -14964,11 +15338,11 @@ mh_numeric_display( cbl_refer_t &destref,
   }
 
 static bool
-mh_little_endian( cbl_refer_t &destref,
-                  cbl_refer_t &sourceref,
-                  TREEPLET    &tsource,
-                  bool check_for_error,
-                  tree size_error)
+mh_little_endian( const cbl_refer_t &destref,
+                  const cbl_refer_t &sourceref,
+                  const TREEPLET    &tsource,
+                        bool check_for_error,
+                        tree size_error)
   {
   bool moved = false;
 
@@ -15036,9 +15410,9 @@ mh_little_endian( cbl_refer_t &destref,
   }
 
 static bool
-mh_source_is_group( cbl_refer_t &destref,
-                    cbl_refer_t &sourceref,
-                    TREEPLET    &tsrc)
+mh_source_is_group( const cbl_refer_t &destref,
+                    const cbl_refer_t &sourceref,
+                    const TREEPLET    &tsrc)
   {
   bool retval = false;
   if( sourceref.field->type == FldGroup && !(destref.field->attr & rjust_e) )
@@ -15103,7 +15477,7 @@ move_helper(tree size_error,        // This is an INT
     {
     // We are creating a copy of the original destination in case we clobber it
     // and have to restore it because of a computational error.
-    bool first_time = true;
+    static bool first_time = true;
     static size_t stash_size = 1024;
     if( first_time )
       {
@@ -15128,7 +15502,7 @@ move_helper(tree size_error,        // This is an INT
     //goto dont_be_clever; this will go through to the default.
     }
 
-  if( !moved )
+  // if( !moved ) // commented out to quiet cppcheck
     {
     moved = mh_source_is_group(destref, sourceref, tsource);
     }
@@ -15197,8 +15571,9 @@ move_helper(tree size_error,        // This is an INT
     if( buffer_size < source_length )
       {
       buffer_size = source_length;
-      buffer = (char *)xrealloc(buffer, buffer_size);
+      buffer = static_cast<char *>(xrealloc(buffer, buffer_size));
       }
+    gcc_assert(buffer);
 
     if( figconst )
       {
@@ -15341,7 +15716,7 @@ move_helper(tree size_error,        // This is an INT
       gg_attribute_bit_clear(destref.field, refmod_e);
       }
 
-    moved = true;
+    // moved = true; // commented out to quiet cppcheck
     }
 
   if( restore_on_error )
@@ -15472,7 +15847,8 @@ binary_initial_from_float128(cbl_field_t *field, int rdigits,
   FIXED_WIDE_INT(128) i
     = FIXED_WIDE_INT(128)::from (real_to_integer (&value, &fail, 128), SIGNED);
 
-  retval = (char *)xmalloc(field->data.capacity);
+  retval = static_cast<char *>(xmalloc(field->data.capacity));
+  gcc_assert(retval);
   switch(field->data.capacity)
     {
       tree type;
@@ -15483,7 +15859,7 @@ binary_initial_from_float128(cbl_field_t *field, int rdigits,
     case 16:
       type = build_nonstandard_integer_type ( field->data.capacity
                                               * BITS_PER_UNIT, 0);
-      native_encode_wide_int (type, i, (unsigned char *)retval,
+      native_encode_wide_int (type, i, PTRCAST(unsigned char, retval),
                               field->data.capacity);
       break;
     default:
@@ -15613,7 +15989,8 @@ initial_from_initial(cbl_field_t *field)
       }
     if( set_return )
       {
-      retval = (char *)xmalloc(field->data.capacity+1);
+      retval = static_cast<char *>(xmalloc(field->data.capacity+1));
+      gcc_assert(retval);
       memset(retval, const_char, field->data.capacity);
       retval[field->data.capacity] = '\0';
       return retval;
@@ -15683,7 +16060,8 @@ initial_from_initial(cbl_field_t *field)
 
     case FldNumericDisplay:
       {
-      retval = (char *)xmalloc(field->data.capacity);
+      retval = static_cast<char *>(xmalloc(field->data.capacity));
+      gcc_assert(retval);
       char *pretval = retval;
       char ach[128];
 
@@ -15763,7 +16141,8 @@ initial_from_initial(cbl_field_t *field)
 
     case FldPacked:
       {
-      retval = (char *)xmalloc(field->data.capacity);
+      retval = static_cast<char *>(xmalloc(field->data.capacity));
+      gcc_assert(retval);
       char *pretval = retval;
       char ach[128];
 
@@ -15830,7 +16209,8 @@ initial_from_initial(cbl_field_t *field)
       {
       if( field->data.initial )
         {
-        retval = (char *)xmalloc(field->data.capacity+1);
+        retval = static_cast<char *>(xmalloc(field->data.capacity+1));
+        gcc_assert(retval);
         if( field->attr & hex_encoded_e)
           {
           memcpy(retval, field->data.initial, field->data.capacity);
@@ -15838,7 +16218,7 @@ initial_from_initial(cbl_field_t *field)
         else
           {
           size_t buffer_size = 0;
-          size_t length = (size_t)field->data.capacity;
+          size_t length = field->data.capacity;
           memset(retval, internal_space, length);
           raw_to_internal(&retval, &buffer_size, field->data.initial, length);
           if( strlen(field->data.initial) < length )
@@ -15854,7 +16234,8 @@ initial_from_initial(cbl_field_t *field)
 
     case FldNumericEdited:
       {
-      retval = (char *)xmalloc(field->data.capacity+1);
+      retval = static_cast<char *>(xmalloc(field->data.capacity+1));
+      gcc_assert(retval);
       if( field->data.initial && field->attr & quoted_e )
         {
         // What the programmer says the value is, the value becomes, no
@@ -15889,7 +16270,6 @@ initial_from_initial(cbl_field_t *field)
         char ach[128];
         memset(ach, 0, sizeof(ach));
         memset(retval, 0, field->data.capacity);
-        size_t ndigits = field->data.capacity;
 
         if( (field->attr & blank_zero_e) && real_iszero (&value) )
           {
@@ -15897,6 +16277,7 @@ initial_from_initial(cbl_field_t *field)
           }
         else
           {
+          size_t ndigits = field->data.capacity;
           digits_from_float128(ach, field, ndigits, rdigits, value);
           /* ???  This resides in libgcobol valconv.cc.  */
           __gg__string_to_numeric_edited( retval,
@@ -15911,23 +16292,24 @@ initial_from_initial(cbl_field_t *field)
 
     case FldFloat:
       {
-      retval = (char *)xmalloc(field->data.capacity);
+      retval = static_cast<char *>(xmalloc(field->data.capacity));
+      gcc_assert(retval);
       switch( field->data.capacity )
         {
         case 4:
           value = real_value_truncate (TYPE_MODE (FLOAT), value);
           native_encode_real (SCALAR_FLOAT_TYPE_MODE (FLOAT), &value,
-                              (unsigned char *)retval, 4, 0);
+                              PTRCAST(unsigned char, retval), 4, 0);
           break;
         case 8:
           value = real_value_truncate (TYPE_MODE (DOUBLE), value);
           native_encode_real (SCALAR_FLOAT_TYPE_MODE (DOUBLE), &value,
-                              (unsigned char *)retval, 8, 0);
+                              PTRCAST(unsigned char, retval), 8, 0);
           break;
         case 16:
           value = real_value_truncate (TYPE_MODE (FLOAT128), value);
           native_encode_real (SCALAR_FLOAT_TYPE_MODE (FLOAT128), &value,
-                              (unsigned char *)retval, 16, 0);
+                              PTRCAST(unsigned char, retval), 16, 0);
           break;
         }
       break;
@@ -16313,12 +16695,13 @@ psa_FldLiteralA(struct cbl_field_t *field )
 
   // First make room
   static size_t buffer_size = 1024;
-  static char *buffer = (char *)xmalloc(buffer_size);
+  static char *buffer = static_cast<char *>(xmalloc(buffer_size));
   if( buffer_size < field->data.capacity+1 )
     {
     buffer_size = field->data.capacity+1;
-    buffer = (char *)xrealloc(buffer, buffer_size);
+    buffer = static_cast<char *>(xrealloc(buffer, buffer_size));
     }
+  gcc_assert(buffer);
 
   cbl_figconst_t figconst = cbl_figconst_of( field->data.initial );
   gcc_assert(figconst == normal_value_e);
@@ -16373,7 +16756,7 @@ psa_FldLiteralA(struct cbl_field_t *field )
                                                   vs_file_static);
     }
   else
-#endif    
+#endif
     {
     // We have not seen that string before
     static int nvar = 0;
@@ -16387,9 +16770,9 @@ psa_FldLiteralA(struct cbl_field_t *field )
                                                 vs_file_static);
     actually_create_the_static_field(
                 field,
-                build_string_literal(field->data.capacity+1,
+                build_string_literal(field->data.capacity,
                                      buffer),
-                field->data.capacity+1,
+                field->data.capacity,
                 field->data.initial,
                 NULL_TREE,
                 field->var_decl_node);
@@ -16417,6 +16800,8 @@ parser_local_add(struct cbl_field_t *new_var )
     SHOW_PARSE_END
     }
 
+  CHECK_FIELD(new_var);
+
   IF( member(new_var->var_decl_node, "data"),
       ne_op,
       gg_cast(UCHAR_P, null_pointer_node) )
@@ -16473,8 +16858,8 @@ parser_symbol_add(struct cbl_field_t *new_var )
       }
     while(0);
 
-    fprintf(stderr, " %2.2d %s<%s> off:" HOST_SIZE_T_PRINT_DEC " "
-                    "msiz:%d cap:%d dig:%d rdig:%d attr:0x" HOST_SIZE_T_PRINT_HEX_PURE " loc:%p",
+    fprintf(stderr, " %2.2u %s<%s> off:" HOST_SIZE_T_PRINT_UNSIGNED " "
+                    "msiz:%u cap:%u dig:%u rdig:%d attr:0x" HOST_SIZE_T_PRINT_HEX_PURE " loc:%p",
             new_var->level,
             new_var->name,
             cbl_field_type_str(new_var->type),
@@ -16484,7 +16869,7 @@ parser_symbol_add(struct cbl_field_t *new_var )
             new_var->data.digits,
             new_var->data.rdigits,
             (fmt_size_t)new_var->attr,
-            (void*)new_var);
+            static_cast<void*>(new_var));
 
     if( is_table(new_var) )
       {
@@ -16524,7 +16909,7 @@ parser_symbol_add(struct cbl_field_t *new_var )
       {
       fprintf(stderr,
               " redefines:(%p)%s",
-              (void*)symbol_redefines(new_var),
+              static_cast<void*>(symbol_redefines(new_var)),
               symbol_redefines(new_var)->name);
       }
 
@@ -16624,10 +17009,12 @@ parser_symbol_add(struct cbl_field_t *new_var )
       TRACE1_TEXT_ABC(" (", cbl_field_type_str(new_var->type) ,")")
       if( new_var->type == FldLiteralN)
         {
+        const void *p1 = (new_var->data.initial);
+        const long *pldata = static_cast<const long *>(p1);
+        long ldata = *pldata;
         gg_fprintf( trace_handle,
                     1, " [%ld]",
-                    build_int_cst_type(LONG,
-                                        *(const long *)new_var->data.initial));
+                    build_int_cst_type(LONG, ldata));
         }
       TRACE1_END
       }
diff --git a/gcc/cobol/genapi.h b/gcc/cobol/genapi.h
index 36d947b..b41b906 100644
--- a/gcc/cobol/genapi.h
+++ b/gcc/cobol/genapi.h
@@ -52,17 +52,26 @@ void parser_division( cbl_division_t division,
 void parser_enter_program(const char *funcname, bool is_function, int *retval);
 void parser_leave_program();
 
-void parser_accept( cbl_refer_t refer, special_name_t special_e);
+void parser_accept( const cbl_refer_t &refer,
+                    special_name_t special_e,
+                    cbl_label_t *error,
+                    cbl_label_t *not_error );
 void parser_accept_exception( cbl_label_t *name );
 void parser_accept_exception_end( cbl_label_t *name );
 
-void parser_accept_envar( cbl_refer_t refer, cbl_refer_t envar,
-                          cbl_label_t *error, cbl_label_t *not_error );
-void parser_set_envar( cbl_refer_t envar, cbl_refer_t refer );
+void parser_accept_under_discussion(struct cbl_refer_t tgt, special_name_t special,
+                                    cbl_label_t *error, cbl_label_t *not_error );
+void parser_accept_envar( const cbl_refer_t &refer,
+                          const cbl_refer_t &envar,
+                          cbl_label_t *error,
+                          cbl_label_t *not_error );
+void parser_set_envar( const cbl_refer_t &envar, const cbl_refer_t &refer );
 
-void parser_accept_command_line( cbl_refer_t tgt, cbl_refer_t src,
-                          cbl_label_t *error, cbl_label_t *not_error );
-void parser_accept_command_line_count( cbl_refer_t tgt );
+void parser_accept_command_line(const cbl_refer_t &tgt,
+                                const cbl_refer_t &src,
+                                cbl_label_t *error,
+                                cbl_label_t *not_error );
+void parser_accept_command_line_count( const cbl_refer_t &tgt );
 
 void parser_accept_date_yymmdd( cbl_field_t *tgt );
 void parser_accept_date_yyyymmdd( cbl_field_t *tgt );
@@ -86,8 +95,7 @@ parser_add( size_t nC, cbl_num_result_t *C,
             size_t nA, cbl_refer_t *A,
             cbl_arith_format_t format,
             cbl_label_t *error,
-            cbl_label_t *not_error,
-            void *compute_error = NULL);  // This has to be cast to a tree pointer to int
+            cbl_label_t *not_error,            void *compute_error = NULL);  // This has to be cast to a tree pointer to int
 
 void parser_arith_error( cbl_label_t *name );
 void parser_arith_error_end( cbl_label_t *name );
@@ -174,7 +182,8 @@ parser_bitwise_op(struct cbl_field_t *tgt,
 
 void
 parser_classify( struct cbl_field_t *tgt,
-                 struct cbl_refer_t  srca,  enum classify_t type );
+           const struct cbl_refer_t &srca,
+                 enum                classify_t type );
 
 void
 parser_op( struct cbl_refer_t cref,
@@ -253,7 +262,7 @@ parser_program_hierarchy( const struct cbl_prog_hier_t& hier );
 void
 parser_end_program(const char *name=NULL);
 
-void parser_sleep(cbl_refer_t seconds);
+void parser_sleep(const cbl_refer_t &seconds);
 
 void parser_exit( const cbl_refer_t& refer, ec_type_t = ec_none_e );
 void parser_exit_section(void);
@@ -262,9 +271,14 @@ void parser_exit_perform( struct cbl_perform_tgt_t *tgt, bool cycle );
 void parser_exit_program(void); // exits back to COBOL only, else continue
 
 void
+parser_exhibit( bool changed, bool named,
+                const std::vector<cbl_refer_t> &args );
+void
 parser_display( const struct cbl_special_name_t *upon,
-                struct cbl_refer_t args[], size_t n,
-                bool advance = DISPLAY_ADVANCE );
+                const std::vector<cbl_refer_t> &args,
+                bool advance = DISPLAY_ADVANCE,
+          const cbl_label_t *not_error = nullptr,
+          const cbl_label_t *compute_error = nullptr );
 
 void parser_display_field(cbl_field_t *fld);
 
@@ -300,7 +314,7 @@ void
 parser_initialize(const cbl_refer_t& refer, bool like_parser_symbol_add=false);
 
 void
-parser_initialize_programs(size_t nprog, struct cbl_refer_t *progs);
+parser_initialize_programs(size_t nprog, const struct cbl_refer_t *progs);
 
 void
 parser_label_label( struct cbl_label_t *label );
@@ -447,7 +461,7 @@ parser_intrinsic_numval_c( cbl_field_t *f,
 
 void
 parser_intrinsic_subst( cbl_field_t *f,
-                        cbl_refer_t& ref1,
+                  const cbl_refer_t& ref1,
                         size_t argc,
                         cbl_substitute_t * argv );
 
diff --git a/gcc/cobol/gengen.cc b/gcc/cobol/gengen.cc
index e42747b..3ad3344 100644
--- a/gcc/cobol/gengen.cc
+++ b/gcc/cobol/gengen.cc
@@ -107,8 +107,6 @@
 // Don't like it?  Cry me a river.
 static const int ARG_LIMIT = 512;
 
-static int sv_current_line_number;
-
 // These are globally useful constants
 tree char_nodes[256];
 
@@ -140,7 +138,7 @@ struct cbl_translation_unit_t gg_trans_unit;
 // the compiler when a source code module makes that mistake.
 static std::unordered_set<std::string> names_we_have_seen;
 
-// This vector is used to process the function_decls at the point we leave 
+// This vector is used to process the function_decls at the point we leave
 // the file.
 static std::vector<tree> finalized_function_decls;
 
@@ -452,7 +450,7 @@ gg_assign(tree dest, const tree source)
 
   if( okay )
     {
-    stmt = build2_loc(location_from_lineno(),
+    stmt = build2_loc(gg_token_location(),
                       MODIFY_EXPR,
                       TREE_TYPE(dest),
                       dest,
@@ -616,7 +614,7 @@ gg_add_field_to_structure(const tree type_of_field, const char *name_of_field, t
   tree id_of_field = get_identifier (name_of_field);
 
   // Create the new field:
-  tree new_field_decl = build_decl(   location_from_lineno(),
+  tree new_field_decl = build_decl(   gg_token_location(),
                                       FIELD_DECL,
                                       id_of_field,
                                       type_of_field);
@@ -893,7 +891,7 @@ gg_create_assembler_name(const char *cobol_name)
 static char *
 gg_unique_in_function(const char *var_name, gg_variable_scope_t vs_scope)
   {
-  char *retval = (char *)xmalloc(strlen(var_name)+32);
+  char *retval = static_cast<char *>(xmalloc(strlen(var_name)+32));
   if( (vs_scope == vs_stack || vs_scope == vs_static) )
     {
     sprintf(retval, "%s." HOST_SIZE_T_PRINT_DEC, var_name,
@@ -1028,10 +1026,7 @@ gg_declare_variable(tree type_decl,
       break;
     }
   DECL_INITIAL(var_decl) = initial_value;
-  if( unique_name )
-    {
-    free(unique_name);
-    }
+  free(unique_name);
   return var_decl;
   }
 
@@ -1046,7 +1041,7 @@ gg_define_from_declaration(tree var_decl)
     {
     // Having made sure the chain of variable declarations is nicely started,
     // it's time to actually define the storage with a decl_expression:
-    tree stmt = build1_loc (location_from_lineno(),
+    tree stmt = build1_loc (gg_token_location(),
                             DECL_EXPR,
                             TREE_TYPE(var_decl),
                             var_decl);
@@ -1777,7 +1772,7 @@ gg_build_relational_expression(tree operand_a,
       compare = LE_EXPR;
       break;
     }
-  tree relational_expression = build2_loc(location_from_lineno(),
+  tree relational_expression = build2_loc(gg_token_location(),
                                           compare,
                                           boolean_type_node,
                                           operand_a,
@@ -1894,7 +1889,7 @@ gg_create_goto_pair(tree *goto_expr,
 void
 gg_goto_label_decl(tree label_decl)
   {
-  tree goto_expr  = build1_loc( location_from_lineno(),
+  tree goto_expr  = build1_loc( gg_token_location(),
                                 GOTO_EXPR,
                                 void_type_node,
                                 label_decl);
@@ -1941,7 +1936,7 @@ gg_create_goto_pair(tree *goto_expr, tree *label_expr, const char *name)
 void
 gg_goto(tree var_decl_pointer)
   {
-  tree go_to = build1_loc(location_from_lineno(),
+  tree go_to = build1_loc(gg_token_location(),
                           GOTO_EXPR,
                           void_type_node,
                           var_decl_pointer);
@@ -2189,7 +2184,7 @@ gg_printf(const char *format_string, ...)
     function = gg_get_function_address(INT, "__gg__fprintf_stderr");
     }
 
-  tree stmt = build_call_array_loc (location_from_lineno(),
+  tree stmt = build_call_array_loc (gg_token_location(),
                                     INT,
                                     function,
                                     nargs,
@@ -2236,7 +2231,7 @@ gg_fprintf(tree fd, int nargs, const char *format_string, ...)
     function = gg_get_function_address(INT, "sprintf");
     }
 
-  tree stmt = build_call_array_loc (location_from_lineno(),
+  tree stmt = build_call_array_loc (gg_token_location(),
                                     INT,
                                     function,
                                     argc,
@@ -2283,7 +2278,7 @@ void
 gg_memset(tree dest, const tree value, tree size)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_MEMSET),
                           3,
                           dest,
@@ -2297,7 +2292,7 @@ gg_memchr(tree buf, tree ch, tree length)
   {
   tree the_call = fold_convert(
       pvoid_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_MEMCHR),
                           3,
                           buf,
@@ -2312,7 +2307,7 @@ void
 gg_memcpy(tree dest, const tree src, tree size)
   {
   tree the_call = build_call_expr_loc(
-        location_from_lineno(),
+        gg_token_location(),
         builtin_decl_explicit (BUILT_IN_MEMCPY),
         3,
         dest,
@@ -2327,7 +2322,7 @@ void
 gg_memmove(tree dest, const tree src, tree size)
   {
   tree the_call = build_call_expr_loc(
-        location_from_lineno(),
+        gg_token_location(),
         builtin_decl_explicit (BUILT_IN_MEMMOVE),
         3,
         dest,
@@ -2360,7 +2355,7 @@ void
 gg_strcpy(tree dest, tree src)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRCPY),
                           2,
                           dest,
@@ -2373,7 +2368,7 @@ gg_strcmp(tree A, tree B)
   {
   tree the_call = fold_convert(
       integer_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRCMP),
                           2,
                           A,
@@ -2405,7 +2400,7 @@ gg_strncmp(tree char_star_A, tree char_star_B, tree size_t_N)
   {
   tree the_call = fold_convert(
       integer_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRNCMP),
                           3,
                           char_star_A,
@@ -2436,7 +2431,7 @@ gg_return(tree operand)
     {
     // When there is no operand, or if the function result is void, then
     // we just generate a return_expr.
-    stmt = build1_loc(location_from_lineno(), RETURN_EXPR, void_type_node, NULL_TREE);
+    stmt = build1_loc(gg_token_location(), RETURN_EXPR, void_type_node, NULL_TREE);
     }
   else
     {
@@ -2446,7 +2441,7 @@ gg_return(tree operand)
                             function_type,
                             DECL_RESULT(current_function->function_decl),
                             gg_cast(function_type, operand));
-    stmt = build1_loc(location_from_lineno(), RETURN_EXPR, void_type_node, modify);
+    stmt = build1_loc(gg_token_location(), RETURN_EXPR, void_type_node, modify);
     }
   gg_append_statement(stmt);
   }
@@ -2454,7 +2449,7 @@ gg_return(tree operand)
 void
 chain_parameter_to_function(tree function_decl, const tree param_type,  const char *name)
   {
-  tree parm = build_decl (location_from_lineno(),
+  tree parm = build_decl (gg_token_location(),
                           PARM_DECL,
                           get_identifier (name),
                           param_type);
@@ -2521,12 +2516,12 @@ gg_peek_fn_decl(const char *funcname, tree fndecl_type)
     }
   return retval;
   }
-  
+
 tree
 gg_build_fn_decl(const char *funcname, tree fndecl_type)
   {
   tree function_decl;
-  
+
   std::string key = function_decl_key(funcname, fndecl_type);
   std::unordered_map<std::string, tree>::const_iterator it =
           map_of_function_decls.find(key);
@@ -2617,13 +2612,13 @@ gg_define_function( tree return_type,
     }
   va_end(params);
 
+  char ach[32];
   std::unordered_set<std::string>::const_iterator it =
           names_we_have_seen.find(funcname);
   if( it != names_we_have_seen.end() )
     {
     static int bum_counter = 1;
     // We have seen this name before.  Replace it with something unique:
-    char ach[32];
     sprintf(ach, "..no_dupes.%d", bum_counter++);
     funcname = ach;
     }
@@ -2689,7 +2684,7 @@ gg_define_function( tree return_type,
     }
 
   // Establish the RESULT_DECL for the function:
-  tree resdecl = build_decl (location_from_lineno(), RESULT_DECL, NULL_TREE, return_type);
+  tree resdecl = build_decl (gg_token_location(), RESULT_DECL, NULL_TREE, return_type);
   DECL_CONTEXT (resdecl) = function_decl;
   DECL_RESULT (function_decl) = resdecl;
 
@@ -2821,7 +2816,7 @@ gg_get_function_decl(tree return_type, const char *funcname, ...)
     }
 
   // Establish the RESULT_DECL for the function:
-  tree resdecl = build_decl (location_from_lineno(), RESULT_DECL, NULL_TREE, return_type);
+  tree resdecl = build_decl (gg_token_location(), RESULT_DECL, NULL_TREE, return_type);
   DECL_CONTEXT (resdecl) = function_decl;
   DECL_RESULT (function_decl) = resdecl;
 
@@ -3058,7 +3053,7 @@ gg_call_expr(tree return_type, const char *function_name, ...)
 
     tree arg = va_arg(ap, tree);
 
-    if( !arg )
+    if( arg == NULL_TREE )
       {
       break;
       }
@@ -3079,7 +3074,7 @@ gg_call_expr(tree return_type, const char *function_name, ...)
   tree the_func_addr = build1(ADDR_EXPR,
                               build_pointer_type (TREE_TYPE(function_decl)),
                               function_decl);
-  tree the_call = build_call_array_loc(location_from_lineno(),
+  tree the_call = build_call_array_loc(gg_token_location(),
                                        return_type,
                                        the_func_addr,
                                        nargs,
@@ -3114,7 +3109,7 @@ gg_call(tree return_type, const char *function_name,  ...)
 
     tree arg = va_arg(ap, tree);
 
-    if( !arg )
+    if( arg == NULL_TREE )
       {
       break;
       }
@@ -3135,7 +3130,7 @@ gg_call(tree return_type, const char *function_name,  ...)
   tree the_func_addr = build1(ADDR_EXPR,
                               build_pointer_type (TREE_TYPE(function_decl)),
                               function_decl);
-  tree the_call = build_call_array_loc(location_from_lineno(),
+  tree the_call = build_call_array_loc(gg_token_location(),
                                        return_type,
                                        the_func_addr,
                                        nargs,
@@ -3160,7 +3155,7 @@ gg_call_expr_list(tree return_type, tree function_pointer, int param_count, tree
   // Avoid that with something like
   //      gg_assign( dest, gg_call_expr_list(...) );
 
-  tree the_call = build_call_array_loc(location_from_lineno(),
+  tree the_call = build_call_array_loc(gg_token_location(),
                                        return_type,
                                        function_pointer,
                                        param_count,
@@ -3195,7 +3190,7 @@ void
 gg_exit(tree exit_code)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_EXIT),
                           1,
                           exit_code);
@@ -3206,7 +3201,7 @@ void
 gg_abort()
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_ABORT),
                           0);
   gg_append_statement(the_call);
@@ -3217,7 +3212,7 @@ gg_strlen(tree psz)
   {
   tree the_call = fold_convert(
       size_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRLEN),
                           1,
                           psz));
@@ -3229,7 +3224,7 @@ gg_strdup(tree psz)
   {
   tree the_call = fold_convert(
       build_pointer_type(char_type_node),
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRDUP),
                           1,
                           psz));
@@ -3243,7 +3238,7 @@ gg_malloc(tree size)
   {
   tree the_call = fold_convert(
       pvoid_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_MALLOC),
                           1,
                           size));
@@ -3255,7 +3250,7 @@ gg_realloc(tree base, tree size)
   {
   tree the_call = fold_convert(
       pvoid_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_REALLOC),
                           2,
                           base,
@@ -3279,7 +3274,7 @@ void
 gg_free(tree pointer)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_FREE),
                           1,
                           pointer);
@@ -3380,18 +3375,6 @@ gg_string_literal(const char *string)
   return build_string_literal(strlen(string)+1, string);
   }
 
-void
-gg_set_current_line_number(int line_number)
-  {
-  sv_current_line_number = line_number;
-  }
-
-int
-gg_get_current_line_number()
-  {
-  return sv_current_line_number;
-  }
-
 tree
 gg_trans_unit_var_decl(const char *var_name)
   {
@@ -3413,7 +3396,7 @@ gg_insert_into_assembler(const char ach[])
   if( !optimize )
     {
     // Create the required generic tag
-    tree asm_expr = build5_loc( location_from_lineno(),
+    tree asm_expr = build5_loc( gg_token_location(),
                             ASM_EXPR,
                             VOID,
                             build_string(strlen(ach), ach),
@@ -3450,5 +3433,28 @@ gg_insert_into_assemblerf(const char *format, ...)
     gg_insert_into_assembler(ach);
     }
   }
+#pragma GCC diagnostic pop
+
+static location_t sv_token_location_override = 0;
 
-#pragma GCC diagnostic pop
\ No newline at end of file
+void
+token_location_override(location_t loc)
+  {
+  sv_token_location_override = loc;
+  }
+
+location_t
+gg_token_location()
+  {
+  location_t retval;
+  if( sv_token_location_override )
+    {
+    retval = sv_token_location_override;
+    sv_token_location_override = 0;
+    }
+  else
+    {
+    retval = current_token_location();
+    }
+  return retval;
+  }
diff --git a/gcc/cobol/gengen.h b/gcc/cobol/gengen.h
index 06b28e06..96e69dd 100644
--- a/gcc/cobol/gengen.h
+++ b/gcc/cobol/gengen.h
@@ -525,11 +525,11 @@ extern tree gg_indirect(tree pointer, tree byte_offset = NULL_TREE);
 extern tree gg_string_literal(const char *string);
 
 #define CURRENT_LINE_NUMBER (cobol_location().first_line)
-extern location_t location_from_lineno();
-
-// When set to true, use UNKNOWN_LOCATION instead of CURRENT_LINE_NUMBER
-extern void gg_set_current_line_number(int line_number);
-extern int  gg_get_current_line_number();
+extern location_t gg_token_location();
+extern location_t current_token_location();
+extern location_t current_location_minus_one();
+extern void current_location_minus_one_clear();
+extern void token_location_override(location_t loc);
 
 extern tree gg_trans_unit_var_decl(const char *var_name);
 
diff --git a/gcc/cobol/genmath.cc b/gcc/cobol/genmath.cc
index 0a1c12d..e7eb971 100644
--- a/gcc/cobol/genmath.cc
+++ b/gcc/cobol/genmath.cc
@@ -52,7 +52,8 @@ set_up_on_exception_label(cbl_label_t *arithmetic_label)
     if( !arithmetic_label->structs.arith_error )
       {
       arithmetic_label->structs.arith_error
-        = (cbl_arith_error_t *)xmalloc(sizeof(struct cbl_arith_error_t) );
+        = static_cast<cbl_arith_error_t *>
+                                  (xmalloc(sizeof(struct cbl_arith_error_t)));
       // Set up the address pairs for this clause
       gg_create_goto_pair(&arithmetic_label->structs.arith_error->over.go_to,
                           &arithmetic_label->structs.arith_error->over.label);
@@ -72,8 +73,8 @@ set_up_compute_error_label(cbl_label_t *compute_label)
     if( !compute_label->structs.compute_error )
       {
       compute_label->structs.compute_error
-        = (cbl_compute_error_t *)
-          xmalloc(sizeof(struct cbl_compute_error_t) );
+        = static_cast<cbl_compute_error_t *>
+          (xmalloc(sizeof(struct cbl_compute_error_t)));
       compute_label->structs.compute_error->compute_error_code
         = gg_define_int(0);
       }
@@ -112,7 +113,6 @@ arithmetic_operation(size_t nC, cbl_num_result_t *C,
     {
     TRACE1_HEADER
     TRACE1_TEXT_ABC("calling ", operation, "")
-    TRACE1_END
     for(size_t ii=0; ii<nA; ii++)
       {
       TRACE1_INDENT
@@ -129,7 +129,6 @@ arithmetic_operation(size_t nC, cbl_num_result_t *C,
                   build_int_cst_type(SIZE_T, ii));
       TRACE1_REFER("", B[ii], "");
       }
-    TRACE1_END
     }
 
   // We need to split up cbl_num_result_t into two arrays, one for the refer_t
@@ -223,7 +222,6 @@ arithmetic_operation(size_t nC, cbl_num_result_t *C,
     {
     for(size_t ii=0; ii<nC; ii++)
       {
-      break;  // Breaks on ADD 1 SUB2 GIVING SUB4 both PIC S9(3) COMP
       TRACE1_INDENT
       gg_fprintf( trace_handle,
                   1, "result: C[%ld]: ",
@@ -612,7 +610,7 @@ static bool
 fast_divide(size_t nC, cbl_num_result_t *C,
             size_t nA, cbl_refer_t *A,
             size_t nB, cbl_refer_t *B,
-            cbl_refer_t             remainder)
+      const cbl_refer_t             &remainder)
   {
   bool retval = false;
   if( all_results_binary(nC, C) )
@@ -663,8 +661,10 @@ fast_divide(size_t nC, cbl_num_result_t *C,
       // We now either divide into C[n] or assign dividend/divisor to C[n]:
       for(size_t i=0; i<nC; i++ )
         {
-        tree dest_type = tree_type_from_size(C[i].refer.field->data.capacity, 0);
-        tree dest_addr = gg_add(member(C[i].refer.field->var_decl_node, "data"),
+        tree dest_type =
+                       tree_type_from_size(C[i].refer.field->data.capacity, 0);
+        tree dest_addr = gg_add(member( C[i].refer.field->var_decl_node,
+                                        "data"),
                                 refer_offset(C[i].refer));
         tree ptr = gg_cast(build_pointer_type(dest_type), dest_addr);
         if( nB )
@@ -680,16 +680,15 @@ fast_divide(size_t nC, cbl_num_result_t *C,
           }
 
         // This is where we handle any remainder, keeping in mind that for
-        // nB != 0, the actual dividend is in the value we have named "divisor".
-        //
-        // And, yes, I hate comments like that, too.
+        // nB != 0, the actual dividend is in the value we have named
+        // "divisor".
 
         // We calculate the remainder by calculating
         //    dividend minus quotient * divisor
         if( remainder.field )
           {
-          tree dest_addr = gg_add(member(remainder.field->var_decl_node, "data"),
-                                  refer_offset(remainder));
+          dest_addr = gg_add( member(remainder.field->var_decl_node, "data"),
+                              refer_offset(remainder));
           dest_type = tree_type_from_size(remainder.field->data.capacity, 0);
           ptr = gg_cast(build_pointer_type(dest_type), dest_addr);
 
diff --git a/gcc/cobol/genutil.cc b/gcc/cobol/genutil.cc
index 1d921a3..a5f69a0 100644
--- a/gcc/cobol/genutil.cc
+++ b/gcc/cobol/genutil.cc
@@ -27,6 +27,9 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+
+// cppcheck-suppress-file duplicateBreak
+
 #include "cobol-system.h"
 #include "coretypes.h"
 #include "tree.h"
@@ -305,8 +308,11 @@ static
 void
 get_and_check_refstart_and_reflen(  tree         refstart,// LONG returned value
                                     tree         reflen,  // LONG returned value
-                                    cbl_refer_t &refer)
+                              const cbl_refer_t &refer)
   {
+  const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
+
   if( !enabled_exceptions.match(ec_bound_ref_mod_e) )
     {
     // This is normal operation -- no exception checking.  Thus, we won't
@@ -458,6 +464,8 @@ get_depending_on_value_from_odo(tree retval, cbl_field_t *odo)
       declarative with a RESUME NEXT STATEMENT, or before the default_condition
       processing can do a controlled exit.
       */
+  const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
   cbl_field_t *depending_on;
   depending_on = cbl_field_of(symbol_at(odo->occurs.depending_on));
 
@@ -471,8 +479,8 @@ get_depending_on_value_from_odo(tree retval, cbl_field_t *odo)
     return;
     }
 
-  // Bounds checking is enabled, so we test the DEPENDING ON value to be between
-  // the lower and upper OCCURS limits:
+  // Bounds checking is enabled, so we test the DEPENDING ON value to be
+  // between the lower and upper OCCURS limits:
   get_integer_value(retval,
                     depending_on,
                     NULL,
@@ -482,23 +490,28 @@ get_depending_on_value_from_odo(tree retval, cbl_field_t *odo)
     {
     // This needs to evaluate to an integer
     set_exception_code(ec_bound_odo_e);
-    gg_assign(retval, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower));
+    gg_assign(retval, build_int_cst_type( TREE_TYPE(retval),
+                                          odo->occurs.bounds.lower));
     gg_assign(var_decl_rdigits, integer_zero_node);
     }
   ELSE
     ENDIF
 
-  IF( retval, gt_op, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.upper) )
+  IF( retval, gt_op, build_int_cst_type(TREE_TYPE(retval),
+                                        odo->occurs.bounds.upper) )
     {
     set_exception_code(ec_bound_odo_e);
-    gg_assign(retval, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower));
+    gg_assign(retval, build_int_cst_type( TREE_TYPE(retval),
+                                          odo->occurs.bounds.lower));
     }
   ELSE
     {
-    IF( retval, lt_op, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower) )
+    IF( retval, lt_op, build_int_cst_type(TREE_TYPE(retval),
+                                          odo->occurs.bounds.lower) )
       {
       set_exception_code(ec_bound_odo_e);
-      gg_assign(retval, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower));
+      gg_assign(retval, build_int_cst_type( TREE_TYPE(retval),
+                                            odo->occurs.bounds.lower));
       }
     ELSE
       ENDIF
@@ -532,8 +545,8 @@ get_depending_on_value(tree retval, const cbl_refer_t &refer)
 
 static
 tree
-get_data_offset(cbl_refer_t &refer,
-                int *pflags = NULL)
+get_data_offset(const cbl_refer_t &refer,
+                      int *pflags = NULL)
   {
   Analyze();
   // This routine returns a tree which is the size_t offset to the data in the
@@ -548,7 +561,6 @@ get_data_offset(cbl_refer_t &refer,
   // We have a refer.
   // At the very least, we have an constant offset
   int all_flags = 0;
-  int all_flag_bit = 1;
 
   if( refer.nsubscript() )
     {
@@ -568,6 +580,7 @@ get_data_offset(cbl_refer_t &refer,
     // Establish the field_t pointer for walking up through our ancestors:
     cbl_field_t *parent = refer.field;
 
+    int all_flag_bit = 1;
     // Note the backwards test, because refer->nsubscript is an unsigned value
     for(size_t i=refer.nsubscript()-1; i<refer.nsubscript(); i-- )
       {
@@ -601,6 +614,8 @@ get_data_offset(cbl_refer_t &refer,
         }
       else
         {
+        const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
         if( !enabled_exceptions.match(ec_bound_subscript_e) )
           {
           // With no exception testing, just pick up the value
@@ -625,21 +640,25 @@ get_data_offset(cbl_refer_t &refer,
             }
           ELSE
             {
-            IF( subscript, lt_op, gg_cast(TREE_TYPE(subscript), integer_one_node) )
+            IF( subscript, lt_op, gg_cast(TREE_TYPE(subscript),
+                                          integer_one_node) )
               {
               // The subscript is too small
               set_exception_code(ec_bound_subscript_e);
-              gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript), 1));
+              gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript),
+                                                      1));
               }
             ELSE
               {
               IF( subscript,
                   ge_op,
-                  build_int_cst_type(TREE_TYPE(subscript), parent->occurs.ntimes()) )
+                  build_int_cst_type( TREE_TYPE(subscript),
+                                      parent->occurs.ntimes()) )
                 {
                 // The subscript is too large
                 set_exception_code(ec_bound_subscript_e);
-                gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript), 1));
+                gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript),
+                                                        1));
                 }
               ELSE
                 {
@@ -654,14 +673,19 @@ get_data_offset(cbl_refer_t &refer,
 
       all_flag_bit <<= 1;
 
-      // Although we strictly don't need to look at the ODO value at this point,
-      // we do want it checked for the purposes of ec-bound-odo
+      // Although we strictly don't need to look at the ODO value at this
+      // point, we do want it checked for the purposes of ec-bound-odo
+
+      const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
 
       if( enabled_exceptions.match(ec_bound_odo_e) )
         {
         if( parent->occurs.depending_on )
           {
-          static tree value64 = gg_define_variable(LONG, ".._gdos_value64", vs_file_static);
+          static tree value64 = gg_define_variable( LONG,
+                                                    ".._gdos_value64",
+                                                    vs_file_static);
           cbl_field_t *odo = symbol_find_odo(parent);
           get_depending_on_value_from_odo(value64, odo);
           }
@@ -1238,18 +1262,13 @@ get_binary_value( tree value,
       break;
       }
 
-    case FldAlphanumeric:
-      {
-
-      }
-
-
     default:
       {
-      fprintf(stderr, "%s(): We know not how to"
-                      " get a binary value from %s\n",
-                      __func__,
-                      cbl_field_type_str(field->type) );
+      char *err = xasprintf("%s(): We know not how to"
+                            " get a binary value from %s\n",
+                            __func__,
+                            cbl_field_type_str(field->type) );
+      cbl_internal_error("%s", err);
       abort();
       break;
       }
@@ -1667,8 +1686,9 @@ set_exception_code_func(ec_type_t ec, int /*line*/, int from_raise_statement)
   }
 
 bool
-process_this_exception(ec_type_t ec)
+process_this_exception(const ec_type_t ec)
   {
+  const cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
   bool retval;
   if( enabled_exceptions.match(ec) || !skip_exception_processing )
     {
@@ -1700,7 +1720,7 @@ copy_little_endian_into_place(cbl_field_t *dest,
                               tree value,
                               int rhs_rdigits,
                               bool check_for_error,
-                              tree &size_error)
+                        const tree &size_error)
   {
   if( check_for_error )
     {
@@ -1926,7 +1946,7 @@ get_literal_string(cbl_field_t *field)
   }
 
 bool
-refer_is_clean(cbl_refer_t &refer)
+refer_is_clean(const cbl_refer_t &refer)
   {
   if( !refer.field || refer.field->type == FldLiteralN )
     {
@@ -1957,7 +1977,7 @@ refer_is_clean(cbl_refer_t &refer)
     */
 static
 tree  // size_t
-refer_refmod_length(cbl_refer_t &refer)
+refer_refmod_length(const cbl_refer_t &refer)
   {
   Analyze();
   REFER("refstart and reflen");
@@ -1973,7 +1993,7 @@ refer_refmod_length(cbl_refer_t &refer)
 
 static
 tree // size_t
-refer_fill_depends(cbl_refer_t &refer)
+refer_fill_depends(const cbl_refer_t &refer)
   {
   REFER("");
   // This returns a positive number which is the amount a depends-limited
@@ -2000,8 +2020,8 @@ refer_fill_depends(cbl_refer_t &refer)
   }
 
 tree  // size_t
-refer_offset(cbl_refer_t &refer,
-                    int *pflags)
+refer_offset(const cbl_refer_t &refer,
+                   int *pflags)
   {
   // This routine calculates the effect of a refer offset on the
   // refer.field->data location.  When there are subscripts, the data location
@@ -2028,7 +2048,7 @@ refer_offset(cbl_refer_t &refer,
 
 static
 tree   // size_t
-refer_size(cbl_refer_t &refer, refer_type_t refer_type)
+refer_size(const cbl_refer_t &refer, refer_type_t refer_type)
   {
   Analyze();
   static tree retval = gg_define_variable(SIZE_T, "..rs_retval", vs_file_static);
@@ -2069,13 +2089,13 @@ refer_size(cbl_refer_t &refer, refer_type_t refer_type)
   }
 
 tree  // size_t
-refer_size_dest(cbl_refer_t &refer)
+refer_size_dest(const cbl_refer_t &refer)
   {
   return refer_size(refer, refer_dest);
   }
 
 tree  // size_t
-refer_size_source(cbl_refer_t &refer)
+refer_size_source(const cbl_refer_t &refer)
   {
   /*  There are oddities involved with refer_size_source and refer_size_dest.
       See the comments in refer_has_depends for some explanation.  There are
@@ -2112,7 +2132,7 @@ refer_size_source(cbl_refer_t &refer)
   }
 
 tree
-qualified_data_location(cbl_refer_t &refer)
+qualified_data_location(const cbl_refer_t &refer)
   {
   return gg_add(member(refer.field->var_decl_node, "data"),
                 refer_offset(refer));
diff --git a/gcc/cobol/genutil.h b/gcc/cobol/genutil.h
index 0d9028e..f12124e 100644
--- a/gcc/cobol/genutil.h
+++ b/gcc/cobol/genutil.h
@@ -118,7 +118,7 @@ void      set_exception_code_func(ec_type_t ec,
                                   int line,
                                   int from_raise_statement=0);
 #define set_exception_code(ec) set_exception_code_func(ec, __LINE__)
-bool      process_this_exception(ec_type_t ec);
+bool      process_this_exception(const ec_type_t ec);
 #define   CHECK_FOR_FRACTIONAL_DIGITS true
 void      get_integer_value(tree value,  // This is always a LONG
                             cbl_field_t *field,
@@ -130,7 +130,7 @@ void      copy_little_endian_into_place(cbl_field_t *dest,
                                         tree value,
                                         int rhs_rdigits,
                                         bool check_for_error,
-                                        tree &size_error);
+                                  const tree &size_error);
 tree      build_array_of_size_t( size_t  N,
                                  const size_t *values);
 void      parser_display_internal_field(tree file_descriptor,
@@ -138,14 +138,14 @@ void      parser_display_internal_field(tree file_descriptor,
                                         bool advance=DISPLAY_NO_ADVANCE);
 char     *get_literal_string(cbl_field_t *field);
 
-bool      refer_is_clean(cbl_refer_t &refer);
+bool      refer_is_clean(const cbl_refer_t &refer);
 
-tree      refer_offset(cbl_refer_t &refer,
+tree      refer_offset(const cbl_refer_t &refer,
                        int *pflags=NULL);
-tree      refer_size_source(cbl_refer_t &refer);
-tree      refer_size_dest(cbl_refer_t &refer);
+tree      refer_size_source(const cbl_refer_t &refer);
+tree      refer_size_dest(const cbl_refer_t &refer);
 
-tree      qualified_data_location(cbl_refer_t &refer);
+tree      qualified_data_location(const cbl_refer_t &refer);
 
 void      build_array_of_treeplets( int ngroup,
                                     size_t N,
diff --git a/gcc/cobol/lexio.cc b/gcc/cobol/lexio.cc
index 2d9fb72..52d1aff 100644
--- a/gcc/cobol/lexio.cc
+++ b/gcc/cobol/lexio.cc
@@ -38,29 +38,24 @@
 
 extern int yy_flex_debug;
 
-static struct {
-  bool first_file, explicitly;
-  int column, right_margin;
-  bool inference_pending() {
-    bool tf = first_file && !explicitly;
-    first_file = false;
-    return tf;
-  }
-  inline bool is_fixed() const { return column == 7; }
-  inline bool is_reffmt() const { return is_fixed() && right_margin == 73; }
-  inline bool is_free() const { return ! is_fixed(); }
-  
-  const char * description() const {
-    if( is_reffmt() ) return "REFERENCE";
-    if( is_fixed() ) return "FIXED";
-    if( is_free() ) return "FREE";
-    gcc_unreachable();
-  }    
-} indicator = { true, false, 0, 0 };
+source_format_t& cdf_source_format();
+
+void
+source_format_t::infer( const char *bol, bool want_reference_format ) {
+  if( bol ) {
+    left = 7;
+    if( want_reference_format ) {
+      right = 73;
+    }
+  }
+  dbgmsg("%s:%d: %s format detected", __func__, __LINE__,
+         description());
+}
+
 
 // public source format test functions
-bool is_fixed_format() { return indicator.is_fixed(); }
-bool is_reference_format() { return indicator.is_reffmt(); }
+bool is_fixed_format() { return cdf_source_format().is_fixed(); }
+bool is_reference_format() { return cdf_source_format().is_reffmt(); }
 
 static bool debug_mode = false;
 
@@ -76,11 +71,10 @@ static bool debug_mode = false;
 */
 
 static inline int left_margin() {
-  return indicator.column == 0? indicator.column : indicator.column - 1;
+  return cdf_source_format().left_margin();
 }
 static inline int right_margin() {
-  return indicator.right_margin == 0?
-    indicator.right_margin : indicator.right_margin - 1;
+  return cdf_source_format().right_margin();
 }
 
 /*
@@ -89,18 +83,9 @@ static inline int right_margin() {
  *   When setting back to 0 (free), the right margin is also reset to 0.
  */
 void
-cobol_set_indicator_column( int column )
-{
-  indicator.explicitly = true;
-  if( column == 0 ) indicator.right_margin = 0;
-  if( column < 0 ) {
-    column = -column;
-    indicator.right_margin = 73;
-  }
-  indicator.column = column;
-}
+cobol_set_indicator_column( int column );
 
-bool include_debug()      { return indicator.column == 7 && debug_mode; }
+bool include_debug()      { return is_fixed_format() && debug_mode; }
 bool set_debug( bool tf ) { return debug_mode = tf && is_fixed_format(); }
 
 static bool nonblank( const char ch ) { return !isblank(ch); }
@@ -114,7 +99,7 @@ start_of_line( char *bol, char *eol ) {
 
 static inline char *
 continues_at( char *bol, char *eol ) {
-  if( indicator.column == 0 ) return NULL;  // cannot continue in free format
+  if( cdf_source_format().is_free() ) return NULL;  // cannot continue in free format
   bol += left_margin();
   if( *bol != '-' ) return NULL; // not a continuation line
   return start_of_line(++bol, eol);
@@ -124,7 +109,7 @@ continues_at( char *bol, char *eol ) {
 // NULL means no indicator column or tested value not present.
 static inline char *
 indicated( char *bol, const char *eol, char ch = '\0' ) {
-  if( indicator.column == 0 && *bol != '*' ) {
+  if( cdf_source_format().left_margin() == 0 && *bol != '*' ) {
     return NULL;  // no indicator column in free format, except for comments
   }
   gcc_assert(bol != NULL);
@@ -336,7 +321,69 @@ recognize_replacements( filespan_t mfile, std::list<replace_t>& pending_replacem
 }
 
 static void
+check_push_pop_directive( filespan_t& mfile ) {
+  char eol = '\0';
+  const char *p = std::find(mfile.cur, mfile.eol, '>');
+  if( ! (p < mfile.eol && p[1] == *p ) ) return;
+
+  const char pattern[] =
+    ">>[[:blank:]]*(push|pop)[[:blank:]]+"
+    "("
+      "all|"
+      "call-convention|"
+      "cobol-words|"
+      "define|"
+      "source[[:blank:]]+format|"
+      "turn"
+    ")";
+  static regex re(pattern, extended_icase);
+
+  // show contents of marked subexpressions within each match
+  cmatch cm;
+
+  std::swap(*mfile.eol, eol); // see implementation for excuses
+  bool ok = regex_search(p, const_cast<const char *>(mfile.eol), cm, re);
+  std::swap(*mfile.eol, eol);
+
+  if( ok ) {
+    gcc_assert(cm.size() > 1);
+    bool push = TOUPPER(cm[1].first[1]) == 'U';
+    switch( TOUPPER(cm[2].first[0]) ) {
+    case 'A': // ALL
+      push? cdf_push() : cdf_pop();
+      break;
+    case 'C':
+      switch( TOUPPER(cm[2].first[1]) ) {
+      case 'A': // CALL-CONVENTION
+        push? cdf_push_call_convention() : cdf_pop_call_convention();
+        break;
+      case 'O': // COBOL-WORDS
+        push? cdf_push_current_tokens() : cdf_pop_current_tokens();
+        break;
+      default:
+        gcc_unreachable();
+      }
+      break;
+    case 'D': // DEFINE
+      push? cdf_push_dictionary() : cdf_pop_dictionary();
+      break;
+    case 'S': // SOURCE FORMAT
+      push? cdf_push_source_format() : cdf_pop_source_format();
+      break;
+    case 'T': // TURN
+      push? cdf_push_enabled_exceptions() : cdf_pop_enabled_exceptions();
+      break;
+    default:
+      gcc_unreachable();
+    }
+    erase_line(const_cast<char*>(cm[0].first),
+               const_cast<char*>(cm[0].second));
+  }
+}
+
+static void
 check_source_format_directive( filespan_t& mfile ) {
+  char eol = '\0';
   const char *p = std::find(mfile.cur, mfile.eol, '>');
   if( ! (p < mfile.eol && p[1] == *p ) ) return;
 
@@ -349,7 +396,12 @@ check_source_format_directive( filespan_t& mfile ) {
 
   // show contents of marked subexpressions within each match
   cmatch cm;
-  if( regex_search(p, const_cast<const char *>(mfile.eol), cm, re) ) {
+
+  std::swap(*mfile.eol, eol); // see implementation for excuses
+  bool ok = regex_search(p, const_cast<const char *>(mfile.eol), cm, re);
+  std::swap(*mfile.eol, eol);
+
+  if( ok ) {
     gcc_assert(cm.size() > 1);
     switch( cm[3].length() ) {
     case 4:
@@ -365,11 +417,11 @@ check_source_format_directive( filespan_t& mfile ) {
 
     dbgmsg( "%s:%d: %s format set, on line " HOST_SIZE_T_PRINT_UNSIGNED,
             __func__, __LINE__,
-            indicator.column == 7? "FIXED" : "FREE",
+            cdf_source_format().description(),
             (fmt_size_t)mfile.lineno() );
-    char *bol = indicator.is_fixed()? mfile.cur : const_cast<char*>(cm[0].first);
+    char *bol = cdf_source_format().is_fixed()? mfile.cur : const_cast<char*>(cm[0].first);
+    gcc_assert(cm[0].second <= mfile.eol);
     erase_line(bol, const_cast<char*>(cm[0].second));
-    mfile.cur = const_cast<char*>(cm[0].second);
   }
 }
 
@@ -889,7 +941,7 @@ location_in( const filespan_t& mfile, const csub_match& cm ) {
   gcc_assert(mfile.cur <= cm.first && cm.second <= mfile.eodata);
   auto nline = std::count(cm.first, cm.second, '\n');
   if( nline ) {
-    gcc_assert(loc.first_line < nline);
+    gcc_assert(nline < loc.first_line);
     loc.first_line -= nline;
     auto p = static_cast<const char*>(memrchr(cm.first, '\n', cm.length()));
     loc.last_column = (cm.second) - p;
@@ -1327,13 +1379,13 @@ lexer_input( char buf[], int max_size, FILE *input ) {
   for( auto p = mfile.cur; p < next; *output.pos++ = *p++ ) {
     static bool at_bol = false;
     if( at_bol ) {
-      auto nonblank = std::find_if( p, next,
+      auto nonblank_l = std::find_if( p, next,
                                     []( char ch ) {
                                       return !isblank(ch); } );
-      if( nonblank + 1 < next ) {
-        if( *nonblank == '\r' ) nonblank++; // Windows
-        if( *nonblank == '\n' ) {
-          p = nonblank;
+      if( nonblank_l + 1 < next ) {
+        if( *nonblank_l == '\r' ) nonblank_l++; // Windows
+        if( *nonblank_l == '\n' ) {
+          p = nonblank_l;
           continue;
         }
       }
@@ -1461,7 +1513,6 @@ cdftext::lex_open( const char filename[] ) {
 
   // Process any files supplied by the -include command-line option.
   for( auto name : included_files ) {
-    int input;
     if( -1 == (input = open(name, O_RDONLY)) ) {
       yyerrorvl(1, "", "cannot open -include file %s", name);
       continue;
@@ -1634,7 +1685,7 @@ bool lexio_dialect_mf();
  */
 static const char *
 valid_sequence_area( const char *data, const char *eodata ) {
-  
+
   for( const char *p = data;
        (p = std::find_if(p, eodata, is_p)) != eodata;
        p++ )
@@ -1657,7 +1708,7 @@ valid_sequence_area( const char *data, const char *eodata ) {
       }
     }
   }
-  return nullptr;  
+  return nullptr;
 }
 
 /*
@@ -1693,22 +1744,17 @@ cdftext::free_form_reference_format( int input ) {
   } current( mfile.data );
 
   /*
-   * Infer source code format. 
+   * Infer source code format.
    */
-  if( indicator.inference_pending()  ) {
+  if( cdf_source_format().inference_pending()  ) {
     const char *bol = valid_sequence_area(mfile.data, mfile.eodata);
     if( bol ) {
-      indicator.column = 7;
-      if( infer_reference_format(bol, mfile.eodata) ) {
-	indicator.right_margin = 73;
-      }
+      cdf_source_format().infer( bol, infer_reference_format(bol, mfile.eodata) );
     }
-
-    dbgmsg("%s:%d: %s format detected", __func__, __LINE__,
-           indicator.description());
   }
 
   while( mfile.next_line() ) {
+    check_push_pop_directive(mfile);
     check_source_format_directive(mfile);
     remove_inline_comment(mfile.cur, mfile.eol);
 
@@ -1857,7 +1903,7 @@ cdftext::process_file( filespan_t mfile, int output, bool second_pass ) {
                    []( char ch ) { return ch == '\n'; } );
       struct { int in, out; filespan_t mfile; } copy;
       dbgmsg("%s:%d: line " HOST_SIZE_T_PRINT_UNSIGNED ", opening %s on fd %d",
-             __func__, __LINE__,mfile.lineno(),
+             __func__, __LINE__, (fmt_size_t)mfile.lineno(),
              copybook.source(), copybook.current()->fd);
       copy.in = copybook.current()->fd;
       copy.mfile = free_form_reference_format( copy.in );
@@ -1936,15 +1982,15 @@ cdftext::segment_line( filespan_t& mfile ) {
     struct { unsigned long ante, post; } lineno = {
       gb4(mfile.lineno()), gb4(mfile.lineno() + segment.after.nlines())
     };
-    char *directive = lineno.ante == lineno.post?
+    const char *directive = lineno.ante == lineno.post?
       nullptr : xasprintf("\n#line %lu \"%s\"\n",
                           lineno.ante, cobol_filename());
 
-    if( directive ) 
+    if( directive )
       output.push_back( span_t(strlen(directive), directive) );
     output.push_back( span_t(mfile.cur, segment.before.p) );
     output.push_back( span_t(segment.after.p, segment.after.pend ) );
-    if( directive ) 
+    if( directive )
       output.push_back( span_t(strlen(directive), directive) );
 
     mfile.cur = const_cast<char*>(segment.before.pend);
diff --git a/gcc/cobol/lexio.h b/gcc/cobol/lexio.h
index eb41068..ba4ef0a 100644
--- a/gcc/cobol/lexio.h
+++ b/gcc/cobol/lexio.h
@@ -244,8 +244,8 @@ struct span_t {
     return span_t(output, eout);
   }
   const char * has_nul() const {
-    auto p = std::find(this->p, pend, '\0');
-    return p != pend? p : NULL;
+    auto p_l = std::find(this->p, pend, '\0');
+    return p_l != pend? p_l : NULL;
   }
 
   bool at_eol() const {
diff --git a/gcc/cobol/parse.y b/gcc/cobol/parse.y
index 74637c9..fae96ed 100644
--- a/gcc/cobol/parse.y
+++ b/gcc/cobol/parse.y
@@ -45,6 +45,7 @@
   };
 
   enum accept_func_t {
+    accept_e,
     accept_done_e,
     accept_command_line_e,
     accept_envar_e,
@@ -349,7 +350,7 @@
 %token  <string>        SECTION
 %token  <number>        STANDARD_ALPHABET "STANDARD ALPHABET"
 %token  <string>        SWITCH
-%token  <string>        UPSI
+%token  <string>        UPSI 
 %token  <number>        ZERO
 
                          /* environment names */
@@ -385,7 +386,10 @@
 			CDF_EVALUATE ">>EVALUATE"
 			CDF_WHEN ">>WHEN"
 			CDF_END_EVALUATE ">>END-EVALUATE"
+                        CALL_CONVENTION ">>CALL-CONVENTION"
 			CALL_COBOL "CALL" CALL_VERBATIM "CALL (as C)"
+                        CDF_PUSH ">>PUSH" CDF_POP ">>POP"
+                        SOURCE_FORMAT ">>SOURCE FORMAT"
 
 			IF THEN ELSE
 			SENTENCE
@@ -399,7 +403,10 @@
 			STRING_kw "STRING" STOP SUBTRACT START
 			UNSTRING WRITE WHEN
 
-			   ABS ACCESS ACOS ACTUAL ADVANCING AFTER ALL
+			ARGUMENT_NUMBER  ARGUMENT_VALUE
+			ENVIRONMENT_NAME ENVIRONMENT_VALUE
+
+			ABS ACCESS ACOS ACTUAL ADVANCING AFTER ALL
 			ALLOCATE
 			ALPHABET ALPHABETIC ALPHABETIC_LOWER "ALPHABETIC-LOWER"
 			ALPHABETIC_UPPER "ALPHABETIC-UPPER"
@@ -793,6 +800,8 @@
 %type   <error_clauses> io_invalids read_eofs write_eops
 %type   <boolean>       io_invalid  read_eof  write_eop
                         global is_global anycase backward
+                        end_display
+                        exh_changed exh_named
 %type   <number>        mistake globally first_last
 %type   <io_mode>   io_mode
 
@@ -848,7 +857,7 @@
            declarative_list_t* dcl_list_t;
            isym_list_t* isym_list;
     struct { radix_t radix; char *string; } numstr;
-    struct { int token; literal_t name; } prog_end;
+    struct { YYLTYPE loc; int token; literal_t name; } prog_end;
     struct { int token; special_name_t id; } special_type;
     struct { cbl_field_type_t type;
              uint32_t capacity; bool signable; } computational;
@@ -902,7 +911,7 @@
     struct refer_pair_t { cbl_refer_t *first, *second; } refer2;
     struct { refer_collection_t *inputs; refer_pair_t into; } str_body;
 
-    struct { accept_func_t func; cbl_refer_t *into, *from; } accept_func;
+    struct { accept_func_t func; cbl_refer_t *into, *from; special_name_t special;} accept_func;
     struct unstring_into_t *uns_into;
     struct unstring_tgt_list_t *uns_tgts;
     struct unstring_tgt_t *uns_tgt;
@@ -1004,7 +1013,9 @@
 %right                  IF THEN ELSE
                         SENTENCE
                         ACCEPT ADD ALTER CALL CANCEL CLOSE COMPUTE CONTINUE
-                        DELETE DISPLAY DIVIDE EVALUATE END EOP EXIT FILLER_kw
+                        DELETE DISPLAY DIVIDE
+                        EVALUATE END EOP EXIT
+                        FILLER_kw
                         GOBACK GOTO
                         INITIALIZE INSPECT
                         MERGE MOVE MULTIPLY OPEN OVERFLOW_kw PARAGRAPH PERFORM
@@ -1464,16 +1475,16 @@ cobol_words:	cobol_words1
 	|	cobol_words cobol_words1
 		;
 cobol_words1:	COBOL_WORDS EQUATE NAME[keyword] WITH NAME[name] {
-		  if( ! tokens.equate(@keyword, $keyword, $name) ) { YYERROR; }
+		  if( ! cdf_tokens.equate(@keyword, $keyword, $name) ) { YYERROR; }
 		}
 	|	COBOL_WORDS UNDEFINE NAME[keyword] {
-		  if( ! tokens.undefine(@keyword, $keyword) ) { YYERROR; }
+		  if( ! cdf_tokens.undefine(@keyword, $keyword) ) { YYERROR; }
 		}
 	|	COBOL_WORDS SUBSTITUTE NAME[keyword] BY NAME[name] {
-		  if( ! tokens.substitute(@keyword, $keyword, $name) ) { YYERROR; }
+		  if( ! cdf_tokens.substitute(@keyword, $keyword, $name) ) { YYERROR; }
 		}
 	|	COBOL_WORDS RESERVE NAME[name] {
-		  if( ! tokens.reserve(@name, $name) ) { YYERROR; }
+		  if( ! cdf_tokens.reserve(@name, $name) ) { YYERROR; }
 		}
 		;
 
@@ -1513,7 +1524,7 @@ program_as:     %empty     { static const literal_t empty {}; $$ = empty; }
         |       AS LITERAL { $$ = $2; }
                 ;
 
-function_id:    FUNCTION '.' NAME program_as program_attrs[attr] '.'
+function_id:    FUNCTION NAME program_as program_attrs[attr] '.'
                 {
                   internal_ebcdic_lock();
                   current_division = identification_div_e;
@@ -1547,7 +1558,7 @@ function_id:    FUNCTION '.' NAME program_as program_attrs[attr] '.'
                   current.udf_add(current_program_index());
                   if( nparse_error > 0 ) YYABORT;
                 }
-        |       FUNCTION '.' NAME program_as is PROTOTYPE '.'
+        |       FUNCTION NAME program_as is PROTOTYPE '.'
                 {
                   cbl_unimplemented("FUNCTION PROTOTYPE");
                 }
@@ -1838,7 +1849,7 @@ select:         SELECT optional NAME[name] select_clauses[clauses] '.'
                   cbl_file_t *file = $clauses.file;
 
                   file->optional = $optional;
-                  file->line = yylineno;
+                  file->line = @name.first_line;
                   if( !namcpy(@clauses, file->name, $name) ) YYERROR;
 
                   if( ! ($clauses.clauses & assign_clause_e) ) {
@@ -1911,7 +1922,7 @@ select:         SELECT optional NAME[name] select_clauses[clauses] '.'
                   cbl_file_t file = protofile;
 
                   file.optional = $optional;
-                  file.line = yylineno;
+                  file.line = @name.first_line;
                   if( !namcpy(@name, file.name, $name) ) YYERROR;
 
                   if( file_add(@name, &file) == NULL ) YYERROR;
@@ -2473,7 +2484,7 @@ special_name:   dev_mnemonic
         |       CLASS NAME is domains
                 {
                   struct cbl_field_t field = { 0,
-                    FldClass, FldInvalid, 0, 0, 0, 0, nonarray, yylineno, "",
+                    FldClass, FldInvalid, 0, 0, 0, 0, nonarray, @NAME.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@NAME, field.name, $2) ) YYERROR;
@@ -2604,6 +2615,10 @@ device_name:	SYSIN           { $$.token = SYSIN; $$.id = SYSIN_e; }
         |       STDIN           { $$.token = STDIN; $$.id = STDIN_e; }
         |       STDOUT          { $$.token = STDOUT; $$.id = STDOUT_e; }
         |       STDERR          { $$.token = STDERR; $$.id = STDERR_e; }
+                /* These cannot be both ctx_name and here. *
+        /*      ARGUMENT_NUMBER   { $$.token=0; $$.id = ARG_NUM_e; } */
+        /*      ENVIRONMENT_NAME  { $$.token=0; $$.id = ENV_NAME_e; } */
+        /*      ENVIRONMENT_VALUE { $$.token=0; $$.id = ENV_VALUE_e; } */
                 ;
 
 alphabet_name:  STANDARD_ALPHABET  { $$ = alphabet_add(@1, ASCII_e); }
@@ -3164,7 +3179,7 @@ depending:      %empty
                     assert(e->type == SymField);
                     odo = symbol_index(e);
                   } else {
-		    e = symbol_field_forward_add(PROGRAM, 0, $NAME, yylineno);
+		    e = symbol_field_forward_add(PROGRAM, 0, $NAME, @NAME.first_line);
                     if( !e ) YYERROR;
 		    symbol_field_location( symbol_index(e), @NAME );
                     odo = field_index(cbl_field_of(e));
@@ -3364,7 +3379,7 @@ level_name:     LEVEL ctx_name
                   }
                   struct cbl_field_t field = { 0,
                     FldInvalid, FldInvalid, 0, 0, 0, capacity_cast($1),
-		    nonarray, yylineno, "",
+		    nonarray, @ctx_name.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@ctx_name, field.name, $2) ) YYERROR;
@@ -3389,7 +3404,7 @@ level_name:     LEVEL ctx_name
                   }
                   struct cbl_field_t field = { 0,
                     FldInvalid, FldInvalid, 0, 0, 0, capacity_cast($1),
-		    nonarray, yylineno, "",
+		    nonarray, @LEVEL.first_line, "",
                     0, {}, {}, NULL };
 
                   $$ = field_add(@1, &field);
@@ -3527,7 +3542,7 @@ data_descr1:    level_name
                   }
                   struct cbl_field_t field = { 0, FldLiteralA, FldInvalid,
                                                constant_e, 0, 0, 78, nonarray,
-                                               yylineno, "", 0, {}, *$data, NULL };
+                                               @name.first_line, "", 0, {}, *$data, NULL };
                   if( !namcpy(@name, field.name, $name) ) YYERROR;
                   if( field.data.initial ) {
                     field.attr |= quoted_e;
@@ -3550,7 +3565,7 @@ data_descr1:    level_name
         |       LEVEL88 NAME /* VALUE */ NULLPTR
                 {
                   struct cbl_field_t field = { 0,
-                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, yylineno, "",
+                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, @NAME.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@NAME, field.name, $2) ) YYERROR;
@@ -3576,7 +3591,7 @@ data_descr1:    level_name
         |       LEVEL88 NAME VALUE domains
                 {
                   struct cbl_field_t field = { 0,
-                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, yylineno, "",
+                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, @NAME.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@NAME, field.name, $2) ) YYERROR;
@@ -4606,7 +4621,7 @@ justified_clause: is JUSTIFIED
 
 redefines_clause: REDEFINES NAME[orig]
                 {
-                  struct symbol_elem_t *e = field_of($orig);
+                  struct symbol_elem_t *e = symbol_field(PROGRAM, 0, $orig);
                   if( !e ) {
                     error_msg(@2, "REDEFINES target not defined");
                     YYERROR;
@@ -4906,6 +4921,7 @@ by_value_arg:   scalar
 declaratives:   %empty
         |       DECLARATIVES '.'
                 <label>{
+		  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
                   current.enabled_exception_cache = enabled_exceptions;
                   enabled_exceptions.clear();
                   current.doing_declaratives(true);
@@ -4924,6 +4940,7 @@ declaratives:   %empty
                    * forward reference, because we haven't yet begun to parse
                    * nondeclarative procedures.
                    */
+		  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
                   parser_label_label($label);
                   enabled_exceptions = current.enabled_exception_cache;
                   current.enabled_exception_cache.clear();
@@ -5038,6 +5055,7 @@ statement:      error {
         |       divide          { $$ =  DIVIDE; }
         |       entry           { $$ =  ENTRY; }
         |       evaluate        { $$ =  EVALUATE; }
+        |       exhibit_stmt    { $$ =  EXHIBIT; }
         |       exit            { $$ =  EXIT; }
         |       free            { $$ =  FREE; }
         |       go_to           { $$ =  GOTO; }
@@ -5068,9 +5086,8 @@ statement:      error {
 
 		/*
 		 * ISO defines ON EXCEPTION only for Format 3 (screen). We
-		 * implement extensions defined by MF and Fujitsu (and us) to
-		 * use ACCEPT to interact with the command line and the
-		 * environment.
+		 * implement extensions defined by MF and Fujitsu to use ACCEPT
+		 * to interact with the command line and the environment.
 		 *
 		 * ISO ACCEPT and some others are implemented in accept_body,
 		 * before the parser sees any ON EXCEPTION.  In those cases
@@ -5085,6 +5102,9 @@ accept:         accept_body end_accept {
 		  switch( $accept_body.func ) {
 		  case accept_done_e:
 		    break;
+		  case accept_e:
+                    parser_accept(*$1.into, $1.special, nullptr, nullptr);
+		    break;
 		  case accept_command_line_e:
 		    if( $1.from->field == NULL ) { // take next command-line arg
 		      parser_accept_command_line(*$1.into, argi, NULL, NULL);
@@ -5108,6 +5128,9 @@ accept:         accept_body end_accept {
 		    error_msg(@ec, "ON EXCEPTION valid only "
 			    "with ENVIRONMENT or COMMAND-LINE(n)");
 		    break;
+		  case accept_e:
+                    parser_accept(*$1.into, $1.special, $ec.on_error, $ec.not_error);
+		    break;
 		  case accept_command_line_e:
 		    if( $1.from->field == NULL ) { // take next command-line arg
 		      parser_accept_command_line(*$1.into, argi,
@@ -5139,7 +5162,7 @@ end_accept:     %empty %prec ACCEPT
 accept_body:    accept_refer
                 {
 		  $$.func = accept_done_e;
-                  parser_accept(*$1, CONSOLE_e);
+                  parser_accept(*$1, CONSOLE_e, nullptr, nullptr);
                 }
         |       accept_refer FROM DATE
                 {
@@ -5198,29 +5221,15 @@ accept_body:    accept_refer
                 }
         |       accept_refer FROM acceptable
                 {
-		  cbl_field_t *argc = register_find("_ARGI");
-		  switch( $acceptable->id ) {
-		  case ARG_NUM_e:
-		    $$.func = accept_command_line_e;
-		    $$.into = $1;
-		    $$.from = new_reference(argc);
-		    break;
-		  case ARG_VALUE_e:
-		    $$.func = accept_command_line_e;
-		    $$.into = $1;
-		    $$.from = cbl_refer_t::empty();
-		    break;
-		  default:
-		    $$.func = accept_done_e;
-		    parser_accept( *$1, $acceptable->id );
-		  }
+		  $$.func = accept_e;
+                  $$.into = $1;
+                  $$.special = $acceptable->id;
                 }
         |       accept_refer FROM ENVIRONMENT envar
                 {
 		  $$.func = accept_envar_e;
 		  $$.into = $1;
 		  $$.from = $envar;
-                  ////  parser_accept_envar( *$1, *$envar );
                 }
         |       accept_refer FROM COMMAND_LINE
                 {
@@ -5232,7 +5241,6 @@ accept_body:    accept_refer
 		  $$.func = accept_command_line_e;
 		  $$.into = $1;
 		  $$.from = $expr;
-                  //// parser_accept_command_line(*$1, $expr->field );
                 }
         |       accept_refer FROM COMMAND_LINE_COUNT {
 		  $$.func = accept_done_e;
@@ -5285,7 +5293,7 @@ accept_except:	EXCEPTION
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblArith,
-                                          uniq_label("accept"), yylineno);
+                                          uniq_label("accept"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_accept_exception( $$.on_error );
 
@@ -5320,15 +5328,54 @@ acceptable:     device_name
                       error_msg(@NAME, "no such special name '%s'", $NAME);
                       YYERROR;
 		    }
+                    if( ENV_NAME_e == *special_type ) { 
+                      error_msg(@NAME, "cannot ACCEPT FROM %qs", $NAME);
+                      YYERROR;
+                    }
 		    // Add the name now, as a convenience. 
-		    cbl_special_name_t special = { 0, *special_type };
+                    int token = 0;
+                    switch(*special_type) {
+                    case ARG_NUM_e:    token = ARGUMENT_NUMBER; break;
+                    case ARG_VALUE_e:  token = ARGUMENT_VALUE; break;
+                    case ENV_VALUE_e:  token = ENVIRONMENT_VALUE; break;
+
+                    case ENV_NAME_e:
+                    default:
+                      error_msg(@NAME, "cannot ACCEPT FROM %qs", $NAME);
+                      YYERROR;
+                      break;
+                    }
+		    cbl_special_name_t special = { token, *special_type };
 		    namcpy(@NAME, special.name, $NAME);
 
 		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
 		    $$ = cbl_special_name_of(e);
+                    cbl_special_name_t& unused(*$$);
+                    assert(unused.id);
 		  }
 		  assert($$);
                 }
+        |       ENVIRONMENT_VALUE {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ENVIRONMENT_VALUE, ENV_VALUE_e, "ENVIRONMENT-VALUE" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ARGUMENT_NUMBER {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ARGUMENT_NUMBER, ARG_NUM_e, "ARGUMENT-NUMBER" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ARGUMENT_VALUE {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ARGUMENT_VALUE, ARG_VALUE_e, "ARGUMENT-VALUE" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
                 ;
 
 add:            add_impl end_add { ast_add($1); }
@@ -5558,46 +5605,18 @@ compute_expr:   '=' {
                 }
                 ;
 
-display:        disp_body end_display
-                {
-		  std::vector <cbl_refer_t> args($1.vargs->args.size());
-		  std::copy( $1.vargs->args.begin(), $1.vargs->args.end(), args.begin() );
-		  if( $1.special && $1.special->id == ARG_NUM_e ) {
-		    if( $1.vargs->args.size() != 1 ) {
-		      error_msg(@1, "ARGUMENT-NUMBER can be set to only one value");
-		    }
-		   const  cbl_refer_t& src( $1.vargs->args.front() );
-		    cbl_field_t *dst = register_find("_ARGI");
-		    parser_move( dst, src );
-		  } else {
-		    parser_display($1.special,
-				   args.empty()? NULL : args.data(), args.size(),
-				   DISPLAY_ADVANCE);
-		  }
-		  current.declaratives_evaluate();
-                }
-        |       disp_body NO ADVANCING end_display
+display:        disp_body end_display[advance]
                 {
-                  std::vector <cbl_refer_t> args($1.vargs->args.size());
-		  std::copy( $1.vargs->args.begin(), $1.vargs->args.end(), args.begin() );
-
-		  if( $1.special && $1.special->id == ARG_NUM_e ) {
-		    if( $1.vargs->args.size() != 1 ) {
-		      error_msg(@1, "ARGUMENT-NUMBER can be set to only one value");
-		    }
-		   const cbl_refer_t& src( $1.vargs->args.front() );
-		    cbl_field_t *dst = register_find("_ARGI");
-		    parser_move( dst, src );
-		  } else {
-		    parser_display($1.special,
-				   args.empty()? NULL : args.data(), args.size(),
-				   DISPLAY_NO_ADVANCE);
-		  }
+		  std::vector <cbl_refer_t> args($1.vargs->args.begin(),
+                                                 $1.vargs->args.end());
+                  parser_display($1.special, args, $advance);
 		  current.declaratives_evaluate();
                 }
                 ;
-end_display:    %empty
-        |       END_DISPLAY
+end_display:    %empty				{ $$ = DISPLAY_ADVANCE; }
+        |                    END_DISPLAY	{ $$ = DISPLAY_ADVANCE; }
+        |       NO ADVANCING			{ $$ = DISPLAY_NO_ADVANCE; }
+        |       NO ADVANCING END_DISPLAY	{ $$ = DISPLAY_NO_ADVANCE; }
                 ;
 disp_body:      disp_vargs[vargs]
                 {
@@ -5628,14 +5647,62 @@ disp_upon:    	device_name {
                       error_msg(@NAME, "no such special name '%s'", $NAME);
                       YYERROR;
 		    }
-		    // Add the name now, as a convenience. 
-		    cbl_special_name_t special = { 0, *special_type };
+		    // Add the name now, as a convenience.
+                    // These may come through as a NAME, depending on how scanned. 
+                    int token = 0;
+                    switch(*special_type) {
+                    case ARG_NUM_e:    token = ARGUMENT_NUMBER; break;
+                    case ENV_NAME_e:   token = ENVIRONMENT_NAME; break;
+                    case ENV_VALUE_e:  token = ENVIRONMENT_VALUE; break;
+
+                    case ARG_VALUE_e:
+                    default:
+                      error_msg(@NAME, "cannot DISPLAY UPON %qs", $NAME);
+                      YYERROR;
+                      break;
+                    }
+		    cbl_special_name_t special = { token, *special_type };
 		    namcpy(@NAME, special.name, $NAME);
 
                     e = symbol_special_add(PROGRAM, &special);
 		  }
 		  $$ = cbl_special_name_of(e);
                 }
+        |       ARGUMENT_NUMBER {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ARGUMENT_NUMBER, ARG_NUM_e, "ARGUMENT-NUMBER" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ENVIRONMENT_NAME {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ENVIRONMENT_NAME, ENV_NAME_e, "ENVIRONMENT-NAME" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ENVIRONMENT_VALUE {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ENVIRONMENT_VALUE, ENV_VALUE_e, "ENVIRONMENT-VALUE" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+                ;
+
+exhibit_stmt:   EXHIBIT exh_changed exh_named vargs {
+                  statement_begin(@1, EXHIBIT);
+                  std::vector<cbl_refer_t> args( $vargs->args.begin(),
+                                                 $vargs->args.end() );
+                  parser_exhibit( $exh_changed, $exh_named, args );
+                }
+                ;
+exh_changed:    %empty  { $$ = false; }
+        |       CHANGED { $$ = true; }
+                ;
+exh_named:      %empty  { $$ = false; }
+        |       NAMED   { $$ = true; }
                 ;
 
 divide:         divide_impl end_divide { ast_divide($1); }
@@ -5733,14 +5800,14 @@ end_program:    end_program1[end] '.'
                     gcc_unreachable();
                   }
                   if( !matches ) {
-                    error_msg(@end, "END %s %s does not match "
+                    error_msg($end.loc, "END %s %s does not match "
                                     "%<IDENTIFICATION DIVISION %s%>",
                               token_name, name, prog->name);
                     YYERROR;
                   }
 
                   if( 0 != strcasecmp(prog->name, name) ) {
-                    error_msg(@end, "END PROGRAM '%s' does not match PROGRAM-ID '%s'",
+                    error_msg($end.loc, "END PROGRAM '%s' does not match PROGRAM-ID '%s'",
                               name, prog->name);
                     YYERROR;
                   }
@@ -5773,20 +5840,24 @@ end_program:    end_program1[end] '.'
                 ;
 end_program1:   END_PROGRAM  namestr[name]
                 {
+                  $$.loc = @name;
                   $$.token = END_PROGRAM;
                   $$.name = $name;
                 }
         |       END_FUNCTION namestr[name]
                 {
+                  $$.loc = @name;
                   $$.token = END_FUNCTION;
                   $$.name = $name;
                 }
 	|	END_PROGRAM  '.' // error
                 {
+                  $$.loc = @1;
                   $$.token = END_PROGRAM;
                 }
         |       END_FUNCTION '.' // error
                 {
+                  $$.loc = @1;
                   $$.token = END_FUNCTION;
                 }
                 ;
@@ -6622,7 +6693,7 @@ name:           qname
                       auto name = names.front();
                       names.pop_front();
                       auto e = symbol_field_forward_add(PROGRAM, parent,
-                                                        name, yylineno);
+                                                        name, @1.first_line);
                       if( !e ) YYERROR;
 		      symbol_field_location( symbol_index(e), @qname );
                       parent = symbol_index(e);
@@ -6652,6 +6723,10 @@ ctx_name:       NAME
 
 context_word:   APPLY                   { static char s[] ="APPLY";
                                          $$ = s; } // screen description entry
+        |       ARGUMENT_NUMBER         { static char s[] ="ARGUMENT-NUMBER";
+                                         $$ = s; } // Display Upon / Accept From
+        |       ARGUMENT_VALUE         { static char s[] ="ARGUMENT-VALUE";
+                                         $$ = s; } // Accept From
         |       ARITHMETIC             { static char s[] ="ARITHMETIC";
                                          $$ = s; } // OPTIONS paragraph
         |       ATTRIBUTE              { static char s[] ="ATTRIBUTE";
@@ -6688,6 +6763,10 @@ context_word:   APPLY                   { static char s[] ="APPLY";
                                          $$ = s; } // ERASE clause in a screen description entry
         |       ENTRY_CONVENTION       { static char s[] ="ENTRY-CONVENTION";
                                          $$ = s; } // OPTIONS paragraph
+	|       ENVIRONMENT_NAME         { static char s[] ="ENVIRONMENT-NAME";
+                                         $$ = s; } // Display Upon
+        |       ENVIRONMENT_VALUE         { static char s[] ="ENVIRONMENT-VALUE";
+                                         $$ = s; } // Display Upon / Accept From
         |       ERASE                  { static char s[] ="ERASE";
                                          $$ = s; } // screen description entry
         |       EXPANDS                { static char s[] ="EXPANDS";
@@ -7036,9 +7115,9 @@ arith_err:      SIZE_ERROR
                     *ptgt = $1 == NOT?
                       current.compute_not_error() : current.compute_on_error();
                   } else {
-                    *ptgt = label_add(LblArith, uniq_label("arith"), yylineno);
+                    *ptgt = label_add(LblArith, uniq_label("arith"), @1.first_line);
                   }
-		  (*ptgt)->lain = yylineno;
+		  (*ptgt)->lain = @1.first_line;
                   parser_arith_error( *ptgt );
                 }
                 ;
@@ -7575,6 +7654,7 @@ perform_cond:   UNTIL { parser_perform_conditional( &perform_current()->tgt); }
 perform_inline: perform_start statements END_PERFORM
                 {
 		  location_set(@END_PERFORM);
+		  parser_sleep(*cbl_refer_t::empty());
 		  $$ = perform_current();
 		  if( $perform_start == LOCATION ) {
 		    error_msg(@1, "LOCATION not valid with PERFORM Format 2");
@@ -7583,6 +7663,7 @@ perform_inline: perform_start statements END_PERFORM
         |       perform_start END_PERFORM
                 {
 		  location_set(@END_PERFORM);
+		  parser_sleep(*cbl_refer_t::empty());
 		  $$ = perform_current();
 		  if( $perform_start == LOCATION ) {
 		    error_msg(@1, "LOCATION not valid with PERFORM Format 2");
@@ -8754,12 +8835,12 @@ search_1_body:  name[table] search_varying[varying]
 
                   cbl_name_t label_name;
 		  auto len = snprintf(label_name, sizeof(label_name),
-				     "linear_search_%d", yylineno);
+				     "linear_search_%d", @1.first_line);
 		  if( ! (0 < len && len < int(sizeof(label_name))) ) {
 		    gcc_unreachable();
 		  }
                   cbl_label_t *name = label_add( LblSearch,
-                                                 label_name, yylineno );
+                                                 label_name, @1.first_line );
 		  auto varying($varying);
                   if( index == varying ) varying = NULL;
                   parser_lsearch_start( name, $table, index, varying );
@@ -8812,9 +8893,9 @@ search_binary:  SEARCH ALL search_2_body search_2_cases
 search_2_body:  name[table]
                 {
                   statement_begin(@$, SEARCH);
-                  char *label_name = xasprintf("binary_search_%d", yylineno);
+                  char *label_name = xasprintf("binary_search_%d", @1.first_line);
                   cbl_label_t *name = label_add( LblSearch,
-                                                 label_name, yylineno );
+                                                 label_name, @1.first_line );
                   parser_bsearch_start( name, $table );
                   search_alloc(name);
                 }
@@ -9759,7 +9840,7 @@ call_except:    EXCEPTION
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblArith,
-                                          uniq_label("call"), yylineno);
+                                          uniq_label("call"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_call_exception( $$.on_error );
 
@@ -9772,7 +9853,7 @@ call_except:    EXCEPTION
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblArith,
-                                          uniq_label("call"), yylineno);
+                                          uniq_label("call"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_call_exception( $$.on_error );
 
@@ -9828,7 +9909,7 @@ go_to:          GOTO labels[args]
                   }
 
 		  for( auto& label : $args->elems ) {
-		    label->used = yylineno;
+		    label->used = @2.first_line;
 		  }
                   cbl_label_t *arg = $args->elems.front();
                   parser_goto( cbl_refer_t(), 1, &arg );
@@ -9840,7 +9921,7 @@ go_to:          GOTO labels[args]
                   std::vector <cbl_label_t *> args($args->elems.size());
 		  std::copy($args->elems.begin(), $args->elems.end(), args.begin());
 		  for( auto& label : $args->elems ) {
-		    label->used = yylineno;
+		    label->used = @2.first_line;
 		  }
                   parser_goto( *$value, args.size(), args.data() );
                 }
@@ -9860,7 +9941,7 @@ resume:         RESUME NEXT STATEMENT
                 {
                   statement_begin(@1, RESUME);
                   parser_clear_exception();
-		  $tgt->used = yylineno;
+		  $tgt->used = @1.first_line;
                   parser_goto( cbl_refer_t(), 1, &$tgt );
                 }
                 ;
@@ -10035,7 +10116,7 @@ on_overflow:    OVERFLOW_kw
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblString,
-                                          uniq_label("string"), yylineno);
+                                          uniq_label("string"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_string_overflow( $$.on_error );
 
@@ -10255,7 +10336,7 @@ intrinsic:      function_udf
                   if( p != NULL ) {
 		    auto loc = symbol_field_location(field_index(p->field));
                     error_msg(loc, "FUNCTION %qs has "
-                              "inconsistent parameter type %zu (%qs)",
+                              "inconsistent parameter type %td (%qs)",
                               keyword_str($1), p - args.data(), name_of(p->field) );
                     YYERROR;
                   }
@@ -11377,6 +11458,7 @@ void ast_call( const YYLTYPE& loc, cbl_refer_t name, const cbl_refer_t& returnin
  */
 static bool
 possible_ec() {
+  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
   bool format_1 = current.declaratives.has_format_1();
       
   bool enabled = 0xFF < (current.declaratives.status()
@@ -11399,6 +11481,7 @@ possible_ec() {
  */
 static void
 statement_epilog( int token ) {
+  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
   if( possible_ec() && token != CONTINUE ) { 
     if( enabled_exceptions.size() ) {
       current.declaratives_evaluate();
@@ -11461,9 +11544,11 @@ keyword_str( int token ) {
     return ascii;
   }
 
-  return tokens.name_of(token);
+  return cdf_tokens.name_of(token);
 }
 
+bool iso_cobol_word( const std::string& name, bool include_context );
+
 /*
  * Return the token for the Cobol name, unless it is a function name.  The
  * lexer uses keyword_tok to determine if what appears to be a NAME is in fact
@@ -11474,15 +11559,14 @@ keyword_str( int token ) {
  */
 
 // tokens.h is generated as needed from parse.h with tokens.h.gen
-tokenset_t::tokenset_t() {
+current_tokens_t::tokenset_t::tokenset_t() {
 #include "token_names.h"
 }
 
-bool iso_cobol_word( const std::string& name, bool include_context );
 
 // Look up the lowercase form of a keyword, excluding some CDF names.
 int
-tokenset_t::find( const cbl_name_t name, bool include_intrinsics ) {
+current_tokens_t::tokenset_t::find( const cbl_name_t name, bool include_intrinsics ) {
   static const cbl_name_t non_names[] = { // including CDF NAMES, and "SWITCH"
     "CHECKING", "LIST", "LOCATION", "MAP", "SWITCH",
   }, * const eonames = non_names + COUNT_OF(non_names);
@@ -11532,7 +11616,7 @@ tokenset_t::find( const cbl_name_t name, bool include_intrinsics ) {
 
 int
 keyword_tok( const char * text, bool include_intrinsics ) {
-  return tokens.find(text, include_intrinsics);
+  return cdf_tokens.find(text, include_intrinsics);
 }
 
 static inline size_t
@@ -11724,7 +11808,7 @@ label_add( const YYLTYPE& loc,
                 name, cbl_label_of(p)->name, cbl_label_of(p)->line);
     }
   }
-  struct cbl_label_t label = { type, parent, loc.last_line };
+  struct cbl_label_t label = { type, parent, loc.first_line };
 
   if( !namcpy(loc, label.name, name) ) return NULL;
   auto p =  symbol_label_add(PROGRAM, &label);
@@ -13074,7 +13158,7 @@ cobol_dialect_set( cbl_dialect_t dialect ) {
     break;
   case dialect_gnu_e:
     if( 0 == (cbl_dialects & dialect) ) { // first time
-      tokens.equate(YYLTYPE(), "BINARY-DOUBLE", "BINARY-C-LONG");
+      cdf_tokens.equate(YYLTYPE(), "BINARY-DOUBLE", "BINARY-C-LONG");
     }
     break;
   }    
diff --git a/gcc/cobol/parse_ante.h b/gcc/cobol/parse_ante.h
index 105afe9..03cb0a0 100644
--- a/gcc/cobol/parse_ante.h
+++ b/gcc/cobol/parse_ante.h
@@ -103,7 +103,7 @@ void input_file_status_notify();
         }                                                               \
       location_dump("parse.c", __LINE__, "current", (Current));         \
       input_file_status_notify();                                       \
-      gcc_location_set( location_set(Current) );                        \
+      location_set(Current);                                            \
   } while (0)
 
 int yylex(void);
@@ -935,165 +935,11 @@ teed_up_names() {
   return name_queue_t::namelist_of( name_queue.peek() );
 }
 
-class tokenset_t {
-  // token_names is initialized from a generated header file. 
-  std::vector<const char *>token_names;  // position indicates token value
-  std::map <std::string, int> tokens;    // aliases
-  std::set<std::string> cobol_words;  // Anything in COBOL-WORDS may appear only once. 
- public:
-  static std::string
-  lowercase( const cbl_name_t name ) {
-    cbl_name_t lname;
-    std::transform(name, name + strlen(name) + 1, lname, ftolower);
-    return lname;
-  }
-  static std::string
-  uppercase( const cbl_name_t name ) {
-    cbl_name_t uname;
-    std::transform(name, name + strlen(name) + 1, uname, ftoupper);
-    return uname;
-  }
-
- public:
-  tokenset_t();
-  int find( const cbl_name_t name, bool include_intrinsics );
-
-  bool equate( const YYLTYPE& loc, int token,
-	       const cbl_name_t name, const cbl_name_t verb = "EQUATE") {
-    auto lname( lowercase(name) );
-    auto cw = cobol_words.insert(lname);
-    if( ! cw.second ) {
-      error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
-      return false;
-    }
-    auto p = tokens.find(lowercase(name));
-    bool fOK = p == tokens.end();
-    if( fOK ) { // name not already in use
-      tokens[lname] = token;
-      dbgmsg("%s:%d: %d has alias %s", __func__, __LINE__, token, name);
-    } else {
-      error_msg(loc, "%s: %s already defined as a token", verb, name);
-    }
-    return fOK;
-  }
-  bool undefine( const YYLTYPE& loc,
-		 const cbl_name_t name, const cbl_name_t verb = "UNDEFINE" ) {
-    auto lname( lowercase(name) );
-    auto cw = cobol_words.insert(lname);
-    if( ! cw.second ) {
-      error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
-      return false;
-    }
-
-    // Do not erase generic, multi-type tokens COMPUTATIONAL and BINARY_INTEGER.
-    if( binary_integer_usage_of(name) ) {
-      dbgmsg("%s:%d: generic %s remains valid as a token", __func__, __LINE__, name);
-      return true;
-    }
-
-    auto p = tokens.find(lname);
-    bool fOK = p != tokens.end();
-    if( fOK ) { // name in use
-      tokens.erase(p);
-    } else {
-      error_msg(loc, "%s: %s not defined as a token", verb, name);
-    }
-    dbgmsg("%s:%d: %s removed as a valid token name", __func__, __LINE__, name);
-    return fOK;
-  }
-  
-  bool substitute( const YYLTYPE& loc,
-		   const cbl_name_t extant, int token, const cbl_name_t name ) {
-    return
-      equate( loc, token, name, "SUBSTITUTE" )
-      &&
-      undefine( loc, extant, "SUBSTITUTE" );
-  }
-  bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
-    auto lname( lowercase(name) );
-    auto cw = cobol_words.insert(lname);
-    if( ! cw.second ) {
-      error_msg(loc, "COBOL-WORDS RESERVE: %s may appear but once", name);
-      return false;
-    }
-    tokens[lname] = -42;
-    return true;
-  }
-  int redefined_as( const cbl_name_t name ) {
-    auto lname( lowercase(name) );
-    if( cobol_words.find(lname) != cobol_words.end() ) {
-      auto p = tokens.find(lname);
-      if( p != tokens.end() ) {
-        return p->second;
-      }
-    }
-    return 0;
-  }
-  const char * name_of( int tok ) const {
-    tok -= (255 + 3);
-    gcc_assert(0 <= tok && size_t(tok) < token_names.size());
-    return tok < 0? "???" : token_names[tok];
-  }
-};
-
-class current_tokens_t {
-  tokenset_t tokens;
- public:
-  current_tokens_t() {}
-  int find( const cbl_name_t name, bool include_intrinsics ) {
-    return tokens.find(name, include_intrinsics);
-  }
-  bool equate( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
-    int token; 
-    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
-      if( 0 == (token = keyword_tok(keyword)) ) {
-	error_msg(loc, "EQUATE %s: not a valid token", keyword);
-	return false;
-      }
-    }
-    auto name = keyword_alias_add(tokens.uppercase(keyword),
-				  tokens.uppercase(alias));
-    if( name != keyword ) {
-      error_msg(loc, "EQUATE: %s is already an alias for %s", alias, name.c_str());
-      return false;
-    } 
-    return tokens.equate(loc, token, alias);
-  }
-  bool undefine( const YYLTYPE& loc, cbl_name_t keyword ) {
-    return tokens.undefine(loc, keyword);
-  }
-  bool substitute( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
-    int token; 
-    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
-      if( 0 == (token = keyword_tok(keyword)) ) {
-	error_msg(loc, "SUBSTITUTE %s: not a valid token", keyword);
-	return false;
-      }
-    }
-    auto name = keyword_alias_add(tokens.uppercase(keyword),
-				  tokens.uppercase(alias));
-    if( name != keyword ) {
-      error_msg(loc, "SUBSTITUTE: %s is already an alias for %s", alias, name.c_str());
-      return false;
-    } 
-
-    dbgmsg("%s:%d: %s (%d) will have alias %s", __func__, __LINE__, keyword, token, alias);
-    return tokens.substitute(loc, keyword, token, alias);
-  }
-  bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
-    return tokens.reserve(loc, name);
-  }
-  int redefined_as( const cbl_name_t name ) {
-    return tokens.redefined_as(name);
-  }
-  const char * name_of( int tok ) const {
-    return tokens.name_of(tok);
-  }
-} tokens;
+#define cdf_tokens cdf_current_tokens()
 
 int
 redefined_token( const cbl_name_t name ) {
-  return tokens.redefined_as(name);
+  return cdf_tokens.redefined_as(name);
 }
 
 struct file_list_t {
@@ -1475,7 +1321,6 @@ class prog_descr_t {
       }
     }
   } locale;
-  cbl_call_convention_t call_convention;
   cbl_options_t options;
 
   explicit prog_descr_t( size_t isymbol )
@@ -1484,9 +1329,7 @@ class prog_descr_t {
     , paragraph(NULL)
     , section(NULL)
     , collating_sequence(NULL)
-  {
-    call_convention = current_call_convention();
-  }
+  {}
 
  std::set<std::string> external_targets() {
    std::set<std::string> externals;
@@ -1575,24 +1418,13 @@ static cbl_label_t *  implicit_section();
 
 class program_stack_t : protected  std::stack<prog_descr_t> {
   struct pending_t {
-    cbl_call_convention_t call_convention;
     bool initial;
-    pending_t()
-      : call_convention(cbl_call_convention_t(0))
-      , initial(false)
-    {}
+    pending_t() : initial(false) {}
   } pending;
  public:
-  cbl_call_convention_t
-  pending_call_convention( cbl_call_convention_t convention ) {
-    return pending.call_convention = convention;
-  }
   bool pending_initial() { return pending.initial = true; }
 
   void push( prog_descr_t descr ) {
-    cbl_call_convention_t call_convention = cbl_call_cobol_e;
-    if( !empty() ) call_convention = top().call_convention;
-    descr.call_convention = call_convention;
     std::stack<prog_descr_t>& me(*this);
     me.push(descr);
   }
@@ -1618,9 +1450,6 @@ class program_stack_t : protected  std::stack<prog_descr_t> {
   }
 
   void apply_pending() {
-    if( size() == 1 && 0 != pending.call_convention ) {
-      top().call_convention = pending.call_convention;
-  }
     if( pending.initial ) {
       auto e = symbol_at(top().program_index);
       auto prog(cbl_label_of(e));
@@ -2027,19 +1856,6 @@ static class current_t {
     return programs.top().options.default_round = mode;
   }
 
-  cbl_call_convention_t
-  call_convention() {
-    return programs.empty()? cbl_call_cobol_e : programs.top().call_convention;
-  }
-  cbl_call_convention_t
-  call_convention( cbl_call_convention_t convention) {
-    if( programs.empty() ) {
-      return programs.pending_call_convention(convention);
-    }
-    auto& prog( programs.top() );
-    return prog.call_convention = convention;
-  }
-
   const char *
   locale() {
     return programs.empty()? NULL : programs.top().locale.os_name;
@@ -2137,6 +1953,7 @@ static class current_t {
    * ISO, in new_program.
    */
   std::set<std::string>  end_program() {
+    cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
     if( enabled_exceptions.size() ) {
       declaratives_evaluate();
     }
@@ -2427,15 +2244,6 @@ current_rounded_mode( cbl_round_t rounded) {
 #endif
 static cbl_round_t current_rounded_mode( int token );
 
-cbl_call_convention_t
-current_call_convention() {
-  return current.call_convention();
-}
-cbl_call_convention_t
-current_call_convention( cbl_call_convention_t convention) {
-  return current.call_convention(convention);
-}
-
 size_t program_level() { return current.program_level(); }
 
 static size_t constant_index( int token );
@@ -2909,17 +2717,6 @@ group_attr( const cbl_field_t * field ) {
   return p->attr;
 }
 
-static struct symbol_elem_t *
-field_of( const char F[], int L, const char name[] ) {
-  struct symbol_elem_t *e = symbol_field(PROGRAM, 0, name);
-  if( !e ) {
-    cbl_internal_error("%s:%d: no symbol '%s' found", F, L, name);
-  }
-  assert( procedure_div_e != current_division  );
-  return e;
-}
-#define field_of( F ) field_of(__func__, __LINE__, (F))
-
 static struct cbl_field_t *
 field_add( const YYLTYPE& loc, cbl_field_t *field ) {
   switch(current_data_section) {
@@ -3696,18 +3493,18 @@ goodnight_gracie() {
 
 // false after USE statement, to enter Declarative with EC intact. 
 static bool statement_cleanup = true;
+static YYLTYPE current_location;
 
 static void statement_epilog( int token );
 
 const char * keyword_str( int token );
 
-static YYLTYPE current_location;
-
 const YYLTYPE& cobol_location() { return current_location; }
 
-static inline YYLTYPE
+static inline void
 location_set( const YYLTYPE& loc ) {
-  return current_location = loc;
+  current_location = loc;
+  gcc_location_set(loc);
 }
 
 static void statement_begin( const YYLTYPE& loc, int token );
diff --git a/gcc/cobol/scan.l b/gcc/cobol/scan.l
index 2fc4aea..ba4c044 100644
--- a/gcc/cobol/scan.l
+++ b/gcc/cobol/scan.l
@@ -83,10 +83,13 @@ NONWORD   [^[:alnum:]$_-]+
 
 SPC  [[:space:]]+
 OSPC [[:space:]]*
+BLANK  [[:blank:]]+
+OBLANK [[:blank:]]*
 EOL  \r?\n
 BLANK_EOL   [[:blank:]]*{EOL}
 BLANK_OEOL  [[:blank:]]*{EOL}?
 
+PICTURE   [^[:space:]]+
 
 DOTSEP    [.]+[[:space:]]
 DOTEOL    [[:blank:]]*[.]{BLANK_EOL}
@@ -160,7 +163,7 @@ COMMA    [,;][[:blank:]]*
 ISNT	 (IS{SPC})?NOT
 
 
-COMMENTARY DATE-COMPILED|DATE-WRITTEN|INSTALLATION|SECURITY
+COMMENTARY AUTHOR|DATE-COMPILED|DATE-WRITTEN|INSTALLATION|SECURITY
 
 SORT_MERGE    SORT(-MERGE)?
 
@@ -174,7 +177,7 @@ SIZE_ERROR (ON[[[:space:]]+)?SIZE[[:space:]]+ERROR
 VARTYPE  NUMERIC|ALPHABETIC|ALPHABETIC_LOWER|ALPHABETIC_UPPER|DBCS|KANJI
 NAMTYP   {NAME}|{VARTYPE}
 
-NL       [[:blank:]]*\r?\n[[:blank:]]*
+NL       [[:blank:]]*{EOL}[[:blank:]]*
 
 PUSH_FILE \f?[#]FILE{SPC}PUSH{SPC}[^\f]+\f
 POP_FILE  \f?[#]FILE{SPC}POP\f
@@ -182,7 +185,7 @@ LINE_DIRECTIVE ^[#]line{SPC}[[:alnum:]]+{SPC}[""''].+\n
 
 %x procedure_div ident_state addr_of function classify
 %x program_id_state comment_entries
-%x author_state date_state field_level field_state dot_state
+%x date_state field_level field_state dot_state
 %x numeric_state name_state
 %x quoted1 quoted2 quoteq
 %x picture picture_count integer_count
@@ -238,30 +241,23 @@ WORKING-STORAGE{SPC}SECTION {
                 yy_push_state(field_state);
                 return WORKING_STORAGE_SECT; }
 LOCAL-STORAGE{SPC}SECTION {
-                yy_push_state(field_state);
-                return LOCAL_STORAGE_SECT; }
-WORKING-STORAGE {
-                return WORKING_STORAGE; }
-LOCAL-STORAGE {
-                return LOCAL_STORAGE; }
-SCREEN {
-                return SCREEN; }
+                  yy_push_state(field_state);
+                  return LOCAL_STORAGE_SECT; }
+WORKING-STORAGE	{ return WORKING_STORAGE; }
+LOCAL-STORAGE 	{ return LOCAL_STORAGE; }
+SCREEN 		{ return SCREEN; }
 
 LINKAGE{SPC}SECTION {
                 yy_push_state(field_state);
                 return LINKAGE_SECT; }
 
-FUNCTION-ID		{ yy_push_state(ident_state);
-                  	  yy_push_state(program_id_state);
-                  	  yy_push_state(name_state); return FUNCTION; }
-
-PROGRAM-ID		{ yy_push_state(ident_state);
-                  	  yy_push_state(program_id_state);
-                  	  yy_push_state(name_state); return PROGRAM_ID; }
+FUNCTION-ID{OSPC}{DOTSEP}?	{ yy_push_state(ident_state);
+                	  	  yy_push_state(program_id_state);
+                  		  yy_push_state(name_state); return FUNCTION; }
 
-PROGRAM-ID/{DOTEOL}	{ yy_push_state(ident_state);
-                  	  yy_push_state(name_state);
-                  	  yy_push_state(dot_state); return PROGRAM_ID; }
+PROGRAM-ID{OSPC}{DOTSEP}?	{ yy_push_state(ident_state);
+	                  	  yy_push_state(program_id_state);
+        	          	  yy_push_state(name_state); return PROGRAM_ID; }
 
 PROCEDURE{SPC}DIVISION	{ yy_push_state(procedure_div);
                                   return PROCEDURE_DIV; }
@@ -272,30 +268,18 @@ PROCEDURE{SPC}DIVISION	{ yy_push_state(procedure_div);
 }
 
 <ident_state>{
+  {BLANK_OEOL}
   ID(ENTIFICATION)?{SPC}DIVISION	{ myless(0); yy_pop_state(); }
+  (ENVIRONMENT|DATA|PROCEDURE){SPC}DIVISION {
+					  myless(0); yy_pop_state(); }
+  OPTIONS				{ myless(0); yy_pop_state(); }
+
   AS{SPC}[""] 	{ yy_push_state(quoted2); return AS; }
   AS{SPC}[''] 	{ yy_push_state(quoted1); return AS; }
   IS		{ pop_return IS; }
 
-  OPTIONS		{ yy_pop_state(); myless(0); }
-  [[:blank:]]*(ENVIRONMENT|DATA|PROCEDURE){SPC}DIVISION/[[:space:].] {
-                                  yy_pop_state(); myless(0); }
-  [[:blank:]]*AUTHOR[[:blank:].]+{EOL}? {
-				  // Might not have an EOL, but stop on one.
-                                  yy_push_state(author_state); }
-
-  {DOTEOL}
-
   {COMMENTARY} { BEGIN(comment_entries); }
 }
-<author_state>{
-    [[:blank:]]+
-    ^{BLANK_EOL}
-    [^\r\n]+                   { yy_pop_state();
-                                  yylval.string = xstrdup(yytext);
-                               }
-}
-
 
 <INITIAL>{
   COBOL		{ return COBOL; }
@@ -307,6 +291,15 @@ PROCEDURE{SPC}DIVISION	{ yy_push_state(procedure_div);
                             yy_push_state(field_state);
                             yy_set_bol(1);
                             myless(0); }
+
+  END{SPC}PROGRAM        { yy_push_state(name_state);
+                                  return program_level() > 1?
+                                         END_SUBPROGRAM : END_PROGRAM; }
+
+  END{SPC}FUNCTION        { yy_push_state(name_state);
+                                  return program_level() > 1?
+                                         END_SUBPROGRAM /*invalid*/ :
+                                         END_FUNCTION; }
 }
 
 <INITIAL,procedure_div,cdf_state>{
@@ -444,6 +437,11 @@ STDOUT		{ return STDOUT; }
 STDERR		{ return STDERR; }
 SYSERR		{ return STDERR; }
 
+ARGUMENT-NUMBER  	{ return ARGUMENT_NUMBER; }
+ARGUMENT-VALUE   	{ return ARGUMENT_VALUE; }
+ENVIRONMENT-NAME 	{ return ENVIRONMENT_NAME; }
+ENVIRONMENT-VALUE	{ return ENVIRONMENT_VALUE; }
+
 CANCEL		{ return CANCEL; }
 COMMIT		{ return COMMIT; }
 COMMON		{ return COMMON; }
@@ -541,7 +539,7 @@ SECTION{SPC}[+-]?{INTEGERZ}/{OSPC}{DOTSEP}	{
 				  auto eotext = yytext + yyleng;
 				  auto p = std::find_if(yytext, eotext, fisspace);
 				  p = std::find_if(p, eotext, nonspace);
-				  yylval.string = p;
+				  yylval.string = xstrdup(p);
 				  return SECTION;
 				}
 
@@ -968,7 +966,9 @@ USE({SPC}FOR)?		{ return USE; }
                               return NUMSTR;
                            }
 
-  PIC(TURE)?({SPC}IS)?[[:space:]]{BLANK_OEOL} {
+  PIC(TURE)?({SPC}IS)?{SPC}{PICTURE} {
+				  auto pos = validate_picture();
+				  myless(pos);
 				  yy_push_state(picture); return PIC; }
 
   ANY				{ return ANY; }
@@ -1150,7 +1150,7 @@ USE({SPC}FOR)?		{ return USE; }
 	    		  yy_push_state(hex_state); }
   N?X{nonseq}		{ dbgmsg("invalid hexadecimal value: %s", yytext);
 			  return NO_CONDITION; }
-  [[:blank:]]*\r?\n     {}
+  [[:blank:]]*{EOL}     {}
 
   WORKING-STORAGE{SPC}SECTION { return WORKING_STORAGE_SECT; }
   LOCAL-STORAGE{SPC}SECTION   { return LOCAL_STORAGE_SECT; }
@@ -1220,7 +1220,7 @@ USE({SPC}FOR)?		{ return USE; }
   {NP}V?/[,.]?     { yylval.number = ndigit(yyleng);     return picset(PIC_P); }
   {N9}*V/{N9}*     { yylval.number = ndigit(yyleng - 1); return picset(NINEV); }
   {N9}/{N9}*[,.]?  { yylval.number = ndigit(yyleng);     return picset(NINES); }
-  P+/[,.]?\r?\n    { yylval.number = yyleng;             return picset(PIC_P); }
+  P+/[,.]?{EOL}    { yylval.number = yyleng;             return picset(PIC_P); }
 
   1{1,31}/({COUNT}|[(]{NAME}[)]) {
                          yy_push_state(picture_count);
@@ -1319,7 +1319,7 @@ USE({SPC}FOR)?		{ return USE; }
   [""]{SPC}[&]{SPC}[""''] {
 		        if( yytext[yyleng - 1] == '\'' ) BEGIN(quoted1);
 		      }
-  [""]-{OSPC}(\r?\n{OSPC})+[""] /* continue ...  */
+  [""]-{OSPC}({EOL}{OSPC})+[""] /* continue ...  */
   [""]                {
       			char *s = xstrdup(tmpstring? tmpstring : "\0");
                         yylval.literal.set_data(strlen(s), s);
@@ -1336,7 +1336,7 @@ USE({SPC}FOR)?		{ return USE; }
   ['']{SPC}[&]{SPC}[""''] {
 		        if( yytext[yyleng - 1] == '"' ) BEGIN(quoted2);
 		      }
-  ['']-{OSPC}(\r?\n{OSPC})+[''] /* continue ...  */
+  ['']-{OSPC}({EOL}{OSPC})+[''] /* continue ...  */
   ['']                {
       			char *s = xstrdup(tmpstring? tmpstring : "\0");
                         yylval.literal.set_data(strlen(s), s);
@@ -1384,45 +1384,36 @@ USE({SPC}FOR)?		{ return USE; }
 }
 
 <program_id_state>{
-  ^[[:blank:]]+
-  ^{BLANK_EOL}
+  {BLANK_OEOL}
   (IS)?[[:space:]]
+  AS/{SPC} 	{ myless(0); yy_pop_state(); } /* => ident_state */
 
-  COMMON/[.]|{SPC}[[:alnum:].] 	{ return COMMON; }
-  INITIAL/[.]|{SPC}[[:alnum:].]	{ return INITIAL_kw; }
-  RECURSIVE			{ return RECURSIVE; }
-  PROGRAM/[.]|{SPC}[[:alnum:].] { return PROGRAM_kw; }
-
-  INITIAL	{ pop_return INITIAL_kw; }
-  COMMON	{ pop_return COMMON; }
-  PROGRAM 	{ pop_return PROGRAM; }
+  INITIAL	{ return INITIAL_kw; }
+  COMMON	{ return COMMON; }
+  RECURSIVE	{ return RECURSIVE; }
+  PROGRAM 	{ return PROGRAM_kw; }
 
-  AS/{SPC} 	{ myless(0); yy_pop_state(); } /* => ident_state */
-  [[:blank:]]*{DOTSEP}[[:blank:].]+{EOL} { pop_return '.'; }
-  {DOTEOL}		 		 { pop_return '.'; }
+  {DOTSEP}	{ pop_return '.'; }
 }
 
-<name_state>{
-  ^[[:blank:]]+
-  ^{BLANK_EOL}
-  {NAME}		|
-  {NAME}/{OSPC}[.]	{ yy_pop_state();
-    			  yylval.string = xstrdup(yytext); return NAME; }
+<name_state>{           /* Either pop from here, or let the quoted state pop */
+  {BLANK_OEOL}
 
+  {NAME}	        { yy_pop_state();
+			  yylval.string = xstrdup(yytext);
+			  return NAME; 
+			}
   Z?['']                { yylval.literal.set_prefix(yytext, yyleng-1);
-			  yy_push_state(quoted1); }
+			  BEGIN(quoted1); }
   Z?[""]                { yylval.literal.set_prefix(yytext, yyleng-1);
-			  yy_push_state(quoted2); }
-
-  [.]/[[:blank:]]+.     { return *yytext; }
+			  BEGIN(quoted2); }
 
-  [[:blank:]]*{DOTSEP}[[:blank:].]+{EOL} {
-			  yy_pop_state(); myless(0); }
-  {DOTEOL}             	{ yy_pop_state(); myless(0); }
+  .             	{ myless(0); yy_pop_state();
+			  /* Should not happen for valid inputs. */ }
 }
 <dot_state>{
   [[:blank:]]*[.][[:blank:].]+{EOL} 	{ pop_return '.'; }
-  [[:blank:]]*[.]			{ pop_return '.'; }
+  [[:blank:]]*[.]+			{ pop_return '.'; }
 }
 
 <date_state>{
@@ -1645,9 +1636,9 @@ B-SHIFT-RC
 
   FUNCTION 			{ yy_push_state(function); return FUNCTION; }
 
-  SECTION{OSPC}[.]{SPC}/USE[[:space:]]	{ yylval.string = NULL; return SECTION; }
+  SECTION{OSPC}[.]+{SPC}/USE[[:space:]]	{ yylval.string = NULL; return SECTION; }
 
-  [.]({SPC}(EJECT|SKIP[123]))*{SPC}EXIT{OSPC}/{DOTSEP} {
+  [.]+({SPC}(EJECT|SKIP[123]))*{SPC}EXIT{OSPC}/{DOTSEP} {
                    // EXIT format-1 is a "continue" statement
                  }
   {NAME}/{OSPC}{DOTSEP} {
@@ -1682,16 +1673,17 @@ B-SHIFT-RC
 		    p += 2;
 		    while( ISSPACE(*p) ) p++;
 		    cbl_name_t name2;
-		    std::transform( p, p + sizeof(name2), name2,
-				    []( char ch ) {
-				      switch(ch) {
-				      case '-':
-				      case '_': return ch;
-				      default:
-					if( ISALNUM(ch) ) return ch;
-				      }
-				      return '\0';
-				    } );
+                    const char *pend = p + sizeof(name2);
+                    char *pout = name2;
+                    while( p < pend ) {
+                      char ch = *p++;
+                      if( ISALNUM(ch) || ch == '-' || ch == '_' )  {
+                        *pout++ = ch;
+                      } else {
+                        *pout++  = '\0';
+                        break;
+                      }
+                    }
 		    symbol_elem_t *e = symbol_file(PROGRAM, name2);
 		    /*
 		     * For NAME IN FILENAME, we want the parser to handle it.
@@ -2052,7 +2044,7 @@ BASIS		{ yy_push_state(basis); return BASIS; }
 		  return symbol_file(PROGRAM, yytext)? FILENAME : NAME;
 		}
   [[:blank:]]+
-  \r?\n		{ yy_pop_state(); }
+  {EOL}		{ yy_pop_state(); }
 }
 
 <raising>{
@@ -2073,49 +2065,62 @@ BASIS		{ yy_push_state(basis); return BASIS; }
 		    if( include_debug() ) myless(7);
 		  }
 		}
-  ^[ ]*>>{OSPC}IF		{ yy_push_state(cdf_state); return CDF_IF; }
-  ^[ ]*>>{OSPC}ELSE	 	{ return CDF_ELSE; }
-  ^[ ]*>>{OSPC}END-IF	 	{ return CDF_END_IF; }
+  ^[ ]*>>{OBLANK}IF		{ yy_push_state(cdf_state); return CDF_IF; }
+  ^[ ]*>>{OBLANK}ELSE	 	{ return CDF_ELSE; }
+  ^[ ]*>>{OBLANK}END-IF	 	{ return CDF_END_IF; }
 
-  ^[ ]*[$]{OSPC}IF		{ if( ! dialect_mf() )  {
+  ^[ ]*[$]{OBLANK}IF		{ if( ! dialect_mf() )  {
 				    dialect_error(yylloc,  yytext, "mf");
 				  }
 				  yy_push_state(cdf_state); return CDF_IF; }
-  ^[ ]*[$]{OSPC}ELSE	 	{ if( ! dialect_mf() )  {
+  ^[ ]*[$]{OBLANK}ELSE	 	{ if( ! dialect_mf() )  {
 				    dialect_error(yylloc,  yytext, "mf");
 				  }
 				  return CDF_ELSE; }
-  ^[ ]*[$]{OSPC}END	 	{ if( ! dialect_mf() ) {
+  ^[ ]*[$]{OBLANK}END	 	{ if( ! dialect_mf() ) {
 				    dialect_error(yylloc,  yytext, "mf");
 				  }
 				  return CDF_END_IF; }
 
-  ^[ ]*[$]{OSPC}SET({SPC}CONSTANT)? {
+  ^[ ]*[$]{OBLANK}SET({SPC}CONSTANT)? {
 				  if( ! dialect_mf() ) dialect_error(yylloc, yytext, "mf");
 				  yy_push_state(cdf_state); return CDF_DEFINE; }
 
-  ^[ ]*>>{OSPC}EVALUATE		{ return CDF_EVALUATE; }
-  ^[ ]*>>{OSPC}WHEN		{ return CDF_WHEN; }
-  ^[ ]*>>{OSPC}END-EVALUATE	{ return CDF_END_EVALUATE; }
+  ^[ ]*>>{OBLANK}EVALUATE		{ return CDF_EVALUATE; }
+  ^[ ]*>>{OBLANK}WHEN		{ return CDF_WHEN; }
+  ^[ ]*>>{OBLANK}END-EVALUATE	{ return CDF_END_EVALUATE; }
+
+  ^[ ]*>>{OBLANK}CALL-CONVENTION{BLANK}C		{ return CALL_VERBATIM; }
+  ^[ ]*>>{OBLANK}CALL-CONVENTION{BLANK}COBOL	{ return CALL_COBOL; }
+  ^[ ]*>>{OBLANK}CALL-CONVENTION{BLANK}VERBATIM	{ return CALL_VERBATIM; }
+
+  ^[ ]*>>{OBLANK}DEFINE	{ yy_push_state(cdf_state); return CDF_DEFINE; }
+  ^[ ]*>>{OBLANK}DISPLAY 	{ return CDF_DISPLAY; }
+  ^[ ]*>>{OBLANK}TURN 	{ yy_push_state(exception); return TURN; }
+  ^[ ]*>>{OBLANK}COBOL-WORDS	{ yy_push_state(cobol_words); return COBOL_WORDS; }
 
-  ^[ ]*>>{OSPC}CALL-CONVENTION{SPC}C		{ return CALL_VERBATIM; }
-  ^[ ]*>>{OSPC}CALL-CONVENTION{SPC}COBOL	{ return CALL_COBOL; }
-  ^[ ]*>>{OSPC}CALL-CONVENTION{SPC}VERBATIM	{ return CALL_VERBATIM; }
+  ^[ ]*>>{OBLANK}SOURCE{BLANK}FORMAT	{ return SOURCE_FORMAT; }
 
-  ^[ ]*>>{OSPC}DEFINE	{ yy_push_state(cdf_state); return CDF_DEFINE; }
-  ^[ ]*>>{OSPC}DISPLAY 	{ return CDF_DISPLAY; }
-  ^[ ]*>>{OSPC}TURN 	{ yy_push_state(exception); return TURN; }
-  ^[ ]*>>{OSPC}COBOL-WORDS	{ yy_push_state(cobol_words); return COBOL_WORDS; }
+  ^[ ]*>>{OBLANK}PUSH 		{ return CDF_PUSH; }
+  ^[ ]*>>{OBLANK}POP 		{ return CDF_POP; }
 
-  ^[ ]*>>{OSPC}{NAME} 	{
+  ^[ ]*>>{OBLANK}{NAME} 	{
 			  error_msg(yylloc, "unknown CDF token: %s", yytext);
 			}
 
+
   OTHER		{ return OTHER; }
   OVERRIDE	{ return OVERRIDE; }
   PARAMETER	{ return PARAMETER_kw; }
   THRU		{ return THRU; }
   TRUE		{ return TRUE_kw; }
+
+  ALL			{ return ALL; }
+  CALL-CONVENTION	{ return CALL_CONVENTION; }
+  COBOL-WORDS		{ return COBOL_WORDS; }
+  DEFINE		{ return CDF_DEFINE; }
+  SOURCE{BLANK}FORMAT	{ return SOURCE_FORMAT; }
+
 }
 
 <cobol_words>{
@@ -2165,10 +2170,10 @@ BASIS		{ yy_push_state(basis); return BASIS; }
 
 <*>OR 				{ return OR; }
 <*>AND				{ return AND; }
-<*>{DOTSEP}[[:blank:].]+$	{ return '.'; }
+<*>{DOTSEP}			{ return '.'; }
 <*>[().=*/+&-]			{ return *yytext; }
 <*>[[:blank:]]+
-<*>\r?\n
+<*>{EOL}
 
 <*>{
   {COMMA}
@@ -2369,7 +2374,7 @@ BASIS		{ yy_push_state(basis); return BASIS; }
   POINTER	{ return POINTER; }
   POSITIVE	{ return POSITIVE; }
   PROCEDURE	{ return PROCEDURE; }
-  PROGRAM	{ return PROGRAM; }
+  PROGRAM	{ return PROGRAM_kw; }
   PROGRAM-ID	{ return PROGRAM_ID; }
   PROPERTY	{ return PROPERTY; }
   PROTOTYPE	{ return PROTOTYPE; }
@@ -2411,7 +2416,7 @@ BASIS		{ yy_push_state(basis); return BASIS; }
   SCREEN	{ return SCREEN; }
   SD		{ return SD; }
   SEARCH	{ return SEARCH; }
-  SECTION	{ return SECTION; }
+  SECTION	{ yylval.string = NULL; return SECTION; }
   SELECT	{ return SELECT; }
   SENTENCE	{ return SENTENCE; }
   SEPARATE	{ return SEPARATE; }
diff --git a/gcc/cobol/scan_ante.h b/gcc/cobol/scan_ante.h
index ea304ba..31093a6 100644
--- a/gcc/cobol/scan_ante.h
+++ b/gcc/cobol/scan_ante.h
@@ -149,7 +149,7 @@ numstr_of( const char string[], radix_t radix = decimal_e ) {
     }
     auto nx = std::count_if(input, p, fisdigit);
     if( 36 < nx ) {
-      error_msg(yylloc, "significand of %s has more than 36 digits (%zu)", input, nx);
+      error_msg(yylloc, "significand of %s has more than 36 digits (%td)", input, nx);
       return NO_CONDITION;
     }
 
@@ -356,6 +356,10 @@ static void level_found() {
   if( scanner_normal() ) parsing.need_level(false);
 }
 
+/*
+ * Trim the scanned location by the amount about to re-scanned. 
+ * Must be a macro because it expands yyless. 
+ */
 #define myless(N)				\
   do {						\
     auto n(N);					\
@@ -486,7 +490,8 @@ trim_location( int nkeep) {
          (fmt_size_t)nline, (fmt_size_t)rescan.size());
   if( nline ) {
     gcc_assert( yylloc.first_line + nline <= yylloc.last_line );
-    yylloc.last_line =- int(nline);
+    yylloc.last_line -= int(nline);
+    gcc_assert( yylloc.first_line <= yylloc.last_line );
     char *p = static_cast<char*>(memrchr(rescan.p, '\n', rescan.size()));
     yylloc.last_column = rescan.pend - ++p;
     return;
@@ -604,7 +609,9 @@ static const std::map <std::string, bint_t > binary_integers {
 
 static int
 binary_integer_usage( const char name[]) {
-  cbl_name_t uname = {};
+  // uname can't be cbl_name_t, because at this point name[] might have more
+  // than sizeof(cbl_name_t) characters.  The length check comes later.
+  char *uname = xstrdup(name);
   std::transform(name, name + strlen(name), uname, ftoupper);
 
   dbgmsg("%s:%d: checking %s in %zu keyword_aliases",
@@ -623,6 +630,7 @@ binary_integer_usage( const char name[]) {
   yylval.computational.signable = p->second.signable;
   dbgmsg("%s:%d: %s has type %d", __func__, __LINE__,
 	 uname, p->second.type );
+  free(uname);
   return p->second.token;
 }
       
@@ -689,6 +697,387 @@ picset( int token ) {
   return token;
 }
 
+/**
+## Script and data to produce picture_t::followers.
+## Based on ISO Table 10. 
+#! /usr/bin/awk -f
+
+BEGIN  {
+  str = "B0/ , . + +- +- CR/DB cs cs Z* Z* + + cs cs 9 AX S V P P 1 N E"
+  split(str, cols)
+}
+
+$1 ~ /CR|DB|cs/ { next }
+
+0 && !nlines++ {
+  for( i=0; i < length(cols); i++ ) {
+    print i, cols[i], "'" $i "'"
+  }
+}
+
+$field == "x" {
+  if( ! nout++ ) {
+    printf "%2d: %5s: \"", field, cols[field - 1]
+  }
+
+  gsub(/^ +| +$/, "", $1) 
+  printf "%s", $1
+}
+
+END {
+  if( ! nout++ ) {
+    printf "%2d: %5s: \"", field, cols[field - 1]
+  }
+  print "\""
+}
+
+B  x x x - x - - x - x x x x x x x x - x - x - x
+0  x x x - x - - x - x x x x x x x x - x - x - x
+/  x x x - x - - x - x x x x x x x x - x - x - x
+,  x x x - x - - x - x x x x x x x - - x - x
+.  x x - - x - - x - x - x - x - x
++  - - - - - - - - - - - - - - - - - - - - - - - x
++
+–
++  x x x - - - - x x x x - - x x x - - x x x
+CR x x x - - - - x x x x - - x x x - - x x x
+DB x x x - - - - x x x x - - x x x - - x x x
+cs - - - - x
+cs x x x - x - - - - x x - - - - x - - x x x
+
+Z  x x - - x - - x - x
+*  x x - - x - - x - x
+Z  x x x - x - - x - x x - - - - - - - x - x
+*  x x x - x - - x - x x - - - - - - - x - x
++  x x - - - - - x - - - x
+–  x x - - - - - x - - - x
++  x x x - - - - x - - - x x - - - - - x
+–  x x x - - - - x - - - x x - - - - - x
+cs x x - - x - - - - - - - - x
+cs x x x - x - - - - - - - - x x - - - x
+
+9  x x x x x - - x - x - x - x - x x x x - x - - x
+A  x - - - - - - - - - - - - - - x x
+X  x - - - - - - - - - - - - - - x x
+S 
+V  x x - - x - - x - x - x - x - x - x - x
+P  x x - - x - - x - x - x - x - x - x - x
+P  - - - - x - - x - - - - - - - - - x x - x
+1  - - - - - - - - - - - - - - - - - - - - - x
+N  x - - - - - - - - - - - - - - - - - - - - - x
+E  x x x - x - - - - - - - - - - x
+**/
+
+class picture_t {
+  static const char dot = '.', comma = ',';
+
+  typedef std::vector<std::string> followings_t;
+  static const std::map <char, followings_t> followers;
+  
+  const char * const begin;
+  const char *p, *pend; 
+  size_t pos;
+  struct exclusions_t { // Nonzero if set, > 1 is false.
+    // crdb means CR/DB or +/-.
+    // pluses means 2 or more consecutive '+'.
+    // minuses means 2 or more consecutive '-'.
+    // "21) The symbol 'Z' and the symbol '*' are mutually exclusive "
+    // stars means '*' or Z.
+    unsigned short int crdb, currency, dot, pluses, minuses, stars, zzz;
+    exclusions_t()
+      : crdb(0), currency(0), dot(0), pluses(0), minuses(0), stars(0)
+    {}
+  } exclusions;
+  YYLTYPE loc;
+  
+  bool is_crdb() const { // input must be uppercase for CR/DB
+    if( p[0] == 'C' || p[0] == 'D' ) {
+      char input[3] = { p[0], p[1] };
+      return ( 0 == strcmp(input, "CR") || 0 == strcmp(input, "DB") );
+    }
+    return false;
+  }
+
+  const char * match_paren( const char *paren ) const {
+    gcc_assert(paren[0] == '(');  // start with opening paren
+    paren = std::find_if( paren, pend,
+                          []( char ch ) {
+                            return ch == '(' || ch == ')';
+                          } );
+    if( *paren == '(' ) return nullptr; // no nesting
+    if( paren == pend ) return nullptr;
+    return ++paren;
+  }
+
+  const char * next_not( char ch ) const {
+    return std::find_if( p, pend,
+                         [ch = TOUPPER(ch)]( char next ) {
+                           return ch != next;
+                         } );
+  }
+
+  const char * valid_next( const char *p, const std::string& valid ) const {
+    if( p == pend || p + 1 == pend ) return pend;
+    if( p[1] == '(' ) {
+      return match_paren(++p);
+    }
+    auto pv = std::find(valid.begin(), valid.end(), TOUPPER(p[1]));
+    return pv != valid.end()? ++p : nullptr;
+  }
+  const char * valid_next( const char *p,
+                           bool first = true, char ch = '\0' ) const {
+    if( p == pend || p + 1 == pend ) return pend;
+    if( p[0] == '(' ) {
+      if( (p = match_paren(p)) == nullptr ) return nullptr;
+    }
+    if( p[0] == '(' ) return nullptr;  // consecutive parentheses
+    
+    int index = first? 0 : 1;
+    if( !ch ) ch = *p;   // use current character unless overridden
+    auto valid = followers.find(TOUPPER(ch));
+    if( valid == followers.end() ) {
+      YYLTYPE loc(yylloc);
+      loc.first_column += int(p - begin);
+      error_msg( loc, "PICTURE: strange character %qc, giving up", ch );
+      return nullptr;
+    }
+    return valid_next(p, valid->second[index]);
+  }
+
+  const char * start() { // start modifies exclusions, but not p
+    auto pnext = p;
+
+    switch(TOUPPER(p[0])) {
+    case comma: case dot:
+      // use decimal_is_comma()
+      //  4:     .: "B0/,+Z*+-9E"
+      exclusions.dot++;
+      pnext = valid_next(p, "B0/,+Z*+-9E");
+      break;
+    case '+': case '-':
+      //  6:    +-: "B0/,.Z*Z*9VPPE"
+      exclusions.crdb++;
+      pnext = next_not(p[0]);
+      if( p + 1 < pnext ) {
+        exclusions.pluses++;
+      }
+      pnext = valid_next(--pnext, "B0/,.Z*Z*9VPPE");
+      break;
+    case 'Z': case '*': 
+      exclusions.stars++;
+      pnext = next_not(p[0]);
+      break;
+    case 'S':
+      // 19:     S: "9VP"
+      pnext = valid_next(p, "9VP");
+      break;
+    }
+
+    /*
+     * "For fixed editing sign control, the currency symbol, when used, shall
+     * be either the leftmost symbol in character-string-1, optionally preceded
+     * by one of the symbols '+' or '-' "
+     */
+    if( pnext ) {
+      if( p == pnext || p[0] == '+' || p[0] == '-' ) {
+        if( symbol_currency(*pnext) ) {
+          exclusions.currency++;
+          pnext = next_not(*pnext);
+          pnext = valid_next(--pnext, true, '$');
+        }
+      }
+    }
+    
+    return pnext;
+  }
+
+  const char * next() { // modify state; do not modify position
+    auto pnext = p;
+    auto loc(picture_t::loc);
+    loc.first_column += int(p - begin);
+
+    if( is_crdb() ) {
+      if( exclusions.crdb++ ) {
+        error_msg( loc, "PICTURE: CR/DB and %c/%c may appear only once", '+', '-' );
+        return nullptr;
+      }
+      if( p + 2 != pend ) { 
+        error_msg( loc, "PICTURE: CR/DB must appear at the end" );
+        return nullptr;
+      }
+      return pend;
+    }
+
+    if( symbol_currency(p[0]) ) {
+      if( false && exclusions.currency++ ) { // not enforced
+        error_msg( loc, "PICTURE: CURRENCY SYMBOL sequence may appear at most once" );
+        return nullptr;
+      }
+      return valid_next(p, ! exclusions.dot, '$');
+    }
+
+    switch(TOUPPER(p[0])) {
+    case '(':
+      return match_paren(p);
+      break;
+    case 'B': case '0': case '/':
+      pnext = valid_next(p);
+      break;
+    case comma: 
+      if( decimal_is_comma() ) {
+        if( exclusions.dot++ ) {
+          error_msg( loc, "PICTURE: %qc: may appear at most once", p[0] );
+          return nullptr;
+        }
+        pnext = valid_next(p, true, dot);
+      } else {
+        pnext = valid_next(p);
+      }
+      break;
+    case dot: 
+      if( p + 1 == pend ) {
+        pnext = pend;
+      } else {
+        if( decimal_is_comma() ) {
+          pnext = valid_next(p, true, comma );
+        } else {
+          if( exclusions.dot++ ) {
+            error_msg( loc, "PICTURE: %qc: may appear at most once", p[0] );
+            return nullptr;
+          }
+          pnext = valid_next(p);
+        }
+      }
+      break;
+
+    case '+': case '-':
+      // 7 is trailing sign; 13 & 14 are numeric.  Leading sign handled by start(). 
+      if( p + 1 == pend ) {
+        if( exclusions.crdb++ ) {
+          error_msg( loc, "PICTURE: %c/%c may appear at most once as a sign", '+', '-' );
+          return nullptr;
+        }
+        pnext = pend;
+      } else {
+        pnext = next_not(p[0]);
+        if( p + 1 < pnext ) {
+          if( false && exclusions.pluses++ )  { // not enforced
+            error_msg( loc, "PICTURE: %qc: sequence may appear at most once", p[0] );
+            return nullptr;
+          }
+        }
+        pnext = valid_next(pnext, ! exclusions.dot);
+      }
+      break;
+
+    case 'Z': case '*':
+      if( false && exclusions.stars++ ) { // not enforced 
+        error_msg( loc, "PICTURE: %qc: sequence may appear at most once", p[0] );
+        return nullptr;
+      }
+      if( (pnext = next_not(p[0])) == nullptr ) return pnext;
+      pnext = valid_next(pnext, ! exclusions.dot);
+      break;
+    case 'P':
+      pnext = valid_next(pnext, ! exclusions.dot);
+      break;
+    case '9':
+    case 'A': case 'X':
+    case 'V':
+    case '1':
+    case 'N':
+      pnext = valid_next(p);
+      break;
+    case 'E':
+      pnext = valid_next(p, "+9");
+      if( pnext && *pnext == '+' ) {
+        pnext = valid_next(p, "9");
+      }
+      break;
+    default:
+      error_msg( loc, "PICTURE: %qc: invalid character", p[0] );
+      return nullptr;
+    }
+    return pnext;
+  }
+  
+ public:
+  picture_t( const char *p, int len )
+    : begin(p)
+    , p(p), pend(p + len)
+    , loc(yylloc)
+  { 
+    assert(TOUPPER(*p) == 'P'); // as in PICTURE (or PICTURE IS)
+    // move p to start of picture string
+    while( (p = std::find_if(p, pend, fisspace)) != pend ) {
+      this->p = p = std::find_if(p, pend,
+                                 []( char ch ) { return ! fisspace(ch); } );
+    }
+    assert(this->p != pend);
+    pos = this->p - begin;
+  }
+
+  bool is_valid() {
+    if( !p ) return false;
+    if( (p = start()) == nullptr ) {
+      return false;
+    }
+
+    while( p && p < pend) {
+      p = next();
+    }
+    return p == pend;
+  }
+
+  int starts_at() const { return pos; }
+};
+
+/*
+ * The Followers map gives 1 or 2 lists of valid characters following a
+ * character, the one in the key. If there are two lists, the correct one is
+ * determined by the caller based on the state of the picture string, i.e.,
+ * what has been seen before.
+ */
+const std::map <char, picture_t::followings_t> picture_t::followers {
+  /*   B0/ */ { 'B', {"B0/,.Z*+-9AXVPNE" } },
+  /*   B0/ */ { '0', {"B0/,.Z*+-9AXVPNE" } },
+  /*   B0/ */ { '/', {"B0/,.Z*+-9AXVPNE" } },
+  /*     , */ { ',', {"B0/,.Z*+-9VPE"} },
+  /*     . */ { '.', {"B0/,Z*+-9E"} },
+  /*     +    { '+', "9" }, */
+  /*    +- */ { '+', {"B0/,.Z*9VPE", "" } },
+  /*    +- */ { '-', {"B0/,.Z*9VPE", "" } },
+  /* CR/DB    { 'C', "" }, */
+  /*    cs    { 'c', "B0/,.Z*+-9VP" }, */
+  /*    cs    { 'c', "+" }, */
+  /*    Z* */ { 'Z', {"B0/,.+Z*9VP", "B0/,+Z*"} },
+  /*    Z* */ { '*', {"B0/,.+Z*9VP", "B0/,+Z*"} },
+  /*     + */ { '+', {"B0/,.+-9VP",  "B0/,+-"} },
+  /*    cs */ { '$', {"B0/,.+9VP",   "B0/,+"} },
+  /*     9 */ { '9', {"B0/,.+9AXVPE"} },
+  /*    AX */ { 'A', {"B0/9AX"} },
+  /*    AX */ { 'X', {"B0/9AX"} },
+  /*     S */ { 'S', {"9VP"} },
+  /*     V */ { 'V', {"B0/,+Z*+-9P"} },
+  /*     P */ { 'P', {"+VP", "B0/,+Z*9P"} },
+  /*     1 */ { '1', {"1"} },
+  /*     N */ { 'N', {"B0/N"} },
+  /*     E */ { 'E', {"+9"} },
+};
+
+/*
+ * Although picture_t::is_valid return a bool, it's not used. The validation
+ * routines emit messages where the error is detected. The entire string is
+ * subsequently parsed by the parser, which might otherwise accept an invalid
+ * string, but will usually emit a message of its own.
+ */
+static int 
+validate_picture() {
+  picture_t picture(yytext, yyleng);
+  picture.is_valid();
+  return  picture.starts_at();
+}
+
 static inline bool
 is_integer_token( int *pvalue = NULL ) {
   int v, n = 0;
diff --git a/gcc/cobol/scan_post.h b/gcc/cobol/scan_post.h
index a273da9..7cf2b98 100644
--- a/gcc/cobol/scan_post.h
+++ b/gcc/cobol/scan_post.h
@@ -34,7 +34,6 @@ start_condition_str( int sc ) {
   switch(sc) {
   case INITIAL: state = "INITIAL"; break;
   case addr_of: state = "addr_of"; break;
-  case author_state: state = "author_state"; break;
   case basis: state = "basis"; break;
   case bool_state: state = "bool_state"; break;
   case cdf_state: state = "cdf_state"; break;
@@ -159,6 +158,8 @@ is_cdf_token( int token ) {
   case CDF_DISPLAY:
   case CDF_IF:       case CDF_ELSE: case CDF_END_IF:
   case CDF_EVALUATE: case CDF_WHEN: case CDF_END_EVALUATE:
+  case CDF_PUSH:
+  case CDF_POP:
     return true;
   case CALL_COBOL:
   case CALL_VERBATIM:
diff --git a/gcc/cobol/show_parse.h b/gcc/cobol/show_parse.h
index db24807..bd0e16f 100644
--- a/gcc/cobol/show_parse.h
+++ b/gcc/cobol/show_parse.h
@@ -176,11 +176,21 @@ extern bool cursor_at_sol;
                 } \
             else \
                 { \
-		fprintf(stderr, " %p:%s (%s)", (void*)b, b->name, b->type_str()); \
+                fprintf(stderr, " %p:%s (%s)", static_cast<void*>(b), b->name, b->type_str()); \
                 } \
             show_parse_sol = false; \
             } while(0);
 
+// Use this version when b is known to be valid.  This is necessary to quiet
+// cppcheck nullPointerRedundantCheck warnings
+#define SHOW_PARSE_LABEL_OK(a, b) \
+        do \
+            { \
+            fprintf(stderr, "%s", a); \
+            fprintf(stderr, " %p:%s (%s)", static_cast<void*>(b), b->name, b->type_str()); \
+            show_parse_sol = false; \
+            } while(0);
+
 #define TRACE1 if(bTRACE1)
 #define TRACE1_HEADER do \
   { \
@@ -211,6 +221,7 @@ extern bool cursor_at_sol;
 #define TRACE1_FIELD_VALUE(a, field, b) \
         do \
             { \
+            gcc_assert(field); \
             cursor_at_sol=false; \
             if ( field->type == FldConditional ) \
               {                                       \
@@ -423,13 +434,39 @@ extern bool cursor_at_sol;
     } while(0);
 
 // Use CHECK_FIELD when a should be non-null, and a->var_decl_node also should
-// by non-null:
+// by non-null.  (The useless calls to abort() are because cppcheck doesn't
+// understand that gcc_unreachable doesn't return);
+
+// Use this after doing any SHOW_PARSE stuff, to avoid cppcheck complaints
+// about nullPointerRedundantCheck 
 #define CHECK_FIELD(a)                                                  \
-        do {                                                                  \
+        do {                                                            \
         if(!a)                                                          \
             {                                                           \
-            yywarn("%s: parameter %<" #a "%> is NULL", __func__);        \
+            yywarn("%s: parameter %<" #a "%> is NULL", __func__);       \
             gcc_unreachable();                                          \
+            abort();                                                    \
+            }                                                           \
+        if( !a->var_decl_node )                                         \
+            {                                                           \
+            yywarn("%s: parameter %<" #a "%> is variable "              \
+                   "%s<%s> with NULL %<var_decl_node%>",                \
+                __func__,                                               \
+                a->name,                                                \
+                cbl_field_type_str(a->type) );                          \
+            gcc_unreachable();                                          \
+            abort();                                                    \
+            }                                                           \
+        } while(0);
+
+// This version is a bit more lax, for special cases
+#define CHECK_FIELD2(a)                                                  \
+        do {                                                            \
+        if(!a)                                                          \
+            {                                                           \
+            yywarn("%s: parameter %<" #a "%> is NULL", __func__);       \
+            gcc_unreachable();                                          \
+            abort();                                                    \
             }                                                           \
         if( !a->var_decl_node && a->type != FldConditional && a->type != FldLiteralA) \
             {                                                           \
@@ -439,15 +476,18 @@ extern bool cursor_at_sol;
                 a->name,                                                \
                 cbl_field_type_str(a->type) );                          \
             gcc_unreachable();                                          \
+            abort();                                                    \
             }                                                           \
         } while(0);
 
+
 #define CHECK_LABEL(a)                                                  \
         do{                                                             \
         if(!a)                                                          \
             {                                                           \
             yywarn("%s: parameter %<" #a "%> is NULL", __func__);       \
             gcc_unreachable();                                          \
+            abort();                                                    \
             }                                                           \
         }while(0);
 
@@ -504,6 +544,7 @@ class ANALYZE
       }
   };
 #else
+// cppcheck-suppress ctuOneDefinitionRuleViolation
 class ANALYZE
   {
   public:
diff --git a/gcc/cobol/symbols.cc b/gcc/cobol/symbols.cc
index 089c9c1..f2cd1b5 100644
--- a/gcc/cobol/symbols.cc
+++ b/gcc/cobol/symbols.cc
@@ -28,6 +28,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+// cppcheck-suppress-file duplicateBreak
+
 #include "config.h"
 #include <fstream> // Before cobol-system because it uses poisoned functions
 #include "cobol-system.h"
@@ -672,7 +674,7 @@ symbol_special( size_t program, const char name[] )
 struct symbol_elem_t *
 symbol_alphabet( size_t program, const char name[] )
 {
-  cbl_alphabet_t alphabet(YYLTYPE(), custom_encoding_e);
+  cbl_alphabet_t alphabet(YYLTYPE(), custom_encoding_e); // cppcheck-suppress syntaxError
   assert(strlen(name) < sizeof alphabet.name);
   strcpy(alphabet.name, name);
 
@@ -931,7 +933,7 @@ end_of_group( size_t igroup ) {
       if( e->program != group->program ) return isym;
       if( e->type == SymLabel ) return isym; // end of data division
       if( e->type == SymField ) {
-        const auto f = cbl_field_of(e);
+        const cbl_field_t * f = cbl_field_of(e);
         if( f->level == LEVEL77 || f->level == 66 ) return isym;
         if( f->level == 1 && f->parent != igroup ) {
           return isym;
@@ -1174,7 +1176,7 @@ static struct symbol_elem_t *
         // If an 01 record exists for the FD/SD, use its capacity as the
         // default_record capacity.
         if( p != symbols_end() ) {
-          const auto record = cbl_field_of(p);
+          const cbl_field_t * record = cbl_field_of(p);
           assert(record->level == 1);
           e = calculate_capacity(p);
           auto record_size = std::max(record->data.memsize,
@@ -1262,7 +1264,7 @@ static struct symbol_elem_t *
 
     // If group has a parent that is a record area, expand it, too.
     if( 0 < group->parent ) {
-      auto redefined = symbol_redefines(group);
+      redefined = symbol_redefines(group);
       if( redefined && is_record_area(redefined) ) {
         if( redefined->data.capacity < group->data.memsize ) {
           redefined->data.capacity = group->data.memsize;
@@ -1434,11 +1436,11 @@ cbl_field_t::attr_str( const std::vector<cbl_field_attr_t>& attrs ) const
   const char *sep = "";
   char *out = NULL;
 
-  for( auto attr : attrs ) {
+  for( auto attr_l : attrs ) {
     char *part = out;
-    if( has_attr(attr) ) {
+    if( has_attr(attr_l) ) {
       int erc = asprintf(&out, "%s%s%s",
-                         part? part : "", sep, cbl_field_attr_str(attr));
+                         part? part : "", sep, cbl_field_attr_str(attr_l));
       if( -1 == erc ) return part;
       free(part);
       sep = ", ";
@@ -1745,7 +1747,7 @@ symbols_update( size_t first, bool parsed_ok ) {
 
     bool size_invalid = field->data.memsize > 0 && symbol_redefines(field);
     if( size_invalid ) { // redefine of record area is ok
-      const auto redefined = symbol_redefines(field);
+      const cbl_field_t * redefined = symbol_redefines(field);
       size_invalid = ! is_record_area(redefined);
     }
     if( !field->is_valid() || size_invalid )
@@ -1768,8 +1770,8 @@ symbols_update( size_t first, bool parsed_ok ) {
         if( e == symbols_end() ) {
           // no field redefines the file's default record
           auto file = cbl_file_of(symbol_at(field->parent));
-          ERROR_FIELD(field, "line %d: %s lacks a file description",
-                      file->line, file->name);
+          ERROR_FIELD(field, "%s lacks a file description",
+                      file->name);
           return 0;
         }
       }
@@ -1828,7 +1830,7 @@ symbols_update( size_t first, bool parsed_ok ) {
     }
 
     // Verify REDEFINing field has no ODO components
-    const auto parent = symbol_redefines(field);
+    const cbl_field_t * parent = symbol_redefines(field);
     if( parent && !is_record_area(parent) && is_variable_length(field) ) {
       ERROR_FIELD(field, "line %d: REDEFINES field %s cannot be variable length",
                field->line, field->name);
@@ -2180,14 +2182,22 @@ symbol_table_init(void) {
   }
 
   static symbol_elem_t environs[] = {
+    { symbol_elem_t{ 0, cbl_special_name_t{0, CONSOLE_e, "CONSOLE", 0, "/dev/stdout"}} }, // stdout in DISPLAY; stdin in ACCEPT
+
+    { symbol_elem_t{ 0, cbl_special_name_t{0, STDIN_e, "STDIN", 0, "/dev/stdin"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSIN_e, "SYSIN", 0, "/dev/stdin"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSIPT_e, "SYSIPT", 0, "/dev/stdout"}} },
+    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSIPT_e, "SYSIPT", 0, "/dev/stdin"}} },
+
+    { symbol_elem_t{ 0, cbl_special_name_t{0, STDOUT_e, "STDOUT", 0, "/dev/stdout"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSOUT_e, "SYSOUT", 0, "/dev/stdout"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSLIST_e, "SYSLIST", 0, "/dev/stdout"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSLST_e, "SYSLST", 0, "/dev/stdout"}} },
+
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSPUNCH_e, "SYSPUNCH", 0, "/dev/stderr"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSPCH_e, "SYSPCH", 0, "/dev/stderr"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, CONSOLE_e, "CONSOLE", 0, "/dev/stdout"}} },
+    { symbol_elem_t{ 0, cbl_special_name_t{0, STDERR_e, "STDERR", 0, "/dev/stderr"}} },
+    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSERR_e, "SYSERR", 0, "/dev/stderr"}} },
+
     { symbol_elem_t{ 0, cbl_special_name_t{0, C01_e, "C01", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, C02_e, "C02", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, C03_e, "C03", 0, "/dev/null"}} },
@@ -2207,10 +2217,6 @@ symbol_table_init(void) {
     { symbol_elem_t{ 0, cbl_special_name_t{0, S04_e, "S04", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, S05_e, "S05", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, AFP_5A_e, "AFP-5A", 0, "/dev/null"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, STDIN_e, "STDIN", 0, "/dev/stdin"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, STDOUT_e, "STDOUT", 0, "/dev/stdout"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, STDERR_e, "STDERR", 0, "/dev/stderr"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSERR_e, "SYSERR", 0, "/dev/stderr"}} },
   };
 
   struct symbol_elem_t *p = table.elems + table.nelem;
@@ -2466,7 +2472,7 @@ symbol_typedef_add( size_t program, struct cbl_field_t *field ) {
   auto e = symbols_end() - 1;
   assert( symbols_begin() < e );
   if( e->type == SymField ) {
-    const auto f = cbl_field_of(e);
+    const cbl_field_t * f = cbl_field_of(e);
     if( f == field ) return e;
   }
 
@@ -2516,7 +2522,8 @@ symbol_field_add( size_t program, struct cbl_field_t *field )
     if( is_numeric(parent->usage) && parent->data.capacity > 0 ) {
       field->type = parent->usage;
       field->data = parent->data;
-      field->data = 0;
+      field->data = 0;  // cppcheck-suppress redundantAssignment
+      //                // cppcheck doesn't understand multiple overloaded operator=
       field->data.initial = NULL;
     }
   }
@@ -3140,7 +3147,6 @@ static cbl_field_t *
 new_temporary_impl( enum cbl_field_type_t type, const cbl_name_t name = nullptr )
 {
   extern int yylineno;
-  static int nstack, nliteral;
   static const struct cbl_field_t empty_alpha = {
                                 0, FldAlphanumeric, FldInvalid,
                                 intermediate_e, 0, 0, 0, nonarray, 0, "",
@@ -3209,8 +3215,10 @@ new_temporary_impl( enum cbl_field_type_t type, const cbl_name_t name = nullptr
 
   f->line = yylineno;
   if( is_literal(type) ) {
+    static int nliteral = 0;
     snprintf(f->name, sizeof(f->name), "_literal%d",++nliteral);
   } else {
+    static int nstack = 0;
     snprintf(f->name, sizeof(f->name), "_stack%d",++nstack);
   }
 
@@ -3724,6 +3732,12 @@ symbol_label_add( size_t program, cbl_label_t *input )
 bool
 symbol_label_section_exists( size_t eval_label_index ) {
   auto eval = symbols_begin(eval_label_index);
+  /*  cppcheck warns that the following statement depends on the order of
+      evaluation of side effects.  Since this isn't my code, and since I don't
+      think the warning can be eliminated without rewriting it, I am just
+      supprressing it.
+      -- Bob Dubner, 2025-07-14 */
+  // cppcheck-suppress unknownEvaluationOrder
   bool has_section = std::any_of( ++eval, symbols_end(),
                                [program = eval->program]( const auto& sym ) {
                                  if( program == sym.program && sym.type == SymLabel ) {
@@ -4183,7 +4197,7 @@ symbol_program_callables( size_t program ) {
     if( e->type != SymLabel ) continue;
     if( e->elem.label.type != LblProgram ) continue;
 
-    const auto prog = cbl_label_of(e);
+    const cbl_label_t * prog = cbl_label_of(e);
     if( program == symbol_index(e) && !prog->recursive ) continue;
 
     if( (self->parent == prog->parent && prog->common) ||
@@ -4237,6 +4251,11 @@ symbol_currency( char sign ) {
   if( currencies.size() == 0 ) {
     currencies['$'] = "$";
   }
+  if( sign == '\0' ) { // default
+    auto result = currencies.begin();
+    gcc_assert(result != currencies.end());
+    return result->second;
+  }
   auto result = currencies.find(sign);
   return result == currencies.end()? NULL : result->second;
 }
@@ -4345,6 +4364,26 @@ cbl_occurs_t::subscript_ok( const cbl_field_t *subscript ) const {
   return bounds.lower <= (size_t)sub && (size_t)sub <= bounds.upper;
 }
 
+const cbl_field_t * 
+symbol_unresolved_file_key( const cbl_file_t * file,
+                            const cbl_name_t key_field_name ) {
+  const symbol_elem_t *file_sym = symbol_elem_of(file);
+  size_t program = file_sym->program;
+  for( const symbol_elem_t *e = file_sym - 1; e->program == program; e-- ) {
+    if( e->type == SymFile ) break;
+    if( e->type == SymField ) {
+      auto f = cbl_field_of(e);
+      if( f->type == FldLiteralA ) break;
+      if( f->type == FldForward ) {
+        if( 0 == strcmp(key_field_name, f->name) ) {
+          return f;
+        }
+      }
+    }
+  }
+  return nullptr;
+}
+
 cbl_file_key_t::
 cbl_file_key_t( cbl_name_t name,
                 const std::list<cbl_field_t *>& fields,
@@ -4486,7 +4525,7 @@ cbl_file_key_t::deforward( size_t ifile ) {
                     if( ifield == fwd ) {
                       ERROR_FIELD(field, "line %d: %s of %s "
                                "is not defined",
-                               file->line, field->name, file->name);
+                               field->line, field->name, file->name);
                       return ifield;
                     }
 
@@ -4515,9 +4554,13 @@ cbl_file_key_t::deforward( size_t ifile ) {
                     // looked-up field must have same file as parent
                     if( ! (parent != NULL &&
                            symbol_index(symbol_elem_of(parent)) == ifile) ) {
-                      ERROR_FIELD(field, "line %d: %s of %s "
-                               "is not defined in file description",
-                               file->line, field->name, file->name);
+                      const cbl_field_t *undefined =
+                        symbol_unresolved_file_key(file, field->name);
+                      int lineno = undefined? undefined->line : file->line;
+                      ERROR_FIELD(undefined? undefined : field,
+                                  "line %d: %s of %s "
+                                  "is not defined in file description",
+                                  lineno, field->name, file->name);
                     }
                     return ifield;
                   } );
@@ -4630,9 +4673,11 @@ file_status_status_of( file_status_t status ) {
   size_t n = COUNT_OF(file_status_fields);
   const file_status_field_t *fs, key { status };
 
-  fs = (file_status_field_t*)lfind( &key, file_status_fields,
-                                    &n, sizeof(*fs), cbl_file_status_cmp );
-
+  fs = static_cast<file_status_field_t*>(lfind( &key,
+                                                file_status_fields,
+                                                &n,
+                                                sizeof(*fs),
+                                                cbl_file_status_cmp ));
   return fs? (long)fs->status : -1;
 }
 
diff --git a/gcc/cobol/symbols.h b/gcc/cobol/symbols.h
index 0b72b5c..c8ae32f 100644
--- a/gcc/cobol/symbols.h
+++ b/gcc/cobol/symbols.h
@@ -1894,6 +1894,10 @@ const cbl_label_t * symbol_program_local( const char called[] );
 
 bool redefine_field( cbl_field_t *field );
 
+const cbl_field_t * 
+symbol_unresolved_file_key( const cbl_file_t * file,
+                            const cbl_name_t key_field_name );
+
 static inline struct cbl_section_t *
 cbl_section_of( struct symbol_elem_t *e ) {
   assert(e && e->type == SymDataSection);
@@ -2387,9 +2391,169 @@ enum cbl_call_convention_t {
   cbl_call_cobol_e = 'N', // native
 };
 
+int keyword_tok( const char * text, bool include_intrinsics = false );
+int redefined_token( const cbl_name_t name );
+
+class current_tokens_t {
+  class tokenset_t {
+    // token_names is initialized from a generated header file. 
+    std::vector<const char *>token_names;  // position indicates token value
+    std::map <std::string, int> tokens;    // aliases
+    std::set<std::string> cobol_words;  // Anything in COBOL-WORDS may appear only once. 
+  public:
+    static std::string
+    lowercase( const cbl_name_t name ) {
+      cbl_name_t lname;
+      std::transform(name, name + strlen(name) + 1, lname, ftolower);
+      return lname;
+    }
+    static std::string
+    uppercase( const cbl_name_t name ) {
+      cbl_name_t uname;
+      std::transform(name, name + strlen(name) + 1, uname, ftoupper);
+      return uname;
+    }
+
+  public:
+    tokenset_t();
+    int find( const cbl_name_t name, bool include_intrinsics );
+
+    bool equate( const YYLTYPE& loc, int token,
+                 const cbl_name_t name, const cbl_name_t verb = "EQUATE") {
+      auto lname( lowercase(name) );
+      auto cw = cobol_words.insert(lname);
+      if( ! cw.second ) {
+        error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
+        return false;
+      }
+      auto p = tokens.find(lowercase(name));
+      bool fOK = p == tokens.end();
+      if( fOK ) { // name not already in use
+        tokens[lname] = token;
+        dbgmsg("%s:%d: %d has alias %s", __func__, __LINE__, token, name);
+      } else {
+        error_msg(loc, "%s: %s already defined as a token", verb, name);
+      }
+      return fOK;
+    }
+    bool undefine( const YYLTYPE& loc,
+                   const cbl_name_t name, const cbl_name_t verb = "UNDEFINE" ) {
+      auto lname( lowercase(name) );
+      auto cw = cobol_words.insert(lname);
+      if( ! cw.second ) {
+        error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
+        return false;
+      }
+
+      // Do not erase generic, multi-type tokens COMPUTATIONAL and BINARY_INTEGER.
+      if( binary_integer_usage_of(name) ) {
+        dbgmsg("%s:%d: generic %s remains valid as a token", __func__, __LINE__, name);
+        return true;
+      }
+
+      auto p = tokens.find(lname);
+      bool fOK = p != tokens.end();
+      if( fOK ) { // name in use
+        tokens.erase(p);
+      } else {
+        error_msg(loc, "%s: %s not defined as a token", verb, name);
+      }
+      dbgmsg("%s:%d: %s removed as a valid token name", __func__, __LINE__, name);
+      return fOK;
+    }
+  
+    bool substitute( const YYLTYPE& loc,
+                     const cbl_name_t extant, int token, const cbl_name_t name ) {
+      return
+        equate( loc, token, name, "SUBSTITUTE" )
+        &&
+        undefine( loc, extant, "SUBSTITUTE" );
+    }
+    bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
+      auto lname( lowercase(name) );
+      auto cw = cobol_words.insert(lname);
+      if( ! cw.second ) {
+        error_msg(loc, "COBOL-WORDS RESERVE: %s may appear but once", name);
+        return false;
+      }
+      tokens[lname] = -42;
+      return true;
+    }
+    int redefined_as( const cbl_name_t name ) {
+      auto lname( lowercase(name) );
+      if( cobol_words.find(lname) != cobol_words.end() ) {
+        auto p = tokens.find(lname);
+        if( p != tokens.end() ) {
+          return p->second;
+        }
+      }
+      return 0;
+    }
+    const char * name_of( int tok ) const {
+      tok -= (255 + 3);
+      gcc_assert(0 <= tok && size_t(tok) < token_names.size());
+      return tok < 0? "???" : token_names[tok];
+    }
+  };
+
+  tokenset_t tokens;
+ public:
+  current_tokens_t() {}
+  int find( const cbl_name_t name, bool include_intrinsics ) {
+    return tokens.find(name, include_intrinsics);
+  }
+  bool equate( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
+    int token; 
+    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
+      if( 0 == (token = keyword_tok(keyword)) ) {
+	error_msg(loc, "EQUATE %s: not a valid token", keyword);
+	return false;
+      }
+    }
+    auto name = keyword_alias_add(tokens.uppercase(keyword),
+				  tokens.uppercase(alias));
+    if( name != keyword ) {
+      error_msg(loc, "EQUATE: %s is already an alias for %s", alias, name.c_str());
+      return false;
+    } 
+    return tokens.equate(loc, token, alias);
+  }
+  bool undefine( const YYLTYPE& loc, cbl_name_t keyword ) {
+    return tokens.undefine(loc, keyword);
+  }
+  bool substitute( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
+    int token; 
+    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
+      if( 0 == (token = keyword_tok(keyword)) ) {
+	error_msg(loc, "SUBSTITUTE %s: not a valid token", keyword);
+	return false;
+      }
+    }
+    auto name = keyword_alias_add(tokens.uppercase(keyword),
+				  tokens.uppercase(alias));
+    if( name != keyword ) {
+      error_msg(loc, "SUBSTITUTE: %s is already an alias for %s", alias, name.c_str());
+      return false;
+    } 
+
+    dbgmsg("%s:%d: %s (%d) will have alias %s", __func__, __LINE__, keyword, token, alias);
+    return tokens.substitute(loc, keyword, token, alias);
+  }
+  bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
+    return tokens.reserve(loc, name);
+  }
+  int redefined_as( const cbl_name_t name ) {
+    return tokens.redefined_as(name);
+  }
+  const char * name_of( int tok ) const {
+    return tokens.name_of(tok);
+  }
+};
+
 cbl_call_convention_t current_call_convention();
+current_tokens_t& cdf_current_tokens();
 
-cbl_call_convention_t
+void
 current_call_convention( cbl_call_convention_t convention);
 
 class procref_base_t {
@@ -2433,9 +2597,6 @@ public:
   int line_number() const { return line; }
 };
 
-int keyword_tok( const char * text, bool include_intrinsics = false );
-int redefined_token( const cbl_name_t name );
-
 void procedure_definition_add( size_t program, const cbl_label_t *procedure );
 void procedure_reference_add( const char *sect, const char *para,
                               int line, size_t context );
@@ -2452,7 +2613,7 @@ symbol_field_same_as( cbl_field_t *tgt, const cbl_field_t *src );
 size_t symbol_file_same_record_area( std::list<cbl_file_t*>& files );
 
 bool symbol_currency_add( const char symbol[], const char sign[] = NULL );
-const char * symbol_currency( char symbol );
+const char * symbol_currency( char symbol = '\0' );
 
 const char * symbol_type_str( enum symbol_type_t type );
 const char * cbl_field_type_str( enum cbl_field_type_t type );
diff --git a/gcc/cobol/symfind.cc b/gcc/cobol/symfind.cc
index 39df2a0..8141b22 100644
--- a/gcc/cobol/symfind.cc
+++ b/gcc/cobol/symfind.cc
@@ -48,7 +48,7 @@ extern int yydebug;
 static bool
 is_data_field( symbol_elem_t& e ) {
   if( e.type != SymField ) return false;
-  const auto f = cbl_field_of(&e);
+  const cbl_field_t *f = cbl_field_of(&e);
   if( f->name[0] == '\0' ) return false;
   if( is_filler(f) ) return false;
 
@@ -129,7 +129,7 @@ finalize_symbol_map2() {
   for( auto& elem : symbol_map2 ) {
     auto& fields( elem.second );
     fields.remove_if( []( auto isym ) {
-			const auto f = cbl_field_of(symbol_at(isym));
+			const cbl_field_t *f = cbl_field_of(symbol_at(isym));
 			return f->type == FldInvalid;
 		      } );
     if( fields.empty() ) empties.insert(elem.first);
@@ -316,9 +316,9 @@ public:
       if( p != item.second.end() ) {
         // Preserve symbol's index at front of ancestor list.
         symbol_map_t::mapped_type shorter(1 + ancestors->size());
-        auto p = shorter.begin();
-        *p = item.second.front();
-        shorter.insert( ++p, ancestors->begin(), ancestors->end() );
+        auto p_l = shorter.begin();
+        *p_l = item.second.front();
+        shorter.insert( ++p_l, ancestors->begin(), ancestors->end() );
         return make_pair(item.first, shorter);
       }
     }
@@ -341,7 +341,7 @@ class in_scope {
   size_t program;
 
   static size_t prog_of( size_t program ) {
-    const auto L = cbl_label_of(symbol_at(program));
+    const cbl_label_t *L = cbl_label_of(symbol_at(program));
     return L->parent;
   }
 
@@ -430,7 +430,7 @@ symbol_match2( size_t program,
   auto plist = symbol_map2.find(key);
   if( plist != symbol_map2.end() ) {
     for( auto candidate : plist->second ) {
-      const auto e = symbol_at(candidate);
+      const symbol_elem_t *e = symbol_at(candidate);
       if( name_has_names( e, names, local ) ) {
         fields.push_back( symbol_index(e) );
       }
@@ -504,7 +504,7 @@ symbol_match( size_t program, const std::list<const char *>& names ) {
     }
     auto inserted = output.insert(*p);
     if( ! inserted.second ) {
-      yyerror("%s is not a unique reference", key.name);
+      error_msg_direct("%s is not a unique reference", key.name);
     }
   }
   return output;
diff --git a/gcc/cobol/token_names.h b/gcc/cobol/token_names.h
index 4b24fc6..ca51510 100644
--- a/gcc/cobol/token_names.h
+++ b/gcc/cobol/token_names.h
@@ -1,5 +1,5 @@
 // generated by ./token_names.h.gen ../../build/gcc/cobol/parse.h
-// Thu May  8 18:53:33 EDT 2025
+// Tue Jul  8 19:21:28 EDT 2025
 tokens = {
 	{ "identification", IDENTIFICATION_DIV }, // 258
 	{ "environment", ENVIRONMENT_DIV }, // 259
@@ -137,558 +137,566 @@ tokens = {
 	{ "cdf-evaluate", CDF_EVALUATE }, // 388
 	{ "cdf-when", CDF_WHEN }, // 389
 	{ "cdf-end-evaluate", CDF_END_EVALUATE }, // 390
-	{ "call-cobol", CALL_COBOL }, // 391
-	{ "call-verbatim", CALL_VERBATIM }, // 392
-	{ "if", IF }, // 393
-	{ "then", THEN }, // 394
-	{ "else", ELSE }, // 395
-	{ "sentence", SENTENCE }, // 396
-	{ "accept", ACCEPT }, // 397
-	{ "add", ADD }, // 398
-	{ "alter", ALTER }, // 399
-	{ "call", CALL }, // 400
-	{ "cancel", CANCEL }, // 401
-	{ "close", CLOSE }, // 402
-	{ "compute", COMPUTE }, // 403
-	{ "continue", CONTINUE }, // 404
-	{ "delete", DELETE }, // 405
-	{ "display", DISPLAY }, // 406
-	{ "divide", DIVIDE }, // 407
-	{ "evaluate", EVALUATE }, // 408
-	{ "exit", EXIT }, // 409
-	{ "filler", FILLER_kw }, // 410
-	{ "goback", GOBACK }, // 411
-	{ "goto", GOTO }, // 412
-	{ "initialize", INITIALIZE }, // 413
-	{ "inspect", INSPECT }, // 414
-	{ "merge", MERGE }, // 415
-	{ "move", MOVE }, // 416
-	{ "multiply", MULTIPLY }, // 417
-	{ "open", OPEN }, // 418
-	{ "paragraph", PARAGRAPH }, // 419
-	{ "read", READ }, // 420
-	{ "release", RELEASE }, // 421
-	{ "return", RETURN }, // 422
-	{ "rewrite", REWRITE }, // 423
-	{ "search", SEARCH }, // 424
-	{ "set", SET }, // 425
-	{ "select", SELECT }, // 426
-	{ "sort", SORT }, // 427
-	{ "sort-merge", SORT_MERGE }, // 428
-	{ "string", STRING_kw }, // 429
-	{ "stop", STOP }, // 430
-	{ "subtract", SUBTRACT }, // 431
-	{ "start", START }, // 432
-	{ "unstring", UNSTRING }, // 433
-	{ "write", WRITE }, // 434
-	{ "when", WHEN }, // 435
-	{ "abs", ABS }, // 436
-	{ "access", ACCESS }, // 437
-	{ "acos", ACOS }, // 438
-	{ "actual", ACTUAL }, // 439
-	{ "advancing", ADVANCING }, // 440
-	{ "after", AFTER }, // 441
-	{ "all", ALL }, // 442
-	{ "allocate", ALLOCATE }, // 443
-	{ "alphabet", ALPHABET }, // 444
-	{ "alphabetic", ALPHABETIC }, // 445
-	{ "alphabetic-lower", ALPHABETIC_LOWER }, // 446
-	{ "alphabetic-upper", ALPHABETIC_UPPER }, // 447
-	{ "alphanumeric", ALPHANUMERIC }, // 448
-	{ "alphanumeric-edited", ALPHANUMERIC_EDITED }, // 449
-	{ "also", ALSO }, // 450
-	{ "alternate", ALTERNATE }, // 451
-	{ "annuity", ANNUITY }, // 452
-	{ "anum", ANUM }, // 453
-	{ "any", ANY }, // 454
-	{ "anycase", ANYCASE }, // 455
-	{ "apply", APPLY }, // 456
-	{ "are", ARE }, // 457
-	{ "area", AREA }, // 458
-	{ "areas", AREAS }, // 459
-	{ "as", AS }, // 460
-	{ "ascending", ASCENDING }, // 461
-	{ "activating", ACTIVATING }, // 462
-	{ "asin", ASIN }, // 463
-	{ "assign", ASSIGN }, // 464
-	{ "at", AT }, // 465
-	{ "atan", ATAN }, // 466
-	{ "based", BASED }, // 467
-	{ "baseconvert", BASECONVERT }, // 468
-	{ "before", BEFORE }, // 469
-	{ "binary", BINARY }, // 470
-	{ "bit", BIT }, // 471
-	{ "bit-of", BIT_OF }, // 472
-	{ "bit-to-char", BIT_TO_CHAR }, // 473
-	{ "blank", BLANK }, // 474
-	{ "block", BLOCK_kw }, // 475
-	{ "boolean-of-integer", BOOLEAN_OF_INTEGER }, // 476
-	{ "bottom", BOTTOM }, // 477
-	{ "by", BY }, // 478
-	{ "byte", BYTE }, // 479
-	{ "byte-length", BYTE_LENGTH }, // 480
-	{ "cf", CF }, // 481
-	{ "ch", CH }, // 482
-	{ "changed", CHANGED }, // 483
-	{ "char", CHAR }, // 484
-	{ "char-national", CHAR_NATIONAL }, // 485
-	{ "character", CHARACTER }, // 486
-	{ "characters", CHARACTERS }, // 487
-	{ "checking", CHECKING }, // 488
-	{ "class", CLASS }, // 489
-	{ "cobol", COBOL }, // 490
-	{ "code", CODE }, // 491
-	{ "code-set", CODESET }, // 492
-	{ "collating", COLLATING }, // 493
-	{ "column", COLUMN }, // 494
-	{ "combined-datetime", COMBINED_DATETIME }, // 495
-	{ "comma", COMMA }, // 496
-	{ "command-line", COMMAND_LINE }, // 497
-	{ "command-line-count", COMMAND_LINE_COUNT }, // 498
-	{ "commit", COMMIT }, // 499
-	{ "common", COMMON }, // 500
-	{ "concat", CONCAT }, // 501
-	{ "condition", CONDITION }, // 502
-	{ "configuration", CONFIGURATION_SECT }, // 503
-	{ "contains", CONTAINS }, // 504
-	{ "content", CONTENT }, // 505
-	{ "control", CONTROL }, // 506
-	{ "controls", CONTROLS }, // 507
-	{ "convert", CONVERT }, // 508
-	{ "converting", CONVERTING }, // 509
-	{ "corresponding", CORRESPONDING }, // 510
-	{ "cos", COS }, // 511
-	{ "count", COUNT }, // 512
-	{ "currency", CURRENCY }, // 513
-	{ "current", CURRENT }, // 514
-	{ "current-date", CURRENT_DATE }, // 515
-	{ "data", DATA }, // 516
-	{ "date", DATE }, // 517
-	{ "date-compiled", DATE_COMPILED }, // 518
-	{ "date-of-integer", DATE_OF_INTEGER }, // 519
-	{ "date-to-yyyymmdd", DATE_TO_YYYYMMDD }, // 520
-	{ "date-written", DATE_WRITTEN }, // 521
-	{ "day", DAY }, // 522
-	{ "day-of-integer", DAY_OF_INTEGER }, // 523
-	{ "day-of-week", DAY_OF_WEEK }, // 524
-	{ "day-to-yyyyddd", DAY_TO_YYYYDDD }, // 525
-	{ "dbcs", DBCS }, // 526
-	{ "de", DE }, // 527
-	{ "debugging", DEBUGGING }, // 528
-	{ "decimal-point", DECIMAL_POINT }, // 529
-	{ "declaratives", DECLARATIVES }, // 530
-	{ "default", DEFAULT }, // 531
-	{ "delimited", DELIMITED }, // 532
-	{ "delimiter", DELIMITER }, // 533
-	{ "depending", DEPENDING }, // 534
-	{ "descending", DESCENDING }, // 535
-	{ "detail", DETAIL }, // 536
-	{ "direct", DIRECT }, // 537
-	{ "direct-access", DIRECT_ACCESS }, // 538
-	{ "down", DOWN }, // 539
-	{ "duplicates", DUPLICATES }, // 540
-	{ "dynamic", DYNAMIC }, // 541
-	{ "e", E }, // 542
-	{ "ebcdic", EBCDIC }, // 543
-	{ "ec", EC }, // 544
-	{ "egcs", EGCS }, // 545
-	{ "entry", ENTRY }, // 546
-	{ "environment", ENVIRONMENT }, // 547
-	{ "equal", EQUAL }, // 548
-	{ "every", EVERY }, // 549
-	{ "examine", EXAMINE }, // 550
-	{ "exhibit", EXHIBIT }, // 551
-	{ "exp", EXP }, // 552
-	{ "exp10", EXP10 }, // 553
-	{ "extend", EXTEND }, // 554
-	{ "external", EXTERNAL }, // 555
-	{ "exception-file", EXCEPTION_FILE }, // 556
-	{ "exception-file-n", EXCEPTION_FILE_N }, // 557
-	{ "exception-location", EXCEPTION_LOCATION }, // 558
-	{ "exception-location-n", EXCEPTION_LOCATION_N }, // 559
-	{ "exception-statement", EXCEPTION_STATEMENT }, // 560
-	{ "exception-status", EXCEPTION_STATUS }, // 561
-	{ "factorial", FACTORIAL }, // 562
-	{ "false", FALSE_kw }, // 563
-	{ "fd", FD }, // 564
-	{ "file-control", FILE_CONTROL }, // 565
-	{ "file", FILE_KW }, // 566
-	{ "file-limit", FILE_LIMIT }, // 567
-	{ "final", FINAL }, // 568
-	{ "finally", FINALLY }, // 569
-	{ "find-string", FIND_STRING }, // 570
-	{ "first", FIRST }, // 571
-	{ "fixed", FIXED }, // 572
-	{ "footing", FOOTING }, // 573
-	{ "for", FOR }, // 574
-	{ "formatted-current-date", FORMATTED_CURRENT_DATE }, // 575
-	{ "formatted-date", FORMATTED_DATE }, // 576
-	{ "formatted-datetime", FORMATTED_DATETIME }, // 577
-	{ "formatted-time", FORMATTED_TIME }, // 578
-	{ "form-overflow", FORM_OVERFLOW }, // 579
-	{ "free", FREE }, // 580
-	{ "fraction-part", FRACTION_PART }, // 581
-	{ "from", FROM }, // 582
-	{ "function", FUNCTION }, // 583
-	{ "generate", GENERATE }, // 584
-	{ "giving", GIVING }, // 585
-	{ "global", GLOBAL }, // 586
-	{ "go", GO }, // 587
-	{ "group", GROUP }, // 588
-	{ "heading", HEADING }, // 589
-	{ "hex", HEX }, // 590
-	{ "hex-of", HEX_OF }, // 591
-	{ "hex-to-char", HEX_TO_CHAR }, // 592
-	{ "high-values", HIGH_VALUES }, // 593
-	{ "highest-algebraic", HIGHEST_ALGEBRAIC }, // 594
-	{ "hold", HOLD }, // 595
-	{ "ibm-360", IBM_360 }, // 596
-	{ "in", IN }, // 597
-	{ "include", INCLUDE }, // 598
-	{ "index", INDEX }, // 599
-	{ "indexed", INDEXED }, // 600
-	{ "indicate", INDICATE }, // 601
-	{ "initial", INITIAL_kw }, // 602
-	{ "initiate", INITIATE }, // 603
-	{ "input", INPUT }, // 604
-	{ "installation", INSTALLATION }, // 605
-	{ "interface", INTERFACE }, // 606
-	{ "integer", INTEGER }, // 607
-	{ "integer-of-boolean", INTEGER_OF_BOOLEAN }, // 608
-	{ "integer-of-date", INTEGER_OF_DATE }, // 609
-	{ "integer-of-day", INTEGER_OF_DAY }, // 610
-	{ "integer-of-formatted-date", INTEGER_OF_FORMATTED_DATE }, // 611
-	{ "integer-part", INTEGER_PART }, // 612
-	{ "into", INTO }, // 613
-	{ "intrinsic", INTRINSIC }, // 614
-	{ "invoke", INVOKE }, // 615
-	{ "i-o", IO }, // 616
-	{ "i-o-control", IO_CONTROL }, // 617
-	{ "is", IS }, // 618
-	{ "isnt", ISNT }, // 619
-	{ "kanji", KANJI }, // 620
-	{ "key", KEY }, // 621
-	{ "label", LABEL }, // 622
-	{ "last", LAST }, // 623
-	{ "leading", LEADING }, // 624
-	{ "left", LEFT }, // 625
-	{ "length", LENGTH }, // 626
-	{ "length-of", LENGTH_OF }, // 627
-	{ "limit", LIMIT }, // 628
-	{ "limits", LIMITS }, // 629
-	{ "line", LINE }, // 630
-	{ "lines", LINES }, // 631
-	{ "line-counter", LINE_COUNTER }, // 632
-	{ "linage", LINAGE }, // 633
-	{ "linkage", LINKAGE }, // 634
-	{ "locale", LOCALE }, // 635
-	{ "locale-compare", LOCALE_COMPARE }, // 636
-	{ "locale-date", LOCALE_DATE }, // 637
-	{ "locale-time", LOCALE_TIME }, // 638
-	{ "locale-time-from-seconds", LOCALE_TIME_FROM_SECONDS }, // 639
-	{ "local-storage", LOCAL_STORAGE }, // 640
-	{ "location", LOCATION }, // 641
-	{ "lock", LOCK }, // 642
-	{ "lock-on", LOCK_ON }, // 643
-	{ "log", LOG }, // 644
-	{ "log10", LOG10 }, // 645
-	{ "lower-case", LOWER_CASE }, // 646
-	{ "low-values", LOW_VALUES }, // 647
-	{ "lowest-algebraic", LOWEST_ALGEBRAIC }, // 648
-	{ "lparen", LPAREN }, // 649
-	{ "manual", MANUAL }, // 650
-	{ "maxx", MAXX }, // 651
-	{ "mean", MEAN }, // 652
-	{ "median", MEDIAN }, // 653
-	{ "midrange", MIDRANGE }, // 654
-	{ "minn", MINN }, // 655
-	{ "multiple", MULTIPLE }, // 656
-	{ "mod", MOD }, // 657
-	{ "mode", MODE }, // 658
-	{ "module-name", MODULE_NAME }, // 659
-	{ "named", NAMED }, // 660
-	{ "nat", NAT }, // 661
-	{ "national", NATIONAL }, // 662
-	{ "national-edited", NATIONAL_EDITED }, // 663
-	{ "national-of", NATIONAL_OF }, // 664
-	{ "native", NATIVE }, // 665
-	{ "nested", NESTED }, // 666
-	{ "next", NEXT }, // 667
-	{ "no", NO }, // 668
-	{ "note", NOTE }, // 669
-	{ "nulls", NULLS }, // 670
-	{ "null", NULLS }, // 670
-	{ "nullptr", NULLPTR }, // 671
-	{ "numeric", NUMERIC }, // 672
-	{ "numeric-edited", NUMERIC_EDITED }, // 673
-	{ "numval", NUMVAL }, // 674
-	{ "numval-c", NUMVAL_C }, // 675
-	{ "numval-f", NUMVAL_F }, // 676
-	{ "occurs", OCCURS }, // 677
-	{ "of", OF }, // 678
-	{ "off", OFF }, // 679
-	{ "omitted", OMITTED }, // 680
-	{ "on", ON }, // 681
-	{ "only", ONLY }, // 682
-	{ "optional", OPTIONAL }, // 683
-	{ "options", OPTIONS }, // 684
-	{ "ord", ORD }, // 685
-	{ "order", ORDER }, // 686
-	{ "ord-max", ORD_MAX }, // 687
-	{ "ord-min", ORD_MIN }, // 688
-	{ "organization", ORGANIZATION }, // 689
-	{ "other", OTHER }, // 690
-	{ "otherwise", OTHERWISE }, // 691
-	{ "output", OUTPUT }, // 692
-	{ "packed-decimal", PACKED_DECIMAL }, // 693
-	{ "padding", PADDING }, // 694
-	{ "page", PAGE }, // 695
-	{ "page-counter", PAGE_COUNTER }, // 696
-	{ "pf", PF }, // 697
-	{ "ph", PH }, // 698
-	{ "pi", PI }, // 699
-	{ "pic", PIC }, // 700
-	{ "picture", PICTURE }, // 701
-	{ "plus", PLUS }, // 702
-	{ "present-value", PRESENT_VALUE }, // 703
-	{ "print-switch", PRINT_SWITCH }, // 704
-	{ "procedure", PROCEDURE }, // 705
-	{ "procedures", PROCEDURES }, // 706
-	{ "proceed", PROCEED }, // 707
-	{ "process", PROCESS }, // 708
-	{ "program-id", PROGRAM_ID }, // 709
-	{ "program", PROGRAM_kw }, // 710
-	{ "property", PROPERTY }, // 711
-	{ "prototype", PROTOTYPE }, // 712
-	{ "pseudotext", PSEUDOTEXT }, // 713
-	{ "quotes", QUOTES }, // 714
-	{ "quote", QUOTES }, // 714
-	{ "random", RANDOM }, // 715
-	{ "random-seed", RANDOM_SEED }, // 716
-	{ "range", RANGE }, // 717
-	{ "raise", RAISE }, // 718
-	{ "raising", RAISING }, // 719
-	{ "rd", RD }, // 720
-	{ "record", RECORD }, // 721
-	{ "recording", RECORDING }, // 722
-	{ "records", RECORDS }, // 723
-	{ "recursive", RECURSIVE }, // 724
-	{ "redefines", REDEFINES }, // 725
-	{ "reel", REEL }, // 726
-	{ "reference", REFERENCE }, // 727
-	{ "relative", RELATIVE }, // 728
-	{ "rem", REM }, // 729
-	{ "remainder", REMAINDER }, // 730
-	{ "remarks", REMARKS }, // 731
-	{ "removal", REMOVAL }, // 732
-	{ "renames", RENAMES }, // 733
-	{ "replace", REPLACE }, // 734
-	{ "replacing", REPLACING }, // 735
-	{ "report", REPORT }, // 736
-	{ "reporting", REPORTING }, // 737
-	{ "reports", REPORTS }, // 738
-	{ "repository", REPOSITORY }, // 739
-	{ "rerun", RERUN }, // 740
-	{ "reserve", RESERVE }, // 741
-	{ "restricted", RESTRICTED }, // 742
-	{ "resume", RESUME }, // 743
-	{ "reverse", REVERSE }, // 744
-	{ "reversed", REVERSED }, // 745
-	{ "rewind", REWIND }, // 746
-	{ "rf", RF }, // 747
-	{ "rh", RH }, // 748
-	{ "right", RIGHT }, // 749
-	{ "rounded", ROUNDED }, // 750
-	{ "run", RUN }, // 751
-	{ "same", SAME }, // 752
-	{ "screen", SCREEN }, // 753
-	{ "sd", SD }, // 754
-	{ "seconds-from-formatted-time", SECONDS_FROM_FORMATTED_TIME }, // 755
-	{ "seconds-past-midnight", SECONDS_PAST_MIDNIGHT }, // 756
-	{ "security", SECURITY }, // 757
-	{ "separate", SEPARATE }, // 758
-	{ "sequence", SEQUENCE }, // 759
-	{ "sequential", SEQUENTIAL }, // 760
-	{ "sharing", SHARING }, // 761
-	{ "simple-exit", SIMPLE_EXIT }, // 762
-	{ "sign", SIGN }, // 763
-	{ "sin", SIN }, // 764
-	{ "size", SIZE }, // 765
-	{ "smallest-algebraic", SMALLEST_ALGEBRAIC }, // 766
-	{ "source", SOURCE }, // 767
-	{ "source-computer", SOURCE_COMPUTER }, // 768
-	{ "special-names", SPECIAL_NAMES }, // 769
-	{ "sqrt", SQRT }, // 770
-	{ "stack", STACK }, // 771
-	{ "standard", STANDARD }, // 772
-	{ "standard-1", STANDARD_1 }, // 773
-	{ "standard-deviation", STANDARD_DEVIATION }, // 774
-	{ "standard-compare", STANDARD_COMPARE }, // 775
-	{ "status", STATUS }, // 776
-	{ "strong", STRONG }, // 777
-	{ "substitute", SUBSTITUTE }, // 778
-	{ "sum", SUM }, // 779
-	{ "symbol", SYMBOL }, // 780
-	{ "symbolic", SYMBOLIC }, // 781
-	{ "synchronized", SYNCHRONIZED }, // 782
-	{ "tally", TALLY }, // 783
-	{ "tallying", TALLYING }, // 784
-	{ "tan", TAN }, // 785
-	{ "terminate", TERMINATE }, // 786
-	{ "test", TEST }, // 787
-	{ "test-date-yyyymmdd", TEST_DATE_YYYYMMDD }, // 788
-	{ "test-day-yyyyddd", TEST_DAY_YYYYDDD }, // 789
-	{ "test-formatted-datetime", TEST_FORMATTED_DATETIME }, // 790
-	{ "test-numval", TEST_NUMVAL }, // 791
-	{ "test-numval-c", TEST_NUMVAL_C }, // 792
-	{ "test-numval-f", TEST_NUMVAL_F }, // 793
-	{ "than", THAN }, // 794
-	{ "time", TIME }, // 795
-	{ "times", TIMES }, // 796
-	{ "to", TO }, // 797
-	{ "top", TOP }, // 798
-	{ "top-level", TOP_LEVEL }, // 799
-	{ "tracks", TRACKS }, // 800
-	{ "track-area", TRACK_AREA }, // 801
-	{ "trailing", TRAILING }, // 802
-	{ "transform", TRANSFORM }, // 803
-	{ "trim", TRIM }, // 804
-	{ "true", TRUE_kw }, // 805
-	{ "try", TRY }, // 806
-	{ "turn", TURN }, // 807
-	{ "type", TYPE }, // 808
-	{ "typedef", TYPEDEF }, // 809
-	{ "ulength", ULENGTH }, // 810
-	{ "unbounded", UNBOUNDED }, // 811
-	{ "unit", UNIT }, // 812
-	{ "units", UNITS }, // 813
-	{ "unit-record", UNIT_RECORD }, // 814
-	{ "until", UNTIL }, // 815
-	{ "up", UP }, // 816
-	{ "upon", UPON }, // 817
-	{ "upos", UPOS }, // 818
-	{ "upper-case", UPPER_CASE }, // 819
-	{ "usage", USAGE }, // 820
-	{ "using", USING }, // 821
-	{ "usubstr", USUBSTR }, // 822
-	{ "usupplementary", USUPPLEMENTARY }, // 823
-	{ "utility", UTILITY }, // 824
-	{ "uuid4", UUID4 }, // 825
-	{ "uvalid", UVALID }, // 826
-	{ "uwidth", UWIDTH }, // 827
-	{ "value", VALUE }, // 828
-	{ "variance", VARIANCE }, // 829
-	{ "varying", VARYING }, // 830
-	{ "volatile", VOLATILE }, // 831
-	{ "when-compiled", WHEN_COMPILED }, // 832
-	{ "with", WITH }, // 833
-	{ "working-storage", WORKING_STORAGE }, // 834
-	{ "xml", XML }, // 835
-	{ "xmlgenerate", XMLGENERATE }, // 836
-	{ "xmlparse", XMLPARSE }, // 837
-	{ "year-to-yyyy", YEAR_TO_YYYY }, // 838
-	{ "yyyyddd", YYYYDDD }, // 839
-	{ "yyyymmdd", YYYYMMDD }, // 840
-	{ "arithmetic", ARITHMETIC }, // 841
-	{ "attribute", ATTRIBUTE }, // 842
-	{ "auto", AUTO }, // 843
-	{ "automatic", AUTOMATIC }, // 844
-	{ "away-from-zero", AWAY_FROM_ZERO }, // 845
-	{ "background-color", BACKGROUND_COLOR }, // 846
-	{ "bell", BELL }, // 847
-	{ "binary-encoding", BINARY_ENCODING }, // 848
-	{ "blink", BLINK }, // 849
-	{ "capacity", CAPACITY }, // 850
-	{ "center", CENTER }, // 851
-	{ "classification", CLASSIFICATION }, // 852
-	{ "cycle", CYCLE }, // 853
-	{ "decimal-encoding", DECIMAL_ENCODING }, // 854
-	{ "entry-convention", ENTRY_CONVENTION }, // 855
-	{ "eol", EOL }, // 856
-	{ "eos", EOS }, // 857
-	{ "erase", ERASE }, // 858
-	{ "expands", EXPANDS }, // 859
-	{ "float-binary", FLOAT_BINARY }, // 860
-	{ "float-decimal", FLOAT_DECIMAL }, // 861
-	{ "foreground-color", FOREGROUND_COLOR }, // 862
-	{ "forever", FOREVER }, // 863
-	{ "full", FULL }, // 864
-	{ "highlight", HIGHLIGHT }, // 865
-	{ "high-order-left", HIGH_ORDER_LEFT }, // 866
-	{ "high-order-right", HIGH_ORDER_RIGHT }, // 867
-	{ "ignoring", IGNORING }, // 868
-	{ "implements", IMPLEMENTS }, // 869
-	{ "initialized", INITIALIZED }, // 870
-	{ "intermediate", INTERMEDIATE }, // 871
-	{ "lc-all", LC_ALL_kw }, // 872
-	{ "lc-collate", LC_COLLATE_kw }, // 873
-	{ "lc-ctype", LC_CTYPE_kw }, // 874
-	{ "lc-messages", LC_MESSAGES_kw }, // 875
-	{ "lc-monetary", LC_MONETARY_kw }, // 876
-	{ "lc-numeric", LC_NUMERIC_kw }, // 877
-	{ "lc-time", LC_TIME_kw }, // 878
-	{ "lowlight", LOWLIGHT }, // 879
-	{ "nearest-away-from-zero", NEAREST_AWAY_FROM_ZERO }, // 880
-	{ "nearest-even", NEAREST_EVEN }, // 881
-	{ "nearest-toward-zero", NEAREST_TOWARD_ZERO }, // 882
-	{ "none", NONE }, // 883
-	{ "normal", NORMAL }, // 884
-	{ "numbers", NUMBERS }, // 885
-	{ "prefixed", PREFIXED }, // 886
-	{ "previous", PREVIOUS }, // 887
-	{ "prohibited", PROHIBITED }, // 888
-	{ "relation", RELATION }, // 889
-	{ "required", REQUIRED }, // 890
-	{ "reverse-video", REVERSE_VIDEO }, // 891
-	{ "rounding", ROUNDING }, // 892
-	{ "seconds", SECONDS }, // 893
-	{ "secure", SECURE }, // 894
-	{ "short", SHORT }, // 895
-	{ "signed", SIGNED_kw }, // 896
-	{ "standard-binary", STANDARD_BINARY }, // 897
-	{ "standard-decimal", STANDARD_DECIMAL }, // 898
-	{ "statement", STATEMENT }, // 899
-	{ "step", STEP }, // 900
-	{ "structure", STRUCTURE }, // 901
-	{ "toward-greater", TOWARD_GREATER }, // 902
-	{ "toward-lesser", TOWARD_LESSER }, // 903
-	{ "truncation", TRUNCATION }, // 904
-	{ "ucs-4", UCS_4 }, // 905
-	{ "underline", UNDERLINE }, // 906
-	{ "unsigned", UNSIGNED_kw }, // 907
-	{ "utf-16", UTF_16 }, // 908
-	{ "utf-8", UTF_8 }, // 909
-	{ "address", ADDRESS }, // 910
-	{ "end-accept", END_ACCEPT }, // 911
-	{ "end-add", END_ADD }, // 912
-	{ "end-call", END_CALL }, // 913
-	{ "end-compute", END_COMPUTE }, // 914
-	{ "end-delete", END_DELETE }, // 915
-	{ "end-display", END_DISPLAY }, // 916
-	{ "end-divide", END_DIVIDE }, // 917
-	{ "end-evaluate", END_EVALUATE }, // 918
-	{ "end-multiply", END_MULTIPLY }, // 919
-	{ "end-perform", END_PERFORM }, // 920
-	{ "end-read", END_READ }, // 921
-	{ "end-return", END_RETURN }, // 922
-	{ "end-rewrite", END_REWRITE }, // 923
-	{ "end-search", END_SEARCH }, // 924
-	{ "end-start", END_START }, // 925
-	{ "end-string", END_STRING }, // 926
-	{ "end-subtract", END_SUBTRACT }, // 927
-	{ "end-unstring", END_UNSTRING }, // 928
-	{ "end-write", END_WRITE }, // 929
-	{ "end-if", END_IF }, // 930
-	{ "thru", THRU }, // 931
-	{ "through", THRU }, // 931
-	{ "or", OR }, // 932
-	{ "and", AND }, // 933
-	{ "not", NOT }, // 934
-	{ "ne", NE }, // 935
-	{ "le", LE }, // 936
-	{ "ge", GE }, // 937
-	{ "pow", POW }, // 938
-	{ "neg", NEG }, // 939
+	{ "call-convention", CALL_CONVENTION }, // 391
+	{ "call-cobol", CALL_COBOL }, // 392
+	{ "call-verbatim", CALL_VERBATIM }, // 393
+	{ "cdf-push", CDF_PUSH }, // 394
+	{ "cdf-pop", CDF_POP }, // 395
+	{ "source-format", SOURCE_FORMAT }, // 396
+	{ "if", IF }, // 397
+	{ "then", THEN }, // 398
+	{ "else", ELSE }, // 399
+	{ "sentence", SENTENCE }, // 400
+	{ "accept", ACCEPT }, // 401
+	{ "add", ADD }, // 402
+	{ "alter", ALTER }, // 403
+	{ "call", CALL }, // 404
+	{ "cancel", CANCEL }, // 405
+	{ "close", CLOSE }, // 406
+	{ "compute", COMPUTE }, // 407
+	{ "continue", CONTINUE }, // 408
+	{ "delete", DELETE }, // 409
+	{ "display", DISPLAY }, // 410
+	{ "divide", DIVIDE }, // 411
+	{ "evaluate", EVALUATE }, // 412
+	{ "exit", EXIT }, // 413
+	{ "filler", FILLER_kw }, // 414
+	{ "goback", GOBACK }, // 415
+	{ "goto", GOTO }, // 416
+	{ "initialize", INITIALIZE }, // 417
+	{ "inspect", INSPECT }, // 418
+	{ "merge", MERGE }, // 419
+	{ "move", MOVE }, // 420
+	{ "multiply", MULTIPLY }, // 421
+	{ "open", OPEN }, // 422
+	{ "paragraph", PARAGRAPH }, // 423
+	{ "read", READ }, // 424
+	{ "release", RELEASE }, // 425
+	{ "return", RETURN }, // 426
+	{ "rewrite", REWRITE }, // 427
+	{ "search", SEARCH }, // 428
+	{ "set", SET }, // 429
+	{ "select", SELECT }, // 430
+	{ "sort", SORT }, // 431
+	{ "sort-merge", SORT_MERGE }, // 432
+	{ "string", STRING_kw }, // 433
+	{ "stop", STOP }, // 434
+	{ "subtract", SUBTRACT }, // 435
+	{ "start", START }, // 436
+	{ "unstring", UNSTRING }, // 437
+	{ "write", WRITE }, // 438
+	{ "when", WHEN }, // 439
+	{ "argument-number", ARGUMENT_NUMBER }, // 440
+	{ "argument-value", ARGUMENT_VALUE }, // 441
+	{ "environment-name", ENVIRONMENT_NAME }, // 442
+	{ "environment-value", ENVIRONMENT_VALUE }, // 443
+	{ "abs", ABS }, // 444
+	{ "access", ACCESS }, // 445
+	{ "acos", ACOS }, // 446
+	{ "actual", ACTUAL }, // 447
+	{ "advancing", ADVANCING }, // 448
+	{ "after", AFTER }, // 449
+	{ "all", ALL }, // 450
+	{ "allocate", ALLOCATE }, // 451
+	{ "alphabet", ALPHABET }, // 452
+	{ "alphabetic", ALPHABETIC }, // 453
+	{ "alphabetic-lower", ALPHABETIC_LOWER }, // 454
+	{ "alphabetic-upper", ALPHABETIC_UPPER }, // 455
+	{ "alphanumeric", ALPHANUMERIC }, // 456
+	{ "alphanumeric-edited", ALPHANUMERIC_EDITED }, // 457
+	{ "also", ALSO }, // 458
+	{ "alternate", ALTERNATE }, // 459
+	{ "annuity", ANNUITY }, // 460
+	{ "anum", ANUM }, // 461
+	{ "any", ANY }, // 462
+	{ "anycase", ANYCASE }, // 463
+	{ "apply", APPLY }, // 464
+	{ "are", ARE }, // 465
+	{ "area", AREA }, // 466
+	{ "areas", AREAS }, // 467
+	{ "as", AS }, // 468
+	{ "ascending", ASCENDING }, // 469
+	{ "activating", ACTIVATING }, // 470
+	{ "asin", ASIN }, // 471
+	{ "assign", ASSIGN }, // 472
+	{ "at", AT }, // 473
+	{ "atan", ATAN }, // 474
+	{ "based", BASED }, // 475
+	{ "baseconvert", BASECONVERT }, // 476
+	{ "before", BEFORE }, // 477
+	{ "binary", BINARY }, // 478
+	{ "bit", BIT }, // 479
+	{ "bit-of", BIT_OF }, // 480
+	{ "bit-to-char", BIT_TO_CHAR }, // 481
+	{ "blank", BLANK }, // 482
+	{ "block", BLOCK_kw }, // 483
+	{ "boolean-of-integer", BOOLEAN_OF_INTEGER }, // 484
+	{ "bottom", BOTTOM }, // 485
+	{ "by", BY }, // 486
+	{ "byte", BYTE }, // 487
+	{ "byte-length", BYTE_LENGTH }, // 488
+	{ "cf", CF }, // 489
+	{ "ch", CH }, // 490
+	{ "changed", CHANGED }, // 491
+	{ "char", CHAR }, // 492
+	{ "char-national", CHAR_NATIONAL }, // 493
+	{ "character", CHARACTER }, // 494
+	{ "characters", CHARACTERS }, // 495
+	{ "checking", CHECKING }, // 496
+	{ "class", CLASS }, // 497
+	{ "cobol", COBOL }, // 498
+	{ "code", CODE }, // 499
+	{ "code-set", CODESET }, // 500
+	{ "collating", COLLATING }, // 501
+	{ "column", COLUMN }, // 502
+	{ "combined-datetime", COMBINED_DATETIME }, // 503
+	{ "comma", COMMA }, // 504
+	{ "command-line", COMMAND_LINE }, // 505
+	{ "command-line-count", COMMAND_LINE_COUNT }, // 506
+	{ "commit", COMMIT }, // 507
+	{ "common", COMMON }, // 508
+	{ "concat", CONCAT }, // 509
+	{ "condition", CONDITION }, // 510
+	{ "configuration", CONFIGURATION_SECT }, // 511
+	{ "contains", CONTAINS }, // 512
+	{ "content", CONTENT }, // 513
+	{ "control", CONTROL }, // 514
+	{ "controls", CONTROLS }, // 515
+	{ "convert", CONVERT }, // 516
+	{ "converting", CONVERTING }, // 517
+	{ "corresponding", CORRESPONDING }, // 518
+	{ "cos", COS }, // 519
+	{ "count", COUNT }, // 520
+	{ "currency", CURRENCY }, // 521
+	{ "current", CURRENT }, // 522
+	{ "current-date", CURRENT_DATE }, // 523
+	{ "data", DATA }, // 524
+	{ "date", DATE }, // 525
+	{ "date-compiled", DATE_COMPILED }, // 526
+	{ "date-of-integer", DATE_OF_INTEGER }, // 527
+	{ "date-to-yyyymmdd", DATE_TO_YYYYMMDD }, // 528
+	{ "date-written", DATE_WRITTEN }, // 529
+	{ "day", DAY }, // 530
+	{ "day-of-integer", DAY_OF_INTEGER }, // 531
+	{ "day-of-week", DAY_OF_WEEK }, // 532
+	{ "day-to-yyyyddd", DAY_TO_YYYYDDD }, // 533
+	{ "dbcs", DBCS }, // 534
+	{ "de", DE }, // 535
+	{ "debugging", DEBUGGING }, // 536
+	{ "decimal-point", DECIMAL_POINT }, // 537
+	{ "declaratives", DECLARATIVES }, // 538
+	{ "default", DEFAULT }, // 539
+	{ "delimited", DELIMITED }, // 540
+	{ "delimiter", DELIMITER }, // 541
+	{ "depending", DEPENDING }, // 542
+	{ "descending", DESCENDING }, // 543
+	{ "detail", DETAIL }, // 544
+	{ "direct", DIRECT }, // 545
+	{ "direct-access", DIRECT_ACCESS }, // 546
+	{ "down", DOWN }, // 547
+	{ "duplicates", DUPLICATES }, // 548
+	{ "dynamic", DYNAMIC }, // 549
+	{ "e", E }, // 550
+	{ "ebcdic", EBCDIC }, // 551
+	{ "ec", EC }, // 552
+	{ "egcs", EGCS }, // 553
+	{ "entry", ENTRY }, // 554
+	{ "environment", ENVIRONMENT }, // 555
+	{ "equal", EQUAL }, // 556
+	{ "every", EVERY }, // 557
+	{ "examine", EXAMINE }, // 558
+	{ "exhibit", EXHIBIT }, // 559
+	{ "exp", EXP }, // 560
+	{ "exp10", EXP10 }, // 561
+	{ "extend", EXTEND }, // 562
+	{ "external", EXTERNAL }, // 563
+	{ "exception-file", EXCEPTION_FILE }, // 564
+	{ "exception-file-n", EXCEPTION_FILE_N }, // 565
+	{ "exception-location", EXCEPTION_LOCATION }, // 566
+	{ "exception-location-n", EXCEPTION_LOCATION_N }, // 567
+	{ "exception-statement", EXCEPTION_STATEMENT }, // 568
+	{ "exception-status", EXCEPTION_STATUS }, // 569
+	{ "factorial", FACTORIAL }, // 570
+	{ "false", FALSE_kw }, // 571
+	{ "fd", FD }, // 572
+	{ "file-control", FILE_CONTROL }, // 573
+	{ "file", FILE_KW }, // 574
+	{ "file-limit", FILE_LIMIT }, // 575
+	{ "final", FINAL }, // 576
+	{ "finally", FINALLY }, // 577
+	{ "find-string", FIND_STRING }, // 578
+	{ "first", FIRST }, // 579
+	{ "fixed", FIXED }, // 580
+	{ "footing", FOOTING }, // 581
+	{ "for", FOR }, // 582
+	{ "formatted-current-date", FORMATTED_CURRENT_DATE }, // 583
+	{ "formatted-date", FORMATTED_DATE }, // 584
+	{ "formatted-datetime", FORMATTED_DATETIME }, // 585
+	{ "formatted-time", FORMATTED_TIME }, // 586
+	{ "form-overflow", FORM_OVERFLOW }, // 587
+	{ "free", FREE }, // 588
+	{ "fraction-part", FRACTION_PART }, // 589
+	{ "from", FROM }, // 590
+	{ "function", FUNCTION }, // 591
+	{ "generate", GENERATE }, // 592
+	{ "giving", GIVING }, // 593
+	{ "global", GLOBAL }, // 594
+	{ "go", GO }, // 595
+	{ "group", GROUP }, // 596
+	{ "heading", HEADING }, // 597
+	{ "hex", HEX }, // 598
+	{ "hex-of", HEX_OF }, // 599
+	{ "hex-to-char", HEX_TO_CHAR }, // 600
+	{ "high-values", HIGH_VALUES }, // 601
+	{ "highest-algebraic", HIGHEST_ALGEBRAIC }, // 602
+	{ "hold", HOLD }, // 603
+	{ "ibm-360", IBM_360 }, // 604
+	{ "in", IN }, // 605
+	{ "include", INCLUDE }, // 606
+	{ "index", INDEX }, // 607
+	{ "indexed", INDEXED }, // 608
+	{ "indicate", INDICATE }, // 609
+	{ "initial", INITIAL_kw }, // 610
+	{ "initiate", INITIATE }, // 611
+	{ "input", INPUT }, // 612
+	{ "installation", INSTALLATION }, // 613
+	{ "interface", INTERFACE }, // 614
+	{ "integer", INTEGER }, // 615
+	{ "integer-of-boolean", INTEGER_OF_BOOLEAN }, // 616
+	{ "integer-of-date", INTEGER_OF_DATE }, // 617
+	{ "integer-of-day", INTEGER_OF_DAY }, // 618
+	{ "integer-of-formatted-date", INTEGER_OF_FORMATTED_DATE }, // 619
+	{ "integer-part", INTEGER_PART }, // 620
+	{ "into", INTO }, // 621
+	{ "intrinsic", INTRINSIC }, // 622
+	{ "invoke", INVOKE }, // 623
+	{ "i-o", IO }, // 624
+	{ "i-o-control", IO_CONTROL }, // 625
+	{ "is", IS }, // 626
+	{ "isnt", ISNT }, // 627
+	{ "kanji", KANJI }, // 628
+	{ "key", KEY }, // 629
+	{ "label", LABEL }, // 630
+	{ "last", LAST }, // 631
+	{ "leading", LEADING }, // 632
+	{ "left", LEFT }, // 633
+	{ "length", LENGTH }, // 634
+	{ "length-of", LENGTH_OF }, // 635
+	{ "limit", LIMIT }, // 636
+	{ "limits", LIMITS }, // 637
+	{ "line", LINE }, // 638
+	{ "lines", LINES }, // 639
+	{ "line-counter", LINE_COUNTER }, // 640
+	{ "linage", LINAGE }, // 641
+	{ "linkage", LINKAGE }, // 642
+	{ "locale", LOCALE }, // 643
+	{ "locale-compare", LOCALE_COMPARE }, // 644
+	{ "locale-date", LOCALE_DATE }, // 645
+	{ "locale-time", LOCALE_TIME }, // 646
+	{ "locale-time-from-seconds", LOCALE_TIME_FROM_SECONDS }, // 647
+	{ "local-storage", LOCAL_STORAGE }, // 648
+	{ "location", LOCATION }, // 649
+	{ "lock", LOCK }, // 650
+	{ "lock-on", LOCK_ON }, // 651
+	{ "log", LOG }, // 652
+	{ "log10", LOG10 }, // 653
+	{ "lower-case", LOWER_CASE }, // 654
+	{ "low-values", LOW_VALUES }, // 655
+	{ "lowest-algebraic", LOWEST_ALGEBRAIC }, // 656
+	{ "lparen", LPAREN }, // 657
+	{ "manual", MANUAL }, // 658
+	{ "maxx", MAXX }, // 659
+	{ "mean", MEAN }, // 660
+	{ "median", MEDIAN }, // 661
+	{ "midrange", MIDRANGE }, // 662
+	{ "minn", MINN }, // 663
+	{ "multiple", MULTIPLE }, // 664
+	{ "mod", MOD }, // 665
+	{ "mode", MODE }, // 666
+	{ "module-name", MODULE_NAME }, // 667
+	{ "named", NAMED }, // 668
+	{ "nat", NAT }, // 669
+	{ "national", NATIONAL }, // 670
+	{ "national-edited", NATIONAL_EDITED }, // 671
+	{ "national-of", NATIONAL_OF }, // 672
+	{ "native", NATIVE }, // 673
+	{ "nested", NESTED }, // 674
+	{ "next", NEXT }, // 675
+	{ "no", NO }, // 676
+	{ "note", NOTE }, // 677
+	{ "nulls", NULLS }, // 678
+	{ "null", NULLS }, // 678
+	{ "nullptr", NULLPTR }, // 679
+	{ "numeric", NUMERIC }, // 680
+	{ "numeric-edited", NUMERIC_EDITED }, // 681
+	{ "numval", NUMVAL }, // 682
+	{ "numval-c", NUMVAL_C }, // 683
+	{ "numval-f", NUMVAL_F }, // 684
+	{ "occurs", OCCURS }, // 685
+	{ "of", OF }, // 686
+	{ "off", OFF }, // 687
+	{ "omitted", OMITTED }, // 688
+	{ "on", ON }, // 689
+	{ "only", ONLY }, // 690
+	{ "optional", OPTIONAL }, // 691
+	{ "options", OPTIONS }, // 692
+	{ "ord", ORD }, // 693
+	{ "order", ORDER }, // 694
+	{ "ord-max", ORD_MAX }, // 695
+	{ "ord-min", ORD_MIN }, // 696
+	{ "organization", ORGANIZATION }, // 697
+	{ "other", OTHER }, // 698
+	{ "otherwise", OTHERWISE }, // 699
+	{ "output", OUTPUT }, // 700
+	{ "packed-decimal", PACKED_DECIMAL }, // 701
+	{ "padding", PADDING }, // 702
+	{ "page", PAGE }, // 703
+	{ "page-counter", PAGE_COUNTER }, // 704
+	{ "pf", PF }, // 705
+	{ "ph", PH }, // 706
+	{ "pi", PI }, // 707
+	{ "pic", PIC }, // 708
+	{ "picture", PICTURE }, // 709
+	{ "plus", PLUS }, // 710
+	{ "present-value", PRESENT_VALUE }, // 711
+	{ "print-switch", PRINT_SWITCH }, // 712
+	{ "procedure", PROCEDURE }, // 713
+	{ "procedures", PROCEDURES }, // 714
+	{ "proceed", PROCEED }, // 715
+	{ "process", PROCESS }, // 716
+	{ "program-id", PROGRAM_ID }, // 717
+	{ "program", PROGRAM_kw }, // 718
+	{ "property", PROPERTY }, // 719
+	{ "prototype", PROTOTYPE }, // 720
+	{ "pseudotext", PSEUDOTEXT }, // 721
+	{ "quotes", QUOTES }, // 722
+	{ "quote", QUOTES }, // 722
+	{ "random", RANDOM }, // 723
+	{ "random-seed", RANDOM_SEED }, // 724
+	{ "range", RANGE }, // 725
+	{ "raise", RAISE }, // 726
+	{ "raising", RAISING }, // 727
+	{ "rd", RD }, // 728
+	{ "record", RECORD }, // 729
+	{ "recording", RECORDING }, // 730
+	{ "records", RECORDS }, // 731
+	{ "recursive", RECURSIVE }, // 732
+	{ "redefines", REDEFINES }, // 733
+	{ "reel", REEL }, // 734
+	{ "reference", REFERENCE }, // 735
+	{ "relative", RELATIVE }, // 736
+	{ "rem", REM }, // 737
+	{ "remainder", REMAINDER }, // 738
+	{ "remarks", REMARKS }, // 739
+	{ "removal", REMOVAL }, // 740
+	{ "renames", RENAMES }, // 741
+	{ "replace", REPLACE }, // 742
+	{ "replacing", REPLACING }, // 743
+	{ "report", REPORT }, // 744
+	{ "reporting", REPORTING }, // 745
+	{ "reports", REPORTS }, // 746
+	{ "repository", REPOSITORY }, // 747
+	{ "rerun", RERUN }, // 748
+	{ "reserve", RESERVE }, // 749
+	{ "restricted", RESTRICTED }, // 750
+	{ "resume", RESUME }, // 751
+	{ "reverse", REVERSE }, // 752
+	{ "reversed", REVERSED }, // 753
+	{ "rewind", REWIND }, // 754
+	{ "rf", RF }, // 755
+	{ "rh", RH }, // 756
+	{ "right", RIGHT }, // 757
+	{ "rounded", ROUNDED }, // 758
+	{ "run", RUN }, // 759
+	{ "same", SAME }, // 760
+	{ "screen", SCREEN }, // 761
+	{ "sd", SD }, // 762
+	{ "seconds-from-formatted-time", SECONDS_FROM_FORMATTED_TIME }, // 763
+	{ "seconds-past-midnight", SECONDS_PAST_MIDNIGHT }, // 764
+	{ "security", SECURITY }, // 765
+	{ "separate", SEPARATE }, // 766
+	{ "sequence", SEQUENCE }, // 767
+	{ "sequential", SEQUENTIAL }, // 768
+	{ "sharing", SHARING }, // 769
+	{ "simple-exit", SIMPLE_EXIT }, // 770
+	{ "sign", SIGN }, // 771
+	{ "sin", SIN }, // 772
+	{ "size", SIZE }, // 773
+	{ "smallest-algebraic", SMALLEST_ALGEBRAIC }, // 774
+	{ "source", SOURCE }, // 775
+	{ "source-computer", SOURCE_COMPUTER }, // 776
+	{ "special-names", SPECIAL_NAMES }, // 777
+	{ "sqrt", SQRT }, // 778
+	{ "stack", STACK }, // 779
+	{ "standard", STANDARD }, // 780
+	{ "standard-1", STANDARD_1 }, // 781
+	{ "standard-deviation", STANDARD_DEVIATION }, // 782
+	{ "standard-compare", STANDARD_COMPARE }, // 783
+	{ "status", STATUS }, // 784
+	{ "strong", STRONG }, // 785
+	{ "substitute", SUBSTITUTE }, // 786
+	{ "sum", SUM }, // 787
+	{ "symbol", SYMBOL }, // 788
+	{ "symbolic", SYMBOLIC }, // 789
+	{ "synchronized", SYNCHRONIZED }, // 790
+	{ "tally", TALLY }, // 791
+	{ "tallying", TALLYING }, // 792
+	{ "tan", TAN }, // 793
+	{ "terminate", TERMINATE }, // 794
+	{ "test", TEST }, // 795
+	{ "test-date-yyyymmdd", TEST_DATE_YYYYMMDD }, // 796
+	{ "test-day-yyyyddd", TEST_DAY_YYYYDDD }, // 797
+	{ "test-formatted-datetime", TEST_FORMATTED_DATETIME }, // 798
+	{ "test-numval", TEST_NUMVAL }, // 799
+	{ "test-numval-c", TEST_NUMVAL_C }, // 800
+	{ "test-numval-f", TEST_NUMVAL_F }, // 801
+	{ "than", THAN }, // 802
+	{ "time", TIME }, // 803
+	{ "times", TIMES }, // 804
+	{ "to", TO }, // 805
+	{ "top", TOP }, // 806
+	{ "top-level", TOP_LEVEL }, // 807
+	{ "tracks", TRACKS }, // 808
+	{ "track-area", TRACK_AREA }, // 809
+	{ "trailing", TRAILING }, // 810
+	{ "transform", TRANSFORM }, // 811
+	{ "trim", TRIM }, // 812
+	{ "true", TRUE_kw }, // 813
+	{ "try", TRY }, // 814
+	{ "turn", TURN }, // 815
+	{ "type", TYPE }, // 816
+	{ "typedef", TYPEDEF }, // 817
+	{ "ulength", ULENGTH }, // 818
+	{ "unbounded", UNBOUNDED }, // 819
+	{ "unit", UNIT }, // 820
+	{ "units", UNITS }, // 821
+	{ "unit-record", UNIT_RECORD }, // 822
+	{ "until", UNTIL }, // 823
+	{ "up", UP }, // 824
+	{ "upon", UPON }, // 825
+	{ "upos", UPOS }, // 826
+	{ "upper-case", UPPER_CASE }, // 827
+	{ "usage", USAGE }, // 828
+	{ "using", USING }, // 829
+	{ "usubstr", USUBSTR }, // 830
+	{ "usupplementary", USUPPLEMENTARY }, // 831
+	{ "utility", UTILITY }, // 832
+	{ "uuid4", UUID4 }, // 833
+	{ "uvalid", UVALID }, // 834
+	{ "uwidth", UWIDTH }, // 835
+	{ "value", VALUE }, // 836
+	{ "variance", VARIANCE }, // 837
+	{ "varying", VARYING }, // 838
+	{ "volatile", VOLATILE }, // 839
+	{ "when-compiled", WHEN_COMPILED }, // 840
+	{ "with", WITH }, // 841
+	{ "working-storage", WORKING_STORAGE }, // 842
+	{ "xml", XML }, // 843
+	{ "xmlgenerate", XMLGENERATE }, // 844
+	{ "xmlparse", XMLPARSE }, // 845
+	{ "year-to-yyyy", YEAR_TO_YYYY }, // 846
+	{ "yyyyddd", YYYYDDD }, // 847
+	{ "yyyymmdd", YYYYMMDD }, // 848
+	{ "arithmetic", ARITHMETIC }, // 849
+	{ "attribute", ATTRIBUTE }, // 850
+	{ "auto", AUTO }, // 851
+	{ "automatic", AUTOMATIC }, // 852
+	{ "away-from-zero", AWAY_FROM_ZERO }, // 853
+	{ "background-color", BACKGROUND_COLOR }, // 854
+	{ "bell", BELL }, // 855
+	{ "binary-encoding", BINARY_ENCODING }, // 856
+	{ "blink", BLINK }, // 857
+	{ "capacity", CAPACITY }, // 858
+	{ "center", CENTER }, // 859
+	{ "classification", CLASSIFICATION }, // 860
+	{ "cycle", CYCLE }, // 861
+	{ "decimal-encoding", DECIMAL_ENCODING }, // 862
+	{ "entry-convention", ENTRY_CONVENTION }, // 863
+	{ "eol", EOL }, // 864
+	{ "eos", EOS }, // 865
+	{ "erase", ERASE }, // 866
+	{ "expands", EXPANDS }, // 867
+	{ "float-binary", FLOAT_BINARY }, // 868
+	{ "float-decimal", FLOAT_DECIMAL }, // 869
+	{ "foreground-color", FOREGROUND_COLOR }, // 870
+	{ "forever", FOREVER }, // 871
+	{ "full", FULL }, // 872
+	{ "highlight", HIGHLIGHT }, // 873
+	{ "high-order-left", HIGH_ORDER_LEFT }, // 874
+	{ "high-order-right", HIGH_ORDER_RIGHT }, // 875
+	{ "ignoring", IGNORING }, // 876
+	{ "implements", IMPLEMENTS }, // 877
+	{ "initialized", INITIALIZED }, // 878
+	{ "intermediate", INTERMEDIATE }, // 879
+	{ "lc-all", LC_ALL_kw }, // 880
+	{ "lc-collate", LC_COLLATE_kw }, // 881
+	{ "lc-ctype", LC_CTYPE_kw }, // 882
+	{ "lc-messages", LC_MESSAGES_kw }, // 883
+	{ "lc-monetary", LC_MONETARY_kw }, // 884
+	{ "lc-numeric", LC_NUMERIC_kw }, // 885
+	{ "lc-time", LC_TIME_kw }, // 886
+	{ "lowlight", LOWLIGHT }, // 887
+	{ "nearest-away-from-zero", NEAREST_AWAY_FROM_ZERO }, // 888
+	{ "nearest-even", NEAREST_EVEN }, // 889
+	{ "nearest-toward-zero", NEAREST_TOWARD_ZERO }, // 890
+	{ "none", NONE }, // 891
+	{ "normal", NORMAL }, // 892
+	{ "numbers", NUMBERS }, // 893
+	{ "prefixed", PREFIXED }, // 894
+	{ "previous", PREVIOUS }, // 895
+	{ "prohibited", PROHIBITED }, // 896
+	{ "relation", RELATION }, // 897
+	{ "required", REQUIRED }, // 898
+	{ "reverse-video", REVERSE_VIDEO }, // 899
+	{ "rounding", ROUNDING }, // 900
+	{ "seconds", SECONDS }, // 901
+	{ "secure", SECURE }, // 902
+	{ "short", SHORT }, // 903
+	{ "signed", SIGNED_kw }, // 904
+	{ "standard-binary", STANDARD_BINARY }, // 905
+	{ "standard-decimal", STANDARD_DECIMAL }, // 906
+	{ "statement", STATEMENT }, // 907
+	{ "step", STEP }, // 908
+	{ "structure", STRUCTURE }, // 909
+	{ "toward-greater", TOWARD_GREATER }, // 910
+	{ "toward-lesser", TOWARD_LESSER }, // 911
+	{ "truncation", TRUNCATION }, // 912
+	{ "ucs-4", UCS_4 }, // 913
+	{ "underline", UNDERLINE }, // 914
+	{ "unsigned", UNSIGNED_kw }, // 915
+	{ "utf-16", UTF_16 }, // 916
+	{ "utf-8", UTF_8 }, // 917
+	{ "address", ADDRESS }, // 918
+	{ "end-accept", END_ACCEPT }, // 919
+	{ "end-add", END_ADD }, // 920
+	{ "end-call", END_CALL }, // 921
+	{ "end-compute", END_COMPUTE }, // 922
+	{ "end-delete", END_DELETE }, // 923
+	{ "end-display", END_DISPLAY }, // 924
+	{ "end-divide", END_DIVIDE }, // 925
+	{ "end-evaluate", END_EVALUATE }, // 926
+	{ "end-multiply", END_MULTIPLY }, // 927
+	{ "end-perform", END_PERFORM }, // 928
+	{ "end-read", END_READ }, // 929
+	{ "end-return", END_RETURN }, // 930
+	{ "end-rewrite", END_REWRITE }, // 931
+	{ "end-search", END_SEARCH }, // 932
+	{ "end-start", END_START }, // 933
+	{ "end-string", END_STRING }, // 934
+	{ "end-subtract", END_SUBTRACT }, // 935
+	{ "end-unstring", END_UNSTRING }, // 936
+	{ "end-write", END_WRITE }, // 937
+	{ "end-if", END_IF }, // 938
+	{ "thru", THRU }, // 939
+	{ "through", THRU }, // 939
+	{ "or", OR }, // 940
+	{ "and", AND }, // 941
+	{ "not", NOT }, // 942
+	{ "ne", NE }, // 943
+	{ "le", LE }, // 944
+	{ "ge", GE }, // 945
+	{ "pow", POW }, // 946
+	{ "neg", NEG }, // 947
 };
 
 // cppcheck-suppress useInitializationList
@@ -826,553 +834,561 @@ token_names = {
 	"CDF-EVALUATE", // 130 (388)
 	"CDF-WHEN", // 131 (389)
 	"CDF-END-EVALUATE", // 132 (390)
-	"CALL-COBOL", // 133 (391)
-	"CALL-VERBATIM", // 134 (392)
-	"IF", // 135 (393)
-	"THEN", // 136 (394)
-	"ELSE", // 137 (395)
-	"SENTENCE", // 138 (396)
-	"ACCEPT", // 139 (397)
-	"ADD", // 140 (398)
-	"ALTER", // 141 (399)
-	"CALL", // 142 (400)
-	"CANCEL", // 143 (401)
-	"CLOSE", // 144 (402)
-	"COMPUTE", // 145 (403)
-	"CONTINUE", // 146 (404)
-	"DELETE", // 147 (405)
-	"DISPLAY", // 148 (406)
-	"DIVIDE", // 149 (407)
-	"EVALUATE", // 150 (408)
-	"EXIT", // 151 (409)
-	"FILLER", // 152 (410)
-	"GOBACK", // 153 (411)
-	"GOTO", // 154 (412)
-	"INITIALIZE", // 155 (413)
-	"INSPECT", // 156 (414)
-	"MERGE", // 157 (415)
-	"MOVE", // 158 (416)
-	"MULTIPLY", // 159 (417)
-	"OPEN", // 160 (418)
-	"PARAGRAPH", // 161 (419)
-	"READ", // 162 (420)
-	"RELEASE", // 163 (421)
-	"RETURN", // 164 (422)
-	"REWRITE", // 165 (423)
-	"SEARCH", // 166 (424)
-	"SET", // 167 (425)
-	"SELECT", // 168 (426)
-	"SORT", // 169 (427)
-	"SORT-MERGE", // 170 (428)
-	"STRING", // 171 (429)
-	"STOP", // 172 (430)
-	"SUBTRACT", // 173 (431)
-	"START", // 174 (432)
-	"UNSTRING", // 175 (433)
-	"WRITE", // 176 (434)
-	"WHEN", // 177 (435)
-	"ABS", // 178 (436)
-	"ACCESS", // 179 (437)
-	"ACOS", // 180 (438)
-	"ACTUAL", // 181 (439)
-	"ADVANCING", // 182 (440)
-	"AFTER", // 183 (441)
-	"ALL", // 184 (442)
-	"ALLOCATE", // 185 (443)
-	"ALPHABET", // 186 (444)
-	"ALPHABETIC", // 187 (445)
-	"ALPHABETIC-LOWER", // 188 (446)
-	"ALPHABETIC-UPPER", // 189 (447)
-	"ALPHANUMERIC", // 190 (448)
-	"ALPHANUMERIC-EDITED", // 191 (449)
-	"ALSO", // 192 (450)
-	"ALTERNATE", // 193 (451)
-	"ANNUITY", // 194 (452)
-	"ANUM", // 195 (453)
-	"ANY", // 196 (454)
-	"ANYCASE", // 197 (455)
-	"APPLY", // 198 (456)
-	"ARE", // 199 (457)
-	"AREA", // 200 (458)
-	"AREAS", // 201 (459)
-	"AS", // 202 (460)
-	"ASCENDING", // 203 (461)
-	"ACTIVATING", // 204 (462)
-	"ASIN", // 205 (463)
-	"ASSIGN", // 206 (464)
-	"AT", // 207 (465)
-	"ATAN", // 208 (466)
-	"BASED", // 209 (467)
-	"BASECONVERT", // 210 (468)
-	"BEFORE", // 211 (469)
-	"BINARY", // 212 (470)
-	"BIT", // 213 (471)
-	"BIT-OF", // 214 (472)
-	"BIT-TO-CHAR", // 215 (473)
-	"BLANK", // 216 (474)
-	"BLOCK", // 217 (475)
-	"BOOLEAN-OF-INTEGER", // 218 (476)
-	"BOTTOM", // 219 (477)
-	"BY", // 220 (478)
-	"BYTE", // 221 (479)
-	"BYTE-LENGTH", // 222 (480)
-	"CF", // 223 (481)
-	"CH", // 224 (482)
-	"CHANGED", // 225 (483)
-	"CHAR", // 226 (484)
-	"CHAR-NATIONAL", // 227 (485)
-	"CHARACTER", // 228 (486)
-	"CHARACTERS", // 229 (487)
-	"CHECKING", // 230 (488)
-	"CLASS", // 231 (489)
-	"COBOL", // 232 (490)
-	"CODE", // 233 (491)
-	"CODE-SET", // 234 (492)
-	"COLLATING", // 235 (493)
-	"COLUMN", // 236 (494)
-	"COMBINED-DATETIME", // 237 (495)
-	"COMMA", // 238 (496)
-	"COMMAND-LINE", // 239 (497)
-	"COMMAND-LINE-COUNT", // 240 (498)
-	"COMMIT", // 241 (499)
-	"COMMON", // 242 (500)
-	"CONCAT", // 243 (501)
-	"CONDITION", // 244 (502)
-	"CONFIGURATION", // 245 (503)
-	"CONTAINS", // 246 (504)
-	"CONTENT", // 247 (505)
-	"CONTROL", // 248 (506)
-	"CONTROLS", // 249 (507)
-	"CONVERT", // 250 (508)
-	"CONVERTING", // 251 (509)
-	"CORRESPONDING", // 252 (510)
-	"COS", // 253 (511)
-	"COUNT", // 254 (512)
-	"CURRENCY", // 255 (513)
-	"CURRENT", // 256 (514)
-	"CURRENT-DATE", // 257 (515)
-	"DATA", // 258 (516)
-	"DATE", // 259 (517)
-	"DATE-COMPILED", // 260 (518)
-	"DATE-OF-INTEGER", // 261 (519)
-	"DATE-TO-YYYYMMDD", // 262 (520)
-	"DATE-WRITTEN", // 263 (521)
-	"DAY", // 264 (522)
-	"DAY-OF-INTEGER", // 265 (523)
-	"DAY-OF-WEEK", // 266 (524)
-	"DAY-TO-YYYYDDD", // 267 (525)
-	"DBCS", // 268 (526)
-	"DE", // 269 (527)
-	"DEBUGGING", // 270 (528)
-	"DECIMAL-POINT", // 271 (529)
-	"DECLARATIVES", // 272 (530)
-	"DEFAULT", // 273 (531)
-	"DELIMITED", // 274 (532)
-	"DELIMITER", // 275 (533)
-	"DEPENDING", // 276 (534)
-	"DESCENDING", // 277 (535)
-	"DETAIL", // 278 (536)
-	"DIRECT", // 279 (537)
-	"DIRECT-ACCESS", // 280 (538)
-	"DOWN", // 281 (539)
-	"DUPLICATES", // 282 (540)
-	"DYNAMIC", // 283 (541)
-	"E", // 284 (542)
-	"EBCDIC", // 285 (543)
-	"EC", // 286 (544)
-	"EGCS", // 287 (545)
-	"ENTRY", // 288 (546)
-	"ENVIRONMENT", // 289 (547)
-	"EQUAL", // 290 (548)
-	"EVERY", // 291 (549)
-	"EXAMINE", // 292 (550)
-	"EXHIBIT", // 293 (551)
-	"EXP", // 294 (552)
-	"EXP10", // 295 (553)
-	"EXTEND", // 296 (554)
-	"EXTERNAL", // 297 (555)
-	"EXCEPTION-FILE", // 298 (556)
-	"EXCEPTION-FILE-N", // 299 (557)
-	"EXCEPTION-LOCATION", // 300 (558)
-	"EXCEPTION-LOCATION-N", // 301 (559)
-	"EXCEPTION-STATEMENT", // 302 (560)
-	"EXCEPTION-STATUS", // 303 (561)
-	"FACTORIAL", // 304 (562)
-	"FALSE", // 305 (563)
-	"FD", // 306 (564)
-	"FILE-CONTROL", // 307 (565)
-	"FILE", // 308 (566)
-	"FILE-LIMIT", // 309 (567)
-	"FINAL", // 310 (568)
-	"FINALLY", // 311 (569)
-	"FIND-STRING", // 312 (570)
-	"FIRST", // 313 (571)
-	"FIXED", // 314 (572)
-	"FOOTING", // 315 (573)
-	"FOR", // 316 (574)
-	"FORMATTED-CURRENT-DATE", // 317 (575)
-	"FORMATTED-DATE", // 318 (576)
-	"FORMATTED-DATETIME", // 319 (577)
-	"FORMATTED-TIME", // 320 (578)
-	"FORM-OVERFLOW", // 321 (579)
-	"FREE", // 322 (580)
-	"FRACTION-PART", // 323 (581)
-	"FROM", // 324 (582)
-	"FUNCTION", // 325 (583)
-	"GENERATE", // 326 (584)
-	"GIVING", // 327 (585)
-	"GLOBAL", // 328 (586)
-	"GO", // 329 (587)
-	"GROUP", // 330 (588)
-	"HEADING", // 331 (589)
-	"HEX", // 332 (590)
-	"HEX-OF", // 333 (591)
-	"HEX-TO-CHAR", // 334 (592)
-	"HIGH-VALUES", // 335 (593)
-	"HIGHEST-ALGEBRAIC", // 336 (594)
-	"HOLD", // 337 (595)
-	"IBM-360", // 338 (596)
-	"IN", // 339 (597)
-	"INCLUDE", // 340 (598)
-	"INDEX", // 341 (599)
-	"INDEXED", // 342 (600)
-	"INDICATE", // 343 (601)
-	"INITIAL", // 344 (602)
-	"INITIATE", // 345 (603)
-	"INPUT", // 346 (604)
-	"INSTALLATION", // 347 (605)
-	"INTERFACE", // 348 (606)
-	"INTEGER", // 349 (607)
-	"INTEGER-OF-BOOLEAN", // 350 (608)
-	"INTEGER-OF-DATE", // 351 (609)
-	"INTEGER-OF-DAY", // 352 (610)
-	"INTEGER-OF-FORMATTED-DATE", // 353 (611)
-	"INTEGER-PART", // 354 (612)
-	"INTO", // 355 (613)
-	"INTRINSIC", // 356 (614)
-	"INVOKE", // 357 (615)
-	"I-O", // 358 (616)
-	"I-O-CONTROL", // 359 (617)
-	"IS", // 360 (618)
-	"ISNT", // 361 (619)
-	"KANJI", // 362 (620)
-	"KEY", // 363 (621)
-	"LABEL", // 364 (622)
-	"LAST", // 365 (623)
-	"LEADING", // 366 (624)
-	"LEFT", // 367 (625)
-	"LENGTH", // 368 (626)
-	"LENGTH-OF", // 369 (627)
-	"LIMIT", // 370 (628)
-	"LIMITS", // 371 (629)
-	"LINE", // 372 (630)
-	"LINES", // 373 (631)
-	"LINE-COUNTER", // 374 (632)
-	"LINAGE", // 375 (633)
-	"LINKAGE", // 376 (634)
-	"LOCALE", // 377 (635)
-	"LOCALE-COMPARE", // 378 (636)
-	"LOCALE-DATE", // 379 (637)
-	"LOCALE-TIME", // 380 (638)
-	"LOCALE-TIME-FROM-SECONDS", // 381 (639)
-	"LOCAL-STORAGE", // 382 (640)
-	"LOCATION", // 383 (641)
-	"LOCK", // 384 (642)
-	"LOCK-ON", // 385 (643)
-	"LOG", // 386 (644)
-	"LOG10", // 387 (645)
-	"LOWER-CASE", // 388 (646)
-	"LOW-VALUES", // 389 (647)
-	"LOWEST-ALGEBRAIC", // 390 (648)
-	"LPAREN", // 391 (649)
-	"MANUAL", // 392 (650)
-	"MAXX", // 393 (651)
-	"MEAN", // 394 (652)
-	"MEDIAN", // 395 (653)
-	"MIDRANGE", // 396 (654)
-	"MINN", // 397 (655)
-	"MULTIPLE", // 398 (656)
-	"MOD", // 399 (657)
-	"MODE", // 400 (658)
-	"MODULE-NAME", // 401 (659)
-	"NAMED", // 402 (660)
-	"NAT", // 403 (661)
-	"NATIONAL", // 404 (662)
-	"NATIONAL-EDITED", // 405 (663)
-	"NATIONAL-OF", // 406 (664)
-	"NATIVE", // 407 (665)
-	"NESTED", // 408 (666)
-	"NEXT", // 409 (667)
-	"NO", // 410 (668)
-	"NOTE", // 411 (669)
-	"NULLS", // 412 (670)
-	"NULLPTR", // 413 (671)
-	"NUMERIC", // 414 (672)
-	"NUMERIC-EDITED", // 415 (673)
-	"NUMVAL", // 416 (674)
-	"NUMVAL-C", // 417 (675)
-	"NUMVAL-F", // 418 (676)
-	"OCCURS", // 419 (677)
-	"OF", // 420 (678)
-	"OFF", // 421 (679)
-	"OMITTED", // 422 (680)
-	"ON", // 423 (681)
-	"ONLY", // 424 (682)
-	"OPTIONAL", // 425 (683)
-	"OPTIONS", // 426 (684)
-	"ORD", // 427 (685)
-	"ORDER", // 428 (686)
-	"ORD-MAX", // 429 (687)
-	"ORD-MIN", // 430 (688)
-	"ORGANIZATION", // 431 (689)
-	"OTHER", // 432 (690)
-	"OTHERWISE", // 433 (691)
-	"OUTPUT", // 434 (692)
-	"PACKED-DECIMAL", // 435 (693)
-	"PADDING", // 436 (694)
-	"PAGE", // 437 (695)
-	"PAGE-COUNTER", // 438 (696)
-	"PF", // 439 (697)
-	"PH", // 440 (698)
-	"PI", // 441 (699)
-	"PIC", // 442 (700)
-	"PICTURE", // 443 (701)
-	"PLUS", // 444 (702)
-	"PRESENT-VALUE", // 445 (703)
-	"PRINT-SWITCH", // 446 (704)
-	"PROCEDURE", // 447 (705)
-	"PROCEDURES", // 448 (706)
-	"PROCEED", // 449 (707)
-	"PROCESS", // 450 (708)
-	"PROGRAM-ID", // 451 (709)
-	"PROGRAM", // 452 (710)
-	"PROPERTY", // 453 (711)
-	"PROTOTYPE", // 454 (712)
-	"PSEUDOTEXT", // 455 (713)
-	"QUOTES", // 456 (714)
-	"RANDOM", // 457 (715)
-	"RANDOM-SEED", // 458 (716)
-	"RANGE", // 459 (717)
-	"RAISE", // 460 (718)
-	"RAISING", // 461 (719)
-	"RD", // 462 (720)
-	"RECORD", // 463 (721)
-	"RECORDING", // 464 (722)
-	"RECORDS", // 465 (723)
-	"RECURSIVE", // 466 (724)
-	"REDEFINES", // 467 (725)
-	"REEL", // 468 (726)
-	"REFERENCE", // 469 (727)
-	"RELATIVE", // 470 (728)
-	"REM", // 471 (729)
-	"REMAINDER", // 472 (730)
-	"REMARKS", // 473 (731)
-	"REMOVAL", // 474 (732)
-	"RENAMES", // 475 (733)
-	"REPLACE", // 476 (734)
-	"REPLACING", // 477 (735)
-	"REPORT", // 478 (736)
-	"REPORTING", // 479 (737)
-	"REPORTS", // 480 (738)
-	"REPOSITORY", // 481 (739)
-	"RERUN", // 482 (740)
-	"RESERVE", // 483 (741)
-	"RESTRICTED", // 484 (742)
-	"RESUME", // 485 (743)
-	"REVERSE", // 486 (744)
-	"REVERSED", // 487 (745)
-	"REWIND", // 488 (746)
-	"RF", // 489 (747)
-	"RH", // 490 (748)
-	"RIGHT", // 491 (749)
-	"ROUNDED", // 492 (750)
-	"RUN", // 493 (751)
-	"SAME", // 494 (752)
-	"SCREEN", // 495 (753)
-	"SD", // 496 (754)
-	"SECONDS-FROM-FORMATTED-TIME", // 497 (755)
-	"SECONDS-PAST-MIDNIGHT", // 498 (756)
-	"SECURITY", // 499 (757)
-	"SEPARATE", // 500 (758)
-	"SEQUENCE", // 501 (759)
-	"SEQUENTIAL", // 502 (760)
-	"SHARING", // 503 (761)
-	"SIMPLE-EXIT", // 504 (762)
-	"SIGN", // 505 (763)
-	"SIN", // 506 (764)
-	"SIZE", // 507 (765)
-	"SMALLEST-ALGEBRAIC", // 508 (766)
-	"SOURCE", // 509 (767)
-	"SOURCE-COMPUTER", // 510 (768)
-	"SPECIAL-NAMES", // 511 (769)
-	"SQRT", // 512 (770)
-	"STACK", // 513 (771)
-	"STANDARD", // 514 (772)
-	"STANDARD-1", // 515 (773)
-	"STANDARD-DEVIATION", // 516 (774)
-	"STANDARD-COMPARE", // 517 (775)
-	"STATUS", // 518 (776)
-	"STRONG", // 519 (777)
-	"SUBSTITUTE", // 520 (778)
-	"SUM", // 521 (779)
-	"SYMBOL", // 522 (780)
-	"SYMBOLIC", // 523 (781)
-	"SYNCHRONIZED", // 524 (782)
-	"TALLY", // 525 (783)
-	"TALLYING", // 526 (784)
-	"TAN", // 527 (785)
-	"TERMINATE", // 528 (786)
-	"TEST", // 529 (787)
-	"TEST-DATE-YYYYMMDD", // 530 (788)
-	"TEST-DAY-YYYYDDD", // 531 (789)
-	"TEST-FORMATTED-DATETIME", // 532 (790)
-	"TEST-NUMVAL", // 533 (791)
-	"TEST-NUMVAL-C", // 534 (792)
-	"TEST-NUMVAL-F", // 535 (793)
-	"THAN", // 536 (794)
-	"TIME", // 537 (795)
-	"TIMES", // 538 (796)
-	"TO", // 539 (797)
-	"TOP", // 540 (798)
-	"TOP-LEVEL", // 541 (799)
-	"TRACKS", // 542 (800)
-	"TRACK-AREA", // 543 (801)
-	"TRAILING", // 544 (802)
-	"TRANSFORM", // 545 (803)
-	"TRIM", // 546 (804)
-	"TRUE", // 547 (805)
-	"TRY", // 548 (806)
-	"TURN", // 549 (807)
-	"TYPE", // 550 (808)
-	"TYPEDEF", // 551 (809)
-	"ULENGTH", // 552 (810)
-	"UNBOUNDED", // 553 (811)
-	"UNIT", // 554 (812)
-	"UNITS", // 555 (813)
-	"UNIT-RECORD", // 556 (814)
-	"UNTIL", // 557 (815)
-	"UP", // 558 (816)
-	"UPON", // 559 (817)
-	"UPOS", // 560 (818)
-	"UPPER-CASE", // 561 (819)
-	"USAGE", // 562 (820)
-	"USING", // 563 (821)
-	"USUBSTR", // 564 (822)
-	"USUPPLEMENTARY", // 565 (823)
-	"UTILITY", // 566 (824)
-	"UUID4", // 567 (825)
-	"UVALID", // 568 (826)
-	"UWIDTH", // 569 (827)
-	"VALUE", // 570 (828)
-	"VARIANCE", // 571 (829)
-	"VARYING", // 572 (830)
-	"VOLATILE", // 573 (831)
-	"WHEN-COMPILED", // 574 (832)
-	"WITH", // 575 (833)
-	"WORKING-STORAGE", // 576 (834)
-	"XML", // 577 (835)
-	"XMLGENERATE", // 578 (836)
-	"XMLPARSE", // 579 (837)
-	"YEAR-TO-YYYY", // 580 (838)
-	"YYYYDDD", // 581 (839)
-	"YYYYMMDD", // 582 (840)
-	"ARITHMETIC", // 583 (841)
-	"ATTRIBUTE", // 584 (842)
-	"AUTO", // 585 (843)
-	"AUTOMATIC", // 586 (844)
-	"AWAY-FROM-ZERO", // 587 (845)
-	"BACKGROUND-COLOR", // 588 (846)
-	"BELL", // 589 (847)
-	"BINARY-ENCODING", // 590 (848)
-	"BLINK", // 591 (849)
-	"CAPACITY", // 592 (850)
-	"CENTER", // 593 (851)
-	"CLASSIFICATION", // 594 (852)
-	"CYCLE", // 595 (853)
-	"DECIMAL-ENCODING", // 596 (854)
-	"ENTRY-CONVENTION", // 597 (855)
-	"EOL", // 598 (856)
-	"EOS", // 599 (857)
-	"ERASE", // 600 (858)
-	"EXPANDS", // 601 (859)
-	"FLOAT-BINARY", // 602 (860)
-	"FLOAT-DECIMAL", // 603 (861)
-	"FOREGROUND-COLOR", // 604 (862)
-	"FOREVER", // 605 (863)
-	"FULL", // 606 (864)
-	"HIGHLIGHT", // 607 (865)
-	"HIGH-ORDER-LEFT", // 608 (866)
-	"HIGH-ORDER-RIGHT", // 609 (867)
-	"IGNORING", // 610 (868)
-	"IMPLEMENTS", // 611 (869)
-	"INITIALIZED", // 612 (870)
-	"INTERMEDIATE", // 613 (871)
-	"LC-ALL", // 614 (872)
-	"LC-COLLATE", // 615 (873)
-	"LC-CTYPE", // 616 (874)
-	"LC-MESSAGES", // 617 (875)
-	"LC-MONETARY", // 618 (876)
-	"LC-NUMERIC", // 619 (877)
-	"LC-TIME", // 620 (878)
-	"LOWLIGHT", // 621 (879)
-	"NEAREST-AWAY-FROM-ZERO", // 622 (880)
-	"NEAREST-EVEN", // 623 (881)
-	"NEAREST-TOWARD-ZERO", // 624 (882)
-	"NONE", // 625 (883)
-	"NORMAL", // 626 (884)
-	"NUMBERS", // 627 (885)
-	"PREFIXED", // 628 (886)
-	"PREVIOUS", // 629 (887)
-	"PROHIBITED", // 630 (888)
-	"RELATION", // 631 (889)
-	"REQUIRED", // 632 (890)
-	"REVERSE-VIDEO", // 633 (891)
-	"ROUNDING", // 634 (892)
-	"SECONDS", // 635 (893)
-	"SECURE", // 636 (894)
-	"SHORT", // 637 (895)
-	"SIGNED", // 638 (896)
-	"STANDARD-BINARY", // 639 (897)
-	"STANDARD-DECIMAL", // 640 (898)
-	"STATEMENT", // 641 (899)
-	"STEP", // 642 (900)
-	"STRUCTURE", // 643 (901)
-	"TOWARD-GREATER", // 644 (902)
-	"TOWARD-LESSER", // 645 (903)
-	"TRUNCATION", // 646 (904)
-	"UCS-4", // 647 (905)
-	"UNDERLINE", // 648 (906)
-	"UNSIGNED", // 649 (907)
-	"UTF-16", // 650 (908)
-	"UTF-8", // 651 (909)
-	"ADDRESS", // 652 (910)
-	"END-ACCEPT", // 653 (911)
-	"END-ADD", // 654 (912)
-	"END-CALL", // 655 (913)
-	"END-COMPUTE", // 656 (914)
-	"END-DELETE", // 657 (915)
-	"END-DISPLAY", // 658 (916)
-	"END-DIVIDE", // 659 (917)
-	"END-EVALUATE", // 660 (918)
-	"END-MULTIPLY", // 661 (919)
-	"END-PERFORM", // 662 (920)
-	"END-READ", // 663 (921)
-	"END-RETURN", // 664 (922)
-	"END-REWRITE", // 665 (923)
-	"END-SEARCH", // 666 (924)
-	"END-START", // 667 (925)
-	"END-STRING", // 668 (926)
-	"END-SUBTRACT", // 669 (927)
-	"END-UNSTRING", // 670 (928)
-	"END-WRITE", // 671 (929)
-	"END-IF", // 672 (930)
-	"THRU", // 673 (931)
-	"OR", // 674 (932)
-	"AND", // 675 (933)
-	"NOT", // 676 (934)
-	"NE", // 677 (935)
-	"LE", // 678 (936)
-	"GE", // 679 (937)
-	"POW", // 680 (938)
-	"NEG", // 681 (939)
+	"CALL-CONVENTION", // 133 (391)
+	"CALL-COBOL", // 134 (392)
+	"CALL-VERBATIM", // 135 (393)
+	"CDF-PUSH", // 136 (394)
+	"CDF-POP", // 137 (395)
+	"SOURCE-FORMAT", // 138 (396)
+	"IF", // 139 (397)
+	"THEN", // 140 (398)
+	"ELSE", // 141 (399)
+	"SENTENCE", // 142 (400)
+	"ACCEPT", // 143 (401)
+	"ADD", // 144 (402)
+	"ALTER", // 145 (403)
+	"CALL", // 146 (404)
+	"CANCEL", // 147 (405)
+	"CLOSE", // 148 (406)
+	"COMPUTE", // 149 (407)
+	"CONTINUE", // 150 (408)
+	"DELETE", // 151 (409)
+	"DISPLAY", // 152 (410)
+	"DIVIDE", // 153 (411)
+	"EVALUATE", // 154 (412)
+	"EXIT", // 155 (413)
+	"FILLER", // 156 (414)
+	"GOBACK", // 157 (415)
+	"GOTO", // 158 (416)
+	"INITIALIZE", // 159 (417)
+	"INSPECT", // 160 (418)
+	"MERGE", // 161 (419)
+	"MOVE", // 162 (420)
+	"MULTIPLY", // 163 (421)
+	"OPEN", // 164 (422)
+	"PARAGRAPH", // 165 (423)
+	"READ", // 166 (424)
+	"RELEASE", // 167 (425)
+	"RETURN", // 168 (426)
+	"REWRITE", // 169 (427)
+	"SEARCH", // 170 (428)
+	"SET", // 171 (429)
+	"SELECT", // 172 (430)
+	"SORT", // 173 (431)
+	"SORT-MERGE", // 174 (432)
+	"STRING", // 175 (433)
+	"STOP", // 176 (434)
+	"SUBTRACT", // 177 (435)
+	"START", // 178 (436)
+	"UNSTRING", // 179 (437)
+	"WRITE", // 180 (438)
+	"WHEN", // 181 (439)
+	"ARGUMENT-NUMBER", // 182 (440)
+	"ARGUMENT-VALUE", // 183 (441)
+	"ENVIRONMENT-NAME", // 184 (442)
+	"ENVIRONMENT-VALUE", // 185 (443)
+	"ABS", // 186 (444)
+	"ACCESS", // 187 (445)
+	"ACOS", // 188 (446)
+	"ACTUAL", // 189 (447)
+	"ADVANCING", // 190 (448)
+	"AFTER", // 191 (449)
+	"ALL", // 192 (450)
+	"ALLOCATE", // 193 (451)
+	"ALPHABET", // 194 (452)
+	"ALPHABETIC", // 195 (453)
+	"ALPHABETIC-LOWER", // 196 (454)
+	"ALPHABETIC-UPPER", // 197 (455)
+	"ALPHANUMERIC", // 198 (456)
+	"ALPHANUMERIC-EDITED", // 199 (457)
+	"ALSO", // 200 (458)
+	"ALTERNATE", // 201 (459)
+	"ANNUITY", // 202 (460)
+	"ANUM", // 203 (461)
+	"ANY", // 204 (462)
+	"ANYCASE", // 205 (463)
+	"APPLY", // 206 (464)
+	"ARE", // 207 (465)
+	"AREA", // 208 (466)
+	"AREAS", // 209 (467)
+	"AS", // 210 (468)
+	"ASCENDING", // 211 (469)
+	"ACTIVATING", // 212 (470)
+	"ASIN", // 213 (471)
+	"ASSIGN", // 214 (472)
+	"AT", // 215 (473)
+	"ATAN", // 216 (474)
+	"BASED", // 217 (475)
+	"BASECONVERT", // 218 (476)
+	"BEFORE", // 219 (477)
+	"BINARY", // 220 (478)
+	"BIT", // 221 (479)
+	"BIT-OF", // 222 (480)
+	"BIT-TO-CHAR", // 223 (481)
+	"BLANK", // 224 (482)
+	"BLOCK", // 225 (483)
+	"BOOLEAN-OF-INTEGER", // 226 (484)
+	"BOTTOM", // 227 (485)
+	"BY", // 228 (486)
+	"BYTE", // 229 (487)
+	"BYTE-LENGTH", // 230 (488)
+	"CF", // 231 (489)
+	"CH", // 232 (490)
+	"CHANGED", // 233 (491)
+	"CHAR", // 234 (492)
+	"CHAR-NATIONAL", // 235 (493)
+	"CHARACTER", // 236 (494)
+	"CHARACTERS", // 237 (495)
+	"CHECKING", // 238 (496)
+	"CLASS", // 239 (497)
+	"COBOL", // 240 (498)
+	"CODE", // 241 (499)
+	"CODE-SET", // 242 (500)
+	"COLLATING", // 243 (501)
+	"COLUMN", // 244 (502)
+	"COMBINED-DATETIME", // 245 (503)
+	"COMMA", // 246 (504)
+	"COMMAND-LINE", // 247 (505)
+	"COMMAND-LINE-COUNT", // 248 (506)
+	"COMMIT", // 249 (507)
+	"COMMON", // 250 (508)
+	"CONCAT", // 251 (509)
+	"CONDITION", // 252 (510)
+	"CONFIGURATION", // 253 (511)
+	"CONTAINS", // 254 (512)
+	"CONTENT", // 255 (513)
+	"CONTROL", // 256 (514)
+	"CONTROLS", // 257 (515)
+	"CONVERT", // 258 (516)
+	"CONVERTING", // 259 (517)
+	"CORRESPONDING", // 260 (518)
+	"COS", // 261 (519)
+	"COUNT", // 262 (520)
+	"CURRENCY", // 263 (521)
+	"CURRENT", // 264 (522)
+	"CURRENT-DATE", // 265 (523)
+	"DATA", // 266 (524)
+	"DATE", // 267 (525)
+	"DATE-COMPILED", // 268 (526)
+	"DATE-OF-INTEGER", // 269 (527)
+	"DATE-TO-YYYYMMDD", // 270 (528)
+	"DATE-WRITTEN", // 271 (529)
+	"DAY", // 272 (530)
+	"DAY-OF-INTEGER", // 273 (531)
+	"DAY-OF-WEEK", // 274 (532)
+	"DAY-TO-YYYYDDD", // 275 (533)
+	"DBCS", // 276 (534)
+	"DE", // 277 (535)
+	"DEBUGGING", // 278 (536)
+	"DECIMAL-POINT", // 279 (537)
+	"DECLARATIVES", // 280 (538)
+	"DEFAULT", // 281 (539)
+	"DELIMITED", // 282 (540)
+	"DELIMITER", // 283 (541)
+	"DEPENDING", // 284 (542)
+	"DESCENDING", // 285 (543)
+	"DETAIL", // 286 (544)
+	"DIRECT", // 287 (545)
+	"DIRECT-ACCESS", // 288 (546)
+	"DOWN", // 289 (547)
+	"DUPLICATES", // 290 (548)
+	"DYNAMIC", // 291 (549)
+	"E", // 292 (550)
+	"EBCDIC", // 293 (551)
+	"EC", // 294 (552)
+	"EGCS", // 295 (553)
+	"ENTRY", // 296 (554)
+	"ENVIRONMENT", // 297 (555)
+	"EQUAL", // 298 (556)
+	"EVERY", // 299 (557)
+	"EXAMINE", // 300 (558)
+	"EXHIBIT", // 301 (559)
+	"EXP", // 302 (560)
+	"EXP10", // 303 (561)
+	"EXTEND", // 304 (562)
+	"EXTERNAL", // 305 (563)
+	"EXCEPTION-FILE", // 306 (564)
+	"EXCEPTION-FILE-N", // 307 (565)
+	"EXCEPTION-LOCATION", // 308 (566)
+	"EXCEPTION-LOCATION-N", // 309 (567)
+	"EXCEPTION-STATEMENT", // 310 (568)
+	"EXCEPTION-STATUS", // 311 (569)
+	"FACTORIAL", // 312 (570)
+	"FALSE", // 313 (571)
+	"FD", // 314 (572)
+	"FILE-CONTROL", // 315 (573)
+	"FILE", // 316 (574)
+	"FILE-LIMIT", // 317 (575)
+	"FINAL", // 318 (576)
+	"FINALLY", // 319 (577)
+	"FIND-STRING", // 320 (578)
+	"FIRST", // 321 (579)
+	"FIXED", // 322 (580)
+	"FOOTING", // 323 (581)
+	"FOR", // 324 (582)
+	"FORMATTED-CURRENT-DATE", // 325 (583)
+	"FORMATTED-DATE", // 326 (584)
+	"FORMATTED-DATETIME", // 327 (585)
+	"FORMATTED-TIME", // 328 (586)
+	"FORM-OVERFLOW", // 329 (587)
+	"FREE", // 330 (588)
+	"FRACTION-PART", // 331 (589)
+	"FROM", // 332 (590)
+	"FUNCTION", // 333 (591)
+	"GENERATE", // 334 (592)
+	"GIVING", // 335 (593)
+	"GLOBAL", // 336 (594)
+	"GO", // 337 (595)
+	"GROUP", // 338 (596)
+	"HEADING", // 339 (597)
+	"HEX", // 340 (598)
+	"HEX-OF", // 341 (599)
+	"HEX-TO-CHAR", // 342 (600)
+	"HIGH-VALUES", // 343 (601)
+	"HIGHEST-ALGEBRAIC", // 344 (602)
+	"HOLD", // 345 (603)
+	"IBM-360", // 346 (604)
+	"IN", // 347 (605)
+	"INCLUDE", // 348 (606)
+	"INDEX", // 349 (607)
+	"INDEXED", // 350 (608)
+	"INDICATE", // 351 (609)
+	"INITIAL", // 352 (610)
+	"INITIATE", // 353 (611)
+	"INPUT", // 354 (612)
+	"INSTALLATION", // 355 (613)
+	"INTERFACE", // 356 (614)
+	"INTEGER", // 357 (615)
+	"INTEGER-OF-BOOLEAN", // 358 (616)
+	"INTEGER-OF-DATE", // 359 (617)
+	"INTEGER-OF-DAY", // 360 (618)
+	"INTEGER-OF-FORMATTED-DATE", // 361 (619)
+	"INTEGER-PART", // 362 (620)
+	"INTO", // 363 (621)
+	"INTRINSIC", // 364 (622)
+	"INVOKE", // 365 (623)
+	"I-O", // 366 (624)
+	"I-O-CONTROL", // 367 (625)
+	"IS", // 368 (626)
+	"ISNT", // 369 (627)
+	"KANJI", // 370 (628)
+	"KEY", // 371 (629)
+	"LABEL", // 372 (630)
+	"LAST", // 373 (631)
+	"LEADING", // 374 (632)
+	"LEFT", // 375 (633)
+	"LENGTH", // 376 (634)
+	"LENGTH-OF", // 377 (635)
+	"LIMIT", // 378 (636)
+	"LIMITS", // 379 (637)
+	"LINE", // 380 (638)
+	"LINES", // 381 (639)
+	"LINE-COUNTER", // 382 (640)
+	"LINAGE", // 383 (641)
+	"LINKAGE", // 384 (642)
+	"LOCALE", // 385 (643)
+	"LOCALE-COMPARE", // 386 (644)
+	"LOCALE-DATE", // 387 (645)
+	"LOCALE-TIME", // 388 (646)
+	"LOCALE-TIME-FROM-SECONDS", // 389 (647)
+	"LOCAL-STORAGE", // 390 (648)
+	"LOCATION", // 391 (649)
+	"LOCK", // 392 (650)
+	"LOCK-ON", // 393 (651)
+	"LOG", // 394 (652)
+	"LOG10", // 395 (653)
+	"LOWER-CASE", // 396 (654)
+	"LOW-VALUES", // 397 (655)
+	"LOWEST-ALGEBRAIC", // 398 (656)
+	"LPAREN", // 399 (657)
+	"MANUAL", // 400 (658)
+	"MAXX", // 401 (659)
+	"MEAN", // 402 (660)
+	"MEDIAN", // 403 (661)
+	"MIDRANGE", // 404 (662)
+	"MINN", // 405 (663)
+	"MULTIPLE", // 406 (664)
+	"MOD", // 407 (665)
+	"MODE", // 408 (666)
+	"MODULE-NAME", // 409 (667)
+	"NAMED", // 410 (668)
+	"NAT", // 411 (669)
+	"NATIONAL", // 412 (670)
+	"NATIONAL-EDITED", // 413 (671)
+	"NATIONAL-OF", // 414 (672)
+	"NATIVE", // 415 (673)
+	"NESTED", // 416 (674)
+	"NEXT", // 417 (675)
+	"NO", // 418 (676)
+	"NOTE", // 419 (677)
+	"NULLS", // 420 (678)
+	"NULLPTR", // 421 (679)
+	"NUMERIC", // 422 (680)
+	"NUMERIC-EDITED", // 423 (681)
+	"NUMVAL", // 424 (682)
+	"NUMVAL-C", // 425 (683)
+	"NUMVAL-F", // 426 (684)
+	"OCCURS", // 427 (685)
+	"OF", // 428 (686)
+	"OFF", // 429 (687)
+	"OMITTED", // 430 (688)
+	"ON", // 431 (689)
+	"ONLY", // 432 (690)
+	"OPTIONAL", // 433 (691)
+	"OPTIONS", // 434 (692)
+	"ORD", // 435 (693)
+	"ORDER", // 436 (694)
+	"ORD-MAX", // 437 (695)
+	"ORD-MIN", // 438 (696)
+	"ORGANIZATION", // 439 (697)
+	"OTHER", // 440 (698)
+	"OTHERWISE", // 441 (699)
+	"OUTPUT", // 442 (700)
+	"PACKED-DECIMAL", // 443 (701)
+	"PADDING", // 444 (702)
+	"PAGE", // 445 (703)
+	"PAGE-COUNTER", // 446 (704)
+	"PF", // 447 (705)
+	"PH", // 448 (706)
+	"PI", // 449 (707)
+	"PIC", // 450 (708)
+	"PICTURE", // 451 (709)
+	"PLUS", // 452 (710)
+	"PRESENT-VALUE", // 453 (711)
+	"PRINT-SWITCH", // 454 (712)
+	"PROCEDURE", // 455 (713)
+	"PROCEDURES", // 456 (714)
+	"PROCEED", // 457 (715)
+	"PROCESS", // 458 (716)
+	"PROGRAM-ID", // 459 (717)
+	"PROGRAM", // 460 (718)
+	"PROPERTY", // 461 (719)
+	"PROTOTYPE", // 462 (720)
+	"PSEUDOTEXT", // 463 (721)
+	"QUOTES", // 464 (722)
+	"RANDOM", // 465 (723)
+	"RANDOM-SEED", // 466 (724)
+	"RANGE", // 467 (725)
+	"RAISE", // 468 (726)
+	"RAISING", // 469 (727)
+	"RD", // 470 (728)
+	"RECORD", // 471 (729)
+	"RECORDING", // 472 (730)
+	"RECORDS", // 473 (731)
+	"RECURSIVE", // 474 (732)
+	"REDEFINES", // 475 (733)
+	"REEL", // 476 (734)
+	"REFERENCE", // 477 (735)
+	"RELATIVE", // 478 (736)
+	"REM", // 479 (737)
+	"REMAINDER", // 480 (738)
+	"REMARKS", // 481 (739)
+	"REMOVAL", // 482 (740)
+	"RENAMES", // 483 (741)
+	"REPLACE", // 484 (742)
+	"REPLACING", // 485 (743)
+	"REPORT", // 486 (744)
+	"REPORTING", // 487 (745)
+	"REPORTS", // 488 (746)
+	"REPOSITORY", // 489 (747)
+	"RERUN", // 490 (748)
+	"RESERVE", // 491 (749)
+	"RESTRICTED", // 492 (750)
+	"RESUME", // 493 (751)
+	"REVERSE", // 494 (752)
+	"REVERSED", // 495 (753)
+	"REWIND", // 496 (754)
+	"RF", // 497 (755)
+	"RH", // 498 (756)
+	"RIGHT", // 499 (757)
+	"ROUNDED", // 500 (758)
+	"RUN", // 501 (759)
+	"SAME", // 502 (760)
+	"SCREEN", // 503 (761)
+	"SD", // 504 (762)
+	"SECONDS-FROM-FORMATTED-TIME", // 505 (763)
+	"SECONDS-PAST-MIDNIGHT", // 506 (764)
+	"SECURITY", // 507 (765)
+	"SEPARATE", // 508 (766)
+	"SEQUENCE", // 509 (767)
+	"SEQUENTIAL", // 510 (768)
+	"SHARING", // 511 (769)
+	"SIMPLE-EXIT", // 512 (770)
+	"SIGN", // 513 (771)
+	"SIN", // 514 (772)
+	"SIZE", // 515 (773)
+	"SMALLEST-ALGEBRAIC", // 516 (774)
+	"SOURCE", // 517 (775)
+	"SOURCE-COMPUTER", // 518 (776)
+	"SPECIAL-NAMES", // 519 (777)
+	"SQRT", // 520 (778)
+	"STACK", // 521 (779)
+	"STANDARD", // 522 (780)
+	"STANDARD-1", // 523 (781)
+	"STANDARD-DEVIATION", // 524 (782)
+	"STANDARD-COMPARE", // 525 (783)
+	"STATUS", // 526 (784)
+	"STRONG", // 527 (785)
+	"SUBSTITUTE", // 528 (786)
+	"SUM", // 529 (787)
+	"SYMBOL", // 530 (788)
+	"SYMBOLIC", // 531 (789)
+	"SYNCHRONIZED", // 532 (790)
+	"TALLY", // 533 (791)
+	"TALLYING", // 534 (792)
+	"TAN", // 535 (793)
+	"TERMINATE", // 536 (794)
+	"TEST", // 537 (795)
+	"TEST-DATE-YYYYMMDD", // 538 (796)
+	"TEST-DAY-YYYYDDD", // 539 (797)
+	"TEST-FORMATTED-DATETIME", // 540 (798)
+	"TEST-NUMVAL", // 541 (799)
+	"TEST-NUMVAL-C", // 542 (800)
+	"TEST-NUMVAL-F", // 543 (801)
+	"THAN", // 544 (802)
+	"TIME", // 545 (803)
+	"TIMES", // 546 (804)
+	"TO", // 547 (805)
+	"TOP", // 548 (806)
+	"TOP-LEVEL", // 549 (807)
+	"TRACKS", // 550 (808)
+	"TRACK-AREA", // 551 (809)
+	"TRAILING", // 552 (810)
+	"TRANSFORM", // 553 (811)
+	"TRIM", // 554 (812)
+	"TRUE", // 555 (813)
+	"TRY", // 556 (814)
+	"TURN", // 557 (815)
+	"TYPE", // 558 (816)
+	"TYPEDEF", // 559 (817)
+	"ULENGTH", // 560 (818)
+	"UNBOUNDED", // 561 (819)
+	"UNIT", // 562 (820)
+	"UNITS", // 563 (821)
+	"UNIT-RECORD", // 564 (822)
+	"UNTIL", // 565 (823)
+	"UP", // 566 (824)
+	"UPON", // 567 (825)
+	"UPOS", // 568 (826)
+	"UPPER-CASE", // 569 (827)
+	"USAGE", // 570 (828)
+	"USING", // 571 (829)
+	"USUBSTR", // 572 (830)
+	"USUPPLEMENTARY", // 573 (831)
+	"UTILITY", // 574 (832)
+	"UUID4", // 575 (833)
+	"UVALID", // 576 (834)
+	"UWIDTH", // 577 (835)
+	"VALUE", // 578 (836)
+	"VARIANCE", // 579 (837)
+	"VARYING", // 580 (838)
+	"VOLATILE", // 581 (839)
+	"WHEN-COMPILED", // 582 (840)
+	"WITH", // 583 (841)
+	"WORKING-STORAGE", // 584 (842)
+	"XML", // 585 (843)
+	"XMLGENERATE", // 586 (844)
+	"XMLPARSE", // 587 (845)
+	"YEAR-TO-YYYY", // 588 (846)
+	"YYYYDDD", // 589 (847)
+	"YYYYMMDD", // 590 (848)
+	"ARITHMETIC", // 591 (849)
+	"ATTRIBUTE", // 592 (850)
+	"AUTO", // 593 (851)
+	"AUTOMATIC", // 594 (852)
+	"AWAY-FROM-ZERO", // 595 (853)
+	"BACKGROUND-COLOR", // 596 (854)
+	"BELL", // 597 (855)
+	"BINARY-ENCODING", // 598 (856)
+	"BLINK", // 599 (857)
+	"CAPACITY", // 600 (858)
+	"CENTER", // 601 (859)
+	"CLASSIFICATION", // 602 (860)
+	"CYCLE", // 603 (861)
+	"DECIMAL-ENCODING", // 604 (862)
+	"ENTRY-CONVENTION", // 605 (863)
+	"EOL", // 606 (864)
+	"EOS", // 607 (865)
+	"ERASE", // 608 (866)
+	"EXPANDS", // 609 (867)
+	"FLOAT-BINARY", // 610 (868)
+	"FLOAT-DECIMAL", // 611 (869)
+	"FOREGROUND-COLOR", // 612 (870)
+	"FOREVER", // 613 (871)
+	"FULL", // 614 (872)
+	"HIGHLIGHT", // 615 (873)
+	"HIGH-ORDER-LEFT", // 616 (874)
+	"HIGH-ORDER-RIGHT", // 617 (875)
+	"IGNORING", // 618 (876)
+	"IMPLEMENTS", // 619 (877)
+	"INITIALIZED", // 620 (878)
+	"INTERMEDIATE", // 621 (879)
+	"LC-ALL", // 622 (880)
+	"LC-COLLATE", // 623 (881)
+	"LC-CTYPE", // 624 (882)
+	"LC-MESSAGES", // 625 (883)
+	"LC-MONETARY", // 626 (884)
+	"LC-NUMERIC", // 627 (885)
+	"LC-TIME", // 628 (886)
+	"LOWLIGHT", // 629 (887)
+	"NEAREST-AWAY-FROM-ZERO", // 630 (888)
+	"NEAREST-EVEN", // 631 (889)
+	"NEAREST-TOWARD-ZERO", // 632 (890)
+	"NONE", // 633 (891)
+	"NORMAL", // 634 (892)
+	"NUMBERS", // 635 (893)
+	"PREFIXED", // 636 (894)
+	"PREVIOUS", // 637 (895)
+	"PROHIBITED", // 638 (896)
+	"RELATION", // 639 (897)
+	"REQUIRED", // 640 (898)
+	"REVERSE-VIDEO", // 641 (899)
+	"ROUNDING", // 642 (900)
+	"SECONDS", // 643 (901)
+	"SECURE", // 644 (902)
+	"SHORT", // 645 (903)
+	"SIGNED", // 646 (904)
+	"STANDARD-BINARY", // 647 (905)
+	"STANDARD-DECIMAL", // 648 (906)
+	"STATEMENT", // 649 (907)
+	"STEP", // 650 (908)
+	"STRUCTURE", // 651 (909)
+	"TOWARD-GREATER", // 652 (910)
+	"TOWARD-LESSER", // 653 (911)
+	"TRUNCATION", // 654 (912)
+	"UCS-4", // 655 (913)
+	"UNDERLINE", // 656 (914)
+	"UNSIGNED", // 657 (915)
+	"UTF-16", // 658 (916)
+	"UTF-8", // 659 (917)
+	"ADDRESS", // 660 (918)
+	"END-ACCEPT", // 661 (919)
+	"END-ADD", // 662 (920)
+	"END-CALL", // 663 (921)
+	"END-COMPUTE", // 664 (922)
+	"END-DELETE", // 665 (923)
+	"END-DISPLAY", // 666 (924)
+	"END-DIVIDE", // 667 (925)
+	"END-EVALUATE", // 668 (926)
+	"END-MULTIPLY", // 669 (927)
+	"END-PERFORM", // 670 (928)
+	"END-READ", // 671 (929)
+	"END-RETURN", // 672 (930)
+	"END-REWRITE", // 673 (931)
+	"END-SEARCH", // 674 (932)
+	"END-START", // 675 (933)
+	"END-STRING", // 676 (934)
+	"END-SUBTRACT", // 677 (935)
+	"END-UNSTRING", // 678 (936)
+	"END-WRITE", // 679 (937)
+	"END-IF", // 680 (938)
+	"THRU", // 681 (939)
+	"OR", // 682 (940)
+	"AND", // 683 (941)
+	"NOT", // 684 (942)
+	"NE", // 685 (943)
+	"LE", // 686 (944)
+	"GE", // 687 (945)
+	"POW", // 688 (946)
+	"NEG", // 689 (947)
 };
diff --git a/gcc/cobol/udf/stored-char-length.cbl b/gcc/cobol/udf/stored-char-length.cbl
index 9ab3b14..66889d0 100644
--- a/gcc/cobol/udf/stored-char-length.cbl
+++ b/gcc/cobol/udf/stored-char-length.cbl
@@ -1,3 +1,6 @@
+        >> PUSH source format
+        >>SOURCE format is fixed
+
       *  This function is in public domain.
       *  Contributed by James K. Lowden of Cobolworx in August 2024
 
@@ -13,3 +16,4 @@
             to Output-Value. 
         End Function STORED-CHAR-LENGTH.
 
+        >> pop source format
\ No newline at end of file
diff --git a/gcc/cobol/util.cc b/gcc/cobol/util.cc
index 23f605d..aed9483 100644
--- a/gcc/cobol/util.cc
+++ b/gcc/cobol/util.cc
@@ -34,29 +34,24 @@
  * header files.
  */
 
-#include "cobol-system.h"
-#include "coretypes.h"
-#include "tree.h"
+#include <cobol-system.h>
+#include <coretypes.h>
+#include <tree.h>
 #undef yy_flex_debug
 
 #include <langinfo.h>
 
-#include "coretypes.h"
-#include "version.h"
-#include "demangle.h"
-#include "intl.h"
-#include "backtrace.h"
-#include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "edit-context.h"
-#include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "opts.h"
+#include <coretypes.h>
+#include <version.h>
+#include <demangle.h>
+#include <intl.h>
+#include <backtrace.h>
+#include <diagnostic.h>
+#include <opts.h>
 #include "util.h"
+
 #include "cbldiag.h"
+#include "cdfval.h"
 #include "lexio.h"
 
 #include "../../libgcobol/ec.h"
@@ -90,7 +85,7 @@ static inline char *
 get_current_dir_name ()
 {
   /* Use libiberty's allocator here.  */
-  char *buf = (char *) xmalloc (PATH_MAX);
+  char *buf = static_cast<char *>(xmalloc (PATH_MAX));
   return getcwd (buf, PATH_MAX);
 }
 #endif
@@ -98,7 +93,7 @@ get_current_dir_name ()
 /*
  * For printing messages, usually the size of the thing is some kind of string
  * length, and doesn't really need a size_t.  For message formatting, use a
- * simple unsigned long, and warn if that's no good.  "gb4" here stands for 
+ * simple unsigned long, and warn if that's no good.  "gb4" here stands for
  * "4 Gigabytes".
  */
 unsigned long
@@ -110,7 +105,159 @@ gb4( size_t input ) {
   }
   return input;
 }
-  
+
+/*
+ * Most CDF Directives -- those that have state -- can be pushed and popped.
+ * This class maintains stacks of them, with each stack having a "default
+ * value" that may be updated, without push/pop, via a CDF directive or
+ * command-line option.  A push to a stack pushes the default value onto it; a
+ * pop copies the top of the stack to the default value.
+ *
+ * Supported:
+ *   CALL-CONVENTION
+ *   COBOL-WORDS
+ *   DEFINE
+ *   DISPLAY
+ *   IF
+ *   POP
+ *   PUSH
+ *   SOURCE FORMAT
+ *   TURN
+ * not supported
+ *   EVALUATE
+ *   FLAG-02
+ *   FLAG-14
+ *   LEAP-SECOND
+ *   LISTING
+ *   PAGE
+ *   PROPAGATE
+ *   REF-MOD-ZERO-LENGTH
+ *
+ * >>PUSH ALL calls the class's push() method.
+ * >>POP ALL calls the class's pop() method.
+ */
+class cdf_directives_t
+{
+  template <typename T>
+  class cdf_stack_t : private std::stack<T> { // cppcheck-suppress noConstructor
+    T default_value;
+    const T& top() const { return std::stack<T>::top(); }
+    bool empty() const { return std::stack<T>::empty(); }
+   public:
+    void value( const T& value ) {
+      T& output( empty()? default_value : std::stack<T>::top() ); // cppcheck-suppress constVariableReference
+      output = value;
+      dbgmsg("cdf_directives_t::%s: %s", __func__, str(output).c_str());
+    }
+    T& value() {
+      return empty()? default_value : std::stack<T>::top();
+    }
+    void push() {
+      std::stack<T>::push(value());
+      dbgmsg("cdf_directives_t::%s: %s", __func__, str(top()).c_str());
+    }
+    void pop() {
+      if( empty() ) {
+        error_msg(YYLTYPE(), "CDF stack empty"); // cppcheck-suppress syntaxError
+        return;
+      }
+      default_value = top();
+      std::stack<T>::pop();
+      dbgmsg("cdf_directives_t::%s: %s", __func__, str(default_value).c_str());
+    }
+  protected:
+    static std::string str(cbl_call_convention_t arg) {
+      char output[2] = { static_cast<char>(arg) };
+      return std::string("call-convention ") + output;
+    }
+    static std::string str(current_tokens_t) {
+      return "<cobol-words>";
+    }
+    static std::string str(cdf_values_t) {
+      return "<dictionary>";
+    }
+    static std::string str(source_format_t arg) {
+      return arg.description();
+    }
+    static std::string str(cbl_enabled_exceptions_t) {
+      return "<enabled_exceptions>";
+    }
+  };
+
+ public:
+  cdf_stack_t<cbl_call_convention_t> call_convention;
+  cdf_stack_t<current_tokens_t> cobol_words;
+  cdf_stack_t<cdf_values_t> dictionary;   // DEFINE
+  cdf_stack_t<source_format_t> source_format;
+  cdf_stack_t<cbl_enabled_exceptions_t> enabled_exceptions;
+
+  cdf_directives_t() {
+    call_convention.value() = cbl_call_cobol_e;
+  }
+
+  void push() {
+    call_convention.push();
+    cobol_words.push();
+    dictionary.push();
+    source_format.push();
+    enabled_exceptions.push();
+  }
+  void pop() {
+    call_convention.pop();
+    cobol_words.pop();
+    dictionary.pop();
+    source_format.pop();
+    enabled_exceptions.pop();
+  }
+};
+static cdf_directives_t cdf_directives;
+
+void
+current_call_convention( cbl_call_convention_t convention) {
+  cdf_directives.call_convention.value(convention);
+}
+cbl_call_convention_t
+current_call_convention() {
+  return cdf_directives.call_convention.value();
+}
+
+current_tokens_t&
+cdf_current_tokens() {
+  return cdf_directives.cobol_words.value();
+}
+
+cdf_values_t&
+cdf_dictionary() {
+  return cdf_directives.dictionary.value();
+}
+
+void
+cobol_set_indicator_column( int column ) {
+  cdf_directives.source_format.value().indicator_column_set(column);
+}
+source_format_t& cdf_source_format() {
+  return cdf_directives.source_format.value();
+}
+
+cbl_enabled_exceptions_t&
+cdf_enabled_exceptions() {
+  return cdf_directives.enabled_exceptions.value();
+}
+
+void cdf_push() { cdf_directives.push(); }
+void cdf_push_call_convention() { cdf_directives.call_convention.push(); }
+void cdf_push_current_tokens() { cdf_directives.cobol_words.push(); }
+void cdf_push_dictionary() { cdf_directives.dictionary.push(); }
+void cdf_push_enabled_exceptions() { cdf_directives.enabled_exceptions.push(); }
+void cdf_push_source_format() { cdf_directives.source_format.push(); }
+
+void cdf_pop() { cdf_directives.pop(); }
+void cdf_pop_call_convention() { cdf_directives.call_convention.pop(); }
+void cdf_pop_current_tokens() { cdf_directives.cobol_words.pop(); }
+void cdf_pop_dictionary() { cdf_directives.dictionary.pop(); }
+void cdf_pop_enabled_exceptions() { cdf_directives.enabled_exceptions.pop(); }
+void cdf_pop_source_format() { cdf_directives.source_format.pop(); }
+
 const char *
 symbol_type_str( enum symbol_type_t type )
 {
@@ -839,7 +986,7 @@ cbl_field_t::report_invalid_initial_value(const YYLTYPE& loc) const {
         // 8 or more, we need do no further testing because we assume
         // everything fits.
         if( data.capacity < 8 ) {
-          const auto p = strchr(data.initial, symbol_decimal_point());
+          const char *p = strchr(data.initial, symbol_decimal_point());
           if( p && atoll(p+1) != 0 ) {
             error_msg(loc, "integer type %s VALUE '%s' "
                      "requires integer VALUE",
@@ -902,7 +1049,7 @@ cbl_field_t::report_invalid_initial_value(const YYLTYPE& loc) const {
                                                  return TOUPPER(ch) == 'E';
                                                } );
               if( !has_exponent && data.precision() < pend - p ) {
-                error_msg(loc, "%s cannot represent VALUE %qs exactly (max %c%zu)",
+                error_msg(loc, "%s cannot represent VALUE %qs exactly (max %c%td)",
                           name, data.initial, '.', pend - p);
               }
             }
@@ -985,7 +1132,7 @@ literal_subscript_oob( const cbl_refer_t& r, size_t& isub /* output */)  {
                            pdim++;
                            return ! occurs.subscript_ok(r.field);
                          } );
-  isub = psub - r.subscripts.begin(); 
+  isub = psub - r.subscripts.begin();
   return psub == r.subscripts.end()? NULL : dims[isub];
 }
 
@@ -998,12 +1145,12 @@ cbl_refer_t::subscripts_set( const std::list<cbl_refer_t>& subs ) {
 
 const char *
 cbl_refer_t::str() const {
-  static char subscripts[64];
-  sprintf(subscripts, "(%u of " HOST_SIZE_T_PRINT_UNSIGNED " dimensions)",
+  static char subscripts_l[64];
+  sprintf(subscripts_l, "(%u of " HOST_SIZE_T_PRINT_UNSIGNED " dimensions)",
           nsubscript(), (fmt_size_t)dimensions(field));
   char *output = xasprintf("%s %s %s",
                            field? field_str(field) : "(none)",
-                           0 < dimensions(field)? subscripts : "",
+                           0 < dimensions(field)? subscripts_l : "",
                            is_refmod_reference()? "(refmod)" : "" );
   return output;
 }
@@ -1705,12 +1852,13 @@ date_time_fmt( const char input[] ) {
     { regex_t(), 'd', "^(" DATE_FMT_B "|" DATE_FMT_E ")$" },
     { regex_t(), 't', "^(" TIME_FMT_B "|" TIME_FMT_E ")$" },
   };
-  int erc, cflags = REG_EXTENDED | REG_ICASE, eflags=0;
+  int cflags = REG_EXTENDED | REG_ICASE, eflags=0;
   regmatch_t m[5];
   char result = 0;
 
   if( ! compiled ) {
     for( auto& fmt : fmts ) {
+      int erc;
       if( (erc = regcomp(&fmt.reg, fmt.pattern, cflags)) != 0 ) {
         char msg[80];
         regerror(erc, &fmt.reg, msg, sizeof(msg));
@@ -1768,7 +1916,7 @@ class unique_stack : public std::stack<input_file_t>
   friend void cobol_set_pp_option(int opt);
   bool option_m;
   std::set<std::string> all_names;
-  
+
   const char *
   no_wd( const char *wd, const char *name ) {
     int i;
@@ -1779,7 +1927,7 @@ class unique_stack : public std::stack<input_file_t>
 
  public:
   unique_stack() : option_m(false) {}
-  
+
   bool push( const value_type& value ) {
     auto ok = std::none_of( c.cbegin(), c.cend(),
                             [value]( const auto& that ) {
@@ -1813,8 +1961,8 @@ class unique_stack : public std::stack<input_file_t>
   const input_file_t& peek( size_t n ) const {
     gcc_assert( n < size() );
     return c.at(size() - ++n);
-  } 
-  
+  }
+
   void option( int opt ) { // capture other preprocessor options eventually
     assert(opt == 'M');
     option_m = true;
@@ -1827,7 +1975,7 @@ class unique_stack : public std::stack<input_file_t>
     std::string input( top().name );
     printf( "%s: ", input.c_str() );
     for( const auto& name : all_names ) {
-      if( name != input ) 
+      if( name != input )
 	printf( "\\\n\t%s ", name.c_str() );
     }
     printf("\n");
@@ -1844,7 +1992,7 @@ void cobol_set_pp_option(int opt) {
   assert(opt == 'M');
   input_filenames.option_m = true;
 }
-				  
+
 /*
  * Maintain a stack of input filenames.  Ensure the files are unique (by
  * inode), to prevent copybook cycles. Before pushing a new name, Record the
@@ -1855,7 +2003,7 @@ void cobol_set_pp_option(int opt) {
  * to enforce uniqueness, and the scanner to maintain line numbers.
  */
 bool cobol_filename( const char *name, ino_t inode ) {
-  const line_map *lines = NULL;
+  //const line_map *lines = NULL;
   if( inode == 0 ) {
     auto p = old_filenames.find(name);
     if( p == old_filenames.end() ) {
@@ -1865,8 +2013,10 @@ bool cobol_filename( const char *name, ino_t inode ) {
       }
       cbl_errx( "logic error: missing inode for %s", name);
     }
-    inode = p->second;
-    assert(inode != 0);
+    else {
+      inode = p->second;
+      assert(inode != 0);
+    }
   }
   linemap_add(line_table, LC_ENTER, sysp, name, 1);
   input_filename_vestige = name;
@@ -1915,20 +2065,50 @@ cobol_filename_restore() {
   input_filenames.pop();
   if( input_filenames.empty() ) return;
 
-  auto& input = input_filenames.top();
+  const auto& input = input_filenames.top();
 
   linemap_add(line_table, LC_LEAVE, sysp, NULL, 0);
 }
 
-static location_t token_location;
+static int first_line_minus_1 = 0;
+static location_t token_location_minus_1 = 0;
+static location_t token_location = 0;
 
-location_t location_from_lineno() { return token_location; }
+location_t current_token_location() { return token_location; }
+location_t current_location_minus_one() { return token_location_minus_1; }
+void current_location_minus_one_clear()
+  {
+  first_line_minus_1 = 0;
+  }
 
 template <typename LOC>
 static void
 gcc_location_set_impl( const LOC& loc ) {
-  token_location = linemap_line_start( line_table, loc.last_line, 80 );
+  // Set the position to the first line & column in the location.
+  if( getenv("KILROY") )
+    {
+    fprintf(stderr, "********** KILROY %d\n", loc.first_line);
+    }
+
+ static location_t loc_m_1 = 0;
+
+  token_location = linemap_line_start( line_table, loc.first_line, 80 );
   token_location = linemap_position_for_column( line_table, loc.first_column);
+
+  if( loc.first_line > first_line_minus_1 )
+    {
+    // In order for GDB-COBOL to be able to step through COBOL code properly,
+    // it is sometimes necessary for the code at the beginning of a COBOL
+    // line to be using the location_t of the previous line.  This is true, for
+    // example, when laying down the infrastructure code between the last
+    // statement of a paragraph and the code created at the beginning of the
+    // following paragragh.  This code assumes that token_location values of
+    // interest are monotonic, and stores that prior value.
+    first_line_minus_1 = loc.first_line;
+    token_location_minus_1 = loc_m_1;
+    loc_m_1 = token_location;
+    }
+
   location_dump(__func__, __LINE__, "parser", loc);
 }
 
@@ -1969,9 +2149,14 @@ verify_format( const char gmsgid[] ) {
 }
 #endif
 
-static const diagnostic_option_id option_zero;
+static const diagnostics::option_id option_zero;
 size_t parse_error_inc();
 
+void gcc_location_dump() {
+    linemap_dump_location( line_table, token_location, stderr );
+}
+
+
 void ydferror( const char gmsgid[], ... ) ATTRIBUTE_GCOBOL_DIAG(1, 2);
 
 void
@@ -1982,8 +2167,9 @@ ydferror( const char gmsgid[], ... ) {
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, token_location);
-  bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_zero,
-                                         gmsgid, &ap, DK_ERROR);
+  /*bool ret =*/ global_dc->diagnostic_impl (&richloc, nullptr, option_zero,
+                                             gmsgid, &ap,
+                                             diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -2008,10 +2194,7 @@ class temp_loc_t {
     gcc_location_set(loc);
   }
   explicit temp_loc_t( const YDFLTYPE& loc) : orig(token_location) {
-    YYLTYPE lloc = {
-      loc.first_line, loc.first_column,
-      loc.last_line,  loc.last_column };
-    gcc_location_set(lloc);
+    gcc_location_set(loc);
   }
   ~temp_loc_t() {
     if( orig != token_location ) {
@@ -2041,7 +2224,8 @@ class temp_loc_t {
   va_start (ap, gmsgid);                                                \
   rich_location richloc (line_table, token_location);                   \
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_zero,        \
-                                         gmsgid, &ap, DK_ERROR);        \
+                                         gmsgid, &ap,                   \
+                                         diagnostics::kind::error);     \
   va_end (ap);                                                          \
   global_dc->end_group();
 
@@ -2057,6 +2241,33 @@ void error_msg( const YDFLTYPE& loc, const char gmsgid[], ... ) {
   ERROR_MSG_BODY
 }
 
+bool
+warn_msg( const YYLTYPE& loc, const char gmsgid[], ... ) {
+  temp_loc_t looker(loc);
+  verify_format(gmsgid);
+  auto_diagnostic_group d;
+  va_list ap;
+  va_start (ap, gmsgid);
+  rich_location richloc (line_table, token_location);
+  auto ret = emit_diagnostic_valist( diagnostics::kind::warning,
+                                     token_location,
+                                     option_zero, gmsgid, &ap );
+  va_end (ap);
+  return ret;
+}
+
+void error_msg_direct( const char gmsgid[], ... ) {
+  verify_format(gmsgid);
+  parse_error_inc();
+  auto_diagnostic_group d;
+  va_list ap;
+  va_start (ap, gmsgid);
+  /*auto ret = */emit_diagnostic_valist( diagnostics::kind::error,
+                                         token_location,
+                                         option_zero, gmsgid, &ap );
+  va_end (ap);
+}
+
 void
 yyerror( const char gmsgid[], ... ) {
   temp_loc_t looker;
@@ -2066,8 +2277,11 @@ yyerror( const char gmsgid[], ... ) {
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, token_location);
-  bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_zero,
-                                         gmsgid, &ap, DK_ERROR);
+  /*bool ret =*/ global_dc->diagnostic_impl ( &richloc,
+                                              nullptr,
+                                              option_zero,
+                                              gmsgid,
+                                              &ap, diagnostics::kind::error);
   va_end (ap);
   global_dc->end_group();
 }
@@ -2078,7 +2292,7 @@ yywarn( const char gmsgid[], ... ) {
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  auto ret = emit_diagnostic_valist( DK_WARNING, token_location,
+  auto ret = emit_diagnostic_valist( diagnostics::kind::warning, token_location,
                                      option_zero, gmsgid, &ap );
   va_end (ap);
   return ret;
@@ -2276,8 +2490,11 @@ cbl_internal_error(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_ICE, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::ice,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
+  abort();  // This unnecessary statement is needed so that [[noreturn]]
+  //        // doesn't cause a warning.
 }
 
 void
@@ -2286,7 +2503,8 @@ cbl_unimplementedw(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_SORRY, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::sorry,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 
@@ -2296,7 +2514,8 @@ cbl_unimplemented(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_SORRY, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::sorry,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 
@@ -2307,12 +2526,13 @@ cbl_unimplemented_at( const YYLTYPE& loc, const char *gmsgid, ... ) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_SORRY, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::sorry,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 
-/* 
- * analogs to err(3) and errx(3). 
+/*
+ * analogs to err(3) and errx(3).
  */
 
 #pragma GCC diagnostic push
@@ -2324,7 +2544,8 @@ cbl_err(const char *fmt, ...) {
   verify_format(gmsgid);
   va_list ap;
   va_start(ap, fmt);
-  emit_diagnostic_valist( DK_FATAL, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::fatal,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 #pragma GCC diagnostic pop
@@ -2335,7 +2556,8 @@ cbl_errx(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_FATAL, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::fatal,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
   }
 
@@ -2447,7 +2669,7 @@ static const std::set<std::string> reserved_words = {
   "VOLATILE",
   "XML",
   "END-START",
-  
+
   // ISO 2023 keywords
   "ACCEPT",
   "ACCESS",
diff --git a/gcc/cobol/util.h b/gcc/cobol/util.h
index 165915a..d478ea2 100644
--- a/gcc/cobol/util.h
+++ b/gcc/cobol/util.h
@@ -33,7 +33,7 @@
 
 void cbl_message(int fd, const char *format_string, ...)
   ATTRIBUTE_PRINTF_2;
-void cbl_internal_error(const char *format_string, ...)
+[[noreturn]] void cbl_internal_error(const char *format_string, ...)
   ATTRIBUTE_GCOBOL_DIAG(1, 2);
 
 void cbl_err(const char *format_string, ...) ATTRIBUTE_GCOBOL_DIAG(1, 2);
@@ -49,7 +49,7 @@ void cobol_set_pp_option(int opt);
 
 void cobol_filename_restore();
 const char * cobol_lineno( int );
-int cobol_lineno();
+int cobol_lineno(void);
 
 unsigned long gb4( size_t input );
 
@@ -59,5 +59,70 @@ as_voidp( P p ) {
   return static_cast<const void *>(p);
 }
 
+/*
+ * The default source format, whether free or fixed, is determined
+ * heuristically by examining the PROGRAM-ID line, if it exists, in the first
+ * input file. If that file does not have such a line, the default is free
+ * format.  Else the format is set to fixed if anything appears on that line
+ * that would prohibit parsing it as free format,
+ */
+class source_format_t {
+  bool first_file, explicitly;
+  int left, right;
+public:
+  source_format_t()
+    : first_file(true), explicitly(false), left(0), right(0)
+  {}
+  void indicator_column_set( int column ) {
+    explicitly = true;
+    if( column == 0 ) right = 0;
+    if( column < 0 ) {
+      column = -column;
+      right = 73;
+    }
+    left = column;
+  }
+  
+  bool inference_pending() {
+    bool tf = first_file && !explicitly;
+    first_file = false;
+    return tf;
+  }
+
+  void infer( const char *bol, bool want_reference_format );
+  
+  inline bool is_fixed() const { return left == 7; }
+  inline bool is_reffmt() const { return is_fixed() && right == 73; }
+  inline bool is_free() const { return ! is_fixed(); }
+  
+  const char * description() const {
+    if( is_reffmt() ) return "REFERENCE";
+    if( is_fixed() ) return "FIXED";
+    if( is_free() ) return "FREE";
+    gcc_unreachable();
+  }    
+
+  inline int left_margin() {
+    return left == 0? left : left - 1;
+  }
+  inline int right_margin() {
+    return right == 0? right : right - 1;
+  }
+}; 
+
+
+void cdf_push();
+void cdf_push_call_convention();
+void cdf_push_current_tokens();
+void cdf_push_dictionary();
+void cdf_push_enabled_exceptions();
+void cdf_push_source_format();
+
+void cdf_pop();
+void cdf_pop_call_convention();
+void cdf_pop_current_tokens();
+void cdf_pop_dictionary();
+void cdf_pop_source_format();
+void cdf_pop_enabled_exceptions();
 
 #endif
diff --git a/gcc/common.opt b/gcc/common.opt
index 8163e03..70659fa 100644
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -547,7 +547,7 @@ Warn if an array is accessed out of bounds.
 
 Wauto-profile
 Common Var(warn_auto_profile) Warning
-Warn about problems with auto-profile data
+Warn about problems with auto-profile data.
 
 Wuse-after-free
 Common Var(warn_use_after_free) Warning
@@ -851,11 +851,17 @@ Common Var(warn_unused) Init(0) Warning
 Enable all -Wunused- warnings.
 
 Wunused-but-set-parameter
-Common Var(warn_unused_but_set_parameter) Warning EnabledBy(Wunused && Wextra)
+Common Alias(Wunused-but-set-parameter=,3,0) Warning
+
+Wunused-but-set-parameter=
+Common Var(warn_unused_but_set_parameter) RejectNegative Joined UInteger Warning IntegerRange(0, 3) EnabledBy(Wunused && Wextra)
 Warn when a function parameter is only set, otherwise unused.
 
 Wunused-but-set-variable
-Common Var(warn_unused_but_set_variable) Warning EnabledBy(Wunused)
+Common Alias(Wunused-but-set-variable=,3,0) Warning
+
+Wunused-but-set-variable=
+Common Var(warn_unused_but_set_variable) RejectNegative Joined UInteger Warning IntegerRange(0, 3) EnabledBy(Wunused)
 Warn when a variable is only set, otherwise unused.
 
 Wunused-function
@@ -1441,7 +1447,7 @@ Driver Common Joined RejectNegative Var(flag_diagnostics_show_color) Enum(diagno
 
 ; Required for these enum values.
 SourceInclude
-diagnostic-color.h
+diagnostics/color.h
 
 Enum
 Name(diagnostic_color_rule) Type(int)
@@ -1465,7 +1471,7 @@ Driver Common Joined RejectNegative Var(flag_diagnostics_show_urls) Enum(diagnos
 
 ; Required for these enum values.
 SourceInclude
-diagnostic-url.h
+diagnostics/url.h
 
 Enum
 Name(diagnostic_url_rule) Type(int)
diff --git a/gcc/common.opt.urls b/gcc/common.opt.urls
index c705089..38dd9d3 100644
--- a/gcc/common.opt.urls
+++ b/gcc/common.opt.urls
@@ -64,6 +64,9 @@ UrlSuffix(gcc/Warning-Options.html#index-Warray-bounds)
 Warray-bounds=
 UrlSuffix(gcc/Warning-Options.html#index-Warray-bounds)
 
+Wauto-profile
+UrlSuffix(gcc/Warning-Options.html#index-Wauto-profile)
+
 Wuse-after-free
 UrlSuffix(gcc/Warning-Options.html#index-Wno-use-after-free)
 
@@ -269,9 +272,15 @@ UrlSuffix(gcc/Warning-Options.html#index-Wno-unused)
 Wunused-but-set-parameter
 UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-parameter)
 
+Wunused-but-set-parameter=
+UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-parameter)
+
 Wunused-but-set-variable
 UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-variable)
 
+Wunused-but-set-variable=
+UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-variable)
+
 Wunused-function
 UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-function)
 
diff --git a/gcc/common/config/aarch64/cpuinfo.h b/gcc/common/config/aarch64/cpuinfo.h
index cd3c2b2..d329d86 100644
--- a/gcc/common/config/aarch64/cpuinfo.h
+++ b/gcc/common/config/aarch64/cpuinfo.h
@@ -39,10 +39,10 @@ enum CPUFeatures {
   FEAT_FP,
   FEAT_SIMD,
   FEAT_CRC,
-  FEAT_SHA1,
+  FEAT_CSSC,
   FEAT_SHA2,
   FEAT_SHA3,
-  FEAT_AES,
+  FEAT_unused5,
   FEAT_PMULL,
   FEAT_FP16,
   FEAT_DIT,
@@ -53,30 +53,30 @@ enum CPUFeatures {
   FEAT_RCPC,
   FEAT_RCPC2,
   FEAT_FRINTTS,
-  FEAT_DGH,
+  FEAT_unused6,
   FEAT_I8MM,
   FEAT_BF16,
-  FEAT_EBF16,
-  FEAT_RPRES,
+  FEAT_unused7,
+  FEAT_unused8,
   FEAT_SVE,
-  FEAT_SVE_BF16,
-  FEAT_SVE_EBF16,
-  FEAT_SVE_I8MM,
+  FEAT_unused9,
+  FEAT_unused10,
+  FEAT_unused11,
   FEAT_SVE_F32MM,
   FEAT_SVE_F64MM,
   FEAT_SVE2,
-  FEAT_SVE_AES,
+  FEAT_unused12,
   FEAT_SVE_PMULL128,
   FEAT_SVE_BITPERM,
   FEAT_SVE_SHA3,
   FEAT_SVE_SM4,
   FEAT_SME,
-  FEAT_MEMTAG,
+  FEAT_unused13,
   FEAT_MEMTAG2,
-  FEAT_MEMTAG3,
+  FEAT_unused14,
   FEAT_SB,
   FEAT_unused1,
-  FEAT_SSBS,
+  FEAT_unused15,
   FEAT_SSBS2,
   FEAT_BTI,
   FEAT_unused2,
@@ -87,6 +87,7 @@ enum CPUFeatures {
   FEAT_SME_I64,
   FEAT_SME2,
   FEAT_RCPC3,
+  FEAT_MOPS,
   FEAT_MAX,
   FEAT_EXT = 62, /* Reserved to indicate presence of additional features field
 		    in __aarch64_cpu_features.  */
diff --git a/gcc/common/config/avr/avr-common.cc b/gcc/common/config/avr/avr-common.cc
index 203a965..d8b982c 100644
--- a/gcc/common/config/avr/avr-common.cc
+++ b/gcc/common/config/avr/avr-common.cc
@@ -38,6 +38,7 @@ static const struct default_options avr_option_optimization_table[] =
     { OPT_LEVELS_1_PLUS, OPT_mmain_is_OS_task, NULL, 1 },
     { OPT_LEVELS_1_PLUS, OPT_mfuse_add_, NULL, 1 },
     { OPT_LEVELS_2_PLUS, OPT_mfuse_add_, NULL, 2 },
+    { OPT_LEVELS_1_PLUS, OPT_mfuse_move2, NULL, 1 },
     { OPT_LEVELS_1_PLUS_NOT_DEBUG, OPT_mfuse_move_, NULL, 3 },
     { OPT_LEVELS_2_PLUS, OPT_mfuse_move_, NULL, 23 },
     { OPT_LEVELS_2_PLUS, OPT_msplit_bit_shift, NULL, 1 },
diff --git a/gcc/common/config/i386/i386-common.cc b/gcc/common/config/i386/i386-common.cc
index dfcd4e9..9e807e4 100644
--- a/gcc/common/config/i386/i386-common.cc
+++ b/gcc/common/config/i386/i386-common.cc
@@ -131,8 +131,7 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA2_AVX10_2_SET \
   (OPTION_MASK_ISA2_AVX10_1_SET | OPTION_MASK_ISA2_AVX10_2)
 #define OPTION_MASK_ISA2_AMX_AVX512_SET \
-  (OPTION_MASK_ISA2_AMX_TILE_SET | OPTION_MASK_ISA2_AVX10_2_SET \
-   | OPTION_MASK_ISA2_AMX_AVX512)
+  (OPTION_MASK_ISA2_AMX_TILE_SET | OPTION_MASK_ISA2_AMX_AVX512)
 #define OPTION_MASK_ISA2_AMX_TF32_SET \
   (OPTION_MASK_ISA2_AMX_TILE_SET | OPTION_MASK_ISA2_AMX_TF32)
 #define OPTION_MASK_ISA2_AMX_TRANSPOSE_SET \
@@ -328,8 +327,7 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA2_USER_MSR_UNSET OPTION_MASK_ISA2_USER_MSR
 #define OPTION_MASK_ISA2_AVX10_1_UNSET \
   (OPTION_MASK_ISA2_AVX10_1 | OPTION_MASK_ISA2_AVX10_2_UNSET)
-#define OPTION_MASK_ISA2_AVX10_2_UNSET \
-  (OPTION_MASK_ISA2_AVX10_2 | OPTION_MASK_ISA2_AMX_AVX512_UNSET)
+#define OPTION_MASK_ISA2_AVX10_2_UNSET OPTION_MASK_ISA2_AVX10_2
 #define OPTION_MASK_ISA2_AMX_AVX512_UNSET OPTION_MASK_ISA2_AMX_AVX512
 #define OPTION_MASK_ISA2_AMX_TF32_UNSET OPTION_MASK_ISA2_AMX_TF32
 #define OPTION_MASK_ISA2_AMX_TRANSPOSE_UNSET OPTION_MASK_ISA2_AMX_TRANSPOSE
@@ -379,7 +377,8 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA2_AVX512F_UNSET \
   (OPTION_MASK_ISA2_AVX512BW_UNSET \
    | OPTION_MASK_ISA2_AVX512VP2INTERSECT_UNSET \
-   | OPTION_MASK_ISA2_AVX10_1_UNSET)
+   | OPTION_MASK_ISA2_AVX10_1_UNSET \
+   | OPTION_MASK_ISA2_AMX_AVX512_UNSET)
 #define OPTION_MASK_ISA2_GENERAL_REGS_ONLY_UNSET \
   OPTION_MASK_ISA2_SSE_UNSET
 #define OPTION_MASK_ISA2_AVX_UNSET \
@@ -1374,8 +1373,8 @@ ix86_handle_option (struct gcc_options *opts,
 	{
 	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA2_AMX_AVX512_SET;
 	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA2_AMX_AVX512_SET;
-	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX10_1_SET;
-	  opts->x_ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX10_1_SET;
+	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512F_SET;
+	  opts->x_ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX512F_SET;
 	}
       else
 	{
diff --git a/gcc/common/config/riscv/riscv-common.cc b/gcc/common/config/riscv/riscv-common.cc
index 82037a3..da3cb9f 100644
--- a/gcc/common/config/riscv/riscv-common.cc
+++ b/gcc/common/config/riscv/riscv-common.cc
@@ -1606,8 +1606,9 @@ bool
 riscv_ext_is_subset (struct cl_target_option *opts,
 		     struct cl_target_option *subset)
 {
-  for (const auto &[ext_name, ext_info] : riscv_ext_infos)
+  for (const auto &riscv_ext_info : riscv_ext_infos)
     {
+      const auto &ext_info = riscv_ext_info.second;
       if (ext_info.check_opts (opts) && !ext_info.check_opts (subset))
 	return false;
     }
diff --git a/gcc/config.gcc b/gcc/config.gcc
index a6f6efe..0d8dbc4 100644
--- a/gcc/config.gcc
+++ b/gcc/config.gcc
@@ -1847,11 +1847,11 @@ moxie-*-moxiebox*)
 	;;
 h8300-*-elf*)
 	tmake_file="h8300/t-h8300"
-	tm_file="h8300/h8300.h elfos.h newlib-stdint.h h8300/elf.h"
+	tm_file="elfos.h h8300/h8300.h newlib-stdint.h h8300/elf.h"
 	;;
 h8300-*-linux*)
 	tmake_file="${tmake_file} h8300/t-h8300 h8300/t-linux"
-	tm_file="h8300/h8300.h elfos.h gnu-user.h linux.h glibc-stdint.h h8300/linux.h"
+	tm_file="elfos.h h8300/h8300.h gnu-user.h linux.h glibc-stdint.h h8300/linux.h"
 	;;
 hppa*64*-*-linux*)
 	target_cpu_default="MASK_PA_11|MASK_PA_20"
@@ -3504,10 +3504,9 @@ sparc*-*-solaris2*)
 	    sparc64-*-* | sparcv9-*-*)
 		tm_file="sparc/default64.h ${tm_file}"
 		;;
-	    *)
-		test x$with_cpu != x || with_cpu=v9
-		;;
 	esac
+	# Match Studio 12.6 cc.
+	with_cpu=${with_cpu:-ultrasparc3}
 	tmake_file="${tmake_file} sparc/t-sparc sparc/t-sol2"
 	;;
 sparc-wrs-vxworks)
@@ -5894,7 +5893,7 @@ esac
 # distinguish VxWorks variants such as VxWorks 7 or 64).
 
 case ${target} in
-arm*-*-* | i[34567]86-*-* | mips*-*-* | powerpc*-*-* | sh*-*-* \
+aarch64*-*-* | arm*-*-* | i[34567]86-*-* | mips*-*-* | powerpc*-*-* | sh*-*-* \
 | sparc*-*-* | x86_64-*-*)
 	tm_file="vxworks-dummy.h ${tm_file}"
 	;;
diff --git a/gcc/config.in b/gcc/config.in
index ab62c15..353d1bc 100644
--- a/gcc/config.in
+++ b/gcc/config.in
@@ -318,6 +318,12 @@
 #endif
 
 
+/* Define to enable -mfentry by default on x86-64. */
+#ifndef USED_FOR_TARGET
+#undef ENABLE_X86_64_MFENTRY
+#endif
+
+
 /* Define to the name of a file containing a list of extra machine modes for
    this architecture. */
 #ifndef USED_FOR_TARGET
diff --git a/gcc/config/aarch64/aarch64-builtin-pairs.def b/gcc/config/aarch64/aarch64-builtin-pairs.def
new file mode 100644
index 0000000..1757e19
--- /dev/null
+++ b/gcc/config/aarch64/aarch64-builtin-pairs.def
@@ -0,0 +1,73 @@
+/* Pairs of AArch64 builtins that can be folded into each other.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Pairs of single and half integer modes.  */
+#define LO_HI_PAIR_V_HSI(T, LO, HI) \
+  LO_HI_PAIR (T##_##LO##v2si, T##_##HI##v4si) \
+  LO_HI_PAIR (T##_##LO##v4hi, T##_##HI##v8hi)
+
+#define LO_HI_PAIR_V_US_HSI(T, LO, HI) \
+  LO_HI_PAIR_V_HSI (T, s##LO, s##HI) \
+  LO_HI_PAIR_V_HSI (T##U, u##LO, u##HI)
+
+/* Pairs of widenable integer modes.  */
+#define LO_HI_PAIR_V_WI(T, LO, HI) \
+  LO_HI_PAIR_V_HSI (T, LO, HI) \
+  LO_HI_PAIR (T##_##LO##v8qi, T##_##HI##v16qi)
+
+#define LO_HI_PAIR_V_US_WI(T, LO, HI) \
+  LO_HI_PAIR_V_WI (T, s##LO, s##HI) \
+  LO_HI_PAIR_V_WI (T##U, u##LO, u##HI)
+
+#define UNOP_LONG_LH_PAIRS \
+  LO_HI_PAIR (UNOP_sxtlv8hi,  UNOP_vec_unpacks_hi_v16qi) \
+  LO_HI_PAIR (UNOP_sxtlv4si,  UNOP_vec_unpacks_hi_v8hi) \
+  LO_HI_PAIR (UNOP_sxtlv2di,  UNOP_vec_unpacks_hi_v4si) \
+  LO_HI_PAIR (UNOPU_uxtlv8hi, UNOPU_vec_unpacku_hi_v16qi) \
+  LO_HI_PAIR (UNOPU_uxtlv4si, UNOPU_vec_unpacku_hi_v8hi) \
+  LO_HI_PAIR (UNOPU_uxtlv2di, UNOPU_vec_unpacku_hi_v4si)
+
+#define BINOP_LONG_LH_PAIRS \
+  LO_HI_PAIR_V_US_WI (BINOP,  addl, addl2) \
+  LO_HI_PAIR_V_US_WI (BINOP,  subl, subl2) \
+  LO_HI_PAIR_V_US_WI (BINOP,  abdl, abdl2) \
+  LO_HI_PAIR_V_WI (BINOP,  intrinsic_vec_smult_lo_, vec_widen_smult_hi_) \
+  LO_HI_PAIR_V_WI (BINOPU, intrinsic_vec_umult_lo_, vec_widen_umult_hi_) \
+  LO_HI_PAIR_V_HSI (BINOP,  sqdmull, sqdmull2) \
+  LO_HI_PAIR (BINOPP_pmullv8qi, BINOPP_pmull_hiv16qi)
+
+#define BINOP_LONG_N_LH_PAIRS \
+  LO_HI_PAIR_V_US_HSI (BINOP,  mull_n, mull_hi_n) \
+  LO_HI_PAIR_V_HSI (BINOP,  sqdmull_n, sqdmull2_n)
+
+#define BINOP_WIDE_LH_PAIRS \
+  LO_HI_PAIR_V_US_WI (BINOP,  subw, subw2) \
+  LO_HI_PAIR_V_US_WI (BINOP,  addw, addw2)
+
+#define TERNOP_LONG_LH_PAIRS \
+  LO_HI_PAIR_V_US_WI (TERNOP,  mlal, mlal_hi) \
+  LO_HI_PAIR_V_US_WI (TERNOP,  mlsl, mlsl_hi) \
+  LO_HI_PAIR_V_US_WI (TERNOP,  abal, abal2) \
+  LO_HI_PAIR_V_HSI (TERNOP, sqdmlal, sqdmlal2) \
+  LO_HI_PAIR_V_HSI (TERNOP, sqdmlsl, sqdmlsl2)
+
+#define TERNOP_LONG_N_LH_PAIRS \
+  LO_HI_PAIR_V_US_HSI (TERNOP,  mlal_n, mlal_hi_n) \
+  LO_HI_PAIR_V_US_HSI (TERNOP,  mlsl_n, mlsl_hi_n) \
+  LO_HI_PAIR_V_HSI (TERNOP,  sqdmlal_n, sqdmlal2_n)
diff --git a/gcc/config/aarch64/aarch64-builtins.cc b/gcc/config/aarch64/aarch64-builtins.cc
index 93f939a..408099a 100644
--- a/gcc/config/aarch64/aarch64-builtins.cc
+++ b/gcc/config/aarch64/aarch64-builtins.cc
@@ -49,6 +49,8 @@
 #include "attribs.h"
 #include "gimple-fold.h"
 #include "builtins.h"
+#include "tree-pass.h"
+#include "tree-vector-builder.h"
 #include "aarch64-builtins.h"
 
 using namespace aarch64;
@@ -738,6 +740,16 @@ static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
   VGET_HIGH_BUILTIN(u64) \
   VGET_HIGH_BUILTIN(bf16)
 
+#include "aarch64-builtin-pairs.def"
+
+#define LO_HI_PAIRINGS \
+  UNOP_LONG_LH_PAIRS \
+  BINOP_LONG_LH_PAIRS \
+  BINOP_LONG_N_LH_PAIRS \
+  BINOP_WIDE_LH_PAIRS \
+  TERNOP_LONG_LH_PAIRS \
+  TERNOP_LONG_N_LH_PAIRS
+
 typedef struct
 {
   const char *name;
@@ -5004,6 +5016,167 @@ aarch64_gimple_fold_pragma_builtin
     }
 }
 
+/* Return the fndecl of the builtin paired with FCODE_LO if one
+   exists (see aarch64-builtin-pairs.def), or NULL_TREE if not.  */
+static inline tree
+aarch64_get_highpart_builtin (unsigned int fcode_lo)
+{
+#undef LO_HI_PAIR
+#define LO_HI_PAIR(A, B) case AARCH64_SIMD_BUILTIN_##A:   \
+  return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_##B];
+
+  switch (fcode_lo)
+    {
+      LO_HI_PAIRINGS
+      default:
+	return NULL_TREE;
+    }
+}
+
+/* If REF describes the high half of a 128-bit vector, return this
+   vector.  Otherwise, return NULL_TREE.  */
+static tree
+aarch64_v128_highpart_ref (const_tree ref)
+{
+  if (TREE_CODE (ref) != SSA_NAME)
+    return NULL_TREE;
+
+  gassign *stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (ref));
+  if (!stmt || gimple_assign_rhs_code (stmt) != BIT_FIELD_REF)
+    return NULL_TREE;
+
+  /* Look for a BIT_FIELD_REF that denotes the most significant 64
+     bits of a 128-bit vector.  */
+  tree bf_ref = gimple_assign_rhs1 (stmt);
+  unsigned int offset = BYTES_BIG_ENDIAN ? 0 : 64;
+
+  if (maybe_ne (bit_field_size (bf_ref), 64u)
+      || maybe_ne (bit_field_offset (bf_ref), offset))
+    return NULL_TREE;
+
+  tree obj = TREE_OPERAND (bf_ref, 0);
+  tree type = TREE_TYPE (obj);
+
+  if (VECTOR_TYPE_P (type) && tree_fits_uhwi_p (TYPE_SIZE (type))
+      && tree_to_uhwi (TYPE_SIZE (type)) == 128)
+    return obj;
+
+  return NULL_TREE;
+}
+
+/* Build and return a new VECTOR_CST of type OUT_TY, using repeated
+   copies of the elements of VEC_IN.  */
+static tree
+aarch64_build_vector_cst (const_tree vec_in, tree out_ty)
+{
+  gcc_assert (TREE_CODE (vec_in) == VECTOR_CST
+	      && VECTOR_TYPE_P (out_ty));
+  unsigned HOST_WIDE_INT nelts
+    = VECTOR_CST_NELTS (vec_in).to_constant ();
+
+  tree_vector_builder vec_out (out_ty, nelts, 1);
+  for (unsigned i = 0; i < nelts; i++)
+    vec_out.quick_push (VECTOR_CST_ELT (vec_in, i));
+
+  return vec_out.build ();
+}
+
+/* Try to fold STMT, a call to to a lowpart-operating builtin, to
+   it's highpart-operating equivalent if doing so would save
+   unnecessary data movement instructions.
+
+   Return the new call if so, otherwise nullptr.  */
+static gcall *
+aarch64_fold_lo_call_to_hi (unsigned int fcode, gcall *stmt,
+			    gimple_stmt_iterator *gsi)
+{
+  /* Punt until as late as possible:
+    1) By folding away BIT_FIELD_REFs we remove information about the
+    operands that may be useful to other optimizers.
+
+    2) For simplicity, we'd like the expression
+
+	x = BIT_FIELD_REF<a, x, y>
+
+    to imply that A is not a VECTOR_CST.  This assumption is unlikely
+    to hold before constant prop/folding.  */
+  if (!(cfun->curr_properties & PROP_last_full_fold))
+    return nullptr;
+
+  tree builtin_hi = aarch64_get_highpart_builtin (fcode);
+  gcc_assert (builtin_hi != NULL_TREE);
+
+  /* Prefer to use the highpart builtin when at least one vector
+     argument is a reference to the high half of a 128b vector, and
+     all others are VECTOR_CSTs that we can extend to 128b.  */
+  auto_vec<unsigned int, 2> vec_constants;
+  auto_vec<unsigned int, 2> vec_highparts;
+  /* The arguments and signature of the new call.  */
+  auto_vec<tree, 4> call_args;
+  auto_vec<tree, 4> call_types;
+
+  /* The interesting args are those that differ between the lo/hi
+     builtins.  Walk the function signatures to find these.  */
+  tree types_hi = TYPE_ARG_TYPES (TREE_TYPE (builtin_hi));
+  tree types_lo = TYPE_ARG_TYPES (gimple_call_fntype (stmt));
+  unsigned int argno = 0;
+  while (types_lo != void_list_node && types_hi != void_list_node)
+    {
+      tree type_lo = TREE_VALUE (types_lo);
+      tree type_hi = TREE_VALUE (types_hi);
+      tree arg = gimple_call_arg (stmt, argno);
+      if (!types_compatible_p (type_lo, type_hi))
+	{
+	  /* Check our assumptions about this pair.  */
+	  gcc_assert (wi::to_widest (TYPE_SIZE (type_lo)) == 64
+		      && wi::to_widest (TYPE_SIZE (type_hi)) == 128);
+
+	  tree vq = aarch64_v128_highpart_ref (arg);
+	  if (vq && is_gimple_reg (vq))
+	    {
+	      vec_highparts.safe_push (argno);
+	      arg = vq;
+	    }
+	  else if (TREE_CODE (arg) == VECTOR_CST)
+	    vec_constants.safe_push (argno);
+	  else
+	    return nullptr;
+	}
+      call_args.safe_push (arg);
+      call_types.safe_push (type_hi);
+
+      argno++;
+      types_hi = TREE_CHAIN (types_hi);
+      types_lo = TREE_CHAIN (types_lo);
+    }
+  gcc_assert (types_lo == void_list_node && types_hi == void_list_node);
+
+  if (vec_highparts.is_empty ())
+    return nullptr;
+
+  /* Build and return a new call to BUILTIN_HI.  */
+  for (auto i : vec_constants)
+    call_args[i] = aarch64_build_vector_cst (call_args[i], call_types[i]);
+
+  for (auto i : vec_highparts)
+    if (!types_compatible_p (TREE_TYPE (call_args[i]), call_types[i]))
+      {
+	tree vce_ssa = make_ssa_name (call_types[i]);
+	tree vce_expr = build1 (VIEW_CONVERT_EXPR,
+				call_types[i], call_args[i]);
+	gsi_insert_before (gsi, gimple_build_assign (vce_ssa, vce_expr),
+			   GSI_SAME_STMT);
+	call_args[i] = vce_ssa;
+      }
+
+  gcall *new_call = gimple_build_call_vec (builtin_hi, call_args);
+  gimple_call_set_lhs (new_call, gimple_call_lhs (stmt));
+  return new_call;
+}
+
+#undef LO_HI_PAIR
+#define LO_HI_PAIR(A, B) case AARCH64_SIMD_BUILTIN_##A:
+
 /* Try to fold STMT, given that it's a call to the built-in function with
    subcode FCODE.  Return the new statement on success and null on
    failure.  */
@@ -5190,6 +5363,9 @@ aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt,
 	    }
 	  break;
 	}
+    LO_HI_PAIRINGS
+      new_stmt = aarch64_fold_lo_call_to_hi (fcode, stmt, gsi);
+      break;
     case AARCH64_SIMD_BUILTIN_LANE_CHECK:
       if (aarch64_fold_builtin_lane_check (args[0], args[1], args[2]))
 	{
diff --git a/gcc/config/aarch64/aarch64-cores.def b/gcc/config/aarch64/aarch64-cores.def
index 8040409..6f11cc0 100644
--- a/gcc/config/aarch64/aarch64-cores.def
+++ b/gcc/config/aarch64/aarch64-cores.def
@@ -224,7 +224,7 @@ AARCH64_CORE("neoverse-v3ae", neoversev3ae, cortexa57, V9_2A, (SVE2_BITPERM, RNG
 AARCH64_CORE("demeter", demeter, cortexa57, V9A, (I8MM, BF16, SVE2_BITPERM, RNG, MEMTAG, PROFILE), neoversev2, 0x41, 0xd4f, -1)
 
 /* NVIDIA ('N') cores. */
-AARCH64_CORE("olympus", olympus, cortexa57, V9_2A, (SVE2_BITPERM, RNG, LS64, MEMTAG, PROFILE, FAMINMAX, FP8FMA, FP8DOT2, FP8DOT4, LUT, SVE2_AES, SVE2_SHA3, SVE2_SM4), neoversev3, 0x4e, 0x10, -1)
+AARCH64_CORE("olympus", olympus, cortexa57, V9_2A, (SVE2_BITPERM, RNG, LS64, MEMTAG, PROFILE, FAMINMAX, FP8FMA, FP8DOT2, FP8DOT4, LUT, SVE2_AES, SVE2_SHA3, SVE2_SM4), olympus, 0x4e, 0x10, -1)
 
 /* Armv9-A big.LITTLE processors.  */
 AARCH64_CORE("gb10",  gb10, cortexa57, V9_2A,  (SVE2_BITPERM, SVE2_AES, SVE2_SHA3, SVE2_SM4, MEMTAG, PROFILE), cortexx925, 0x41, AARCH64_BIG_LITTLE (0xd85, 0xd87), -1)
diff --git a/gcc/config/aarch64/aarch64-cost-tables.h b/gcc/config/aarch64/aarch64-cost-tables.h
index c49ff7f..e7926eb 100644
--- a/gcc/config/aarch64/aarch64-cost-tables.h
+++ b/gcc/config/aarch64/aarch64-cost-tables.h
@@ -125,9 +125,9 @@ const struct cpu_cost_table qdf24xx_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -233,9 +233,9 @@ const struct cpu_cost_table thunderx_extra_costs =
   {
     COSTS_N_INSNS (1),	/* Alu.  */
     COSTS_N_INSNS (4),	/* mult.  */
-    COSTS_N_INSNS (1),	/* movi.  */
-    COSTS_N_INSNS (2),	/* dup.  */
-    COSTS_N_INSNS (2)	/* extract.  */
+    COSTS_N_INSNS (0),	/* movi.  */
+    COSTS_N_INSNS (1),	/* dup.  */
+    COSTS_N_INSNS (1)	/* extract.  */
   }
 };
 
@@ -340,9 +340,9 @@ const struct cpu_cost_table thunderx2t99_extra_costs =
   {
     COSTS_N_INSNS (1),	/* Alu.  */
     COSTS_N_INSNS (4),	/* Mult.  */
-    COSTS_N_INSNS (1),	/* movi.  */
-    COSTS_N_INSNS (2),	/* dup.  */
-    COSTS_N_INSNS (2)	/* extract.  */
+    COSTS_N_INSNS (0),	/* movi.  */
+    COSTS_N_INSNS (1),	/* dup.  */
+    COSTS_N_INSNS (1)	/* extract.  */
   }
 };
 
@@ -447,9 +447,9 @@ const struct cpu_cost_table thunderx3t110_extra_costs =
   {
     COSTS_N_INSNS (1),	/* Alu.  */
     COSTS_N_INSNS (4),	/* Mult.  */
-    COSTS_N_INSNS (1),	/* movi.  */
-    COSTS_N_INSNS (2),	/* dup.  */
-    COSTS_N_INSNS (2)	/* extract.  */
+    COSTS_N_INSNS (0),	/* movi.  */
+    COSTS_N_INSNS (1),	/* dup.  */
+    COSTS_N_INSNS (1)	/* extract.  */
   }
 };
 
@@ -555,9 +555,9 @@ const struct cpu_cost_table tsv110_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -662,9 +662,9 @@ const struct cpu_cost_table a64fx_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -769,9 +769,9 @@ const struct cpu_cost_table ampere1_extra_costs =
   {
     COSTS_N_INSNS (3),  /* alu.  */
     COSTS_N_INSNS (3),  /* mult.  */
-    COSTS_N_INSNS (2),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -876,9 +876,9 @@ const struct cpu_cost_table ampere1a_extra_costs =
   {
     COSTS_N_INSNS (3),  /* alu.  */
     COSTS_N_INSNS (3),  /* mult.  */
-    COSTS_N_INSNS (2),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -983,9 +983,9 @@ const struct cpu_cost_table ampere1b_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (2),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (1),  /* dup.  */
-    COSTS_N_INSNS (1)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (0),  /* dup.  */
+    COSTS_N_INSNS (0)   /* extract.  */
   }
 };
 
diff --git a/gcc/config/aarch64/aarch64-option-extensions.def b/gcc/config/aarch64/aarch64-option-extensions.def
index 1c3e697..db88df0 100644
--- a/gcc/config/aarch64/aarch64-option-extensions.def
+++ b/gcc/config/aarch64/aarch64-option-extensions.def
@@ -128,7 +128,9 @@ AARCH64_OPT_FMV_EXTENSION("sha2", SHA2, (SIMD), (), (), "sha1 sha2")
 
 AARCH64_FMV_FEATURE("sha3", SHA3, (SHA3))
 
-AARCH64_OPT_FMV_EXTENSION("aes", AES, (SIMD), (), (), "aes")
+AARCH64_OPT_EXTENSION("aes", AES, (SIMD), (), (), "aes")
+
+AARCH64_FMV_FEATURE("aes", PMULL, (AES))
 
 /* +nocrypto disables AES, SHA2 and SM4, and anything that depends on them
    (such as SHA3 and the SVE2 crypto extensions).  */
@@ -171,8 +173,6 @@ AARCH64_OPT_FMV_EXTENSION("i8mm", I8MM, (SIMD), (), (), "i8mm")
    instructions.  */
 AARCH64_OPT_FMV_EXTENSION("bf16", BF16, (FP), (SIMD), (), "bf16")
 
-AARCH64_FMV_FEATURE("rpres", RPRES, ())
-
 AARCH64_OPT_FMV_EXTENSION("sve", SVE, (SIMD, F16, FCMA), (), (), "sve")
 
 /* This specifically does not imply +sve.  */
@@ -190,7 +190,7 @@ AARCH64_OPT_FMV_EXTENSION("sve2", SVE2, (SVE), (), (), "sve2")
 
 AARCH64_OPT_EXTENSION("sve2-aes", SVE2_AES, (SVE2, AES), (), (), "sveaes")
 
-AARCH64_FMV_FEATURE("sve2-aes", SVE_AES, (SVE2_AES))
+AARCH64_FMV_FEATURE("sve2-aes", SVE_PMULL128, (SVE2_AES))
 
 AARCH64_OPT_EXTENSION("sve2-bitperm", SVE2_BITPERM, (SVE2), (), (),
 		      "svebitperm")
@@ -245,9 +245,9 @@ AARCH64_OPT_EXTENSION("sme-b16b16", SME_B16B16, (SME2, SVE_B16B16), (), (), "sme
 
 AARCH64_OPT_EXTENSION("sme-f16f16", SME_F16F16, (SME2), (), (), "smef16f16")
 
-AARCH64_OPT_EXTENSION("mops", MOPS, (), (), (), "mops")
+AARCH64_OPT_FMV_EXTENSION("mops", MOPS, (), (), (), "mops")
 
-AARCH64_OPT_EXTENSION("cssc", CSSC, (), (), (), "cssc")
+AARCH64_OPT_FMV_EXTENSION("cssc", CSSC, (), (), (), "cssc")
 
 AARCH64_OPT_EXTENSION("cmpbr", CMPBR, (), (), (), "cmpbr")
 
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index e946e8d..36bd885 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1020,7 +1020,7 @@ void aarch64_err_no_fpadvsimd (machine_mode);
 void aarch64_expand_epilogue (rtx_call_insn *);
 rtx aarch64_ptrue_all (unsigned int);
 opt_machine_mode aarch64_ptrue_all_mode (rtx);
-rtx aarch64_convert_sve_data_to_pred (rtx, machine_mode, rtx);
+rtx aarch64_convert_sve_data_to_pred (rtx, rtx);
 rtx aarch64_expand_sve_dupq (rtx, machine_mode, rtx);
 void aarch64_expand_mov_immediate (rtx, rtx);
 rtx aarch64_stack_protect_canary_mem (machine_mode, rtx, aarch64_salt_type);
@@ -1031,6 +1031,7 @@ rtx aarch64_pfalse_reg (machine_mode);
 bool aarch64_sve_same_pred_for_ptest_p (rtx *, rtx *);
 rtx aarch64_sve_packed_pred (machine_mode);
 rtx aarch64_sve_fp_pred (machine_mode, rtx *);
+rtx aarch64_sve_emit_masked_fp_pred (machine_mode, rtx);
 void aarch64_emit_load_store_through_mode (rtx, rtx, machine_mode);
 bool aarch64_expand_maskloadstore (rtx *, machine_mode);
 void aarch64_emit_sve_pred_move (rtx, rtx, rtx);
@@ -1038,6 +1039,7 @@ void aarch64_expand_sve_mem_move (rtx, rtx, machine_mode);
 bool aarch64_maybe_expand_sve_subreg_move (rtx, rtx);
 rtx aarch64_replace_reg_mode (rtx, machine_mode);
 void aarch64_split_sve_subreg_move (rtx, rtx, rtx);
+void aarch64_emit_sve_pred_vec_duplicate (machine_mode, rtx, rtx);
 void aarch64_expand_prologue (void);
 void aarch64_decompose_vec_struct_index (machine_mode, rtx *, rtx *, bool);
 void aarch64_expand_vector_init (rtx, rtx);
diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index 8de79ca..8b75c3d 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -1190,13 +1190,16 @@
   [(set_attr "type" "neon_ins<q>, neon_from_gp<q>, neon_load1_one_lane<q>")]
 )
 
+;; Inserting from the zero register into a vector lane is treated as an
+;; expensive GP->FP move on all CPUs.  Avoid it when optimizing for speed.
 (define_insn "aarch64_simd_vec_set_zero<mode>"
   [(set (match_operand:VALL_F16 0 "register_operand" "=w")
 	(vec_merge:VALL_F16
 	    (match_operand:VALL_F16 1 "register_operand" "0")
 	    (match_operand:VALL_F16 3 "aarch64_simd_imm_zero" "")
 	    (match_operand:SI 2 "immediate_operand" "i")))]
-  "TARGET_SIMD && aarch64_exact_log2_inverse (<nunits>, operands[2]) >= 0"
+  "TARGET_SIMD && aarch64_exact_log2_inverse (<nunits>, operands[2]) >= 0
+   && optimize_function_for_size_p (cfun)"
   {
     int elt = ENDIAN_LANE_N (<nunits>,
 			     aarch64_exact_log2_inverse (<nunits>,
@@ -9180,12 +9183,12 @@
 ;; sha3
 
 (define_insn "eor3q<mode>4"
-  [(set (match_operand:VQ_I 0 "register_operand" "=w")
-	(xor:VQ_I
-	 (xor:VQ_I
-	  (match_operand:VQ_I 2 "register_operand" "w")
-	  (match_operand:VQ_I 3 "register_operand" "w"))
-	 (match_operand:VQ_I 1 "register_operand" "w")))]
+  [(set (match_operand:VDQ_I 0 "register_operand" "=w")
+	(xor:VDQ_I
+	 (xor:VDQ_I
+	  (match_operand:VDQ_I 2 "register_operand" "w")
+	  (match_operand:VDQ_I 3 "register_operand" "w"))
+	 (match_operand:VDQ_I 1 "register_operand" "w")))]
   "TARGET_SHA3"
   "eor3\\t%0.16b, %1.16b, %2.16b, %3.16b"
   [(set_attr "type" "crypto_sha3")]
@@ -9241,17 +9244,46 @@
 )
 
 (define_insn "bcaxq<mode>4"
-  [(set (match_operand:VQ_I 0 "register_operand" "=w")
-	(xor:VQ_I
-	 (and:VQ_I
-	  (not:VQ_I (match_operand:VQ_I 3 "register_operand" "w"))
-	  (match_operand:VQ_I 2 "register_operand" "w"))
-	 (match_operand:VQ_I 1 "register_operand" "w")))]
+  [(set (match_operand:VDQ_I 0 "register_operand" "=w")
+	(xor:VDQ_I
+	 (and:VDQ_I
+	  (not:VDQ_I (match_operand:VDQ_I 3 "register_operand" "w"))
+	  (match_operand:VDQ_I 2 "register_operand" "w"))
+	 (match_operand:VDQ_I 1 "register_operand" "w")))]
   "TARGET_SHA3"
   "bcax\\t%0.16b, %1.16b, %2.16b, %3.16b"
   [(set_attr "type" "crypto_sha3")]
 )
 
+(define_insn_and_split "*bcaxqdi4"
+  [(set (match_operand:DI 0 "register_operand")
+	(xor:DI
+	  (and:DI
+	    (not:DI (match_operand:DI 3 "register_operand"))
+	    (match_operand:DI 2 "register_operand"))
+	  (match_operand:DI 1 "register_operand")))]
+  "TARGET_SHA3"
+  {@ [ cons: =0, 1, 2 , 3  ; attrs: type ]
+     [ w       , w, w , w  ; crypto_sha3 ] bcax\t%0.16b, %1.16b, %2.16b, %3.16b
+     [ &r      , r, r0, r0 ; multiple    ] #
+  }
+  "&& REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
+  [(set (match_dup 4)
+	(and:DI (not:DI (match_dup 3))
+		(match_dup 2)))
+   (set (match_dup 0)
+	(xor:DI (match_dup 4)
+		(match_dup 1)))]
+  {
+    if (reload_completed)
+      operands[4] = operands[0];
+    else if (can_create_pseudo_p ())
+      operands[4] = gen_reg_rtx (DImode);
+    else
+      FAIL;
+  }
+)
+
 ;; SM3
 
 (define_insn "aarch64_sm3ss1qv4si"
diff --git a/gcc/config/aarch64/aarch64-sme.md b/gcc/config/aarch64/aarch64-sme.md
index b8bb4cc..6b1a747 100644
--- a/gcc/config/aarch64/aarch64-sme.md
+++ b/gcc/config/aarch64/aarch64-sme.md
@@ -38,6 +38,7 @@
 ;; ---- Binary arithmetic on ZA tile
 ;; ---- Binary arithmetic on ZA slice
 ;; ---- Binary arithmetic, writing to ZA slice
+;; ---- Absolute minimum/maximum
 ;;
 ;; == Ternary arithmetic
 ;; ---- [INT] Dot product
@@ -61,6 +62,10 @@
 ;; (b) they are sometimes used conditionally, particularly in streaming-
 ;; compatible code.
 ;;
+;; To prevent the latter from upsetting the assembler, we emit the literal
+;; encodings of "SMSTART SM" and "SMSTOP SM" when compiling without
+;; TARGET_SME.
+;;
 ;; =========================================================================
 
 ;; -------------------------------------------------------------------------
@@ -160,7 +165,9 @@
    (clobber (reg:VNx16BI P14_REGNUM))
    (clobber (reg:VNx16BI P15_REGNUM))]
   ""
-  "smstart\tsm"
+  {
+    return TARGET_SME ? "smstart\tsm" : ".inst 0xd503437f // smstart sm";
+  }
 )
 
 ;; Turn off streaming mode.  This clobbers all SVE state.
@@ -195,7 +202,9 @@
    (clobber (reg:VNx16BI P14_REGNUM))
    (clobber (reg:VNx16BI P15_REGNUM))]
   ""
-  "smstop\tsm"
+  {
+    return TARGET_SME ? "smstop\tsm" : ".inst 0xd503427f // smstop sm";
+  }
 )
 
 ;; -------------------------------------------------------------------------
@@ -1264,6 +1273,23 @@
   "<sme_int_op>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2, %3.<Vetype>"
 )
 
+;; -------------------------------------------------------------------------
+;; ---- Absolute minimum/maximum
+;; -------------------------------------------------------------------------
+;; Includes:
+;; - FAMIN (SME2+FAMINMAX)
+;; - FAMAX (SME2+FAMINMAX)
+;; -------------------------------------------------------------------------
+
+(define_insn "@aarch64_sme_<faminmax_uns_op><mode>"
+  [(set (match_operand:SVE_Fx24 0 "register_operand" "=Uw<vector_count>")
+	(unspec:SVE_Fx24 [(match_operand:SVE_Fx24 1 "register_operand" "%0")
+			  (match_operand:SVE_Fx24 2 "register_operand" "Uw<vector_count>")]
+	 FAMINMAX_UNS))]
+  "TARGET_STREAMING_SME2 && TARGET_FAMINMAX"
+  "<faminmax_uns_op>\t%0, %1, %2"
+)
+
 ;; =========================================================================
 ;; == Ternary arithmetic
 ;; =========================================================================
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
index b439683..ecc0687 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
@@ -214,7 +214,8 @@ public:
   expand (function_expander &e) const override
   {
     e.add_ptrue_hint (0, e.gp_mode (0));
-    insn_code icode = code_for_aarch64_pred_fac (m_unspec, e.vector_mode (0));
+    insn_code icode = code_for_aarch64_pred_fac_acle (m_unspec,
+						      e.vector_mode (0));
     return e.use_exact_insn (icode);
   }
 
@@ -497,10 +498,10 @@ public:
       {
 	bool unsigned_p = e.type_suffix (0).unsigned_p;
 	rtx_code code = get_rtx_code (m_code, unsigned_p);
-	return e.use_exact_insn (code_for_aarch64_pred_cmp (code, mode));
+	return e.use_exact_insn (code_for_aarch64_pred_cmp_acle (code, mode));
       }
 
-    insn_code icode = code_for_aarch64_pred_fcm (m_unspec_for_fp, mode);
+    insn_code icode = code_for_aarch64_pred_fcm_acle (m_unspec_for_fp, mode);
     return e.use_exact_insn (icode);
   }
 
@@ -542,7 +543,7 @@ public:
 
     /* If the argument is a constant that the unwidened comparisons
        can handle directly, use them instead.  */
-    insn_code icode = code_for_aarch64_pred_cmp (code, mode);
+    insn_code icode = code_for_aarch64_pred_cmp_acle (code, mode);
     rtx op2 = unwrap_const_vec_duplicate (e.args[3]);
     if (CONSTANT_P (op2)
 	&& insn_data[icode].operand[4].predicate (op2, DImode))
@@ -581,7 +582,8 @@ public:
   expand (function_expander &e) const override
   {
     e.add_ptrue_hint (0, e.gp_mode (0));
-    return e.use_exact_insn (code_for_aarch64_pred_fcmuo (e.vector_mode (0)));
+    auto mode = e.vector_mode (0);
+    return e.use_exact_insn (code_for_aarch64_pred_fcmuo_acle (mode));
   }
 };
 
@@ -1048,6 +1050,23 @@ public:
   rtx
   expand (function_expander &e) const override
   {
+    machine_mode mode = e.vector_mode (0);
+    if (GET_MODE_CLASS (mode) == MODE_VECTOR_BOOL)
+      {
+	gcc_assert (e.pred == PRED_none);
+
+	rtx src = e.args[0];
+	if (GET_CODE (src) == CONST_INT)
+	  return (src == const0_rtx
+		  ? CONST0_RTX (VNx16BImode)
+		  : aarch64_ptrue_all (e.type_suffix (0).element_bytes));
+
+	rtx dest = e.get_reg_target ();
+	src = force_reg (GET_MODE (src), src);
+	aarch64_emit_sve_pred_vec_duplicate (mode, dest, src);
+	return dest;
+      }
+
     if (e.pred == PRED_none || e.pred == PRED_x)
       /* There's no benefit to using predicated instructions for _x here.  */
       return e.use_unpred_insn (e.direct_optab_handler (vec_duplicate_optab));
@@ -1056,7 +1075,6 @@ public:
        the duplicate of the function argument and the "false" value
        is the value of inactive lanes.  */
     insn_code icode;
-    machine_mode mode = e.vector_mode (0);
     if (valid_for_const_vector_p (GET_MODE_INNER (mode), e.args.last ()))
       /* Duplicate the constant to fill a vector.  The pattern optimizes
 	 various cases involving constant operands, falling back to SEL
@@ -1197,8 +1215,7 @@ public:
     if (mode != e.vector_mode (0))
       {
 	rtx data_dupq = aarch64_expand_sve_dupq (NULL, mode, vq_reg);
-	return aarch64_convert_sve_data_to_pred (e.possible_target,
-						 e.vector_mode (0), data_dupq);
+	return aarch64_convert_sve_data_to_pred (e.possible_target, data_dupq);
       }
 
     return aarch64_expand_sve_dupq (e.possible_target, mode, vq_reg);
@@ -1259,9 +1276,10 @@ public:
 	index = target;
       }
 
-    e.args[0] = gen_lowpart (VNx2DImode, e.args[0]);
+    e.args[0] = aarch64_sve_reinterpret (VNx2DImode, e.args[0]);
     e.args[1] = index;
-    return e.use_exact_insn (CODE_FOR_aarch64_sve_tblvnx2di);
+    rtx res = e.use_exact_insn (CODE_FOR_aarch64_sve_tblvnx2di);
+    return aarch64_sve_reinterpret (mode, res);
   }
 };
 
@@ -2857,7 +2875,10 @@ public:
   rtx
   expand (function_expander &e) const override
   {
-    return e.use_exact_insn (code_for_aarch64_sve_rev (e.vector_mode (0)));
+    auto mode = e.vector_mode (0);
+    return e.use_exact_insn (e.type_suffix (0).bool_p
+			     ? code_for_aarch64_sve_rev_acle (mode)
+			     : code_for_aarch64_sve_rev (mode));
   }
 };
 
@@ -3248,7 +3269,7 @@ public:
     unsigned int unpacks = m_high_p ? UNSPEC_UNPACKSHI : UNSPEC_UNPACKSLO;
     insn_code icode;
     if (GET_MODE_CLASS (mode) == MODE_VECTOR_BOOL)
-      icode = code_for_aarch64_sve_punpk (unpacku, mode);
+      icode = code_for_aarch64_sve_punpk_acle (unpacku);
     else
       {
 	int unspec = e.type_suffix (0).unsigned_p ? unpacku : unpacks;
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-functions.h b/gcc/config/aarch64/aarch64-sve-builtins-functions.h
index 6f1c694..c05946d 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-functions.h
+++ b/gcc/config/aarch64/aarch64-sve-builtins-functions.h
@@ -630,7 +630,10 @@ public:
   rtx
   expand (function_expander &e) const override
   {
-    insn_code icode = code_for_aarch64_sve (m_unspec, e.vector_mode (0));
+    auto mode = e.vector_mode (0);
+    insn_code icode = (e.type_suffix (0).bool_p
+		       ? code_for_aarch64_sve_acle (m_unspec, mode)
+		       : code_for_aarch64_sve (m_unspec, mode));
     return e.use_exact_insn (icode);
   }
 
@@ -838,7 +841,8 @@ public:
 
     machine_mode pred_mode = e.vector_mode (0);
     scalar_mode reg_mode = GET_MODE_INNER (e.vector_mode (1));
-    return e.use_exact_insn (code_for_while (unspec, reg_mode, pred_mode));
+    auto icode = code_for_aarch64_sve_while_acle (unspec, reg_mode, pred_mode);
+    return e.use_exact_insn (icode);
   }
 
   /* The unspec codes associated with signed and unsigned operations
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-sme.def b/gcc/config/aarch64/aarch64-sve-builtins-sme.def
index f75c0a5..117b70e 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-sme.def
+++ b/gcc/config/aarch64/aarch64-sve-builtins-sme.def
@@ -92,6 +92,12 @@ DEF_SME_FUNCTION (svstr_zt, str_zt, none, none)
 DEF_SME_FUNCTION (svzero_zt, inherent_zt, none, none)
 #undef REQUIRED_EXTENSIONS
 
+#define REQUIRED_EXTENSIONS streaming_only (AARCH64_FL_SME2 \
+					    | AARCH64_FL_FAMINMAX)
+DEF_SME_FUNCTION_GS (svamin, binary_opt_single_n, all_float, x24, none)
+DEF_SME_FUNCTION_GS (svamax, binary_opt_single_n, all_float, x24, none)
+#undef REQUIRED_EXTENSIONS
+
 /* The d_za entries in this section just declare C _za64 overloads,
    which will then be resolved to either an integer function or a
    floating-point function.  They are needed because the integer and
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc b/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
index d9922de..95c5ed8 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
@@ -316,7 +316,8 @@ public:
   expand (function_expander &e) const override
   {
     e.prepare_gather_address_operands (1, false);
-    return e.use_exact_insn (CODE_FOR_aarch64_gather_ld1q);
+    auto icode = code_for_aarch64_gather_ld1q (e.tuple_mode (0));
+    return e.use_exact_insn (icode);
   }
 };
 
@@ -722,7 +723,7 @@ public:
   expand (function_expander &e) const override
   {
     rtx data = e.args.last ();
-    e.args.last () = force_lowpart_subreg (VNx2DImode, data, GET_MODE (data));
+    e.args.last () = aarch64_sve_reinterpret (VNx2DImode, data);
     e.prepare_gather_address_operands (1, false);
     return e.use_exact_insn (CODE_FOR_aarch64_scatter_st1q);
   }
@@ -880,7 +881,9 @@ public:
   {
     for (unsigned int i = 0; i < 2; ++i)
       e.args[i] = e.convert_to_pmode (e.args[i]);
-    return e.use_exact_insn (code_for_while (m_unspec, Pmode, e.gp_mode (0)));
+    auto icode = code_for_aarch64_sve_while_acle (m_unspec, Pmode,
+						  e.gp_mode (0));
+    return e.use_exact_insn (icode);
   }
 
   int m_unspec;
@@ -929,6 +932,44 @@ public:
   unsigned int m_bits;
 };
 
+/* The same as cond_or_uncond_unspec_function but the intrinsics with vector
+   modes are SME2 extensions instead of SVE.  */
+class faminmaximpl : public function_base
+{
+public:
+  CONSTEXPR faminmaximpl (int cond_unspec, int uncond_unspec)
+    : m_cond_unspec (cond_unspec), m_uncond_unspec (uncond_unspec)
+    {}
+
+  rtx
+  expand (function_expander &e) const override
+  {
+    if (e.group_suffix ().vectors_per_tuple > 1)
+      {
+	/* SME2+faminmax intrinsics.  */
+	gcc_assert (e.pred == PRED_none);
+	auto mode = e.tuple_mode (0);
+	auto icode = (code_for_aarch64_sme (m_uncond_unspec, mode));
+	return e.use_exact_insn (icode);
+      }
+    /* SVE+faminmax intrinsics.  */
+    else if (e.pred == PRED_none)
+      {
+	auto mode = e.tuple_mode (0);
+	auto icode = (e.mode_suffix_id == MODE_single
+		      ? code_for_aarch64_sve_single (m_uncond_unspec, mode)
+		      : code_for_aarch64_sve (m_uncond_unspec, mode));
+	return e.use_exact_insn (icode);
+      }
+    return e.map_to_unspecs (m_cond_unspec, m_cond_unspec, m_cond_unspec);
+  }
+
+  /* The unspecs for the conditional and unconditional instructions,
+     respectively.  */
+  int m_cond_unspec;
+  int m_uncond_unspec;
+};
+
 } /* end anonymous namespace */
 
 namespace aarch64_sve {
@@ -957,10 +998,8 @@ FUNCTION (svaesd, fixed_insn_function, (CODE_FOR_aarch64_sve2_aesd))
 FUNCTION (svaese, fixed_insn_function, (CODE_FOR_aarch64_sve2_aese))
 FUNCTION (svaesimc, fixed_insn_function, (CODE_FOR_aarch64_sve2_aesimc))
 FUNCTION (svaesmc, fixed_insn_function, (CODE_FOR_aarch64_sve2_aesmc))
-FUNCTION (svamax, cond_or_uncond_unspec_function,
-	  (UNSPEC_COND_FAMAX, UNSPEC_FAMAX))
-FUNCTION (svamin, cond_or_uncond_unspec_function,
-	  (UNSPEC_COND_FAMIN, UNSPEC_FAMIN))
+FUNCTION (svamax, faminmaximpl, (UNSPEC_COND_FAMAX, UNSPEC_FAMAX))
+FUNCTION (svamin, faminmaximpl, (UNSPEC_COND_FAMIN, UNSPEC_FAMIN))
 FUNCTION (svandqv, reduction, (UNSPEC_ANDQV, UNSPEC_ANDQV, -1))
 FUNCTION (svbcax, CODE_FOR_MODE0 (aarch64_sve2_bcax),)
 FUNCTION (svbdep, unspec_based_function, (UNSPEC_BDEP, UNSPEC_BDEP, -1))
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.cc b/gcc/config/aarch64/aarch64-sve-builtins.cc
index 2b627a9..e394c9a 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins.cc
@@ -4004,7 +4004,8 @@ rtx
 function_expander::get_reg_target ()
 {
   machine_mode target_mode = result_mode ();
-  if (!possible_target || GET_MODE (possible_target) != target_mode)
+  if (!possible_target
+      || !register_operand (possible_target, target_mode))
     possible_target = gen_reg_rtx (target_mode);
   return possible_target;
 }
@@ -4592,7 +4593,27 @@ function_expander::expand ()
       gcc_assert (args.last ()->mode == DImode);
       emit_move_insn (gen_rtx_REG (DImode, FPM_REGNUM), args.last ());
     }
-  return base->expand (*this);
+  rtx result = base->expand (*this);
+  if (function_returns_void_p ())
+    gcc_assert (result == const0_rtx);
+  else
+    {
+      auto expected_mode = result_mode ();
+      if (GET_MODE_CLASS (expected_mode) == MODE_INT)
+	/* Scalar integer constants don't store a mode.
+
+	   It's OK for a variable result to have a different mode from the
+	   function return type.  In particular, some functions that return int
+	   expand into instructions that have a DImode result, with all 64 bits
+	   of the DImode being well-defined (usually zero).  */
+	gcc_assert (CONST_SCALAR_INT_P (result)
+		    || GET_MODE_CLASS (GET_MODE (result)) == MODE_INT);
+      else
+	/* In other cases, the return value should have the same mode
+	   as the return type.  */
+	gcc_assert (GET_MODE (result) == expected_mode);
+    }
+  return result;
 }
 
 /* Return a structure type that contains a single field of type FIELD_TYPE.
diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
index 4aecb3a..88d323a 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -2990,10 +2990,7 @@
 	(vec_duplicate:PRED_ALL (match_operand:QI 1 "register_operand")))]
   "TARGET_SVE"
   {
-    rtx tmp = gen_reg_rtx (DImode);
-    rtx op1 = gen_lowpart (DImode, operands[1]);
-    emit_insn (gen_ashldi3 (tmp, op1, gen_int_mode (63, DImode)));
-    emit_insn (gen_while_ultdi<mode> (operands[0], const0_rtx, tmp));
+    aarch64_emit_sve_pred_vec_duplicate (<MODE>mode, operands[0], operands[1]);
     DONE;
   }
 )
@@ -3752,9 +3749,9 @@
 
 ;; Unpredicated floating-point unary operations.
 (define_insn "@aarch64_sve_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand" "w")]
+  [(set (match_operand:SVE_F 0 "register_operand" "=w")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand" "w")]
 	  SVE_FP_UNARY))]
   "TARGET_SVE"
   "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>"
@@ -3762,25 +3759,41 @@
 
 ;; Unpredicated floating-point unary operations.
 (define_expand "<optab><mode>2"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_dup 2)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F 1 "register_operand")]
+	   (match_dup 3)
+	   (match_operand:SVE_F 1 "register_operand")]
 	  SVE_COND_FP_UNARY_OPTAB))]
   "TARGET_SVE"
   {
+    operands[2] = aarch64_sve_fp_pred (<MODE>mode, &operands[3]);
+  }
+)
+
+;; FABS and FNEG are non-trapping, so we can always expand with a <VPRED>
+;; predicate.  It doesn't matter whether the padding bits of a partial
+;; vector mode are active or inactive.
+(define_expand "<optab><mode>2"
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_dup 2)
+	   (const_int SVE_RELAXED_GP)
+	   (match_operand:SVE_F 1 "register_operand")]
+	  SVE_COND_FP_UNARY_BITWISE))]
+  "TARGET_SVE"
+  {
     operands[2] = aarch64_ptrue_reg (<VPRED>mode);
   }
 )
 
 ;; Predicated floating-point unary operations.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 3 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")]
+	   (match_operand:SVE_F 2 "register_operand")]
 	  SVE_COND_FP_UNARY))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 2 ; attrs: movprfx ]
@@ -3806,13 +3819,13 @@
 
 ;; Predicated floating-point unary arithmetic, merging with the first input.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 3)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")]
 	     SVE_COND_FP_UNARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -3854,15 +3867,15 @@
 ;; as earlyclobber helps to make the instruction more regular to the
 ;; register allocator.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")]
 	     SVE_COND_FP_UNARY)
-	   (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[3])"
   {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
@@ -3966,7 +3979,7 @@
 )
 
 ;; Predicated predicate inverse.
-(define_insn "*one_cmpl<mode>3"
+(define_insn "@aarch64_pred_one_cmpl<mode>_z"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(and:PRED_ALL
 	  (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa"))
@@ -5495,27 +5508,25 @@
 ;; Split a predicated instruction whose predicate is unused into an
 ;; unpredicated instruction.
 (define_split
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	   (const_int SVE_RELAXED_GP)
+	   (match_operand:SVE_F_B16B16 2 "register_operand")
+	   (match_operand:SVE_F_B16B16 3 "register_operand")]
 	  <SVE_COND_FP>))]
-  "TARGET_SVE
-   && reload_completed
-   && INTVAL (operands[4]) == SVE_RELAXED_GP"
+  "TARGET_SVE && reload_completed"
   [(set (match_dup 0)
-	(SVE_UNPRED_FP_BINARY:SVE_FULL_F_B16B16 (match_dup 2) (match_dup 3)))]
+	(SVE_UNPRED_FP_BINARY:SVE_F_B16B16 (match_dup 2) (match_dup 3)))]
 )
 
 ;; Unpredicated floating-point binary operations (post-RA only).
 ;; These are generated by the split above.
 (define_insn "*post_ra_<sve_fp_op><mode>3"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand" "=w")
-	(SVE_UNPRED_FP_BINARY:SVE_FULL_F_B16B16
-	  (match_operand:SVE_FULL_F_B16B16 1 "register_operand" "w")
-	  (match_operand:SVE_FULL_F_B16B16 2 "register_operand" "w")))]
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand" "=w")
+	(SVE_UNPRED_FP_BINARY:SVE_F_B16B16
+	  (match_operand:SVE_F_B16B16 1 "register_operand" "w")
+	  (match_operand:SVE_F_B16B16 2 "register_operand" "w")))]
   "TARGET_SVE && reload_completed"
   "<b><sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>")
 
@@ -5547,10 +5558,10 @@
 
 ;; Unpredicated floating-point binary operations.
 (define_insn "@aarch64_sve_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand" "w")
-	   (match_operand:SVE_FULL_F 2 "register_operand" "w")]
+  [(set (match_operand:SVE_F 0 "register_operand" "=w")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand" "w")
+	   (match_operand:SVE_F 2 "register_operand" "w")]
 	  SVE_FP_BINARY))]
   "TARGET_SVE"
   "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
@@ -5559,27 +5570,27 @@
 ;; Unpredicated floating-point binary operations that need to be predicated
 ;; for SVE.
 (define_expand "<optab><mode>3"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_dup 3)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F_B16B16 1 "<sve_pred_fp_rhs1_operand>")
-	   (match_operand:SVE_FULL_F_B16B16 2 "<sve_pred_fp_rhs2_operand>")]
+	   (match_dup 4)
+	   (match_operand:SVE_F_B16B16 1 "<sve_pred_fp_rhs1_operand>")
+	   (match_operand:SVE_F_B16B16 2 "<sve_pred_fp_rhs2_operand>")]
 	  SVE_COND_FP_BINARY_OPTAB))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
   {
-    operands[3] = aarch64_ptrue_reg (<VPRED>mode);
+    operands[3] = aarch64_sve_fp_pred (<MODE>mode, &operands[4]);
   }
 )
 
 ;; Predicated floating-point binary operations that have no immediate forms.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "register_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "register_operand")]
 	  SVE_COND_FP_BINARY_REG))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 2 , 3 ; attrs: movprfx ]
@@ -5591,30 +5602,33 @@
 
 ;; Predicated floating-point operations with merging.
 (define_expand "@cond_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "<sve_pred_fp_rhs1_operand>")
-	      (match_operand:SVE_FULL_F_B16B16 3 "<sve_pred_fp_rhs2_operand>")]
+	      (match_operand:SVE_F_B16B16 2 "<sve_pred_fp_rhs1_operand>")
+	      (match_operand:SVE_F_B16B16 3 "<sve_pred_fp_rhs2_operand>")]
 	     SVE_COND_FP_BINARY)
-	   (match_operand:SVE_FULL_F_B16B16 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
+  {
+    operands[1] = aarch64_sve_emit_masked_fp_pred (<MODE>mode, operands[1]);
+  }
 )
 
 ;; Predicated floating-point operations, merging with the first input.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5630,14 +5644,14 @@
 )
 
 (define_insn "*cond_<optab><mode>_2_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5650,14 +5664,14 @@
 
 ;; Same for operations that take a 1-bit constant.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5673,14 +5687,14 @@
 )
 
 (define_insn "*cond_<optab><mode>_2_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5693,14 +5707,14 @@
 
 ;; Predicated floating-point operations, merging with the second input.
 (define_insn_and_rewrite "*cond_<optab><mode>_3_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -5716,14 +5730,14 @@
 )
 
 (define_insn "*cond_<optab><mode>_3_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -5736,16 +5750,16 @@
 
 ;; Predicated floating-point operations, merging with an independent value.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
-	   (match_operand:SVE_FULL_F_B16B16 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -5780,16 +5794,16 @@
 )
 
 (define_insn_and_rewrite "*cond_<optab><mode>_any_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
-	   (match_operand:SVE_FULL_F_B16B16 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -5818,16 +5832,16 @@
 
 ;; Same for operations that take a 1-bit constant.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 4   ]
@@ -5854,16 +5868,16 @@
 )
 
 (define_insn_and_rewrite "*cond_<optab><mode>_any_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 4   ]
@@ -5892,12 +5906,12 @@
 
 ;; Predicated floating-point addition.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand")]
 	  SVE_COND_FP_ADD))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , %2 , 3   , 4   ; attrs: movprfx ]
@@ -5914,14 +5928,14 @@
 ;; Predicated floating-point addition of a constant, merging with the
 ;; first input.
 (define_insn_and_rewrite "*cond_add<mode>_2_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5939,14 +5953,14 @@
 )
 
 (define_insn "*cond_add<mode>_2_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5962,16 +5976,16 @@
 ;; Predicated floating-point addition of a constant, merging with an
 ;; independent value.
 (define_insn_and_rewrite "*cond_add<mode>_any_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 3   , 4   ]
@@ -6001,16 +6015,16 @@
 )
 
 (define_insn_and_rewrite "*cond_add<mode>_any_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 3   , 4   ]
@@ -6208,12 +6222,12 @@
 
 ;; Predicated floating-point subtraction.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_operand")
-	   (match_operand:SVE_FULL_F 3 "register_operand")]
+	   (match_operand:SVE_F 2 "aarch64_sve_float_arith_operand")
+	   (match_operand:SVE_F 3 "register_operand")]
 	  SVE_COND_FP_SUB))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 2   , 3 , 4   ; attrs: movprfx ]
@@ -6229,14 +6243,14 @@
 ;; Predicated floating-point subtraction from a constant, merging with the
 ;; second input.
 (define_insn_and_rewrite "*cond_sub<mode>_3_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -6252,14 +6266,14 @@
 )
 
 (define_insn "*cond_sub<mode>_3_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -6273,16 +6287,16 @@
 ;; Predicated floating-point subtraction from a constant, merging with an
 ;; independent value.
 (define_insn_and_rewrite "*cond_sub<mode>_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[3], operands[4])"
   {@ [ cons: =0 , 1   , 3 , 4   ]
@@ -6309,16 +6323,16 @@
 )
 
 (define_insn_and_rewrite "*cond_sub<mode>_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[3], operands[4])"
   {@ [ cons: =0 , 1   , 3 , 4   ]
@@ -6631,12 +6645,12 @@
 
 ;; Predicated floating-point multiplication.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_sve_float_mul_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_sve_float_mul_operand")]
 	  SVE_COND_FP_MUL))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , %2 , 3   , 4   ; attrs: movprfx ]
@@ -6671,12 +6685,12 @@
 ;; -------------------------------------------------------------------------
 
 (define_expand "div<mode>3"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_dup 3)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F 1 "nonmemory_operand")
-	   (match_operand:SVE_FULL_F 2 "register_operand")]
+	   (match_dup 4)
+	   (match_operand:SVE_F 1 "nonmemory_operand")
+	   (match_operand:SVE_F 2 "register_operand")]
 	  UNSPEC_COND_FDIV))]
   "TARGET_SVE"
   {
@@ -6684,23 +6698,23 @@
       DONE;
 
     operands[1] = force_reg (<MODE>mode, operands[1]);
-    operands[3] = aarch64_ptrue_reg (<VPRED>mode);
+    operands[3] = aarch64_sve_fp_pred (<MODE>mode, &operands[4]);
   }
 )
 
 (define_expand "@aarch64_frecpe<mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand")]
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand")]
 	  UNSPEC_FRECPE))]
   "TARGET_SVE"
 )
 
 (define_expand "@aarch64_frecps<mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand")
-	   (match_operand:SVE_FULL_F 2 "register_operand")]
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand")
+	   (match_operand:SVE_F 2 "register_operand")]
 	  UNSPEC_FRECPS))]
   "TARGET_SVE"
 )
@@ -6865,12 +6879,12 @@
 
 ;; Predicated floating-point maximum/minimum.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_sve_float_maxmin_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_sve_float_maxmin_operand")]
 	  SVE_COND_FP_MAXMIN))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , %2 , 3   ; attrs: movprfx ]
@@ -6899,7 +6913,7 @@
 ;; Predicate AND.  We can reuse one of the inputs as the GP.
 ;; Doubling the second operand is the preferred implementation
 ;; of the MOV alias, so we use that instead of %1/z, %1, %2.
-(define_insn "and<mode>3"
+(define_insn "@and<mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand")
 	(and:PRED_ALL (match_operand:PRED_ALL 1 "register_operand")
 		      (match_operand:PRED_ALL 2 "register_operand")))]
@@ -7581,29 +7595,29 @@
 
 ;; Unpredicated floating-point ternary operations.
 (define_expand "<optab><mode>4"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_dup 4)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F_B16B16 1 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	   (match_dup 5)
+	   (match_operand:SVE_F_B16B16 1 "register_operand")
+	   (match_operand:SVE_F_B16B16 2 "register_operand")
+	   (match_operand:SVE_F_B16B16 3 "register_operand")]
 	  SVE_COND_FP_TERNARY))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
   {
-    operands[4] = aarch64_ptrue_reg (<VPRED>mode);
+    operands[4] = aarch64_sve_fp_pred (<MODE>mode, &operands[5]);
   }
 )
 
 ;; Predicated floating-point ternary operations.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 5 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	   (match_operand:SVE_F_B16B16 2 "register_operand")
+	   (match_operand:SVE_F_B16B16 3 "register_operand")
+	   (match_operand:SVE_F_B16B16 4 "register_operand")]
 	  SVE_COND_FP_TERNARY))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
   {@ [ cons: =0 , 1   , %2  , 3 , 4 ; attrs: movprfx , is_rev ]
@@ -7617,17 +7631,17 @@
 
 ;; Predicated floating-point ternary operations with merging.
 (define_expand "@cond_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
-	   (match_operand:SVE_FULL_F_B16B16 5 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 5 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
 {
@@ -7635,20 +7649,22 @@
      second of the two.  */
   if (rtx_equal_p (operands[3], operands[5]))
     std::swap (operands[2], operands[3]);
+
+  operands[1] = aarch64_sve_emit_masked_fp_pred (<MODE>mode, operands[1]);
 })
 
 ;; Predicated floating-point ternary operations, merging with the
 ;; first input.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "register_operand")
-	      (match_operand:SVE_FULL_F 4 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "register_operand")
+	      (match_operand:SVE_F 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -7664,15 +7680,15 @@
 )
 
 (define_insn "*cond_<optab><mode>_2_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "register_operand")
-	      (match_operand:SVE_FULL_F 4 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "register_operand")
+	      (match_operand:SVE_F 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -7686,15 +7702,15 @@
 ;; Predicated floating-point ternary operations, merging with the
 ;; third input.
 (define_insn_and_rewrite "*cond_<optab><mode>_4_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 4)]
 	  UNSPEC_SEL))]
@@ -7710,15 +7726,15 @@
 )
 
 (define_insn "*cond_<optab><mode>_4_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 4)]
 	  UNSPEC_SEL))]
@@ -7732,17 +7748,17 @@
 ;; Predicated floating-point ternary operations, merging with an
 ;; independent value.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 6)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
-	   (match_operand:SVE_FULL_F_B16B16 5 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 5 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -7778,17 +7794,17 @@
 )
 
 (define_insn_and_rewrite "*cond_<optab><mode>_any_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
-	   (match_operand:SVE_FULL_F_B16B16 5 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 5 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -8187,20 +8203,23 @@
 ;;
 ;; For unpacked vectors, it doesn't really matter whether SEL uses the
 ;; the container size or the element size.  If SEL used the container size,
-;; it would ignore undefined bits of the predicate but would copy the
-;; upper (undefined) bits of each container along with the defined bits.
-;; If SEL used the element size, it would use undefined bits of the predicate
-;; to select between undefined elements in each input vector.  Thus the only
-;; difference is whether the undefined bits in a container always come from
-;; the same input as the defined bits, or whether the choice can vary
-;; independently of the defined bits.
+;; it would would copy the upper (undefined) bits of each container along
+;; with the corresponding defined bits.  If SEL used the element size,
+;; it would use separate predicate bits to select between the undefined
+;; elements in each input vector; these seperate predicate bits might
+;; themselves be undefined, depending on the mode of the predicate.
+;;
+;; Thus the only difference is whether the undefined bits in a container
+;; always come from the same input as the defined bits, or whether the
+;; choice can vary independently of the defined bits.
 ;;
 ;; For the other instructions, using the element size is more natural,
 ;; so we do that for SEL as well.
+;;
 (define_insn "*vcond_mask_<mode><vpred>"
   [(set (match_operand:SVE_ALL 0 "register_operand")
 	(unspec:SVE_ALL
-	  [(match_operand:<VPRED> 3 "register_operand")
+	  [(match_operand:<VPRED> 3 "aarch64_predicate_operand")
 	   (match_operand:SVE_ALL 1 "aarch64_sve_reg_or_dup_imm")
 	   (match_operand:SVE_ALL 2 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
@@ -8339,6 +8358,71 @@
   }
 )
 
+;; Likewise, but yield a VNx16BI result regardless of the element width.
+;; The .b case is equivalent to the above.
+(define_expand "@aarch64_pred_cmp<cmp_op><mode>_acle"
+  [(parallel
+     [(set (match_operand:<VPRED> 0 "register_operand")
+	   (unspec:<VPRED>
+	     [(match_operand:<VPRED> 1 "register_operand")
+	      (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	      (SVE_INT_CMP:<VPRED>
+		(match_operand:VNx16QI_ONLY 3 "register_operand")
+		(match_operand:VNx16QI_ONLY 4 "aarch64_sve_cmp_<sve_imm_con>_operand"))]
+	     UNSPEC_PRED_Z))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  "TARGET_SVE"
+)
+
+;; For wider elements, bitcast the predicate result to a VNx16BI and use
+;; an (and ...) to indicate that only every second, fourth, or eighth bit
+;; is set.
+(define_expand "@aarch64_pred_cmp<cmp_op><mode>_acle"
+  [(parallel
+     [(set (match_operand:VNx16BI 0 "register_operand")
+	   (and:VNx16BI
+	     (subreg:VNx16BI
+	       (unspec:<VPRED>
+		 [(match_operand:<VPRED> 1 "register_operand")
+		  (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+		  (SVE_INT_CMP:<VPRED>
+		    (match_operand:SVE_FULL_HSDI 3 "register_operand")
+		    (match_operand:SVE_FULL_HSDI 4 "aarch64_sve_cmp_<sve_imm_con>_operand"))]
+		 UNSPEC_PRED_Z)
+	       0)
+	     (match_dup 5)))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  "TARGET_SVE"
+  {
+    operands[5] = aarch64_ptrue_all (GET_MODE_UNIT_SIZE (<MODE>mode));
+  }
+)
+
+(define_insn "*aarch64_pred_cmp<cmp_op><mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "register_operand")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (SVE_INT_CMP:<VPRED>
+		 (match_operand:SVE_FULL_HSDI 3 "register_operand")
+		 (match_operand:SVE_FULL_HSDI 4 "aarch64_sve_cmp_<sve_imm_con>_operand"))]
+	      UNSPEC_PRED_Z)
+	    0)
+	  (match_operand:<VPRED> 5 "aarch64_ptrue_all_operand")))
+   (clobber (reg:CC_NZC CC_REGNUM))]
+  "TARGET_SVE"
+  {@ [ cons: =0 , 1  , 3 , 4            ; attrs: pred_clobber ]
+     [ &Upa     , Upl, w , <sve_imm_con>; yes                 ] cmp<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, #%4
+     [ ?Upl     , 0  , w , <sve_imm_con>; yes                 ] ^
+     [ Upa      , Upl, w , <sve_imm_con>; no                  ] ^
+     [ &Upa     , Upl, w , w            ; yes                 ] cmp<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>
+     [ ?Upl     , 0  , w , w            ; yes                 ] ^
+     [ Upa      , Upl, w , w            ; no                  ] ^
+  }
+)
+
 ;; Predicated integer comparisons in which both the flag and predicate
 ;; results are interesting.
 (define_insn_and_rewrite "*cmp<cmp_op><mode>_cc"
@@ -8380,6 +8464,49 @@
   }
 )
 
+(define_insn_and_rewrite "*cmp<cmp_op><mode>_acle_cc"
+  [(set (reg:CC_NZC CC_REGNUM)
+	(unspec:CC_NZC
+	  [(match_operand:VNx16BI 1 "register_operand")
+	   (match_operand 4)
+	   (match_operand:SI 5 "aarch64_sve_ptrue_flag")
+	   (unspec:<VPRED>
+	     [(match_operand 6)
+	      (match_operand:SI 7 "aarch64_sve_ptrue_flag")
+	      (SVE_INT_CMP:<VPRED>
+		(match_operand:SVE_FULL_HSDI 2 "register_operand")
+		(match_operand:SVE_FULL_HSDI 3 "aarch64_sve_cmp_<sve_imm_con>_operand"))]
+	     UNSPEC_PRED_Z)]
+	  UNSPEC_PTEST))
+   (set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_dup 6)
+	       (match_dup 7)
+	       (SVE_INT_CMP:<VPRED>
+		 (match_dup 2)
+		 (match_dup 3))]
+	      UNSPEC_PRED_Z)
+	    0)
+	  (match_operand:<VPRED> 8 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE
+   && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
+  {@ [ cons: =0 , 1   , 2 , 3            ; attrs: pred_clobber ]
+     [ &Upa     ,  Upl, w , <sve_imm_con>; yes                 ] cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
+     [ ?Upl     ,  0  , w , <sve_imm_con>; yes                 ] ^
+     [ Upa      ,  Upl, w , <sve_imm_con>; no                  ] ^
+     [ &Upa     ,  Upl, w , w            ; yes                 ] cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>
+     [ ?Upl     ,  0  , w , w            ; yes                 ] ^
+     [ Upa      ,  Upl, w , w            ; no                  ] ^
+  }
+  "&& !rtx_equal_p (operands[4], operands[6])"
+  {
+    operands[6] = copy_rtx (operands[4]);
+    operands[7] = operands[5];
+  }
+)
+
 ;; Predicated integer comparisons in which only the flags result is
 ;; interesting.
 (define_insn_and_rewrite "*cmp<cmp_op><mode>_ptest"
@@ -8445,14 +8572,52 @@
       (clobber (reg:CC_NZC CC_REGNUM))])]
 )
 
+(define_insn_and_split "*cmp<cmp_op><mode>_acle_and"
+  [(set (match_operand:VNx16BI 0 "register_operand" "=Upa, Upa")
+	(and:VNx16BI
+	  (and:VNx16BI
+	    (subreg:VNx16BI
+	      (unspec:<VPRED>
+		[(match_operand 4)
+		 (const_int SVE_KNOWN_PTRUE)
+		 (SVE_INT_CMP:<VPRED>
+		   (match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w")
+		   (match_operand:SVE_FULL_HSDI 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
+		UNSPEC_PRED_Z)
+	      0)
+	    (match_operand:VNx16BI 1 "register_operand" "Upl, Upl"))
+          (match_operand:<VPRED> 5 "aarch64_ptrue_all_operand")))
+   (clobber (reg:CC_NZC CC_REGNUM))]
+  "TARGET_SVE"
+  "#"
+  "&& 1"
+  [(parallel
+     [(set (match_dup 0)
+	   (and:VNx16BI
+	     (subreg:VNx16BI
+	       (unspec:<VPRED>
+		 [(match_dup 1)
+		  (const_int SVE_MAYBE_NOT_PTRUE)
+		  (SVE_INT_CMP:<VPRED>
+		    (match_dup 2)
+		    (match_dup 3))]
+		 UNSPEC_PRED_Z)
+	       0)
+	     (match_dup 5)))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  {
+    operands[1] = gen_lowpart (<VPRED>mode, operands[1]);
+  }
+)
+
 ;; Predicated integer wide comparisons.
 (define_insn "@aarch64_pred_cmp<cmp_op><mode>_wide"
   [(set (match_operand:<VPRED> 0 "register_operand")
 	(unspec:<VPRED>
-	  [(match_operand:VNx16BI 1 "register_operand")
+	  [(match_operand:<VPRED> 1 "register_operand")
 	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
 	   (unspec:<VPRED>
-	     [(match_operand:SVE_FULL_BHSI 3 "register_operand")
+	     [(match_operand:VNx16QI_ONLY 3 "register_operand")
 	      (match_operand:VNx2DI 4 "register_operand")]
 	     SVE_COND_INT_CMP_WIDE)]
 	  UNSPEC_PRED_Z))
@@ -8465,16 +8630,61 @@
   }
 )
 
+(define_expand "@aarch64_pred_cmp<cmp_op><mode>_wide"
+  [(parallel
+     [(set (match_operand:VNx16BI 0 "register_operand")
+	   (and:VNx16BI
+	     (subreg:VNx16BI
+	       (unspec:<VPRED>
+		 [(match_operand:<VPRED> 1 "register_operand")
+		  (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+		  (unspec:<VPRED>
+		    [(match_operand:SVE_FULL_HSI 3 "register_operand")
+		     (match_operand:VNx2DI 4 "register_operand")]
+		    SVE_COND_INT_CMP_WIDE)]
+		 UNSPEC_PRED_Z)
+	       0)
+	     (match_dup 5)))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  "TARGET_SVE"
+  {
+    operands[5] = aarch64_ptrue_all (GET_MODE_UNIT_SIZE (<MODE>mode));
+  }
+)
+
+(define_insn "*aarch64_pred_cmp<cmp_op><mode>_wide"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "register_operand")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (unspec:<VPRED>
+		 [(match_operand:SVE_FULL_HSI 3 "register_operand")
+		  (match_operand:VNx2DI 4 "register_operand")]
+		 SVE_COND_INT_CMP_WIDE)]
+	      UNSPEC_PRED_Z)
+	    0)
+	  (match_operand:<VPRED> 5 "aarch64_ptrue_all_operand")))
+   (clobber (reg:CC_NZC CC_REGNUM))]
+  "TARGET_SVE"
+  {@ [ cons: =0, 1   , 2, 3, 4; attrs: pred_clobber ]
+     [ &Upa    ,  Upl,  , w, w; yes                 ] cmp<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.d
+     [ ?Upl    ,  0  ,  , w, w; yes                 ] ^
+     [ Upa     ,  Upl,  , w, w; no                  ] ^
+  }
+)
+
 ;; Predicated integer wide comparisons in which both the flag and
 ;; predicate results are interesting.
-(define_insn "*aarch64_pred_cmp<cmp_op><mode>_wide_cc"
+(define_insn_and_rewrite "*aarch64_pred_cmp<cmp_op><mode>_wide_cc"
   [(set (reg:CC_NZC CC_REGNUM)
 	(unspec:CC_NZC
 	  [(match_operand:VNx16BI 1 "register_operand")
 	   (match_operand 4)
 	   (match_operand:SI 5 "aarch64_sve_ptrue_flag")
 	   (unspec:<VPRED>
-	     [(match_operand:VNx16BI 6 "register_operand")
+	     [(match_operand:<VPRED> 6 "register_operand")
 	      (match_operand:SI 7 "aarch64_sve_ptrue_flag")
 	      (unspec:<VPRED>
 		[(match_operand:SVE_FULL_BHSI 2 "register_operand")
@@ -8498,18 +8708,65 @@
      [ ?Upl    ,  0  , w, w, Upl; yes                 ] ^
      [ Upa     ,  Upl, w, w, Upl; no                  ] ^
   }
+  "&& !rtx_equal_p (operands[4], operands[6])"
+  {
+    operands[6] = copy_rtx (operands[4]);
+    operands[7] = operands[5];
+  }
+)
+
+(define_insn_and_rewrite "*aarch64_pred_cmp<cmp_op><mode>_wide_cc"
+  [(set (reg:CC_NZC CC_REGNUM)
+	(unspec:CC_NZC
+	  [(match_operand:VNx16BI 1 "register_operand")
+	   (match_operand 4)
+	   (match_operand:SI 5 "aarch64_sve_ptrue_flag")
+	   (unspec:<VPRED>
+	     [(match_operand:<VPRED> 6 "register_operand")
+	      (match_operand:SI 7 "aarch64_sve_ptrue_flag")
+	      (unspec:<VPRED>
+		[(match_operand:SVE_FULL_HSI 2 "register_operand")
+		 (match_operand:VNx2DI 3 "register_operand")]
+		SVE_COND_INT_CMP_WIDE)]
+	     UNSPEC_PRED_Z)]
+	  UNSPEC_PTEST))
+   (set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_dup 6)
+	       (match_dup 7)
+	       (unspec:<VPRED>
+		 [(match_dup 2)
+		  (match_dup 3)]
+		 SVE_COND_INT_CMP_WIDE)]
+	      UNSPEC_PRED_Z)
+	    0)
+	  (match_operand:<VPRED> 8 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE
+   && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
+  {@ [ cons: =0, 1   , 2, 3, 6  ; attrs: pred_clobber ]
+     [ &Upa    ,  Upl, w, w, Upl; yes                 ] cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.d
+     [ ?Upl    ,  0  , w, w, Upl; yes                 ] ^
+     [ Upa     ,  Upl, w, w, Upl; no                  ] ^
+  }
+  "&& !rtx_equal_p (operands[4], operands[6])"
+  {
+    operands[6] = copy_rtx (operands[4]);
+    operands[7] = operands[5];
+  }
 )
 
 ;; Predicated integer wide comparisons in which only the flags result
 ;; is interesting.
-(define_insn "*aarch64_pred_cmp<cmp_op><mode>_wide_ptest"
+(define_insn_and_rewrite "*aarch64_pred_cmp<cmp_op><mode>_wide_ptest"
   [(set (reg:CC_NZC CC_REGNUM)
 	(unspec:CC_NZC
 	  [(match_operand:VNx16BI 1 "register_operand")
 	   (match_operand 4)
 	   (match_operand:SI 5 "aarch64_sve_ptrue_flag")
 	   (unspec:<VPRED>
-	     [(match_operand:VNx16BI 6 "register_operand")
+	     [(match_operand:<VPRED> 6 "register_operand")
 	      (match_operand:SI 7 "aarch64_sve_ptrue_flag")
 	      (unspec:<VPRED>
 		[(match_operand:SVE_FULL_BHSI 2 "register_operand")
@@ -8525,6 +8782,11 @@
      [ ?Upl     ,  0  , w, w, Upl; yes                 ] ^
      [ Upa      ,  Upl, w, w, Upl; no                  ] ^
   }
+  "&& !rtx_equal_p (operands[4], operands[6])"
+  {
+    operands[6] = copy_rtx (operands[4]);
+    operands[7] = operands[5];
+  }
 )
 
 ;; -------------------------------------------------------------------------
@@ -8562,6 +8824,58 @@
   "while<cmp_op>\t%0.<PRED_ALL:Vetype>, %<w>1, %<w>2"
 )
 
+;; Likewise, but yield a VNx16BI result regardless of the element width.
+;; The .b case is equivalent to the above.
+(define_expand "@aarch64_sve_while_<while_optab_cmp><GPI:mode><VNx16BI_ONLY:mode>_acle"
+  [(parallel
+     [(set (match_operand:VNx16BI_ONLY 0 "register_operand")
+	   (unspec:VNx16BI_ONLY
+	     [(const_int SVE_WHILE_B)
+	      (match_operand:GPI 1 "aarch64_reg_or_zero")
+	      (match_operand:GPI 2 "aarch64_reg_or_zero")]
+	     SVE_WHILE))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  "TARGET_SVE"
+)
+
+;; For wider elements, bitcast the predicate result to a VNx16BI and use
+;; an (and ...) to indicate that only every second, fourth, or eighth bit
+;; is set.
+(define_expand "@aarch64_sve_while_<while_optab_cmp><GPI:mode><PRED_HSD:mode>_acle"
+  [(parallel
+     [(set (match_operand:VNx16BI 0 "register_operand")
+	   (and:VNx16BI
+	     (subreg:VNx16BI
+	       (unspec:PRED_HSD
+		 [(const_int SVE_WHILE_B)
+		  (match_operand:GPI 1 "aarch64_reg_or_zero")
+		  (match_operand:GPI 2 "aarch64_reg_or_zero")]
+		 SVE_WHILE)
+	       0)
+	     (match_dup 3)))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  "TARGET_SVE"
+  {
+    operands[3] = aarch64_ptrue_all (<data_bytes>);
+  }
+)
+
+(define_insn "*aarch64_sve_while_<while_optab_cmp><GPI:mode><PRED_HSD:mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:PRED_HSD
+	      [(const_int SVE_WHILE_B)
+	       (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
+	       (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")]
+	      SVE_WHILE)
+	    0)
+	  (match_operand:PRED_HSD 3 "aarch64_ptrue_all_operand")))
+   (clobber (reg:CC_NZC CC_REGNUM))]
+  "TARGET_SVE"
+  "while<cmp_op>\t%0.<PRED_HSD:Vetype>, %<w>1, %<w>2"
+)
+
 ;; The WHILE instructions set the flags in the same way as a PTEST with
 ;; a PTRUE GP.  Handle the case in which both results are useful.  The GP
 ;; operands to the PTEST aren't needed, so we allow them to be anything.
@@ -8593,6 +8907,38 @@
   }
 )
 
+(define_insn_and_rewrite "*while_<while_optab_cmp><GPI:mode><PRED_HSD:mode>_acle_cc"
+  [(set (reg:CC_NZC CC_REGNUM)
+	(unspec:CC_NZC
+	  [(match_operand 3)
+	   (match_operand 4)
+	   (const_int SVE_KNOWN_PTRUE)
+	   (unspec:PRED_HSD
+	     [(const_int SVE_WHILE_B)
+	      (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
+	      (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")]
+	     SVE_WHILE)]
+	  UNSPEC_PTEST))
+   (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:PRED_HSD [(const_int SVE_WHILE_B)
+			      (match_dup 1)
+			      (match_dup 2)]
+			     SVE_WHILE)
+	    0)
+	  (match_operand:PRED_HSD 5 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE"
+  "while<cmp_op>\t%0.<PRED_HSD:Vetype>, %<w>1, %<w>2"
+  ;; Force the compiler to drop the unused predicate operand, so that we
+  ;; don't have an unnecessary PTRUE.
+  "&& (!CONSTANT_P (operands[3]) || !CONSTANT_P (operands[4]))"
+  {
+    operands[3] = CONSTM1_RTX (VNx16BImode);
+    operands[4] = CONSTM1_RTX (<PRED_HSD:MODE>mode);
+  }
+)
+
 ;; Same, but handle the case in which only the flags result is useful.
 (define_insn_and_rewrite "@while_<while_optab_cmp><GPI:mode><PRED_ALL:mode>_ptest"
   [(set (reg:CC_NZC CC_REGNUM)
@@ -8637,8 +8983,8 @@
 (define_expand "vec_cmp<mode><vpred>"
   [(set (match_operand:<VPRED> 0 "register_operand")
 	(match_operator:<VPRED> 1 "comparison_operator"
-	  [(match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero")]))]
+	  [(match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero")]))]
   "TARGET_SVE"
   {
     aarch64_expand_sve_vec_cmp_float (operands[0], GET_CODE (operands[1]),
@@ -8651,10 +8997,10 @@
 (define_insn "@aarch64_pred_fcm<cmp_op><mode>"
   [(set (match_operand:<VPRED> 0 "register_operand")
 	(unspec:<VPRED>
-	  [(match_operand:<VPRED> 1 "register_operand")
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
-	   (match_operand:SVE_FULL_F 3 "register_operand")
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 3 "register_operand")
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  SVE_COND_FP_CMP_I0))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 3 , 4   ]
@@ -8663,19 +9009,90 @@
   }
 )
 
+(define_expand "@aarch64_pred_fcm<cmp_op><mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (match_operand:SVE_F 3 "register_operand")
+	       (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
+	      SVE_COND_FP_CMP_I0)
+	    0)
+	  (match_dup 5)))]
+  "TARGET_SVE"
+  {
+    operands[5] = aarch64_ptrue_all (GET_MODE_UNIT_SIZE (<MODE>mode));
+  }
+)
+
+(define_insn "*aarch64_pred_fcm<cmp_op><mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (match_operand:SVE_F 3 "register_operand")
+	       (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
+	      SVE_COND_FP_CMP_I0)
+	    0)
+	  (match_operand:<VPRED> 5 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE"
+  {@ [ cons: =0 , 1   , 3 , 4   ]
+     [ Upa      , Upl , w , Dz  ] fcm<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, #0.0
+     [ Upa      , Upl , w , w   ] fcm<cmp_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>
+  }
+)
+
 ;; Same for unordered comparisons.
 (define_insn "@aarch64_pred_fcmuo<mode>"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
 	(unspec:<VPRED>
-	  [(match_operand:<VPRED> 1 "register_operand" "Upl")
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand" "Upl")
 	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
-	   (match_operand:SVE_FULL_F 3 "register_operand" "w")
-	   (match_operand:SVE_FULL_F 4 "register_operand" "w")]
+	   (match_operand:SVE_F 3 "register_operand" "w")
+	   (match_operand:SVE_F 4 "register_operand" "w")]
 	  UNSPEC_COND_FCMUO))]
   "TARGET_SVE"
   "fcmuo\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>"
 )
 
+(define_expand "@aarch64_pred_fcmuo<mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (match_operand:SVE_F 3 "register_operand")
+	       (match_operand:SVE_F 4 "register_operand")]
+	      UNSPEC_COND_FCMUO)
+	    0)
+	  (match_dup 5)))]
+  "TARGET_SVE"
+  {
+    operands[5] = aarch64_ptrue_all (GET_MODE_UNIT_SIZE (<MODE>mode));
+  }
+)
+
+(define_insn "*aarch64_pred_fcmuo<mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "aarch64_predicate_operand" "Upl")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (match_operand:SVE_F 3 "register_operand" "w")
+	       (match_operand:SVE_F 4 "register_operand" "w")]
+	      UNSPEC_COND_FCMUO)
+	    0)
+	  (match_operand:<VPRED> 5 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE"
+  "fcmuo\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>"
+)
+
 ;; Floating-point comparisons predicated on a PTRUE, with the results ANDed
 ;; with another predicate P.  This does not have the same trapping behavior
 ;; as predicating the comparison itself on P, but it's a legitimate fold,
@@ -8690,8 +9107,8 @@
 	  (unspec:<VPRED>
 	    [(match_operand:<VPRED> 1)
 	     (const_int SVE_KNOWN_PTRUE)
-	     (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
-	     (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
+	     (match_operand:SVE_F 2 "register_operand" "w, w")
+	     (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
 	    SVE_COND_FP_CMP_I0)
 	  (match_operand:<VPRED> 4 "register_operand" "Upl, Upl")))]
   "TARGET_SVE"
@@ -8713,8 +9130,8 @@
 	  (unspec:<VPRED>
 	    [(match_operand:<VPRED> 1)
 	     (const_int SVE_KNOWN_PTRUE)
-	     (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	     (match_operand:SVE_FULL_F 3 "register_operand" "w")]
+	     (match_operand:SVE_F 2 "register_operand" "w")
+	     (match_operand:SVE_F 3 "register_operand" "w")]
 	    UNSPEC_COND_FCMUO)
 	  (match_operand:<VPRED> 4 "register_operand" "Upl")))]
   "TARGET_SVE"
@@ -8740,8 +9157,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "wDz")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "wDz")]
 	        SVE_COND_FP_CMP_I0))
 	    (match_operand:<VPRED> 4 "register_operand" "Upa"))
 	  (match_dup:<VPRED> 1)))
@@ -8777,8 +9194,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "wDz")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "wDz")]
 	        SVE_COND_FP_CMP_I0))
 	    (not:<VPRED>
 	      (match_operand:<VPRED> 4 "register_operand" "Upa")))
@@ -8808,6 +9225,7 @@
 }
 )
 
+;; Same for unordered comparisons.
 (define_insn_and_split "*fcmuo<mode>_bic_combine"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
 	(and:<VPRED>
@@ -8816,8 +9234,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "register_operand" "w")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "register_operand" "w")]
 	        UNSPEC_COND_FCMUO))
 	    (match_operand:<VPRED> 4 "register_operand" "Upa"))
 	  (match_dup:<VPRED> 1)))
@@ -8843,7 +9261,6 @@
 }
 )
 
-;; Same for unordered comparisons.
 (define_insn_and_split "*fcmuo<mode>_nor_combine"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
 	(and:<VPRED>
@@ -8852,8 +9269,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "register_operand" "w")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "register_operand" "w")]
 	        UNSPEC_COND_FCMUO))
 	    (not:<VPRED>
 	      (match_operand:<VPRED> 4 "register_operand" "Upa")))
@@ -8894,23 +9311,30 @@
 ;; -------------------------------------------------------------------------
 
 ;; Predicated floating-point absolute comparisons.
-(define_expand "@aarch64_pred_fac<cmp_op><mode>"
-  [(set (match_operand:<VPRED> 0 "register_operand")
-	(unspec:<VPRED>
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
-	   (unspec:SVE_FULL_F
-	     [(match_dup 1)
-	      (match_dup 2)
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
-	     UNSPEC_COND_FABS)
-	   (unspec:SVE_FULL_F
-	     [(match_dup 1)
-	      (match_dup 2)
-	      (match_operand:SVE_FULL_F 4 "register_operand")]
-	     UNSPEC_COND_FABS)]
-	  SVE_COND_FP_ABS_CMP))]
+(define_expand "@aarch64_pred_fac<cmp_op><mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "register_operand")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (unspec:SVE_FULL_F
+		 [(match_dup 1)
+		  (match_dup 2)
+		  (match_operand:SVE_FULL_F 3 "register_operand")]
+		 UNSPEC_COND_FABS)
+	       (unspec:SVE_FULL_F
+		 [(match_dup 1)
+		  (match_dup 2)
+		  (match_operand:SVE_FULL_F 4 "register_operand")]
+		 UNSPEC_COND_FABS)]
+	      SVE_COND_FP_ABS_CMP)
+	    0)
+	  (match_dup 5)))]
   "TARGET_SVE"
+  {
+    operands[5] = aarch64_ptrue_all (GET_MODE_UNIT_SIZE (<MODE>mode));
+  }
 )
 
 (define_insn_and_rewrite "*aarch64_pred_fac<cmp_op><mode>_relaxed"
@@ -8959,6 +9383,30 @@
   "fac<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
 )
 
+(define_insn "*aarch64_pred_fac<cmp_op><mode>_strict_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "register_operand" "Upl")
+	       (match_operand:SI 4 "aarch64_sve_ptrue_flag")
+	       (unspec:SVE_FULL_F
+		 [(match_dup 1)
+		  (match_operand:SI 5 "aarch64_sve_gp_strictness")
+		  (match_operand:SVE_FULL_F 2 "register_operand" "w")]
+		 UNSPEC_COND_FABS)
+	       (unspec:SVE_FULL_F
+		 [(match_dup 1)
+		  (match_operand:SI 6 "aarch64_sve_gp_strictness")
+		  (match_operand:SVE_FULL_F 3 "register_operand" "w")]
+		 UNSPEC_COND_FABS)]
+	      SVE_COND_FP_ABS_CMP)
+	    0)
+	  (match_operand:<VPRED> 7 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE"
+  "fac<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [PRED] Select
 ;; -------------------------------------------------------------------------
@@ -9407,7 +9855,30 @@
 	(unspec:PRED_ALL [(match_operand:PRED_ALL 1 "register_operand" "Upa")]
 			 UNSPEC_REV))]
   "TARGET_SVE"
-  "rev\t%0.<Vetype>, %1.<Vetype>")
+  "rev\t%0.<Vetype>, %1.<Vetype>"
+)
+
+(define_expand "@aarch64_sve_rev<mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(unspec:VNx16BI
+	  [(match_operand:VNx16BI 1 "register_operand")
+	   (match_dup:PRED_ALL 2)]
+	  UNSPEC_REV_PRED))]
+  "TARGET_SVE"
+  {
+    operands[2] = CONST0_RTX (<MODE>mode);
+  }
+)
+
+(define_insn "*aarch64_sve_rev<mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(unspec:VNx16BI
+	  [(match_operand:VNx16BI 1 "register_operand" "Upa")
+	   (match_operand:PRED_ALL 2 "aarch64_simd_imm_zero")]
+	  UNSPEC_REV_PRED))]
+  "TARGET_SVE"
+  "rev\t%0.<Vetype>, %1.<Vetype>"
+)
 
 ;; -------------------------------------------------------------------------
 ;; ---- [PRED] Special-purpose binary permutes
@@ -9432,18 +9903,39 @@
   "<perm_insn>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
 )
 
-;; Special purpose permute used by the predicate generation instructions.
-;; Unlike the normal permute patterns, these instructions operate on VNx16BI
-;; regardless of the element size, so that all input and output bits are
-;; well-defined.  Operand 3 then indicates the size of the permute.
-(define_insn "@aarch64_sve_trn1_conv<mode>"
+;; Special-purpose permutes used by the ACLE intrinsics and predicate
+;; generation instructions.  Unlike the normal permute patterns, these
+;; instructions operate on VNx16BI regardless of the element size, so that
+;; all input and output bits are well-defined.  Operand 3 then indicates
+;; the size of the permute.
+;;
+;; To make generation easier, this pattern embeds the permute type as the
+;; fourth operand to the unspec.  On the one hand, this avoids overloading
+;; unspecs like UNSPEC_ZIP1 to represent two different operations.  On the
+;; other hand, it avoids having a separate unspec for each variant, and
+;; having to map from one kind of unspec to the other.
+(define_expand "@aarch64_sve_<perm_insn><mode>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(unspec:VNx16BI [(match_operand:VNx16BI 1 "register_operand")
+			 (match_operand:VNx16BI 2 "register_operand")
+			 (match_dup:PRED_ALL 3)
+			 (const_int PERMUTE)]
+			UNSPEC_PERMUTE_PRED))]
+  "TARGET_SVE"
+  {
+    operands[3] = CONST0_RTX (<MODE>mode);
+  }
+)
+
+(define_insn "*aarch64_sve_<perm_insn><mode>_acle"
   [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
 	(unspec:VNx16BI [(match_operand:VNx16BI 1 "register_operand" "Upa")
 			 (match_operand:VNx16BI 2 "register_operand" "Upa")
-			 (match_operand:PRED_ALL 3 "aarch64_simd_imm_zero")]
-			UNSPEC_TRN1_CONV))]
+			 (match_operand:PRED_ALL 3 "aarch64_simd_imm_zero")
+			 (const_int PERMUTE)]
+			UNSPEC_PERMUTE_PRED))]
   "TARGET_SVE"
-  "trn1\t%0.<PRED_ALL:Vetype>, %1.<PRED_ALL:Vetype>, %2.<PRED_ALL:Vetype>"
+  "<perm_insn>\t%0.<PRED_ALL:Vetype>, %1.<PRED_ALL:Vetype>, %2.<PRED_ALL:Vetype>"
 )
 
 ;; =========================================================================
@@ -9653,6 +10145,31 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_nontrunc<SVE_PARTIAL_F:mode><SVE_HSDI:mode>_relaxed"
+  [(set (match_operand:SVE_HSDI 0 "register_operand")
+	(unspec:SVE_HSDI
+	  [(match_operand:<SVE_HSDI:VPRED> 1 "register_operand")
+	   (unspec:SVE_HSDI
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_PARTIAL_F 2 "register_operand")]
+	     SVE_COND_FCVTI)
+	   (match_operand:SVE_HSDI 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE
+  && (~(<SVE_HSDI:self_mask> | <SVE_HSDI:narrower_mask>) & <SVE_PARTIAL_F:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ &w       , Upl , w , 0  ; *              ] fcvtz<su>\t%0.<SVE_HSDI:Vetype>, %1/m, %2.<SVE_PARTIAL_F:Vetype>
+     [ &w       , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_HSDI:Vetype>, %1/z, %2.<SVE_HSDI:Vetype>\;fcvtz<su>\t%0.<SVE_HSDI:Vetype>, %1/m, %2.<SVE_PARTIAL_F:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvtz<su>\t%0.<SVE_HSDI:Vetype>, %1/m, %2.<SVE_PARTIAL_F:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 (define_insn "*cond_<optab>_nontrunc<SVE_FULL_F:mode><SVE_FULL_HSDI:mode>_strict"
   [(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
 	(unspec:SVE_FULL_HSDI
@@ -9706,6 +10223,29 @@
   }
 )
 
+(define_insn_and_rewrite "*cond_<optab>_trunc<VNx2DF_ONLY:mode><VNx2SI_ONLY:mode>_relaxed"
+  [(set (match_operand:VNx2SI_ONLY 0 "register_operand")
+	(unspec:VNx2SI_ONLY
+	  [(match_operand:VNx2BI 1 "register_operand")
+	   (unspec:VNx2SI_ONLY
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:VNx2DF_ONLY 2 "register_operand")]
+	     SVE_COND_FCVTI)
+	   (match_operand:VNx2SI_ONLY 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ &w       , Upl , w , 0  ; *              ] fcvtz<su>\t%0.<VNx2SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
+     [ &w       , Upl , w , Dz ; yes            ] movprfx\t%0.<VNx2DF_ONLY:Vetype>, %1/z, %2.<VNx2DF_ONLY:Vetype>\;fcvtz<su>\t%0.<VNx2SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvtz<su>\t%0.<VNx2SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [INT<-FP] Packs
 ;; -------------------------------------------------------------------------
@@ -9857,6 +10397,31 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_nonextend<SVE_HSDI:mode><SVE_PARTIAL_F:mode>_relaxed"
+  [(set (match_operand:SVE_PARTIAL_F 0 "register_operand")
+	(unspec:SVE_PARTIAL_F
+	  [(match_operand:<SVE_HSDI:VPRED> 1 "register_operand")
+	   (unspec:SVE_PARTIAL_F
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_HSDI 2 "register_operand")]
+	     SVE_COND_ICVTF)
+	   (match_operand:SVE_PARTIAL_F 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE
+   && (~(<SVE_HSDI:self_mask> | <SVE_HSDI:narrower_mask>) & <SVE_PARTIAL_F:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ &w       , Upl , w , 0  ; *              ] <su>cvtf\t%0.<SVE_PARTIAL_F:Vetype>, %1/m, %2.<SVE_HSDI:Vetype>
+     [ &w       , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_HSDI:Vetype>, %1/z, %2.<SVE_HSDI:Vetype>\;<su>cvtf\t%0.<SVE_PARTIAL_F:Vetype>, %1/m, %2.<SVE_HSDI:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;<su>cvtf\t%0.<SVE_PARTIAL_F:Vetype>, %1/m, %2.<SVE_HSDI:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 (define_insn "*cond_<optab>_nonextend<SVE_FULL_HSDI:mode><SVE_FULL_F:mode>_strict"
   [(set (match_operand:SVE_FULL_F 0 "register_operand")
 	(unspec:SVE_FULL_F
@@ -10066,6 +10631,30 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_trunc<SVE_SDF:mode><SVE_PARTIAL_HSF:mode>"
+  [(set (match_operand:SVE_PARTIAL_HSF 0 "register_operand")
+	(unspec:SVE_PARTIAL_HSF
+	  [(match_operand:<SVE_SDF:VPRED> 1 "register_operand")
+	   (unspec:SVE_PARTIAL_HSF
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_SDF 2 "register_operand")]
+	     SVE_COND_FCVT)
+	   (match_operand:SVE_PARTIAL_HSF 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE && (~<SVE_SDF:narrower_mask> & <SVE_PARTIAL_HSF:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ w        , Upl , w , 0  ; *              ] fcvt\t%0.<SVE_PARTIAL_HSF:Vetype>, %1/m, %2.<SVE_SDF:Vetype>
+     [ ?&w      , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_SDF:Vetype>, %1/z, %2.<SVE_SDF:Vetype>\;fcvt\t%0.<SVE_PARTIAL_HSF:Vetype>, %1/m, %2.<SVE_SDF:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvt\t%0.<SVE_PARTIAL_HSF:Vetype>, %1/m, %2.<SVE_SDF:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [FP<-FP] Packs (bfloat16)
 ;; -------------------------------------------------------------------------
@@ -10259,6 +10848,30 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_nontrunc<SVE_PARTIAL_HSF:mode><SVE_SDF:mode>_relaxed"
+  [(set (match_operand:SVE_SDF 0 "register_operand")
+	(unspec:SVE_SDF
+	  [(match_operand:<SVE_SDF:VPRED> 1 "register_operand")
+	   (unspec:SVE_SDF
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_PARTIAL_HSF 2 "register_operand")]
+	     SVE_COND_FCVT)
+	   (match_operand:SVE_SDF 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE && (~<SVE_SDF:narrower_mask> & <SVE_PARTIAL_HSF:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ w        , Upl , w , 0  ; *              ] fcvt\t%0.<SVE_SDF:Vetype>, %1/m, %2.<SVE_PARTIAL_HSF:Vetype>
+     [ ?&w      , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_SDF:Vetype>, %1/z, %2.<SVE_SDF:Vetype>\;fcvt\t%0.<SVE_SDF:Vetype>, %1/m, %2.<SVE_PARTIAL_HSF:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvt\t%0.<SVE_SDF:Vetype>, %1/m, %2.<SVE_PARTIAL_HSF:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [PRED<-PRED] Packs
 ;; -------------------------------------------------------------------------
@@ -10311,6 +10924,34 @@
   "punpk<perm_hilo>\t%0.h, %1.b"
 )
 
+(define_expand "@aarch64_sve_punpk<perm_hilo>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:VNx8BI
+	      [(match_operand:VNx16BI 1 "register_operand")]
+	      UNPACK_UNSIGNED)
+	    0)
+	  (match_dup 2)))]
+  "TARGET_SVE"
+  {
+    operands[2] = aarch64_ptrue_all (2);
+  }
+)
+
+(define_insn "*aarch64_sve_punpk<perm_hilo>_acle"
+  [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:VNx8BI
+	      [(match_operand:VNx16BI 1 "register_operand" "Upa")]
+	      UNPACK_UNSIGNED)
+	    0)
+	  (match_operand:VNx8BI 2 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE"
+  "punpk<perm_hilo>\t%0.h, %1.b"
+)
+
 ;; =========================================================================
 ;; == Vector partitioning
 ;; =========================================================================
@@ -10535,14 +11176,49 @@
 ;; -------------------------------------------------------------------------
 
 (define_insn "@aarch64_sve_<sve_pred_op><mode>"
-  [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
-	(unspec:PRED_ALL
-	  [(match_operand:PRED_ALL 1 "register_operand" "Upa")
+  [(set (match_operand:VNx16BI_ONLY 0 "register_operand" "=Upa")
+	(unspec:VNx16BI_ONLY
+	  [(match_operand:VNx16BI_ONLY 1 "register_operand" "Upa")
 	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
-	   (match_operand:PRED_ALL 3 "register_operand" "0")]
+	   (match_operand:VNx16BI_ONLY 3 "register_operand" "0")]
 	  SVE_PITER))
    (clobber (reg:CC_NZC CC_REGNUM))]
-  "TARGET_SVE && <max_elem_bits> >= <elem_bits>"
+  "TARGET_SVE"
+  "<sve_pred_op>\t%0.<Vetype>, %1, %0.<Vetype>"
+)
+
+(define_expand "@aarch64_sve_<sve_pred_op><mode>"
+  [(parallel
+     [(set (match_operand:VNx16BI 0 "register_operand")
+	   (and:VNx16BI
+	     (subreg:VNx16BI
+	       (unspec:PRED_HSD
+		 [(match_operand:PRED_HSD 1 "register_operand")
+		  (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+		  (match_operand:PRED_HSD 3 "register_operand")]
+		 PNEXT_ONLY)
+	       0)
+	     (match_dup 4)))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  "TARGET_SVE"
+  {
+    operands[4] = aarch64_ptrue_all (<data_bytes>);
+  }
+)
+
+(define_insn "*aarch64_sve_<sve_pred_op><mode>"
+  [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:PRED_HSD
+	      [(match_operand:PRED_HSD 1 "register_operand" "Upa")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (match_operand:PRED_HSD 3 "register_operand" "0")]
+	      PNEXT_ONLY)
+	    0)
+	  (match_operand:PRED_HSD 4 "aarch64_ptrue_all_operand")))
+   (clobber (reg:CC_NZC CC_REGNUM))]
+  "TARGET_SVE"
   "<sve_pred_op>\t%0.<Vetype>, %1, %0.<Vetype>"
 )
 
@@ -10576,6 +11252,38 @@
   }
 )
 
+(define_insn_and_rewrite "*aarch64_sve_<sve_pred_op><mode>_cc"
+  [(set (reg:CC_NZC CC_REGNUM)
+	(unspec:CC_NZC
+	  [(match_operand:VNx16BI 1 "register_operand" "Upa")
+	   (match_operand 2)
+	   (match_operand:SI 3 "aarch64_sve_ptrue_flag")
+	   (unspec:PRED_HSD
+	     [(match_operand 4)
+	      (match_operand:SI 5 "aarch64_sve_ptrue_flag")
+	      (match_operand:PRED_HSD 6 "register_operand" "0")]
+	     PNEXT_ONLY)]
+	  UNSPEC_PTEST))
+   (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:PRED_HSD
+	      [(match_dup 4)
+	       (match_dup 5)
+	       (match_dup 6)]
+	      PNEXT_ONLY)
+	    0)
+	  (match_operand:PRED_HSD 7 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE
+   && aarch64_sve_same_pred_for_ptest_p (&operands[2], &operands[4])"
+  "<sve_pred_op>\t%0.<Vetype>, %1, %0.<Vetype>"
+  "&& !rtx_equal_p (operands[2], operands[4])"
+  {
+    operands[4] = operands[2];
+    operands[5] = operands[3];
+  }
+)
+
 ;; Same, but with only the flags result being interesting.
 (define_insn_and_rewrite "*aarch64_sve_<sve_pred_op><mode>_ptest"
   [(set (reg:CC_NZC CC_REGNUM)
@@ -11437,16 +12145,12 @@
 
 (define_insn "@aarch64_sve_set_neonq_<mode>"
   [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
-      (unspec:SVE_FULL
-	[(match_operand:SVE_FULL 1 "register_operand" "w")
-	(match_operand:<V128> 2 "register_operand" "w")
-	(match_operand:<VPRED> 3 "register_operand" "Upl")]
-	UNSPEC_SET_NEONQ))]
+	(unspec:SVE_FULL
+	  [(match_operand:SVE_FULL 1 "register_operand" "w")
+	   (match_operand:<V128> 2 "register_operand" "w")
+	   (match_operand:<VPRED> 3 "register_operand" "Upl")]
+	  UNSPEC_SET_NEONQ))]
   "TARGET_SVE
    && BYTES_BIG_ENDIAN"
-  {
-    operands[2] = lowpart_subreg (<MODE>mode, operands[2],
-                                  GET_MODE (operands[2]));
-    return "sel\t%0.<Vetype>, %3, %2.<Vetype>, %1.<Vetype>";
-  }
+  "sel\t%0.<Vetype>, %3, %Z2.<Vetype>, %1.<Vetype>"
 )
diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md
index 62524f3..a4c3257 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -334,12 +334,21 @@
 ;; - LD1Q (SVE2p1)
 ;; -------------------------------------------------------------------------
 
-;; Model this as operating on the largest valid element size, which is DI.
-;; This avoids having to define move patterns & more for VNx1TI, which would
-;; be difficult without a non-gather form of LD1Q.
-(define_insn "aarch64_gather_ld1q"
-  [(set (match_operand:VNx2DI 0 "register_operand")
-	(unspec:VNx2DI
+;; For little-endian targets, it would be enough to use a single pattern,
+;; with a subreg to bitcast the result to whatever mode is needed.
+;; However, on big-endian targets, the bitcast would need to be an
+;; aarch64_sve_reinterpret instruction.  That would interact badly
+;; with the "&" and "?" constraints in this pattern: if the result
+;; of the reinterpret needs to be in the same register as the index,
+;; the RA would tend to prefer to allocate a separate register for the
+;; intermediate (uncast) result, even if the reinterpret prefers tying.
+;;
+;; The index is logically VNx1DI rather than VNx2DI, but introducing
+;; and using VNx1DI would just create more bitcasting.  The ACLE intrinsic
+;; uses svuint64_t, which corresponds to VNx2DI.
+(define_insn "@aarch64_gather_ld1q<mode>"
+  [(set (match_operand:SVE_FULL 0 "register_operand")
+	(unspec:SVE_FULL
 	  [(match_operand:VNx2BI 1 "register_operand")
 	   (match_operand:DI 2 "aarch64_reg_or_zero")
 	   (match_operand:VNx2DI 3 "register_operand")
@@ -1337,12 +1346,12 @@
 
 ;; Predicated B16B16 binary operations.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:VNx8BF_ONLY 0 "register_operand")
-	(unspec:VNx8BF_ONLY
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_BF 0 "register_operand")
+	(unspec:SVE_BF
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:VNx8BF_ONLY 2 "register_operand")
-	   (match_operand:VNx8BF_ONLY 3 "register_operand")]
+	   (match_operand:SVE_BF 2 "register_operand")
+	   (match_operand:SVE_BF 3 "register_operand")]
 	  SVE_COND_FP_BINARY_OPTAB))]
   "TARGET_SSVE_B16B16 && <supports_bf16>"
   {@ [ cons: =0 , 1   , 2 , 3 ; attrs: movprfx , is_rev ]
@@ -1628,7 +1637,7 @@
   "TARGET_SVE2"
   {@ [ cons: =0 , %1 , 2 ; attrs: movprfx ]
      [ w        , 0  , w ; *              ] nbsl\t%0.d, %0.d, %2.d, %0.d
-     [ ?&w      , w  , w ; yes            ] movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %0.d
+     [ ?&w      , w  , w ; yes            ] movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %1.d
   }
   "&& !CONSTANT_P (operands[3])"
   {
@@ -1636,6 +1645,20 @@
   }
 )
 
+(define_insn "*aarch64_sve2_unpred_nor<mode>"
+  [(set (match_operand:VDQ_I 0 "register_operand")
+	(and:VDQ_I
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 1 "register_operand"))
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 2 "register_operand"))))]
+  "TARGET_SVE2"
+  {@ [ cons: =0 , %1 , 2 ; attrs: movprfx ]
+     [ w        , 0  , w ; *              ] nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z0.d
+     [ ?&w      , w  , w ; yes            ] movprfx\t%Z0, %Z1\;nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z1.d
+  }
+)
+
 ;; Use NBSL for vector NAND.
 (define_insn_and_rewrite "*aarch64_sve2_nand<mode>"
   [(set (match_operand:SVE_FULL_I 0 "register_operand")
@@ -1658,6 +1681,21 @@
   }
 )
 
+;; Same as above but unpredicated and including Advanced SIMD modes.
+(define_insn "*aarch64_sve2_nand_unpred<mode>"
+  [(set (match_operand:VDQ_I 0 "register_operand")
+	(ior:VDQ_I
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 1 "register_operand"))
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 2 "register_operand"))))]
+  "TARGET_SVE2"
+  {@ [ cons: =0 , %1 , 2 ; attrs: movprfx ]
+     [ w        , 0  , w ; *              ] nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z2.d
+     [ ?&w      , w  , w ; yes            ] movprfx\t%Z0, %Z1\;nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z2.d
+  }
+)
+
 ;; Unpredicated bitwise select.
 ;; (op3 ? bsl_mov : bsl_dup) == (((bsl_mov ^ bsl_dup) & op3) ^ bsl_dup)
 (define_expand "@aarch64_sve2_bsl<mode>"
@@ -1918,6 +1956,40 @@
   }
 )
 
+;; Vector EON (~(x, y)) using BSL2N.
+(define_insn_and_rewrite "*aarch64_sve2_bsl2n_eon<mode>"
+  [(set (match_operand:SVE_FULL_I 0 "register_operand")
+	(unspec:SVE_FULL_I
+	  [(match_operand 3)
+	   (not:SVE_FULL_I
+	     (xor:SVE_FULL_I
+		(match_operand:SVE_FULL_I 1 "register_operand")
+		(match_operand:SVE_FULL_I 2 "register_operand")))]
+	    UNSPEC_PRED_X))]
+  "TARGET_SVE2"
+  {@ [ cons: =0, 1, 2 ; attrs: movprfx ]
+     [ w  ,      0, w ; *              ] bsl2n\t%0.d, %0.d, %0.d, %2.d
+     [ ?&w,      w, w ; yes            ] movprfx\t%0, %1\;bsl2n\t%0.d, %0.d, %1.d, %2.d
+  }
+  "&& !CONSTANT_P (operands[3])"
+  {
+    operands[3] = CONSTM1_RTX (<VPRED>mode);
+  }
+)
+
+(define_insn "*aarch64_sve2_eon_bsl2n_unpred<mode>"
+  [(set (match_operand:VDQ_I 0 "register_operand")
+       (not:VDQ_I
+         (xor:VDQ_I
+           (match_operand:VDQ_I 1 "register_operand")
+           (match_operand:VDQ_I 2 "register_operand"))))]
+  "TARGET_SVE2"
+  {@ [ cons: =0, 1, 2 ; attrs: movprfx ]
+     [ w  ,      0, w ; *              ] bsl2n\t%Z0.d, %Z0.d, %Z0.d, %Z2.d
+     [ ?&w,      w, w ; yes            ] movprfx\t%Z0, %Z1\;bsl2n\t%Z0.d, %Z0.d, %Z1.d, %Z2.d
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [INT] Shift-and-accumulate operations
 ;; -------------------------------------------------------------------------
@@ -3996,8 +4068,8 @@
 	  [(match_operand:<VPRED> 1 "register_operand")
 	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
 	   (unspec:<VPRED>
-	     [(match_operand:SVE_FULL_BHI 3 "register_operand")
-	      (match_operand:SVE_FULL_BHI 4 "register_operand")]
+	     [(match_operand:VNx16QI_ONLY 3 "register_operand")
+	      (match_operand:VNx16QI_ONLY 4 "register_operand")]
 	     SVE2_MATCH)]
 	  UNSPEC_PRED_Z))
    (clobber (reg:CC_NZC CC_REGNUM))]
@@ -4009,6 +4081,51 @@
   }
 )
 
+(define_expand "@aarch64_pred_<sve_int_op><mode>"
+  [(parallel
+     [(set (match_operand:VNx16BI 0 "register_operand")
+	   (and:VNx16BI
+	     (subreg:VNx16BI
+	       (unspec:<VPRED>
+		 [(match_operand:<VPRED> 1 "register_operand")
+		  (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+		  (unspec:<VPRED>
+		    [(match_operand:VNx8HI_ONLY 3 "register_operand")
+		     (match_operand:VNx8HI_ONLY 4 "register_operand")]
+		    SVE2_MATCH)]
+		 UNSPEC_PRED_Z)
+	       0)
+	     (match_dup 5)))
+      (clobber (reg:CC_NZC CC_REGNUM))])]
+  "TARGET_SVE2 && TARGET_NON_STREAMING"
+  {
+    operands[5] = aarch64_ptrue_all (GET_MODE_UNIT_SIZE (<MODE>mode));
+  }
+)
+
+(define_insn "*aarch64_pred_<sve_int_op><mode>"
+  [(set (match_operand:VNx16BI 0 "register_operand")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_operand:<VPRED> 1 "register_operand")
+	       (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+	       (unspec:<VPRED>
+		 [(match_operand:VNx8HI_ONLY 3 "register_operand")
+		  (match_operand:VNx8HI_ONLY 4 "register_operand")]
+		 SVE2_MATCH)]
+	      UNSPEC_PRED_Z)
+	    0)
+	  (match_operand:<VPRED> 5 "aarch64_ptrue_all_operand")))
+   (clobber (reg:CC_NZC CC_REGNUM))]
+  "TARGET_SVE2 && TARGET_NON_STREAMING"
+  {@ [ cons: =0, 1  , 3, 4; attrs: pred_clobber ]
+     [ &Upa    , Upl, w, w; yes                 ] <sve_int_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>
+     [ ?Upl    , 0  , w, w; yes                 ] ^
+     [ Upa     , Upl, w, w; no                  ] ^
+  }
+)
+
 ;; Predicated string matching in which both the flag and predicate results
 ;; are interesting.
 (define_insn_and_rewrite "*aarch64_pred_<sve_int_op><mode>_cc"
@@ -4046,6 +4163,45 @@
   }
 )
 
+(define_insn_and_rewrite "*aarch64_pred_<sve_int_op><mode>_cc"
+  [(set (reg:CC_NZC CC_REGNUM)
+	(unspec:CC_NZC
+	  [(match_operand:VNx16BI 1 "register_operand" "Upl")
+	   (match_operand 4)
+	   (match_operand:SI 5 "aarch64_sve_ptrue_flag")
+	   (unspec:<VPRED>
+	     [(match_operand 6)
+	      (match_operand:SI 7 "aarch64_sve_ptrue_flag")
+	      (unspec:<VPRED>
+		[(match_operand:VNx8HI_ONLY 2 "register_operand" "w")
+		 (match_operand:VNx8HI_ONLY 3 "register_operand" "w")]
+		SVE2_MATCH)]
+	     UNSPEC_PRED_Z)]
+	  UNSPEC_PTEST))
+   (set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+	(and:VNx16BI
+	  (subreg:VNx16BI
+	    (unspec:<VPRED>
+	      [(match_dup 6)
+	       (match_dup 7)
+	       (unspec:<VPRED>
+		 [(match_dup 2)
+		  (match_dup 3)]
+		 SVE2_MATCH)]
+	      UNSPEC_PRED_Z)
+	    0)
+	  (match_operand:<VPRED> 8 "aarch64_ptrue_all_operand")))]
+  "TARGET_SVE2
+   && TARGET_NON_STREAMING
+   && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])"
+  "<sve_int_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
+  "&& !rtx_equal_p (operands[4], operands[6])"
+  {
+    operands[6] = copy_rtx (operands[4]);
+    operands[7] = operands[5];
+  }
+)
+
 ;; Predicated string matching in which only the flags result is interesting.
 (define_insn_and_rewrite "*aarch64_pred_<sve_int_op><mode>_ptest"
   [(set (reg:CC_NZC CC_REGNUM)
diff --git a/gcc/config/aarch64/aarch64-tuning-flags.def b/gcc/config/aarch64/aarch64-tuning-flags.def
index f2c916e..dd91324 100644
--- a/gcc/config/aarch64/aarch64-tuning-flags.def
+++ b/gcc/config/aarch64/aarch64-tuning-flags.def
@@ -44,6 +44,8 @@ AARCH64_EXTRA_TUNING_OPTION ("avoid_cross_loop_fma", AVOID_CROSS_LOOP_FMA)
 
 AARCH64_EXTRA_TUNING_OPTION ("fully_pipelined_fma", FULLY_PIPELINED_FMA)
 
+AARCH64_EXTRA_TUNING_OPTION ("avoid_ldapur", AVOID_LDAPUR)
+
 /* Enable is the target prefers to use a fresh register for predicate outputs
    rather than re-use an input predicate register.  */
 AARCH64_EXTRA_TUNING_OPTION ("avoid_pred_rmw", AVOID_PRED_RMW)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index f3ce3a1..d30c9c7 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -356,7 +356,8 @@ static int aarch64_address_cost (rtx, machine_mode, addr_space_t, bool);
 static bool aarch64_builtin_support_vector_misalignment (machine_mode mode,
 							 const_tree type,
 							 int misalignment,
-							 bool is_packed);
+							 bool is_packed,
+							 bool is_gather_scatter);
 static machine_mode aarch64_simd_container_mode (scalar_mode, poly_int64);
 static bool aarch64_print_address_internal (FILE*, machine_mode, rtx,
 					    aarch64_addr_query_type);
@@ -429,6 +430,7 @@ static const struct aarch64_flag_desc aarch64_tuning_flags[] =
 #include "tuning_models/neoversev2.h"
 #include "tuning_models/neoversev3.h"
 #include "tuning_models/neoversev3ae.h"
+#include "tuning_models/olympus.h"
 #include "tuning_models/a64fx.h"
 #include "tuning_models/fujitsu_monaka.h"
 
@@ -3931,18 +3933,53 @@ aarch64_sve_fp_pred (machine_mode data_mode, rtx *strictness)
    return aarch64_ptrue_reg (aarch64_sve_pred_mode (data_mode));
 }
 
+/* PRED is a predicate that governs an operation on DATA_MODE.  If DATA_MODE
+   is a partial vector mode, and if exceptions must be suppressed for its
+   undefined elements, convert PRED from a container-level predicate to
+   an element-level predicate and ensure that the undefined elements
+   are inactive.  Make no changes otherwise.
+
+   Return the resultant predicate.  */
+rtx
+aarch64_sve_emit_masked_fp_pred (machine_mode data_mode, rtx pred)
+{
+  unsigned int vec_flags = aarch64_classify_vector_mode (data_mode);
+  if (flag_trapping_math && (vec_flags & VEC_PARTIAL))
+    {
+      /* Generate an element-level mask.  */
+      rtx mask = aarch64_sve_packed_pred (data_mode);
+      machine_mode pmode = GET_MODE (mask);
+
+      /* Apply the existing predicate.  */
+      rtx dst = gen_reg_rtx (pmode);
+      emit_insn (gen_and3 (pmode, dst, mask,
+			   gen_lowpart (pmode, pred)));
+      return dst;
+    }
+
+  return pred;
+}
+
 /* Emit a comparison CMP between OP0 and OP1, both of which have mode
    DATA_MODE, and return the result in a predicate of mode PRED_MODE.
-   Use TARGET as the target register if nonnull and convenient.  */
+   Use TARGET as the target register if nonnull and convenient.
+
+   PRED_MODE can be either VNx16BI or the natural predicate mode for
+   DATA_MODE.  */
 
 static rtx
 aarch64_sve_emit_int_cmp (rtx target, machine_mode pred_mode, rtx_code cmp,
 			  machine_mode data_mode, rtx op1, rtx op2)
 {
-  insn_code icode = code_for_aarch64_pred_cmp (cmp, data_mode);
+  auto src_pred_mode = aarch64_sve_pred_mode (data_mode);
+  insn_code icode;
+  if (known_eq (GET_MODE_NUNITS (pred_mode), GET_MODE_NUNITS (data_mode)))
+    icode = code_for_aarch64_pred_cmp (cmp, data_mode);
+  else
+    icode = code_for_aarch64_pred_cmp_acle (cmp, data_mode);
   expand_operand ops[5];
   create_output_operand (&ops[0], target, pred_mode);
-  create_input_operand (&ops[1], CONSTM1_RTX (pred_mode), pred_mode);
+  create_input_operand (&ops[1], CONSTM1_RTX (src_pred_mode), src_pred_mode);
   create_integer_operand (&ops[2], SVE_KNOWN_PTRUE);
   create_input_operand (&ops[3], op1, data_mode);
   create_input_operand (&ops[4], op2, data_mode);
@@ -3950,15 +3987,14 @@ aarch64_sve_emit_int_cmp (rtx target, machine_mode pred_mode, rtx_code cmp,
   return ops[0].value;
 }
 
-/* Use a comparison to convert integer vector SRC into MODE, which is
-   the corresponding SVE predicate mode.  Use TARGET for the result
-   if it's nonnull and convenient.  */
+/* Use a comparison to convert integer vector SRC into VNx16BI.
+   Use TARGET for the result if it's nonnull and convenient.  */
 
 rtx
-aarch64_convert_sve_data_to_pred (rtx target, machine_mode mode, rtx src)
+aarch64_convert_sve_data_to_pred (rtx target, rtx src)
 {
   machine_mode src_mode = GET_MODE (src);
-  return aarch64_sve_emit_int_cmp (target, mode, NE, src_mode,
+  return aarch64_sve_emit_int_cmp (target, VNx16BImode, NE, src_mode,
 				   src, CONST0_RTX (src_mode));
 }
 
@@ -6040,9 +6076,9 @@ aarch64_sve_move_pred_via_while (rtx target, machine_mode mode,
 				 unsigned int vl)
 {
   rtx limit = force_reg (DImode, gen_int_mode (vl, DImode));
-  target = aarch64_target_reg (target, mode);
-  emit_insn (gen_while (UNSPEC_WHILELO, DImode, mode,
-			target, const0_rtx, limit));
+  target = aarch64_target_reg (target, VNx16BImode);
+  emit_insn (gen_aarch64_sve_while_acle (UNSPEC_WHILELO, DImode, mode,
+					 target, const0_rtx, limit));
   return target;
 }
 
@@ -6188,8 +6224,7 @@ aarch64_expand_sve_const_pred_trn (rtx target, rtx_vector_builder &builder,
      operands but permutes them as though they had mode MODE.  */
   machine_mode mode = aarch64_sve_pred_mode (permute_size).require ();
   target = aarch64_target_reg (target, GET_MODE (a));
-  rtx type_reg = CONST0_RTX (mode);
-  emit_insn (gen_aarch64_sve_trn1_conv (mode, target, a, b, type_reg));
+  emit_insn (gen_aarch64_sve_acle (UNSPEC_TRN1, mode, target, a, b));
   return target;
 }
 
@@ -6271,8 +6306,7 @@ aarch64_expand_sve_const_pred (rtx target, rtx_vector_builder &builder)
   for (unsigned int i = 0; i < builder.encoded_nelts (); ++i)
     int_builder.quick_push (INTVAL (builder.elt (i))
 			    ? constm1_rtx : const0_rtx);
-  return aarch64_convert_sve_data_to_pred (target, VNx16BImode,
-					   int_builder.build ());
+  return aarch64_convert_sve_data_to_pred (target, int_builder.build ());
 }
 
 /* Set DEST to immediate IMM.  */
@@ -6724,6 +6758,27 @@ aarch64_split_sve_subreg_move (rtx dest, rtx ptrue, rtx src)
 			       dest, ptrue, src));
 }
 
+/* Set predicate register DEST such that every element has the scalar
+   boolean value in SRC, with any nonzero source counting as "true".
+   MODE is a MODE_VECTOR_BOOL that determines the element size;
+   DEST can have this mode or VNx16BImode.  In the latter case,
+   the upper bits of each element are defined to be zero, as for
+   the .H, .S, and .D forms of PTRUE.  */
+
+void
+aarch64_emit_sve_pred_vec_duplicate (machine_mode mode, rtx dest, rtx src)
+{
+  rtx tmp = gen_reg_rtx (DImode);
+  emit_insn (gen_ashldi3 (tmp, gen_lowpart (DImode, src),
+			  gen_int_mode (63, DImode)));
+  if (GET_MODE (dest) == VNx16BImode)
+    emit_insn (gen_aarch64_sve_while_acle (UNSPEC_WHILELO, DImode, mode,
+					   dest, const0_rtx, tmp));
+  else
+    emit_insn (gen_while (UNSPEC_WHILELO, DImode, mode,
+			  dest, const0_rtx, tmp));
+}
+
 static bool
 aarch64_function_ok_for_sibcall (tree, tree exp)
 {
@@ -15854,11 +15909,14 @@ cost_plus:
 	break;
     case CONST_VECTOR:
 	{
-	  /* Load using MOVI/MVNI.  */
-	  if (aarch64_simd_valid_mov_imm (x))
-	    *cost = extra_cost->vect.movi;
-	  else /* Load using constant pool.  */
-	    *cost = extra_cost->ldst.load;
+	  if (speed)
+	    {
+	      /* Load using MOVI/MVNI.  */
+	      if (aarch64_simd_valid_mov_imm (x))
+		*cost += extra_cost->vect.movi;
+	      else /* Load using constant pool.  */
+		*cost += extra_cost->ldst.load;
+	    }
 	  break;
 	}
     case VEC_CONCAT:
@@ -15867,7 +15925,8 @@ cost_plus:
 	break;
     case VEC_DUPLICATE:
 	/* Load using a DUP.  */
-	*cost = extra_cost->vect.dup;
+	if (speed)
+	  *cost += extra_cost->vect.dup;
 	return false;
     case VEC_SELECT:
 	{
@@ -15875,13 +15934,16 @@ cost_plus:
 	  *cost = rtx_cost (op0, GET_MODE (op0), VEC_SELECT, 0, speed);
 
 	  /* cost subreg of 0 as free, otherwise as DUP */
-	  rtx op1 = XEXP (x, 1);
-	  if (vec_series_lowpart_p (mode, GET_MODE (op1), op1))
-	    ;
-	  else if (vec_series_highpart_p (mode, GET_MODE (op1), op1))
-	    *cost = extra_cost->vect.dup;
-	  else
-	    *cost = extra_cost->vect.extract;
+	  if (speed)
+	    {
+	      rtx op1 = XEXP (x, 1);
+	      if (vec_series_lowpart_p (mode, GET_MODE (op1), op1))
+		;
+	      else if (vec_series_highpart_p (mode, GET_MODE (op1), op1))
+		*cost += extra_cost->vect.dup;
+	      else
+		*cost += extra_cost->vect.extract;
+	    }
 	  return true;
 	}
     default:
@@ -17157,8 +17219,8 @@ aarch64_ld234_st234_vectors (vect_cost_for_stmt kind, stmt_vec_info stmt_info,
       && STMT_VINFO_DATA_REF (stmt_info))
     {
       stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
-      if (stmt_info
-	  && vect_mem_access_type (stmt_info, node) == VMAT_LOAD_STORE_LANES)
+      if (node
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_LOAD_STORE_LANES)
 	return DR_GROUP_SIZE (stmt_info);
     }
   return 0;
@@ -17429,8 +17491,9 @@ aarch64_detect_vector_stmt_subtype (vec_info *vinfo, vect_cost_for_stmt kind,
      for each element.  We therefore need to divide the full-instruction
      cost by the number of elements in the vector.  */
   if (kind == scalar_load
+      && node
       && sve_costs
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     {
       unsigned int nunits = vect_nunits_for_cost (vectype);
       /* Test for VNx2 modes, which have 64-bit containers.  */
@@ -17442,8 +17505,9 @@ aarch64_detect_vector_stmt_subtype (vec_info *vinfo, vect_cost_for_stmt kind,
   /* Detect cases in which a scalar_store is really storing one element
      in a scatter operation.  */
   if (kind == scalar_store
+      && node
       && sve_costs
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     return sve_costs->scatter_store_elt_cost;
 
   /* Detect cases in which vec_to_scalar represents an in-loop reduction.  */
@@ -17699,7 +17763,7 @@ aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind,
   if (stmt_info
       && kind == vec_to_scalar
       && (m_vec_flags & VEC_ADVSIMD)
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     {
       auto dr = STMT_VINFO_DATA_REF (stmt_info);
       tree dr_ref = DR_REF (dr);
@@ -17712,7 +17776,7 @@ aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind,
 		{
 		  if (gimple_vuse (SSA_NAME_DEF_STMT (offset)))
 		    {
-		      if (STMT_VINFO_TYPE (stmt_info) == load_vec_info_type)
+		      if (SLP_TREE_TYPE (node) == load_vec_info_type)
 			ops->loads += count - 1;
 		      else
 			  /* Stores want to count both the index to array and data to
@@ -17814,7 +17878,7 @@ aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind,
   if (stmt_info
       && sve_issue
       && (kind == scalar_load || kind == scalar_store)
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     {
       unsigned int pairs = CEIL (count, 2);
       ops->pred_ops += sve_issue->gather_scatter_pair_pred_ops * pairs;
@@ -17932,7 +17996,7 @@ aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 
   /* Do one-time initialization based on the vinfo.  */
   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
-  if (!m_analyzed_vinfo)
+  if (!m_analyzed_vinfo && !m_costing_for_scalar)
     {
       if (loop_vinfo)
 	analyze_loop_vinfo (loop_vinfo);
@@ -17969,8 +18033,10 @@ aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 
       /* Check if we've seen an SVE gather/scatter operation and which size.  */
       if (kind == scalar_load
+	  && node
+	  && vectype
 	  && aarch64_sve_mode_p (TYPE_MODE (vectype))
-	  && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
 	{
 	  const sve_vec_cost *sve_costs = aarch64_tune_params.vec_costs->sve;
 	  if (sve_costs)
@@ -18793,6 +18859,8 @@ aarch64_adjust_generic_arch_tuning (struct tune_params &current_tune)
   if (TARGET_SVE2)
     current_tune.extra_tuning_flags
       &= ~AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS;
+  if (!AARCH64_HAVE_ISA(V8_8A))
+    aarch64_tune_params.extra_tuning_flags |= AARCH64_EXTRA_TUNE_AVOID_LDAPUR;
 }
 
 static void
@@ -18857,7 +18925,10 @@ aarch64_override_options_internal (struct gcc_options *opts)
   /* Make a copy of the tuning parameters attached to the core, which
      we may later overwrite.  */
   aarch64_tune_params = *(tune->tune);
-  if (tune->tune == &generic_tunings)
+
+  if (tune->tune == &generic_tunings
+      || tune->tune == &generic_armv8_a_tunings
+      || tune->tune == &generic_armv9_a_tunings)
     aarch64_adjust_generic_arch_tuning (aarch64_tune_params);
 
   if (opts->x_aarch64_override_tune_string)
@@ -19953,8 +20024,9 @@ aarch64_process_one_target_attr (char *arg_str)
 	      if (valid)
 		{
 		  set_option (&global_options, NULL, p_attr->opt_num, value,
-			      NULL, DK_UNSPECIFIED, input_location,
-			      global_dc);
+			      NULL,
+			      static_cast<int> (diagnostics::kind::unspecified),
+			      input_location, global_dc);
 		}
 	      else
 		{
@@ -20466,6 +20538,8 @@ aarch64_compare_version_priority (tree decl1, tree decl2)
      unsigned long _size; // Size of the struct, so it can grow.
      unsigned long _hwcap;
      unsigned long _hwcap2;
+     unsigned long _hwcap3;
+     unsigned long _hwcap4;
    }
  */
 
@@ -20482,14 +20556,24 @@ build_ifunc_arg_type ()
   tree field3 = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
 			    get_identifier ("_hwcap2"),
 			    long_unsigned_type_node);
+  tree field4 = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
+			    get_identifier ("_hwcap3"),
+			    long_unsigned_type_node);
+  tree field5 = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
+			    get_identifier ("_hwcap4"),
+			    long_unsigned_type_node);
 
   DECL_FIELD_CONTEXT (field1) = ifunc_arg_type;
   DECL_FIELD_CONTEXT (field2) = ifunc_arg_type;
   DECL_FIELD_CONTEXT (field3) = ifunc_arg_type;
+  DECL_FIELD_CONTEXT (field4) = ifunc_arg_type;
+  DECL_FIELD_CONTEXT (field5) = ifunc_arg_type;
 
   TYPE_FIELDS (ifunc_arg_type) = field1;
   DECL_CHAIN (field1) = field2;
   DECL_CHAIN (field2) = field3;
+  DECL_CHAIN (field3) = field4;
+  DECL_CHAIN (field4) = field5;
 
   layout_type (ifunc_arg_type);
 
@@ -23074,6 +23158,58 @@ aarch64_sve_index_immediate_p (rtx base_or_step)
 	  && IN_RANGE (INTVAL (base_or_step), -16, 15));
 }
 
+/* Return true if SERIES is a constant vector that can be loaded using
+   an immediate SVE INDEX, considering both SVE and Advanced SIMD modes.
+   When returning true, store the base in *BASE_OUT and the step
+   in *STEP_OUT.  */
+
+static bool
+aarch64_sve_index_series_p (rtx series, rtx *base_out, rtx *step_out)
+{
+  rtx base, step;
+  if (!const_vec_series_p (series, &base, &step)
+      || !CONST_INT_P (base)
+      || !CONST_INT_P (step))
+    return false;
+
+  auto mode = GET_MODE (series);
+  auto elt_mode = as_a<scalar_int_mode> (GET_MODE_INNER (mode));
+  unsigned int vec_flags = aarch64_classify_vector_mode (mode);
+  if (BYTES_BIG_ENDIAN && (vec_flags & VEC_ADVSIMD))
+    {
+      /* On big-endian targets, architectural lane 0 holds the last element
+	 for Advanced SIMD and the first element for SVE; see the comment at
+	 the head of aarch64-sve.md for details.  This means that, from an SVE
+	 point of view, an Advanced SIMD series goes from the last element to
+	 the first.  */
+      auto i = GET_MODE_NUNITS (mode).to_constant () - 1;
+      base = gen_int_mode (UINTVAL (base) + i * UINTVAL (step), elt_mode);
+      step = gen_int_mode (-UINTVAL (step), elt_mode);
+    }
+
+  if (!aarch64_sve_index_immediate_p (base)
+      || !aarch64_sve_index_immediate_p (step))
+    return false;
+
+  /* If the mode spans multiple registers, check that each subseries is
+     in range.  */
+  unsigned int nvectors = aarch64_ldn_stn_vectors (mode);
+  if (nvectors != 1)
+    {
+      unsigned int nunits;
+      if (!GET_MODE_NUNITS (mode).is_constant (&nunits))
+	return false;
+      nunits /= nvectors;
+      for (unsigned int i = 1; i < nvectors; ++i)
+	if (!IN_RANGE (INTVAL (base) + i * nunits * INTVAL (step), -16, 15))
+	  return false;
+    }
+
+  *base_out = base;
+  *step_out = step;
+  return true;
+}
+
 /* Return true if X is a valid immediate for the SVE ADD and SUB instructions
    when applied to mode MODE.  Negate X first if NEGATE_P is true.  */
 
@@ -23522,13 +23658,8 @@ aarch64_simd_valid_imm (rtx op, simd_immediate_info *info,
     n_elts = CONST_VECTOR_NPATTERNS (op);
   else if (which == AARCH64_CHECK_MOV
 	   && TARGET_SVE
-	   && const_vec_series_p (op, &base, &step))
+	   && aarch64_sve_index_series_p (op, &base, &step))
     {
-      gcc_assert (GET_MODE_CLASS (mode) == MODE_VECTOR_INT);
-      if (!aarch64_sve_index_immediate_p (base)
-	  || !aarch64_sve_index_immediate_p (step))
-	return false;
-
       if (info)
 	{
 	  /* Get the corresponding container mode.  E.g. an INDEX on V2SI
@@ -23640,6 +23771,8 @@ aarch64_simd_valid_imm (rtx op, simd_immediate_info *info,
       long int as_long_ints[2];
       as_long_ints[0] = ival & 0xFFFFFFFF;
       as_long_ints[1] = (ival >> 32) & 0xFFFFFFFF;
+      if (imode == DImode && FLOAT_WORDS_BIG_ENDIAN)
+	std::swap (as_long_ints[0], as_long_ints[1]);
 
       REAL_VALUE_TYPE r;
       real_from_target (&r, as_long_ints, fmode);
@@ -24352,10 +24485,14 @@ aarch64_simd_vector_alignment_reachable (const_tree type, bool is_packed)
 static bool
 aarch64_builtin_support_vector_misalignment (machine_mode mode,
 					     const_tree type, int misalignment,
-					     bool is_packed)
+					     bool is_packed,
+					     bool is_gather_scatter)
 {
   if (TARGET_SIMD && STRICT_ALIGNMENT)
     {
+      if (is_gather_scatter)
+	return true;
+
       /* Return if movmisalign pattern is not supported for this mode.  */
       if (optab_handler (movmisalign_optab, mode) == CODE_FOR_nothing)
         return false;
@@ -24365,7 +24502,8 @@ aarch64_builtin_support_vector_misalignment (machine_mode mode,
 	return false;
     }
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* If VALS is a vector constant that can be loaded into a register
@@ -24830,6 +24968,13 @@ aarch64_expand_vector_init (rtx target, rtx vals)
       emit_insn (rec_seq);
     }
 
+  /* The two halves should (by induction) be individually endian-correct.
+     However, in the memory layout provided by VALS, the nth element of
+     HALVES[0] comes immediately before the nth element HALVES[1].
+     This means that, on big-endian targets, the nth element of HALVES[0]
+     is more significant than the nth element HALVES[1].  */
+  if (BYTES_BIG_ENDIAN)
+    std::swap (halves[0], halves[1]);
   rtvec v = gen_rtvec (2, halves[0], halves[1]);
   rtx_insn *zip1_insn
     = emit_set_insn (target, gen_rtx_UNSPEC (mode, v, UNSPEC_ZIP1));
@@ -26745,7 +26890,6 @@ aarch64_evpc_hvla (struct expand_vec_perm_d *d)
   machine_mode vmode = d->vmode;
   if (!TARGET_SVE2p1
       || !TARGET_NON_STREAMING
-      || BYTES_BIG_ENDIAN
       || d->vec_flags != VEC_SVE_DATA
       || GET_MODE_UNIT_BITSIZE (vmode) > 64)
     return false;
@@ -26905,12 +27049,23 @@ aarch64_evpc_tbl (struct expand_vec_perm_d *d)
 static bool
 aarch64_evpc_sve_tbl (struct expand_vec_perm_d *d)
 {
-  unsigned HOST_WIDE_INT nelt;
+  if (!d->one_vector_p)
+    {
+      /* aarch64_expand_sve_vec_perm does not yet handle variable-length
+	 vectors.  */
+      if (!d->perm.length ().is_constant ())
+	return false;
 
-  /* Permuting two variable-length vectors could overflow the
-     index range.  */
-  if (!d->one_vector_p && !d->perm.length ().is_constant (&nelt))
-    return false;
+      /* This permutation reduces to the vec_perm optab if the elements are
+	 large enough to hold all selector indices.  Do not handle that case
+	 here, since the general TBL+SUB+TBL+ORR sequence is too expensive to
+	 be considered a "native" constant permutation.
+
+	 Not doing this would undermine code that queries can_vec_perm_const_p
+	 with allow_variable_p set to false.  See PR121027.  */
+      if (selector_fits_mode_p (d->vmode, d->perm))
+	return false;
+    }
 
   if (d->testing_p)
     return true;
@@ -27300,7 +27455,7 @@ aarch64_emit_sve_fp_cond (rtx target, rtx_code code, rtx pred,
 			  bool known_ptrue_p, rtx op0, rtx op1)
 {
   rtx flag = gen_int_mode (known_ptrue_p, SImode);
-  rtx unspec = gen_rtx_UNSPEC (GET_MODE (pred),
+  rtx unspec = gen_rtx_UNSPEC (GET_MODE (target),
 			       gen_rtvec (4, pred, flag, op0, op1),
 			       aarch64_unspec_cond_code (code));
   emit_set_insn (target, unspec);
@@ -27319,10 +27474,10 @@ static void
 aarch64_emit_sve_or_fp_conds (rtx target, rtx_code code1, rtx_code code2,
 			      rtx pred, bool known_ptrue_p, rtx op0, rtx op1)
 {
-  machine_mode pred_mode = GET_MODE (pred);
-  rtx tmp1 = gen_reg_rtx (pred_mode);
+  machine_mode target_mode = GET_MODE (target);
+  rtx tmp1 = gen_reg_rtx (target_mode);
   aarch64_emit_sve_fp_cond (tmp1, code1, pred, known_ptrue_p, op0, op1);
-  rtx tmp2 = gen_reg_rtx (pred_mode);
+  rtx tmp2 = gen_reg_rtx (target_mode);
   aarch64_emit_sve_fp_cond (tmp2, code2, pred, known_ptrue_p, op0, op1);
   aarch64_emit_binop (target, ior_optab, tmp1, tmp2);
 }
@@ -27339,8 +27494,7 @@ static void
 aarch64_emit_sve_invert_fp_cond (rtx target, rtx_code code, rtx pred,
 				 bool known_ptrue_p, rtx op0, rtx op1)
 {
-  machine_mode pred_mode = GET_MODE (pred);
-  rtx tmp = gen_reg_rtx (pred_mode);
+  rtx tmp = gen_reg_rtx (GET_MODE (target));
   aarch64_emit_sve_fp_cond (tmp, code, pred, known_ptrue_p, op0, op1);
   aarch64_emit_unop (target, one_cmpl_optab, tmp);
 }
@@ -27352,10 +27506,25 @@ aarch64_emit_sve_invert_fp_cond (rtx target, rtx_code code, rtx pred,
 void
 aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
 {
-  machine_mode pred_mode = GET_MODE (target);
   machine_mode data_mode = GET_MODE (op0);
+  rtx pred = aarch64_sve_fp_pred (data_mode, nullptr);
 
-  rtx ptrue = aarch64_ptrue_reg (pred_mode);
+  /* The governing and destination modes.  */
+  machine_mode pred_mode = GET_MODE (pred);
+  machine_mode target_mode = GET_MODE (target);
+
+  /* For partial vector modes, the choice of predicate mode depends
+     on whether we need to suppress exceptions for inactive elements.
+     If we do need to suppress exceptions, the predicate mode matches
+     the element size rather than the container size and the predicate
+     marks the upper bits in each container as inactive.  The predicate
+     is then a ptrue wrt TARGET_MODE but not wrt PRED_MODE.  It is the
+     latter which matters here.
+
+     If we don't need to suppress exceptions, the predicate mode matches
+     the container size, PRED_MODE == TARGET_MODE, and the predicate is
+     thus a ptrue wrt both TARGET_MODE and PRED_MODE.  */
+  bool known_ptrue_p = pred_mode == target_mode;
   switch (code)
     {
     case UNORDERED:
@@ -27369,12 +27538,13 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
     case EQ:
     case NE:
       /* There is native support for the comparison.  */
-      aarch64_emit_sve_fp_cond (target, code, ptrue, true, op0, op1);
+      aarch64_emit_sve_fp_cond (target, code, pred, known_ptrue_p, op0, op1);
       return;
 
     case LTGT:
       /* This is a trapping operation (LT or GT).  */
-      aarch64_emit_sve_or_fp_conds (target, LT, GT, ptrue, true, op0, op1);
+      aarch64_emit_sve_or_fp_conds (target, LT, GT,
+				    pred, known_ptrue_p, op0, op1);
       return;
 
     case UNEQ:
@@ -27383,7 +27553,7 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
 	  /* This would trap for signaling NaNs.  */
 	  op1 = force_reg (data_mode, op1);
 	  aarch64_emit_sve_or_fp_conds (target, UNORDERED, EQ,
-					ptrue, true, op0, op1);
+					pred, known_ptrue_p, op0, op1);
 	  return;
 	}
       /* fall through */
@@ -27393,11 +27563,19 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
     case UNGE:
       if (flag_trapping_math)
 	{
-	  /* Work out which elements are ordered.  */
-	  rtx ordered = gen_reg_rtx (pred_mode);
 	  op1 = force_reg (data_mode, op1);
-	  aarch64_emit_sve_invert_fp_cond (ordered, UNORDERED,
-					   ptrue, true, op0, op1);
+
+	  /* Work out which elements are unordered.  */
+	  rtx uo_tmp = gen_reg_rtx (target_mode);
+	  aarch64_emit_sve_fp_cond (uo_tmp, UNORDERED,
+				    pred, known_ptrue_p, op0, op1);
+
+	  /* Invert the result.  Governered by PRED so that we only
+	     flip the active bits.  */
+	  rtx ordered = gen_reg_rtx (pred_mode);
+	  uo_tmp = gen_lowpart (pred_mode, uo_tmp);
+	  emit_insn (gen_aarch64_pred_one_cmpl_z (pred_mode, ordered,
+						  pred, uo_tmp));
 
 	  /* Test the opposite condition for the ordered elements,
 	     then invert the result.  */
@@ -27422,7 +27600,8 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
 
   /* There is native support for the inverse comparison.  */
   code = reverse_condition_maybe_unordered (code);
-  aarch64_emit_sve_invert_fp_cond (target, code, ptrue, true, op0, op1);
+  aarch64_emit_sve_invert_fp_cond (target, code,
+				   pred, known_ptrue_p, op0, op1);
 }
 
 /* Return true if:
@@ -31853,9 +32032,43 @@ aarch64_test_sysreg_encoding_clashes (void)
 static void
 aarch64_test_sve_folding ()
 {
+  aarch64_target_switcher switcher (AARCH64_FL_SVE);
+
   tree res = fold_unary (BIT_NOT_EXPR, ssizetype,
 			 ssize_int (poly_int64 (1, 1)));
   ASSERT_TRUE (operand_equal_p (res, ssize_int (poly_int64 (-2, -1))));
+
+  auto build_v16bi = [](bool a, bool b)
+    {
+      rtx_vector_builder builder (VNx16BImode, 2, 1);
+      builder.quick_push (a ? const1_rtx : const0_rtx);
+      builder.quick_push (b ? const1_rtx : const0_rtx);
+      return builder.build ();
+    };
+  rtx v16bi_10 = build_v16bi (1, 0);
+  rtx v16bi_01 = build_v16bi (0, 1);
+
+  for (auto mode : { VNx8BImode, VNx4BImode, VNx2BImode })
+    {
+      rtx reg = gen_rtx_REG (mode, LAST_VIRTUAL_REGISTER + 1);
+      rtx subreg = lowpart_subreg (VNx16BImode, reg, mode);
+      rtx and1 = simplify_gen_binary (AND, VNx16BImode, subreg, v16bi_10);
+      ASSERT_EQ (lowpart_subreg (mode, and1, VNx16BImode), reg);
+      rtx and0 = simplify_gen_binary (AND, VNx16BImode, subreg, v16bi_01);
+      ASSERT_EQ (lowpart_subreg (mode, and0, VNx16BImode), CONST0_RTX (mode));
+
+      rtx ior1 = simplify_gen_binary (IOR, VNx16BImode, subreg, v16bi_10);
+      ASSERT_EQ (lowpart_subreg (mode, ior1, VNx16BImode), CONSTM1_RTX (mode));
+      rtx ior0 = simplify_gen_binary (IOR, VNx16BImode, subreg, v16bi_01);
+      ASSERT_EQ (lowpart_subreg (mode, ior0, VNx16BImode), reg);
+
+      rtx xor1 = simplify_gen_binary (XOR, VNx16BImode, subreg, v16bi_10);
+      ASSERT_RTX_EQ (lowpart_subreg (mode, xor1, VNx16BImode),
+		     lowpart_subreg (mode, gen_rtx_NOT (VNx16BImode, subreg),
+				     VNx16BImode));
+      rtx xor0 = simplify_gen_binary (XOR, VNx16BImode, subreg, v16bi_01);
+      ASSERT_EQ (lowpart_subreg (mode, xor0, VNx16BImode), reg);
+    }
 }
 
 /* Run all target-specific selftests.  */
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index d5c4a42..096c853 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -493,6 +493,11 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED
   (bool (aarch64_tune_params.extra_tuning_flags \
 	 & AARCH64_EXTRA_TUNE_CHEAP_FPMR_WRITE))
 
+/* Enable folding address computation into LDAPUR when RCPC2 is available.  */
+#define TARGET_ENABLE_LDAPUR (TARGET_RCPC2 \
+			      && !(aarch64_tune_params.extra_tuning_flags \
+				   & AARCH64_EXTRA_TUNE_AVOID_LDAPUR))
+
 /* Combinatorial tests.  */
 
 #define TARGET_SVE2_OR_SME2 \
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 509ef4c..dc2be81 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -280,6 +280,7 @@
     UNSPEC_PACIBSP
     UNSPEC_PRLG_STK
     UNSPEC_REV
+    UNSPEC_REV_PRED
     UNSPEC_SADALP
     UNSPEC_SCVTF
     UNSPEC_SET_LANE
@@ -482,6 +483,8 @@
 ;; clobber for SVE predicates.
 (define_attr "pred_clobber" "any,no,yes" (const_string "any"))
 
+(define_attr "enable_ldapur" "any,no,yes" (const_string "any"))
+
 ;; [For compatibility with Arm in pipeline models]
 ;; Attribute that specifies whether or not the instruction touches fp
 ;; registers.
@@ -506,7 +509,14 @@
 	  (eq_attr "pred_clobber" "yes")
 	  (match_test "TARGET_SVE_PRED_CLOBBER"))
 	(eq_attr "pred_clobber" "any"))
-
+      (ior
+	(and
+	  (eq_attr "enable_ldapur" "yes")
+	  (match_test "TARGET_ENABLE_LDAPUR"))
+	(and
+	  (eq_attr "enable_ldapur" "no")
+	  (match_test "!TARGET_ENABLE_LDAPUR"))
+	(eq_attr "enable_ldapur" "any"))
       (ior
 	(eq_attr "arch" "any")
 
@@ -5771,6 +5781,19 @@
    (match_operand:TI 1 "register_operand")]
   "TARGET_SIMD && !TARGET_CSSC"
 {
+  /* For SVE we can do popcount on DImode chunks of the TImode argument
+     and then use a cheap ADDP reduction.  The SVE CNT instruction requires
+     materializing a PTRUE so don't do this if optimizing for size.  */
+  if (TARGET_SVE && !optimize_function_for_size_p (cfun))
+    {
+      rtx v = gen_reg_rtx (V2DImode);
+      rtx v1 = gen_reg_rtx (V2DImode);
+      emit_move_insn (v, gen_lowpart (V2DImode, operands[1]));
+      rtx p = aarch64_ptrue_reg (VNx2BImode, 16);
+      emit_insn (gen_aarch64_pred_popcountv2di (v1, p, v));
+      emit_insn (gen_reduc_plus_scal_v2di (operands[0], v1));
+      DONE;
+    }
   rtx v = gen_reg_rtx (V16QImode);
   rtx v1 = gen_reg_rtx (V16QImode);
   emit_move_insn (v, gen_lowpart (V16QImode, operands[1]));
diff --git a/gcc/config/aarch64/atomics.md b/gcc/config/aarch64/atomics.md
index 36b0dbd..ea4a936 100644
--- a/gcc/config/aarch64/atomics.md
+++ b/gcc/config/aarch64/atomics.md
@@ -679,13 +679,16 @@
 )
 
 (define_insn "aarch64_atomic_load<mode>_rcpc"
-  [(set (match_operand:ALLI 0 "register_operand" "=r")
+  [(set (match_operand:ALLI 0 "register_operand")
     (unspec_volatile:ALLI
-      [(match_operand:ALLI 1 "aarch64_sync_memory_operand" "Q")
+      [(match_operand:ALLI 1 "aarch64_rcpc_memory_operand")
        (match_operand:SI 2 "const_int_operand")]			;; model
       UNSPECV_LDAP))]
   "TARGET_RCPC"
-  "ldapr<atomic_sfx>\t%<w>0, %1"
+  {@ [ cons: =0 , 1   ; attrs: enable_ldapur  ]
+     [ r        , Q   ; any                   ] ldapr<atomic_sfx>\t%<w>0, %1
+     [ r        , Ust ; yes                   ] ldapur<atomic_sfx>\t%<w>0, %1
+  }
 )
 
 (define_insn "aarch64_atomic_load<mode>"
@@ -705,21 +708,24 @@
 )
 
 (define_insn "*aarch64_atomic_load<ALLX:mode>_rcpc_zext"
-  [(set (match_operand:SD_HSDI 0 "register_operand" "=r")
+  [(set (match_operand:SD_HSDI 0 "register_operand")
     (zero_extend:SD_HSDI
       (unspec_volatile:ALLX
-        [(match_operand:ALLX 1 "aarch64_sync_memory_operand" "Q")
+        [(match_operand:ALLX 1 "aarch64_rcpc_memory_operand")
          (match_operand:SI 2 "const_int_operand")]			;; model
        UNSPECV_LDAP)))]
   "TARGET_RCPC && (<SD_HSDI:sizen> > <ALLX:sizen>)"
-  "ldapr<ALLX:atomic_sfx>\t%w0, %1"
+  {@ [ cons: =0 , 1   ; attrs: enable_ldapur ]
+     [ r        , Q   ; any                  ] ldapr<ALLX:atomic_sfx>\t%w0, %1
+     [ r        , Ust ; yes                  ] ldapur<ALLX:atomic_sfx>\t%w0, %1
+  }
 )
 
 (define_insn "*aarch64_atomic_load<ALLX:mode>_rcpc_sext"
   [(set (match_operand:GPI  0 "register_operand" "=r")
     (sign_extend:GPI
       (unspec_volatile:ALLX
-        [(match_operand:ALLX 1 "aarch64_sync_memory_operand" "Q")
+        [(match_operand:ALLX 1 "aarch64_rcpc_memory_operand" "Ust")
          (match_operand:SI 2 "const_int_operand")]			;; model
        UNSPECV_LDAP)))]
   "TARGET_RCPC2 && (<GPI:sizen> > <ALLX:sizen>)"
diff --git a/gcc/config/aarch64/cortex-a57-fma-steering.cc b/gcc/config/aarch64/cortex-a57-fma-steering.cc
index fd6da66..f7675be 100644
--- a/gcc/config/aarch64/cortex-a57-fma-steering.cc
+++ b/gcc/config/aarch64/cortex-a57-fma-steering.cc
@@ -948,6 +948,11 @@ func_fma_steering::analyze ()
 
 	  /* Search the chain where this instruction is (one of) the root.  */
 	  dest_op_info = insn_rr[INSN_UID (insn)].op_info;
+
+	  /* Register rename could fail. */
+	  if (!dest_op_info)
+	    continue;
+
 	  dest_regno = REGNO (SET_DEST (PATTERN (insn)));
 	  for (i = 0; i < dest_op_info->n_chains; i++)
 	    {
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index c59fcd6..8f8237e 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -455,6 +455,7 @@
 (define_mode_iterator VCVTFPM [V4HF V8HF V4SF])
 
 ;; Iterators for single modes, for "@" patterns.
+(define_mode_iterator VNx16BI_ONLY [VNx16BI])
 (define_mode_iterator VNx16QI_ONLY [VNx16QI])
 (define_mode_iterator VNx16SI_ONLY [VNx16SI])
 (define_mode_iterator VNx8HI_ONLY [VNx8HI])
@@ -463,6 +464,7 @@
 (define_mode_iterator VNx8SI_ONLY [VNx8SI])
 (define_mode_iterator VNx8SF_ONLY [VNx8SF])
 (define_mode_iterator VNx8DI_ONLY [VNx8DI])
+(define_mode_iterator VNx2SI_ONLY [VNx2SI])
 (define_mode_iterator VNx4SI_ONLY [VNx4SI])
 (define_mode_iterator VNx4SF_ONLY [VNx4SF])
 (define_mode_iterator VNx2DI_ONLY [VNx2DI])
@@ -929,7 +931,6 @@
     UNSPEC_UZP2Q	; Used in aarch64-sve.md.
     UNSPEC_ZIP1Q	; Used in aarch64-sve.md.
     UNSPEC_ZIP2Q	; Used in aarch64-sve.md.
-    UNSPEC_TRN1_CONV	; Used in aarch64-sve.md.
     UNSPEC_COND_CMPEQ_WIDE ; Used in aarch64-sve.md.
     UNSPEC_COND_CMPGE_WIDE ; Used in aarch64-sve.md.
     UNSPEC_COND_CMPGT_WIDE ; Used in aarch64-sve.md.
@@ -1184,6 +1185,9 @@
     UNSPEC_LUTI2	; Used in aarch64-simd.md.
     UNSPEC_LUTI4	; Used in aarch64-simd.md.
 
+    ;; All used in aarch64-sve.md
+    UNSPEC_PERMUTE_PRED
+
     ;; All used in aarch64-sve2.md
     UNSPEC_ADDQV
     UNSPEC_ANDQV
@@ -3366,6 +3370,10 @@
 (define_int_iterator SVE_INT_UNARY [UNSPEC_REVB
 				    UNSPEC_REVH UNSPEC_REVW])
 
+;; This iterator is currently only used for estimation instructions,
+;; which are never generated automatically when -ftrapping-math is true.
+;; The iterator is therefore applied unconditionally to partial FP modes.
+;; This might need to be revisited if new operations are added in future.
 (define_int_iterator SVE_FP_UNARY [UNSPEC_FRECPE UNSPEC_RSQRTE])
 
 (define_int_iterator SVE_FP_UNARY_INT [(UNSPEC_FEXPA "TARGET_NON_STREAMING")])
@@ -3378,6 +3386,10 @@
 (define_int_iterator SVE_INT_BINARY_MULTI [UNSPEC_SQDMULH
 					   UNSPEC_SRSHL UNSPEC_URSHL])
 
+;; This iterator is currently only used for estimation instructions,
+;; which are never generated automatically when -ftrapping-math is true.
+;; The iterator is therefore applied unconditionally to partial FP modes.
+;; This might need to be revisited if new operations are added in future.
 (define_int_iterator SVE_FP_BINARY [UNSPEC_FRECPS UNSPEC_RSQRTS])
 
 (define_int_iterator SVE_FP_BINARY_INT [UNSPEC_FTSMUL UNSPEC_FTSSEL])
@@ -3429,9 +3441,10 @@
 					   UNSPEC_FMINQV
 					   UNSPEC_FMINNMQV])
 
-(define_int_iterator SVE_COND_FP_UNARY [UNSPEC_COND_FABS
-					UNSPEC_COND_FNEG
-					UNSPEC_COND_FRECPX
+(define_int_iterator SVE_COND_FP_UNARY_BITWISE [UNSPEC_COND_FABS
+						UNSPEC_COND_FNEG])
+
+(define_int_iterator SVE_COND_FP_UNARY [UNSPEC_COND_FRECPX
 					UNSPEC_COND_FRINTA
 					UNSPEC_COND_FRINTI
 					UNSPEC_COND_FRINTM
@@ -3439,13 +3452,12 @@
 					UNSPEC_COND_FRINTP
 					UNSPEC_COND_FRINTX
 					UNSPEC_COND_FRINTZ
-					UNSPEC_COND_FSQRT])
+					UNSPEC_COND_FSQRT
+					SVE_COND_FP_UNARY_BITWISE])
 
 ;; Same as SVE_COND_FP_UNARY, but without codes that have a dedicated
 ;; <optab><mode>2 expander.
-(define_int_iterator SVE_COND_FP_UNARY_OPTAB [UNSPEC_COND_FABS
-					      UNSPEC_COND_FNEG
-					      UNSPEC_COND_FRECPX
+(define_int_iterator SVE_COND_FP_UNARY_OPTAB [UNSPEC_COND_FRECPX
 					      UNSPEC_COND_FRINTA
 					      UNSPEC_COND_FRINTI
 					      UNSPEC_COND_FRINTM
@@ -3868,6 +3880,8 @@
 
 (define_int_iterator SVE_PITER [UNSPEC_PFIRST UNSPEC_PNEXT])
 
+(define_int_iterator PNEXT_ONLY [UNSPEC_PNEXT])
+
 (define_int_iterator MATMUL [UNSPEC_SMATMUL UNSPEC_UMATMUL
 			     UNSPEC_USMATMUL])
 
diff --git a/gcc/config/aarch64/predicates.md b/gcc/config/aarch64/predicates.md
index 32056da..4d5d57f 100644
--- a/gcc/config/aarch64/predicates.md
+++ b/gcc/config/aarch64/predicates.md
@@ -1078,3 +1078,9 @@
 (define_predicate "aarch64_maskload_else_operand"
   (and (match_code "const_vector")
        (match_test "op == CONST0_RTX (GET_MODE (op))")))
+
+;; Check for a VNx16BI predicate that is a canonical PTRUE for the given
+;; predicate mode.
+(define_special_predicate "aarch64_ptrue_all_operand"
+  (and (match_code "const_vector")
+       (match_test "aarch64_ptrue_all_mode (op) == mode")))
diff --git a/gcc/config/aarch64/tuning_models/cortexx925.h b/gcc/config/aarch64/tuning_models/cortexx925.h
index 7d0162e..f448493 100644
--- a/gcc/config/aarch64/tuning_models/cortexx925.h
+++ b/gcc/config/aarch64/tuning_models/cortexx925.h
@@ -222,7 +222,8 @@ static const struct tune_params cortexx925_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/generic_armv9_a.h b/gcc/config/aarch64/tuning_models/generic_armv9_a.h
index f76a250..9eb1a20 100644
--- a/gcc/config/aarch64/tuning_models/generic_armv9_a.h
+++ b/gcc/config/aarch64/tuning_models/generic_armv9_a.h
@@ -26,7 +26,7 @@
 static const struct cpu_addrcost_table generic_armv9_a_addrcost_table =
 {
     {
-      1, /* hi  */
+      0, /* hi  */
       0, /* si  */
       0, /* di  */
       1, /* ti  */
diff --git a/gcc/config/aarch64/tuning_models/neoversev2.h b/gcc/config/aarch64/tuning_models/neoversev2.h
index b000fb4..266d8f1 100644
--- a/gcc/config/aarch64/tuning_models/neoversev2.h
+++ b/gcc/config/aarch64/tuning_models/neoversev2.h
@@ -220,7 +220,8 @@ static const struct tune_params neoversev2_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/neoversev3.h b/gcc/config/aarch64/tuning_models/neoversev3.h
index ad3cd22..f5566d2 100644
--- a/gcc/config/aarch64/tuning_models/neoversev3.h
+++ b/gcc/config/aarch64/tuning_models/neoversev3.h
@@ -220,7 +220,8 @@ static const struct tune_params neoversev3_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/neoversev3ae.h b/gcc/config/aarch64/tuning_models/neoversev3ae.h
index a0adef0..5796e52 100644
--- a/gcc/config/aarch64/tuning_models/neoversev3ae.h
+++ b/gcc/config/aarch64/tuning_models/neoversev3ae.h
@@ -220,7 +220,8 @@ static const struct tune_params neoversev3ae_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/olympus.h b/gcc/config/aarch64/tuning_models/olympus.h
new file mode 100644
index 0000000..268789d
--- /dev/null
+++ b/gcc/config/aarch64/tuning_models/olympus.h
@@ -0,0 +1,210 @@
+/* Tuning model description for the NVIDIA Olympus core.
+   Copyright The GNU Toolchain Authors.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_AARCH64_H_OLYMPUS
+#define GCC_AARCH64_H_OLYMPUS
+
+#include "generic.h"
+
+static struct cpu_regmove_cost olympus_regmove_cost =
+{
+  1, /* GP2GP  */
+  /* Spilling to int<->fp instead of memory is recommended so set
+     realistic costs compared to memmov_cost.  */
+  3, /* GP2FP  */
+  3, /* FP2GP  */
+  2 /* FP2FP  */
+};
+
+static advsimd_vec_cost olympus_advsimd_vector_cost =
+{
+  2, /* int_stmt_cost  */
+  2, /* fp_stmt_cost  */
+  2, /* ld2_st2_permute_cost */
+  2, /* ld3_st3_permute_cost  */
+  3, /* ld4_st4_permute_cost  */
+  2, /* permute_cost  */
+  5, /* reduc_i8_cost  */
+  3, /* reduc_i16_cost  */
+  3, /* reduc_i32_cost  */
+  2, /* reduc_i64_cost  */
+  4, /* reduc_f16_cost  */
+  4, /* reduc_f32_cost  */
+  4, /* reduc_f64_cost  */
+  2, /* store_elt_extra_cost  */
+  8, /* vec_to_scalar_cost  */
+  4, /* scalar_to_vec_cost  */
+  6, /* align_load_cost  */
+  6, /* unalign_load_cost  */
+  1, /* unalign_store_cost  */
+  1  /* store_cost  */
+};
+
+static sve_vec_cost olympus_sve_vector_cost =
+{
+  {
+    2, /* int_stmt_cost  */
+    2, /* fp_stmt_cost  */
+    2, /* ld2_st2_permute_cost  */
+    3, /* ld3_st3_permute_cost  */
+    3, /* ld4_st4_permute_cost  */
+    2, /* permute_cost  */
+    9, /* reduc_i8_cost  */
+    8, /* reduc_i16_cost  */
+    6, /* reduc_i32_cost  */
+    2, /* reduc_i64_cost  */
+    8, /* reduc_f16_cost  */
+    6, /* reduc_f32_cost  */
+    4, /* reduc_f64_cost  */
+    2, /* store_elt_extra_cost  */
+    8, /* vec_to_scalar_cost  */
+    4, /* scalar_to_vec_cost  */
+    4, /* align_load_cost  */
+    6, /* unalign_load_cost  */
+    1, /* unalign_store_cost  */
+    1  /* store_cost  */
+  },
+  3, /* clast_cost  */
+  10, /* fadda_f16_cost  */
+  6, /* fadda_f32_cost  */
+  4, /* fadda_f64_cost  */
+  14, /* gather_load_x32_cost  */
+  12, /* gather_load_x64_cost  */
+  42, /* gather_load_x32_init_cost  */
+  24, /* gather_load_x64_init_cost  */
+  1 /* scatter_store_elt_cost  */
+};
+
+static aarch64_scalar_vec_issue_info olympus_scalar_issue_info =
+{
+  4, /* loads_stores_per_cycle  */
+  2, /* stores_per_cycle  */
+  8, /* general_ops_per_cycle  */
+  0, /* fp_simd_load_general_ops  */
+  1 /* fp_simd_store_general_ops  */
+};
+
+static aarch64_advsimd_vec_issue_info olympus_advsimd_issue_info =
+{
+  {
+    3, /* loads_stores_per_cycle  */
+    2, /* stores_per_cycle  */
+    6, /* general_ops_per_cycle  */
+    0, /* fp_simd_load_general_ops  */
+    1 /* fp_simd_store_general_ops  */
+  },
+  2, /* ld2_st2_general_ops  */
+  2, /* ld3_st3_general_ops  */
+  3 /* ld4_st4_general_ops  */
+};
+
+static aarch64_sve_vec_issue_info olympus_sve_issue_info =
+{
+  {
+    {
+      3, /* loads_stores_per_cycle  */
+      2, /* stores_per_cycle  */
+      6, /* general_ops_per_cycle  */
+      0, /* fp_simd_load_general_ops  */
+      1 /* fp_simd_store_general_ops  */
+    },
+    2, /* ld2_st2_general_ops  */
+    2, /* ld3_st3_general_ops  */
+    3 /* ld4_st4_general_ops  */
+  },
+  2, /* pred_ops_per_cycle  */
+  1, /* while_pred_ops  */
+  0, /* int_cmp_pred_ops  */
+  0, /* fp_cmp_pred_ops  */
+  1, /* gather_scatter_pair_general_ops  */
+  1 /* gather_scatter_pair_pred_ops  */
+};
+
+static aarch64_vec_issue_info olympus_vec_issue_info =
+{
+  &olympus_scalar_issue_info,
+  &olympus_advsimd_issue_info,
+  &olympus_sve_issue_info
+};
+
+/* Olympus costs for vector insn classes.  */
+static struct cpu_vector_cost olympus_vector_cost =
+{
+  1, /* scalar_int_stmt_cost  */
+  2, /* scalar_fp_stmt_cost  */
+  4, /* scalar_load_cost  */
+  1, /* scalar_store_cost  */
+  1, /* cond_taken_branch_cost  */
+  1, /* cond_not_taken_branch_cost  */
+  &olympus_advsimd_vector_cost, /* advsimd  */
+  &olympus_sve_vector_cost, /* sve  */
+  &olympus_vec_issue_info /* issue_info  */
+};
+
+/* Olympus prefetch settings (which disable prefetch).  */
+static cpu_prefetch_tune olympus_prefetch_tune =
+{
+  0,			/* num_slots  */
+  -1,			/* l1_cache_size  */
+  64,			/* l1_cache_line_size  */
+  -1,			/* l2_cache_size  */
+  true,			/* prefetch_dynamic_strides */
+  -1,			/* minimum_stride */
+  -1			/* default_opt_level  */
+};
+
+static struct tune_params olympus_tunings =
+{
+  &cortexa76_extra_costs,
+  &generic_armv9_a_addrcost_table,
+  &olympus_regmove_cost,
+  &olympus_vector_cost,
+  &generic_branch_cost,
+  &generic_approx_modes,
+  SVE_128, /* sve_width  */
+  { 4, /* load_int.  */
+    1, /* store_int.  */
+    6, /* load_fp.  */
+    3, /* store_fp.  */
+    5, /* load_pred.  */
+    1 /* store_pred.  */
+  }, /* memmov_cost.  */
+  10, /* issue_rate  */
+  AARCH64_FUSE_NEOVERSE_BASE, /* fusible_ops  */
+  "32:16",	/* function_align.  */
+  "4",		/* jump_align.  */
+  "32:16",	/* loop_align.  */
+  8,	/* int_reassoc_width.  */
+  6,	/* fp_reassoc_width.  */
+  4,	/* fma_reassoc_width.  */
+  6,	/* vec_reassoc_width.  */
+  2,	/* min_div_recip_mul_sf.  */
+  2,	/* min_div_recip_mul_df.  */
+  0,	/* max_case_values.  */
+  tune_params::AUTOPREFETCHER_WEAK,	/* autoprefetcher_model.  */
+  (AARCH64_EXTRA_TUNE_BASE
+   | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
+   | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+  &olympus_prefetch_tune,
+  AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
+  AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
+};
+
+#endif /* GCC_AARCH64_H_OLYMPUS.  */
diff --git a/gcc/config/arm/aarch-cost-tables.h b/gcc/config/arm/aarch-cost-tables.h
index c7a14b3..0600e59 100644
--- a/gcc/config/arm/aarch-cost-tables.h
+++ b/gcc/config/arm/aarch-cost-tables.h
@@ -123,9 +123,9 @@ const struct cpu_cost_table generic_extra_costs =
   {
     COSTS_N_INSNS (1),	/* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -230,9 +230,9 @@ const struct cpu_cost_table cortexa53_extra_costs =
   {
     COSTS_N_INSNS (1),	/* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -337,9 +337,9 @@ const struct cpu_cost_table cortexa57_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -444,9 +444,9 @@ const struct cpu_cost_table cortexa76_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -551,9 +551,9 @@ const struct cpu_cost_table exynosm1_extra_costs =
   {
     COSTS_N_INSNS (0),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -658,9 +658,9 @@ const struct cpu_cost_table xgene1_extra_costs =
   {
     COSTS_N_INSNS (2),  /* alu.  */
     COSTS_N_INSNS (8),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
diff --git a/gcc/config/arm/arm.cc b/gcc/config/arm/arm.cc
index bde06f3..29b45ae 100644
--- a/gcc/config/arm/arm.cc
+++ b/gcc/config/arm/arm.cc
@@ -289,7 +289,8 @@ static bool arm_vector_alignment_reachable (const_tree type, bool is_packed);
 static bool arm_builtin_support_vector_misalignment (machine_mode mode,
 						     const_tree type,
 						     int misalignment,
-						     bool is_packed);
+						     bool is_packed,
+						     bool is_gather_scatter);
 static void arm_conditional_register_usage (void);
 static enum flt_eval_method arm_excess_precision (enum excess_precision_type);
 static reg_class_t arm_preferred_rename_class (reg_class_t rclass);
@@ -30661,12 +30662,16 @@ arm_vector_alignment_reachable (const_tree type, bool is_packed)
 static bool
 arm_builtin_support_vector_misalignment (machine_mode mode,
 					 const_tree type, int misalignment,
-					 bool is_packed)
+					 bool is_packed,
+					 bool is_gather_scatter)
 {
   if (TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access)
     {
       HOST_WIDE_INT align = TYPE_ALIGN_UNIT (type);
 
+      if (is_gather_scatter)
+	return true;
+
       if (is_packed)
         return align == 1;
 
@@ -30683,7 +30688,8 @@ arm_builtin_support_vector_misalignment (machine_mode mode,
     }
 
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 static void
diff --git a/gcc/config/arm/arm_neon.h b/gcc/config/arm/arm_neon.h
index cba50de..105385f 100644
--- a/gcc/config/arm/arm_neon.h
+++ b/gcc/config/arm/arm_neon.h
@@ -20938,11 +20938,6 @@ vbfdotq_lane_f32 (float32x4_t __r, bfloat16x8_t __a, bfloat16x4_t __b,
   return __builtin_neon_vbfdot_lanev4bfv4sf (__r, __a, __b, __index);
 }
 
-#pragma GCC pop_options
-
-#pragma GCC push_options
-#pragma GCC target ("arch=armv8.2-a+bf16")
-
 typedef struct bfloat16x4x2_t
 {
   bfloat16x4_t val[2];
diff --git a/gcc/config/avr/avr-dimode.md b/gcc/config/avr/avr-dimode.md
index 903bfbf..66ba5a9 100644
--- a/gcc/config/avr/avr-dimode.md
+++ b/gcc/config/avr/avr-dimode.md
@@ -101,10 +101,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (plus:ALL8 (reg:ALL8 ACC_A)
-                              (reg:ALL8 ACC_B)))
-   (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3_insn"
   [(set (reg:ALL8 ACC_A)
@@ -122,10 +120,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:DI ACC_A)
-                   (plus:DI (reg:DI ACC_A)
-                            (sign_extend:DI (reg:QI REG_X))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*adddi3_const8_insn"
   [(set (reg:DI ACC_A)
@@ -146,12 +142,10 @@
                    (match_operand:ALL8 0 "const_operand" "n Ynn")))]
   "avr_have_dimode
    && !s8_operand (operands[0], VOIDmode)"
-   "#"
-   "&& reload_completed"
-   [(parallel [(set (reg:ALL8 ACC_A)
-                    (plus:ALL8 (reg:ALL8 ACC_A)
-                               (match_dup 0)))
-               (clobber (reg:CC REG_CC))])])
+  "#"
+  "&& reload_completed"
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3_const_insn"
   [(set (reg:ALL8 ACC_A)
@@ -211,10 +205,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (minus:ALL8 (reg:ALL8 ACC_A)
-                               (reg:ALL8 ACC_B)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3_insn"
   [(set (reg:ALL8 ACC_A)
@@ -236,10 +228,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (minus:ALL8 (reg:ALL8 ACC_A)
-                               (match_dup 0)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3_const_insn"
   [(set (reg:ALL8 ACC_A)
@@ -288,10 +278,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8S ACC_A)
-                   (ss_addsub:ALL8S (reg:ALL8S ACC_A)
-                                    (reg:ALL8S ACC_B)))
-             (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_insn"
   [(set (reg:ALL8S ACC_A)
@@ -309,10 +297,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8S ACC_A)
-                   (ss_addsub:ALL8S (reg:ALL8S ACC_A)
-                                    (match_dup 0)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_const_insn"
   [(set (reg:ALL8S ACC_A)
@@ -361,10 +347,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8U ACC_A)
-                   (us_addsub:ALL8U (reg:ALL8U ACC_A)
-                                    (reg:ALL8U ACC_B)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_insn"
   [(set (reg:ALL8U ACC_A)
@@ -382,10 +366,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8U ACC_A)
-                   (us_addsub:ALL8U (reg:ALL8U ACC_A)
-                                    (match_operand:ALL8U 0 "const_operand" "n Ynn")))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_const_insn"
   [(set (reg:ALL8U ACC_A)
@@ -421,9 +403,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:DI ACC_A)
-                   (neg:DI (reg:DI ACC_A)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negdi2_insn"
   [(set (reg:DI ACC_A)
@@ -500,7 +481,7 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     emit_insn (gen_compare_<mode>2 ());
     emit_jump_insn (gen_conditional_jump (operands[0], operands[1]));
@@ -529,7 +510,7 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     emit_insn (gen_compare_const8_di2 ());
     emit_jump_insn (gen_conditional_jump (operands[0], operands[1]));
@@ -556,7 +537,7 @@
    && !s8_operand (operands[1], VOIDmode)"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     emit_insn (gen_compare_const_<mode>2 (operands[1], operands[3]));
     emit_jump_insn (gen_conditional_jump (operands[0], operands[2]));
@@ -629,10 +610,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (di_shifts:ALL8 (reg:ALL8 ACC_A)
-                                   (reg:QI 16)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_insn"
   [(set (reg:ALL8 ACC_A)
@@ -674,14 +653,10 @@
    (clobber (reg:HI REG_Z))]
   "avr_have_dimode
    && AVR_HAVE_MUL"
-   "#"
-   "&& reload_completed"
-   [(parallel [(set (reg:DI ACC_A)
-                    (mult:DI (any_extend:DI (reg:SI 18))
-                             (any_extend:DI (reg:SI 22))))
-               (clobber (reg:HI REG_X))
-               (clobber (reg:HI REG_Z))
-               (clobber (reg:CC REG_CC))])])
+  "#"
+  "&& reload_completed"
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>mulsidi3_insn"
   [(set (reg:DI ACC_A)
diff --git a/gcc/config/avr/avr-fixed.md b/gcc/config/avr/avr-fixed.md
index ce46beb..22061fc 100644
--- a/gcc/config/avr/avr-fixed.md
+++ b/gcc/config/avr/avr-fixed.md
@@ -62,10 +62,8 @@
   "<FIXED_B:MODE>mode != <FIXED_A:MODE>mode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (fract_convert:FIXED_A
-                    (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fract<FIXED_B:mode><FIXED_A:mode>2"
   [(set (match_operand:FIXED_A 0 "register_operand" "=r")
@@ -86,10 +84,8 @@
   "<FIXED_B:MODE>mode != <FIXED_A:MODE>mode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unsigned_fract_convert:FIXED_A
-                    (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fractuns<FIXED_B:mode><FIXED_A:mode>2"
   [(set (match_operand:FIXED_A 0 "register_operand" "=r")
@@ -124,10 +120,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ss_addsub:ALL124S (match_dup 1)
-                                      (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3"
   [(set (match_operand:ALL124S 0 "register_operand"                          "=??d,d")
@@ -149,10 +143,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (us_addsub:ALL124U (match_dup 1)
-                                      (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3"
   [(set (match_operand:ALL124U 0 "register_operand"                          "=??r,d")
@@ -189,9 +181,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ss_neg:QQ (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ssnegqq2"
   [(set (match_operand:QQ 0 "register_operand"            "=r")
@@ -207,9 +198,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ss_abs:QQ (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ssabsqq2"
   [(set (match_operand:QQ 0 "register_operand"            "=r")
@@ -241,9 +231,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2S 24)
-                   (ss_abs_neg:ALL2S (reg:ALL2S 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>2"
   [(set (reg:ALL2S 24)
@@ -261,9 +250,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4S 22)
-                   (ss_abs_neg:ALL4S (reg:ALL4S 22)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>2"
   [(set (reg:ALL4S 22)
@@ -296,10 +284,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:QQ (match_dup 1)
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqq3_enh"
   [(set (match_operand:QQ 0 "register_operand"         "=r")
@@ -317,10 +303,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:UQQ (match_dup 1)
-                             (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*muluqq3_enh"
   [(set (match_operand:UQQ 0 "register_operand"          "=r")
@@ -377,12 +361,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QQ 23)
-                   (mult:QQ (reg:QQ 24)
-                            (reg:QQ 25)))
-              (clobber (reg:QI 22))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqq3.call"
   [(set (reg:QQ 23)
@@ -425,11 +405,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2QA 24)
-                   (mult:ALL2QA (reg:ALL2QA 18)
-                                (reg:ALL2QA 26)))
-              (clobber (reg:HI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mul<mode>3.call"
   [(set (reg:ALL2QA 24)
@@ -468,10 +445,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4A 24)
-                   (mult:ALL4A (reg:ALL4A 16)
-                               (reg:ALL4A 20)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mul<mode>3.call"
   [(set (reg:ALL4A 24)
@@ -514,11 +489,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL1Q 24)
-                   (usdiv:ALL1Q (reg:ALL1Q 25)
-                                (reg:ALL1Q 22)))
-              (clobber (reg:QI 25))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code><mode>3.call"
   [(set (reg:ALL1Q 24)
@@ -560,12 +532,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2QA 24)
-                   (usdiv:ALL2QA (reg:ALL2QA 26)
-                                 (reg:ALL2QA 22)))
-              (clobber (reg:HI 26))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code><mode>3.call"
   [(set (reg:ALL2QA 24)
@@ -608,12 +576,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4A 22)
-                   (usdiv:ALL4A (reg:ALL4A 24)
-                                (reg:ALL4A 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code><mode>3.call"
   [(set (reg:ALL4A 22)
@@ -684,12 +648,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:ALL124QA [(match_dup 1)
-                                     (match_dup 2)
-                                     (const_int 0)]
-                                    UNSPEC_ROUND))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3_const"
   [(set (match_operand:ALL124QA 0 "register_operand"                  "=d")
@@ -714,11 +674,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL1Q 24)
-                   (unspec:ALL1Q [(reg:ALL1Q 22)
-                                  (reg:QI 24)] UNSPEC_ROUND))
-              (clobber (reg:ALL1Q 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3.libgcc"
   [(set (reg:ALL1Q 24)
@@ -740,11 +697,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2QA 24)
-                   (unspec:ALL2QA [(reg:ALL2QA 22)
-                                   (reg:QI 24)] UNSPEC_ROUND))
-              (clobber (reg:ALL2QA 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3.libgcc"
   [(set (reg:ALL2QA 24)
@@ -766,11 +720,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4QA 22)
-                   (unspec:ALL4QA [(reg:ALL4QA 18)
-                                   (reg:QI 24)] UNSPEC_ROUND))
-              (clobber (reg:ALL4QA 18))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3.libgcc"
   [(set (reg:ALL4QA 22)
diff --git a/gcc/config/avr/avr-log.cc b/gcc/config/avr/avr-log.cc
index fadb3ca..972ba6b 100644
--- a/gcc/config/avr/avr-log.cc
+++ b/gcc/config/avr/avr-log.cc
@@ -373,7 +373,6 @@ avr_log_set_avr_log (void)
       SET_DUMP_DETAIL (insn_addresses);
       SET_DUMP_DETAIL (legitimate_address_p);
       SET_DUMP_DETAIL (legitimize_address);
-      SET_DUMP_DETAIL (legitimize_reload_address);
       SET_DUMP_DETAIL (progmem);
       SET_DUMP_DETAIL (rtx_costs);
 
diff --git a/gcc/config/avr/avr-mcus.def b/gcc/config/avr/avr-mcus.def
index ad64050..2e7c8ac 100644
--- a/gcc/config/avr/avr-mcus.def
+++ b/gcc/config/avr/avr-mcus.def
@@ -313,6 +313,10 @@ AVR_MCU ("avr64da28",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR
 AVR_MCU ("avr64da32",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA32__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64da48",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA48__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64da64",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA64__",   0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da28s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA28S__",  0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da32s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA32S__",  0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da48s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA48S__",  0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da64s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA64S__",  0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64db28",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DB28__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64db32",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DB32__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64db48",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DB48__",   0x6000, 0x0, 0x10000, 0)
@@ -389,6 +393,9 @@ AVR_MCU ("avr16du32",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR
 AVR_MCU ("avr32da28",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA28__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32da32",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA32__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32da48",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA48__",   0x7000, 0x0, 0x8000, 0x8000)
+AVR_MCU ("avr32da28s",       ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA28S__",  0x7000, 0x0, 0x8000, 0x8000)
+AVR_MCU ("avr32da32s",       ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA32S__",  0x7000, 0x0, 0x8000, 0x8000)
+AVR_MCU ("avr32da48s",       ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA48S__",  0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32db28",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DB28__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32db32",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DB32__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32db48",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DB48__",   0x7000, 0x0, 0x8000, 0x8000)
@@ -427,6 +434,10 @@ AVR_MCU ("avr128da28",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR
 AVR_MCU ("avr128da32",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA32__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128da48",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA48__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128da64",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA64__",  0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da28s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA28S__", 0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da32s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA32S__", 0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da48s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA48S__", 0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da64s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA64S__", 0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128db28",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DB28__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128db32",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DB32__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128db48",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DB48__",  0x4000, 0x0, 0x20000, 0)
diff --git a/gcc/config/avr/avr-passes.cc b/gcc/config/avr/avr-passes.cc
index 284f49d..69df6d2 100644
--- a/gcc/config/avr/avr-passes.cc
+++ b/gcc/config/avr/avr-passes.cc
@@ -4120,9 +4120,8 @@ avr_optimize_casesi (rtx_insn *insns[5], rtx *xop)
   JUMP_LABEL (cbranch) = xop[4];
   ++LABEL_NUSES (xop[4]);
 
-  rtx_insn *seq1 = get_insns ();
   rtx_insn *last1 = get_last_insn ();
-  end_sequence ();
+  rtx_insn *seq1 = end_sequence ();
 
   emit_insn_after (seq1, insns[2]);
 
@@ -4141,9 +4140,8 @@ avr_optimize_casesi (rtx_insn *insns[5], rtx *xop)
 
   emit_insn (pat_4);
 
-  rtx_insn *seq2 = get_insns ();
   rtx_insn *last2 = get_last_insn ();
-  end_sequence ();
+  rtx_insn *seq2 = end_sequence ();
 
   emit_insn_after (seq2, insns[3]);
 
@@ -4845,6 +4843,137 @@ avr_pass_fuse_add::execute1 (function *func)
 
 
 //////////////////////////////////////////////////////////////////////////////
+// Fuse 2 move insns after combine.
+
+static const pass_data avr_pass_data_2moves =
+{
+  RTL_PASS,	    // type
+  "",		    // name (will be patched)
+  OPTGROUP_NONE,    // optinfo_flags
+  TV_DF_SCAN,	    // tv_id
+  0,		    // properties_required
+  0,		    // properties_provided
+  0,		    // properties_destroyed
+  0,		    // todo_flags_start
+  0		    // todo_flags_finish
+};
+
+class avr_pass_2moves : public rtl_opt_pass
+{
+public:
+  avr_pass_2moves (gcc::context *ctxt, const char *name)
+    : rtl_opt_pass (avr_pass_data_2moves, ctxt)
+  {
+    this->name = name;
+  }
+
+  unsigned int execute (function *func) final override
+  {
+    if (optimize && avropt_fuse_move2)
+      {
+	bool changed = false;
+	basic_block bb;
+
+	FOR_EACH_BB_FN (bb, func)
+	  {
+	    changed |= optimize_2moves_bb (bb);
+	  }
+
+	if (changed)
+	  {
+	    df_note_add_problem ();
+	    df_analyze ();
+	  }
+      }
+
+    return 0;
+  }
+
+  bool optimize_2moves (rtx_insn *, rtx_insn *);
+  bool optimize_2moves_bb (basic_block);
+}; // avr_pass_2moves
+
+bool
+avr_pass_2moves::optimize_2moves_bb (basic_block bb)
+{
+  bool changed = false;
+  rtx_insn *insn1 = nullptr;
+  rtx_insn *insn2 = nullptr;
+  rtx_insn *curr;
+
+  FOR_BB_INSNS (bb, curr)
+    {
+      if (insn1 && INSN_P (insn1)
+	  && insn2 && INSN_P (insn2))
+	changed |= optimize_2moves (insn1, insn2);
+
+      insn1 = insn2;
+      insn2 = curr;
+    }
+
+  return changed;
+}
+
+bool
+avr_pass_2moves::optimize_2moves (rtx_insn *insn1, rtx_insn *insn2)
+{
+  bool good = false;
+  bool bad = false;
+  rtx set1, dest1, src1;
+  rtx set2, dest2, src2;
+
+  if ((set1 = single_set (insn1))
+      && (set2 = single_set (insn2))
+      && (src1 = SET_SRC (set1))
+      && REG_P (src2 = SET_SRC (set2))
+      && REG_P (dest1 = SET_DEST (set1))
+      && REG_P (dest2 = SET_DEST (set2))
+      && rtx_equal_p (dest1, src2)
+      // Now we have:
+      // insn1: dest1 = src1
+      // insn2: dest2 = dest1
+      && REGNO (dest1) >= FIRST_PSEUDO_REGISTER
+      // Paranoia.
+      && GET_CODE (PATTERN (insn1)) != PARALLEL
+      && GET_CODE (PATTERN (insn2)) != PARALLEL
+      && (rtx_equal_p (dest2, src1)
+	  || !reg_overlap_mentioned_p (dest2, src1)))
+    {
+      avr_dump ("\n;; Found 2moves:\n%r\n%r\n", insn1, insn2);
+      avr_dump (";; reg %d: insn uses uids:", REGNO (dest1));
+
+      // Go check that dest1 is used exactly once, namely by insn2.
+
+      df_ref use = DF_REG_USE_CHAIN (REGNO (dest1));
+      for (; use; use = DF_REF_NEXT_REG (use))
+	{
+	  rtx_insn *user = DF_REF_INSN (use);
+	  avr_dump (" %d", INSN_UID (user));
+	  good |= INSN_UID (user) == INSN_UID (insn2);
+	  bad |= INSN_UID (user) != INSN_UID (insn2);
+	}
+      avr_dump (".\n");
+
+      if (good && !bad
+	  // Propagate src1 to insn2:
+	  // insn1: # Deleted
+	  // insn2: dest2 = src1
+	  && validate_change (insn2, &SET_SRC (set2), src1, false))
+	{
+	  SET_INSN_DELETED (insn1);
+	  return true;
+	}
+    }
+
+  if (good && !bad)
+    avr_dump (";; Failed\n");
+
+  return false;
+}
+
+
+
+//////////////////////////////////////////////////////////////////////////////
 // Split insns with nonzero_bits() after combine.
 
 static const pass_data avr_pass_data_split_nzb =
@@ -5706,6 +5835,14 @@ make_avr_pass_casesi (gcc::context *ctxt)
   return new avr_pass_casesi (ctxt, "avr-casesi");
 }
 
+// Optimize 2 consecutive moves after combine.
+
+rtl_opt_pass *
+make_avr_pass_2moves (gcc::context *ctxt)
+{
+  return new avr_pass_2moves (ctxt, "avr-2moves");
+}
+
 rtl_opt_pass *
 make_avr_pass_split_nzb (gcc::context *ctxt)
 {
diff --git a/gcc/config/avr/avr-passes.def b/gcc/config/avr/avr-passes.def
index eb60a93..d668c7f 100644
--- a/gcc/config/avr/avr-passes.def
+++ b/gcc/config/avr/avr-passes.def
@@ -74,6 +74,14 @@ INSERT_PASS_BEFORE (pass_free_cfg, 1, avr_pass_recompute_notes);
 
 INSERT_PASS_AFTER (pass_expand, 1, avr_pass_casesi);
 
+/* Insn combine may come up with superfluous reg-reg moves, where the combine
+   people say that these are no problem since reg-alloc is supposed to optimize
+   them.  The issue is that the lower-subreg pass sitting between combine and
+   reg-alloc may split such moves, coming up with a zoo of subregs which are
+   only handled poorly by the register allocator.  */
+
+INSERT_PASS_AFTER (pass_combine, 1, avr_pass_2moves);
+
 /* Some combine insns have nonzero_bits() in their condition, though insns
    should not use such stuff in their condition.  Therefore, we split such
    insn into something without nonzero_bits() in their condition right after
diff --git a/gcc/config/avr/avr-protos.h b/gcc/config/avr/avr-protos.h
index ca30136..8ba1945 100644
--- a/gcc/config/avr/avr-protos.h
+++ b/gcc/config/avr/avr-protos.h
@@ -146,7 +146,6 @@ extern void out_shift_with_cnt (const char *templ, rtx_insn *insn,
 extern enum reg_class avr_mode_code_base_reg_class (machine_mode, addr_space_t, rtx_code, rtx_code);
 extern bool avr_regno_mode_code_ok_for_base_p (int, machine_mode, addr_space_t, rtx_code, rtx_code);
 extern rtx avr_incoming_return_addr_rtx (void);
-extern rtx avr_legitimize_reload_address (rtx*, machine_mode, int, int, int, int, rtx (*)(rtx,int));
 extern bool avr_adiw_reg_p (rtx);
 extern bool avr_mem_flash_p (rtx);
 extern bool avr_mem_flashx_p (rtx);
@@ -168,6 +167,8 @@ regmask (machine_mode mode, unsigned regno)
 
 extern void avr_fix_inputs (rtx*, unsigned, unsigned);
 extern bool avr_emit3_fix_outputs (rtx (*)(rtx,rtx,rtx), rtx*, unsigned, unsigned);
+extern rtx avr_add_ccclobber (rtx_insn *);
+#define DONE_ADD_CCC emit (avr_add_ccclobber (curr_insn)); DONE;
 
 extern rtx lpm_reg_rtx;
 extern rtx lpm_addr_reg_rtx;
@@ -208,6 +209,7 @@ extern rtl_opt_pass *make_avr_pass_casesi (gcc::context *);
 extern rtl_opt_pass *make_avr_pass_ifelse (gcc::context *);
 extern rtl_opt_pass *make_avr_pass_split_nzb (gcc::context *);
 extern rtl_opt_pass *make_avr_pass_split_after_peephole2 (gcc::context *);
+extern rtl_opt_pass *make_avr_pass_2moves (gcc::context *);
 #ifdef RTX_CODE
 extern bool avr_casei_sequence_check_operands (rtx *xop);
 extern bool avr_split_fake_addressing_move (rtx_insn *insn, rtx *operands);
@@ -238,7 +240,6 @@ typedef struct
   unsigned insn_addresses :1;
   unsigned legitimate_address_p :1;
   unsigned legitimize_address :1;
-  unsigned legitimize_reload_address :1;
   unsigned progmem :1;
   unsigned rtx_costs :1;
 } avr_log_t;
diff --git a/gcc/config/avr/avr.cc b/gcc/config/avr/avr.cc
index c469297..ae49d4d 100644
--- a/gcc/config/avr/avr.cc
+++ b/gcc/config/avr/avr.cc
@@ -411,6 +411,29 @@ avr_to_int_mode (rtx x)
 }
 
 
+/* Return the pattern of INSN, but with added (clobber (reg:CC REG_CC)).
+   The pattern of INSN must be a PARALLEL or a SET.  INSN is unchanged.  */
+
+rtx
+avr_add_ccclobber (rtx_insn *insn)
+{
+  rtx pat = PATTERN (insn);
+  gcc_assert (GET_CODE (pat) == SET || GET_CODE (pat) == PARALLEL);
+
+  int newlen = GET_CODE (pat) == SET ? 2 : 1 + XVECLEN (pat, 0);
+  rtx newpat = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (newlen));
+  rtx elt0 = GET_CODE (pat) == SET ? pat : XVECEXP (pat, 0, 0);
+
+  XVECEXP (newpat, 0, 0) = copy_rtx (elt0);
+  XVECEXP (newpat, 0, newlen - 1) = gen_rtx_CLOBBER (VOIDmode, cc_reg_rtx);
+
+  for (int i = 1; i < newlen - 1; ++i)
+    XVECEXP (newpat, 0, i) = copy_rtx (XVECEXP (pat, 0, i));
+
+  return newpat;
+}
+
+
 /* Return true if hard register REG supports the ADIW and SBIW instructions.  */
 
 bool
@@ -430,13 +453,6 @@ avr_ld_regno_p (int regno)
 }
 
 
-static bool
-ra_in_progress ()
-{
-  return avropt_lra_p ? lra_in_progress : reload_in_progress;
-}
-
-
 /* Set `avr_arch' as specified by `-mmcu='.
    Return true on success.  */
 
@@ -2324,8 +2340,8 @@ avr_legitimate_address_p (machine_mode mode, rtx x, bool strict)
   if (avr_log.legitimate_address_p)
     {
       avr_edump ("\n%?: ret=%d, mode=%m strict=%d "
-		 "reload_completed=%d ra_in_progress=%d %s:",
-		 ok, mode, strict, reload_completed, ra_in_progress (),
+		 "reload_completed=%d lra_in_progress=%d %s:",
+		 ok, mode, strict, reload_completed, lra_in_progress,
 		 reg_renumber ? "(reg_renumber)" : "");
 
       if (GET_CODE (x) == PLUS
@@ -2395,88 +2411,6 @@ avr_legitimize_address (rtx x, rtx oldx, machine_mode mode)
 }
 
 
-/* Implement `LEGITIMIZE_RELOAD_ADDRESS'.  */
-/* This will allow register R26/27 to be used where it is no worse than normal
-   base pointers R28/29 or R30/31.  For example, if base offset is greater
-   than 63 bytes or for R++ or --R addressing.  */
-
-rtx
-avr_legitimize_reload_address (rtx *px, machine_mode mode, int opnum,
-			       int type, int addr_type, int /*ind_levels*/,
-			       rtx (*mk_memloc)(rtx,int))
-{
-  rtx x = *px;
-
-  if (avr_log.legitimize_reload_address)
-    avr_edump ("\n%?:%m %r\n", mode, x);
-
-  if (1 && (GET_CODE (x) == POST_INC
-	    || GET_CODE (x) == PRE_DEC))
-    {
-      push_reload (XEXP (x, 0), XEXP (x, 0), &XEXP (x, 0), &XEXP (x, 0),
-		   POINTER_REGS, GET_MODE (x), GET_MODE (x), 0, 0,
-		   opnum, RELOAD_OTHER);
-
-      if (avr_log.legitimize_reload_address)
-	avr_edump (" RCLASS.1 = %R\n IN = %r\n OUT = %r\n",
-		   POINTER_REGS, XEXP (x, 0), XEXP (x, 0));
-
-      return x;
-    }
-
-  if (GET_CODE (x) == PLUS
-      && REG_P (XEXP (x, 0))
-      && reg_equiv_constant (REGNO (XEXP (x, 0))) == 0
-      && CONST_INT_P (XEXP (x, 1))
-      && INTVAL (XEXP (x, 1)) >= 1)
-    {
-      bool fit = INTVAL (XEXP (x, 1)) <= MAX_LD_OFFSET (mode);
-
-      if (fit)
-	{
-	  if (reg_equiv_address (REGNO (XEXP (x, 0))) != 0)
-	    {
-	      int regno = REGNO (XEXP (x, 0));
-	      rtx mem = mk_memloc (x, regno);
-
-	      push_reload (XEXP (mem, 0), NULL_RTX, &XEXP (mem, 0), NULL,
-			   POINTER_REGS, Pmode, VOIDmode, 0, 0,
-			   1, (enum reload_type) addr_type);
-
-	      if (avr_log.legitimize_reload_address)
-		avr_edump (" RCLASS.2 = %R\n IN = %r\n OUT = %r\n",
-			   POINTER_REGS, XEXP (mem, 0), NULL_RTX);
-
-	      push_reload (mem, NULL_RTX, &XEXP (x, 0), NULL,
-			   BASE_POINTER_REGS, GET_MODE (x), VOIDmode, 0, 0,
-			   opnum, (enum reload_type) type);
-
-	      if (avr_log.legitimize_reload_address)
-		avr_edump (" RCLASS.2 = %R\n IN = %r\n OUT = %r\n",
-			   BASE_POINTER_REGS, mem, NULL_RTX);
-
-	      return x;
-	    }
-	}
-      else if (! (frame_pointer_needed
-		  && XEXP (x, 0) == frame_pointer_rtx))
-	{
-	  push_reload (x, NULL_RTX, px, NULL,
-		       POINTER_REGS, GET_MODE (x), VOIDmode, 0, 0,
-		       opnum, (enum reload_type) type);
-
-	  if (avr_log.legitimize_reload_address)
-	    avr_edump (" RCLASS.3 = %R\n IN = %r\n OUT = %r\n",
-		       POINTER_REGS, x, NULL_RTX);
-
-	  return x;
-	}
-    }
-
-  return NULL_RTX;
-}
-
-
 /* Helper function to print assembler resp. track instruction
    sequence lengths.  Always return "".
 
@@ -12824,6 +12758,16 @@ avr_rtx_costs_1 (rtx x, machine_mode mode, int outer_code,
       return true;
 
     case SIGN_EXTEND:
+      if (GET_CODE (XEXP (x, 0)) == ASHIFT
+	  && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
+	{
+	  // "*sext.ashift<QIPSI:mode><HISI:mode>2_split"
+	  int m0 = GET_MODE_SIZE (GET_MODE (XEXP (x, 0)));
+	  int m1 = GET_MODE_SIZE (mode);
+	  *total = COSTS_N_INSNS (m0 * INTVAL (XEXP (XEXP (x, 0), 1))
+				  + m1 - m0);
+	  return true;
+	}
       *total = COSTS_N_INSNS (n_bytes + 2
 			      - GET_MODE_SIZE (GET_MODE (XEXP (x, 0))));
       *total += avr_operand_rtx_cost (XEXP (x, 0), GET_MODE (XEXP (x, 0)),
@@ -13936,8 +13880,8 @@ extra_constraint_Q (rtx x)
 	    || xx == arg_pointer_rtx);
 
       if (avr_log.constraints)
-	avr_edump ("\n%?=%d reload_completed=%d ra_in_progress=%d\n %r\n",
-		   ok, reload_completed, ra_in_progress (), x);
+	avr_edump ("\n%?=%d reload_completed=%d lra_in_progress=%d\n %r\n",
+		   ok, reload_completed, lra_in_progress, x);
     }
 
   return ok;
@@ -14142,17 +14086,6 @@ avr_hard_regno_mode_ok (unsigned int regno, machine_mode mode)
   if (GET_MODE_SIZE (mode) == 1)
     return true;
 
-  /* FIXME: Ideally, the following test is not needed.
-	However, it turned out that it can reduce the number
-	of spill fails.  AVR and it's poor endowment with
-	address registers is extreme stress test for reload.  */
-
-  if (GET_MODE_SIZE (mode) >= 4
-      && regno + GET_MODE_SIZE (mode) >= REG_30
-      // This problem only concerned the old reload.
-      && ! avropt_lra_p)
-    return false;
-
   /* All modes larger than 8 bits should start in an even register.  */
 
   return !(regno & 1);
@@ -14418,6 +14351,13 @@ avr_output_addr_vec (rtx_insn *labl, rtx table)
   // Output the label that precedes the table.
 
   ASM_OUTPUT_ALIGN (stream, 1);
+
+  char s_labl[40];
+  targetm.asm_out.generate_internal_label (s_labl, "L",
+					   CODE_LABEL_NUMBER (labl));
+  ASM_OUTPUT_TYPE_DIRECTIVE (stream, s_labl,
+			     AVR_HAVE_JMP_CALL ? "object" : "function");
+
   targetm.asm_out.internal_label (stream, "L", CODE_LABEL_NUMBER (labl));
 
   // Output the table's content.
@@ -14907,8 +14847,8 @@ avr_addr_space_legitimate_address_p (machine_mode mode, rtx x, bool strict,
   if (avr_log.legitimate_address_p)
     {
       avr_edump ("\n%?: ret=%b, mode=%m strict=%d "
-		 "reload_completed=%d ra_in_progress=%d %s:",
-		 ok, mode, strict, reload_completed, ra_in_progress (),
+		 "reload_completed=%d lra_in_progress=%d %s:",
+		 ok, mode, strict, reload_completed, lra_in_progress,
 		 reg_renumber ? "(reg_renumber)" : "");
 
       if (GET_CODE (x) == PLUS
@@ -14984,10 +14924,11 @@ avr_addr_space_convert (rtx src, tree type_old, tree type_new)
 
       /* Linearize memory: RAM has bit 23 set.  When as_new = __flashx then
 	 this is basically UB since __flashx mistreats RAM addresses, but there
-	 is no way to bail out.  (Though -Waddr-space-convert will tell.)  */
+	 is no way to bail out.  (Though -Waddr-space-convert will tell.)
+	 ...but PR121277 is confusing, in particular when NULL is coming in. */
 
       int msb = ADDR_SPACE_GENERIC_P (as_old)
-	? 0x80
+	? as_new == ADDR_SPACE_MEMX ? 0x80 : 0x00
 	: avr_addrspace[as_old].segment;
 
       src = force_reg (Pmode, src);
@@ -15085,10 +15026,16 @@ avr_convert_to_type (tree type, tree expr)
 	  const char *name_old = avr_addrspace[as_old].name;
 	  const char *name_new = avr_addrspace[as_new].name;
 
-	  warning (OPT_Waddr_space_convert,
-		   "conversion from address space %qs to address space %qs",
-		   ADDR_SPACE_GENERIC_P (as_old) ? "generic" : name_old,
-		   ADDR_SPACE_GENERIC_P (as_new) ? "generic" : name_new);
+	  // Be relaxed when NULL is used, and when 0x0 stands for
+	  // address 0x0.
+	  bool nowarn = (expr == null_pointer_node
+			 && (as_new == ADDR_SPACE_FLASHX
+			     || as_new == ADDR_SPACE_FLASH));
+	  if (!nowarn)
+	    warning (OPT_Waddr_space_convert,
+		     "conversion from address space %qs to address space %qs",
+		     ADDR_SPACE_GENERIC_P (as_old) ? "generic" : name_old,
+		     ADDR_SPACE_GENERIC_P (as_new) ? "generic" : name_new);
 
 	  return fold_build1_loc (loc, ADDR_SPACE_CONVERT_EXPR, type, expr);
 	}
@@ -16679,15 +16626,6 @@ avr_unwind_word_mode ()
   return Pmode;
 }
 
-
-/* Implement `TARGET_LRA_P'.  */
-
-static bool
-avr_use_lra_p ()
-{
-  return avropt_lra_p;
-}
-
 
 
 /* Initialize the GCC target structure.  */
@@ -16829,9 +16767,6 @@ avr_use_lra_p ()
 #undef  TARGET_CONVERT_TO_TYPE
 #define TARGET_CONVERT_TO_TYPE avr_convert_to_type
 
-#undef TARGET_LRA_P
-#define TARGET_LRA_P avr_use_lra_p
-
 #undef  TARGET_ADDR_SPACE_SUBSET_P
 #define TARGET_ADDR_SPACE_SUBSET_P avr_addr_space_subset_p
 
diff --git a/gcc/config/avr/avr.h b/gcc/config/avr/avr.h
index cb818c3..335f9fa5 100644
--- a/gcc/config/avr/avr.h
+++ b/gcc/config/avr/avr.h
@@ -309,12 +309,6 @@ enum reg_class {
 
 #define STATIC_CHAIN_REGNUM ((AVR_TINY) ? 18 :2)
 
-#define RELOAD_ELIMINABLE_REGS {				\
-    { ARG_POINTER_REGNUM, STACK_POINTER_REGNUM },               \
-    { ARG_POINTER_REGNUM, FRAME_POINTER_REGNUM },               \
-    { FRAME_POINTER_REGNUM, STACK_POINTER_REGNUM },             \
-    { FRAME_POINTER_REGNUM + 1, STACK_POINTER_REGNUM + 1 } }
-
 #define ELIMINABLE_REGS						\
   {								\
     { ARG_POINTER_REGNUM, STACK_POINTER_REGNUM },		\
@@ -358,18 +352,6 @@ typedef struct avr_args
 
 #define MAX_REGS_PER_ADDRESS 1
 
-#define LEGITIMIZE_RELOAD_ADDRESS(X,MODE,OPNUM,TYPE,IND_L,WIN)          \
-  do {                                                                  \
-    rtx new_x = avr_legitimize_reload_address (&(X), MODE, OPNUM, TYPE, \
-                                               ADDR_TYPE (TYPE),        \
-                                               IND_L, make_memloc);     \
-    if (new_x)                                                          \
-      {                                                                 \
-        X = new_x;                                                      \
-        goto WIN;                                                       \
-      }                                                                 \
-  } while (0)
-
 /* We increase branch costs after reload in order to keep basic-block
    reordering from introducing out-of-line jumps and to prefer fall-through
    edges instead.  The default branch costs are 0, mainly because otherwise
diff --git a/gcc/config/avr/avr.md b/gcc/config/avr/avr.md
index f8bbdc7..60b1f60 100644
--- a/gcc/config/avr/avr.md
+++ b/gcc/config/avr/avr.md
@@ -574,9 +574,8 @@
    && REG_Z == REGNO (XEXP (operands[0], 0))"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:MOVMODE 22)
-                   (match_dup 0))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*load_<mode>_libgcc"
   [(set (reg:MOVMODE 22)
@@ -716,14 +715,8 @@
     || avr_load_libgcc_insn_p (insn, ADDR_SPACE_FLASHX, true)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:MOVMODE REG_22)
-                   (match_dup 0))
-              (clobber (reg:QI REG_21))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:CC REG_CC))])]
-  {
-    operands[0] = SET_SRC (single_set (curr_insn));
-  })
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fxload_<mode>_libgcc"
   [(set (reg:MOVMODE REG_22)
@@ -853,9 +846,8 @@
     || reg_or_0_operand (operands[1], <MODE>mode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; "movqi_insn"
 ;; "movqq_insn" "movuqq_insn"
@@ -964,9 +956,8 @@
    || reg_or_0_operand (operands[1], <MODE>mode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mov<mode>"
   [(set (match_operand:ALL2 0 "nonimmediate_operand" "=r,r  ,r,m    ,d,*r,q,r")
@@ -1137,9 +1128,8 @@
    || const0_rtx == operands[1]"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*movpsi"
   [(set (match_operand:PSI 0 "nonimmediate_operand" "=r,r,r ,Qm,!d,r")
@@ -1197,9 +1187,8 @@
    || reg_or_0_operand (operands[1], <MODE>mode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mov<mode>"
   [(set (match_operand:ALL4 0 "nonimmediate_operand" "=r,r  ,r ,Qm   ,!d,r")
@@ -1245,9 +1234,8 @@
    || reg_or_0_operand (operands[1], SFmode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*movsf"
   [(set (match_operand:SF 0 "nonimmediate_operand" "=r,r,r ,Qm,!d,r")
@@ -1326,16 +1314,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (reg:HI REG_X))
-                   (mem:BLK (reg:HI REG_Z)))
-              (unspec [(match_dup 0)]
-                      UNSPEC_CPYMEM)
-              (use (match_dup 1))
-              (clobber (reg:HI REG_X))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:QI LPM_REGNO))
-              (clobber (match_dup 2))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*cpymem_<mode>"
   [(set (mem:BLK (reg:HI REG_X))
@@ -1382,22 +1362,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (reg:HI REG_X))
-                   (match_dup 2))
-              (unspec [(match_dup 0)]
-                      UNSPEC_CPYMEM)
-              (use (reg:QIHI 24))
-              (clobber (reg:HI REG_X))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:QI LPM_REGNO))
-              (clobber (reg:HI 24))
-              (clobber (reg:QI 23))
-              (clobber (mem:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])]
-  {
-    rtx xset = XVECEXP (PATTERN (curr_insn), 0, 0);
-    operands[2] = SET_SRC (xset);
-  })
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*cpymemx_<mode>"
   [(set (mem:BLK (reg:HI REG_X))
@@ -1461,13 +1427,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (match_dup 0))
-                   (const_int 0))
-              (use (match_dup 1))
-              (use (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*clrmemqi"
   [(set (mem:BLK (match_operand:HI 0 "register_operand" "e"))
@@ -1492,14 +1453,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (match_dup 0))
-                   (const_int 0))
-              (use (match_dup 1))
-              (use (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "adiw,*")])
 
 
@@ -1550,13 +1505,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel
-      [(set (match_dup 0)
-            (unspec:HI [(mem:BLK (match_dup 1))
-                        (const_int 0)
-                        (match_dup 2)]
-                       UNSPEC_STRLEN))
-       (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*strlenhi"
   [(set (match_operand:HI 0 "register_operand"                      "=e")
@@ -1581,10 +1531,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:ALL1 (match_dup 1)
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"            "=r,d    ,r  ,r  ,r  ,r")
@@ -1640,10 +1588,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (zero_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend"
   [(set (match_operand:HI 0 "register_operand"                         "=r,*?r")
@@ -1663,10 +1609,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (match_dup 1)
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend1"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -1684,10 +1628,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (zero_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend.const"
   [(set (match_operand:HI 0 "register_operand"                         "=d")
@@ -1723,11 +1665,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (ashift:HI (zero_extend:HI (match_dup 1))
-                                       (const_int 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend.ashift1"
   [(set (match_operand:HI 0 "register_operand"                                    "=r")
@@ -1752,11 +1691,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (zero_extend:HI (match_dup 1))
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
-
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*usum_widenqihi3"
   [(set (match_operand:HI 0 "register_operand"                          "=r")
@@ -1774,10 +1710,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HI (zero_extend:HI (match_dup 1))
-                             (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udiff_widenqihi3"
   [(set (match_operand:HI 0 "register_operand"                           "=r")
@@ -1797,7 +1731,7 @@
     return avr_out_addto_sp (operands, NULL);
   }
   ""
-  [(const_int 0)]
+  [(scratch)]
   {
     // Do not attempt to split this pattern. This FAIL is necessary
     // to prevent the splitter from matching *add<ALL2>3_split, splitting
@@ -1909,11 +1843,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:ALL2 (match_dup 1)
-                              (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; "*addhi3_clobber"
 ;; "*addhq3_clobber"  "*adduhq3_clobber"
@@ -1943,11 +1874,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:ALL4 (match_dup 1)
-                              (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3"
   [(set (match_operand:ALL4 0 "register_operand"          "=??r,d ,r")
@@ -1979,10 +1907,8 @@
    && (<HISI:SIZE> > 2 || <CODE> == SIGN_EXTEND)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HISI (any_extend:HISI (match_dup 1))
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;;                            "*addhi3.sign_extend.qi"
 ;; "*addpsi3.zero_extend.qi"  "*addpsi3.sign_extend.qi"
@@ -2019,10 +1945,8 @@
   "<HISI:SIZE> > <QIPSI:SIZE>"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HISI (match_dup 1)
-                               (any_extend:HISI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; "*subhi3.zero_extend.qi"   "*subhi3.sign_extend.qi"
 ;; "*subpsi3.zero_extend.qi"  "*subpsi3.sign_extend.qi"
@@ -2053,11 +1977,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:PSI (match_dup 1)
-                             (match_dup 2)))
-              (clobber (match_dup 3 ))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addpsi3"
   [(set (match_operand:PSI 0 "register_operand"         "=??r,d ,d,r")
@@ -2079,10 +2000,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:PSI (match_dup 1)
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*subpsi3"
   [(set (match_operand:PSI 0 "register_operand"           "=r")
@@ -2106,10 +2025,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:ALL1 (match_dup 1)
-                               (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"                    "=??r,d    ,r  ,r  ,r  ,r")
@@ -2137,11 +2054,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:ALL2 (match_dup 1)
-                               (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3"
   [(set (match_operand:ALL2 0 "register_operand"                    "=??r,d    ,*r")
@@ -2167,11 +2081,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:ALL4 (match_dup 1)
-                               (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3"
   [(set (match_operand:ALL4 0 "register_operand"                    "=??r,d    ,r")
@@ -2209,10 +2120,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:QI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqi3_enh"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -2243,10 +2152,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24)
-                   (mult:QI (reg:QI 24) (reg:QI 22)))
-              (clobber (reg:QI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqi3_call"
   [(set (reg:QI 24)
@@ -2269,12 +2176,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (truncate:QI
-                    (lshiftrt:HI (mult:HI (any_extend:HI (match_dup 1))
-                                          (any_extend:HI (match_dup 2)))
-                                 (const_int 8))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_su>mulqi3_highpart"
   [(set (match_operand:QI 0 "register_operand"                                       "=r")
@@ -2361,21 +2264,21 @@
                                         (const_int 0))))
               (clobber (reg:CC REG_CC))])])
 
-;; *subqi3.lt0   *subqi3.ge0
-;; *subhi3.lt0   *subhi3.ge0
-;; *subpsi3.lt0  *subpsi3.ge0
-;; *subsi3.lt0   *subsi3.ge0
-(define_insn "*sub<QISI:mode>3.<code>0"
-  [(set (match_operand:QISI 0 "register_operand"                        "=r")
-        (minus:QISI (match_operand:QISI 1 "register_operand"             "0")
-                    (gelt:QISI (match_operand:QISI2 2 "register_operand" "r")
-                               (const_int 0))))
-   (clobber (reg:CC REG_CC))]
-  "reload_completed"
-  {
-    return avr_out_add_msb (insn, operands, <CODE>, nullptr);
-  }
-  [(set_attr "adjust_len" "add_<code>0")])
+;; *addqi3.lt0_split   *addqi3.ge0_split
+;; *addhi3.lt0_split   *addhi3.ge0_split
+;; *addpsi3.lt0_split  *addpsi3.ge0_split
+;; *addsi3.lt0_split   *addsi3.ge0_split
+(define_insn_and_split "*add<QISI:mode>3.<code>0_split"
+  [(set (match_operand:QISI 0 "register_operand"                       "=r")
+        (plus:QISI (gelt:QISI (match_operand:QISI2 1 "register_operand" "r")
+                              (const_int 0))
+                   (match_operand:QISI 2 "register_operand"             "0")))]
+  ""
+  "#"
+  "&& reload_completed"
+  ; *add<QISI:mode>3.<code>0
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; *addqi3.lt0   *addqi3.ge0
 ;; *addhi3.lt0   *addhi3.ge0
@@ -2393,25 +2296,6 @@
   }
   [(set_attr "adjust_len" "add_<code>0")])
 
-;; *addqi3.lt0_split   *addqi3.ge0_split
-;; *addhi3.lt0_split   *addhi3.ge0_split
-;; *addpsi3.lt0_split  *addpsi3.ge0_split
-;; *addsi3.lt0_split   *addsi3.ge0_split
-(define_insn_and_split "*add<QISI:mode>3.<code>0_split"
-  [(set (match_operand:QISI 0 "register_operand"                       "=r")
-        (plus:QISI (gelt:QISI (match_operand:QISI2 1 "register_operand" "r")
-                              (const_int 0))
-                   (match_operand:QISI 2 "register_operand"             "0")))]
-  ""
-  "#"
-  "&& reload_completed"
-  [; *add<QISI:mode>3.<code>0
-   (parallel [(set (match_dup 0)
-                   (plus:QISI (gelt:QISI (match_dup 1)
-                                         (const_int 0))
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
-
 ;; *subqi3.lt0_split   *subqi3.ge0_split
 ;; *subhi3.lt0_split   *subhi3.ge0_split
 ;; *subpsi3.lt0_split  *subpsi3.ge0_split
@@ -2424,13 +2308,25 @@
   ""
   "#"
   "&& reload_completed"
-  [; *sub<QISI:mode>3.<code>0
-   (parallel [(set (match_dup 0)
-                   (minus:QISI (match_dup 1)
-                               (gelt:QISI (match_dup 2)
-                                          (const_int 0))))
-              (clobber (reg:CC REG_CC))])])
+  ; *sub<QISI:mode>3.<code>0
+  [(scratch)]
+  { DONE_ADD_CCC })
 
+;; *subqi3.lt0   *subqi3.ge0
+;; *subhi3.lt0   *subhi3.ge0
+;; *subpsi3.lt0  *subpsi3.ge0
+;; *subsi3.lt0   *subsi3.ge0
+(define_insn "*sub<QISI:mode>3.<code>0"
+  [(set (match_operand:QISI 0 "register_operand"                        "=r")
+        (minus:QISI (match_operand:QISI 1 "register_operand"             "0")
+                    (gelt:QISI (match_operand:QISI2 2 "register_operand" "r")
+                               (const_int 0))))
+   (clobber (reg:CC REG_CC))]
+  "reload_completed"
+  {
+    return avr_out_add_msb (insn, operands, <CODE>, nullptr);
+  }
+  [(set_attr "adjust_len" "add_<code>0")])
 
 (define_insn_and_split "*umulqihi3.call_split"
   [(set (reg:HI 24)
@@ -2441,12 +2337,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (mult:HI (zero_extend:HI (reg:QI 22))
-                            (zero_extend:HI (reg:QI 24))))
-              (clobber (reg:QI 21))
-              (clobber (reg:HI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*umulqihi3.call"
   [(set (reg:HI 24)
@@ -2469,10 +2361,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (any_extend:HI (match_dup 1))
-                            (any_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "<extend_u>mulqihi3"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -2492,10 +2382,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (zero_extend:HI (match_dup 1))
-                            (sign_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*usmulqihi3"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -2517,10 +2405,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (sign_extend:HI (match_dup 1))
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sumulqihi3"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -2542,10 +2428,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (not:HI (zero_extend:HI (not:QI (match_dup 1))))
-                            (sign_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*osmulqihi3"
   [(set (match_operand:HI 0 "register_operand"                                        "=&r")
@@ -2566,10 +2450,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (not:HI (zero_extend:HI (not:QI (match_dup 1))))
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*oumulqihi3"
   [(set (match_operand:HI 0 "register_operand"                                        "=&r")
@@ -2596,11 +2478,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:QI (mult:QI (match_dup 1)
-                                     (match_dup 2))
-                            (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*maddqi4"
   [(set (match_operand:QI 0 "register_operand"                  "=r")
@@ -2622,11 +2501,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:QI (match_dup 3)
-                             (mult:QI (match_dup 1)
-                                      (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*msubqi4"
   [(set (match_operand:QI 0 "register_operand"                   "=r")
@@ -2705,11 +2581,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (mult:HI (any_extend:HI (match_dup 1))
-                                     (any_extend:HI (match_dup 2)))
-                            (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>maddqihi4"
   [(set (match_operand:HI 0 "register_operand"                                  "=r")
@@ -2734,11 +2607,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HI (match_dup 3)
-                             (mult:HI (any_extend:HI (match_dup 1))
-                                      (any_extend:HI (match_dup 2)))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>msubqihi4"
   [(set (match_operand:HI 0 "register_operand"                                  "=r")
@@ -2765,11 +2635,8 @@
    && <any_extend:CODE> != <any_extend2:CODE>"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (mult:HI (any_extend:HI  (match_dup 1))
-                                     (any_extend2:HI (match_dup 2)))
-                            (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<any_extend:extend_su><any_extend2:extend_su>msubqihi4"
   [(set (match_operand:HI 0 "register_operand"                                  "=r")
@@ -2800,11 +2667,8 @@
    && <any_extend:CODE> != <any_extend2:CODE>"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HI (match_dup 3)
-                             (mult:HI (any_extend:HI  (match_dup 1))
-                                      (any_extend2:HI (match_dup 2)))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<any_extend:extend_su><any_extend2:extend_su>msubqihi4"
   [(set (match_operand:HI 0 "register_operand"                                   "=r")
@@ -3072,16 +2936,14 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:HI (sign_extend:HI (match_dup 1))
-                              (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ashiftqihi2.signx.1"
   [(set (match_operand:HI 0 "register_operand"                           "=r,*r")
         (ashift:HI (sign_extend:HI (match_operand:QI 1 "register_operand" "0,r"))
                    (const_int 1)))
-   (clobber (reg:CC REG_CC)) ]
+   (clobber (reg:CC REG_CC))]
   "reload_completed"
   "@
 	lsl %A0\;sbc %B0,%B0
@@ -3142,6 +3004,41 @@
     operands[2] = gen_int_mode (1 << INTVAL (operands[2]), QImode);
   })
 
+(define_insn_and_split "*sext.ashift<QIPSI:mode><HISI:mode>2_split"
+  [(set (match_operand:HISI 0 "register_operand"                                 "=r")
+        (sign_extend:HISI (ashift:QIPSI (match_operand:QIPSI 1 "register_operand" "0")
+                                        (match_operand:QI 2 "const_int_operand" "PKC03"))))]
+  "<HISI:SIZE> > <QIPSI:SIZE>
+   && IN_RANGE (INTVAL (operands[2]), 1, 2 + (<QIPSI:SIZE> <= 2))"
+  "#"
+  "&& reload_completed"
+  [(scratch)]
+  { DONE_ADD_CCC })
+
+(define_insn "*sext.ashift<QIPSI:mode><HISI:mode>2"
+  [(set (match_operand:HISI 0 "register_operand"                                 "=r")
+        (sign_extend:HISI (ashift:QIPSI (match_operand:QIPSI 1 "register_operand" "0")
+                                        (match_operand:QI 2 "const_int_operand" "PKC03"))))
+   (clobber (reg:CC REG_CC))]
+  "reload_completed
+   && <HISI:SIZE> > <QIPSI:SIZE>
+   && IN_RANGE (INTVAL (operands[2]), 1, 2 + (<QIPSI:SIZE> <= 2))"
+  {
+    const int regno = REGNO (operands[0]);
+    // The shift.
+    for (int s = 0; s < (int) INTVAL (operands[2]); ++s)
+      for (int b = 0; b < <QIPSI:SIZE>; ++b)
+        output_asm_insn (b == 0 ? "lsl %0" : "rol %0",
+                         &all_regs_rtx[regno + b]);
+    // Sign-extend can use carry.
+    for (int b = <QIPSI:SIZE>; b < <HISI:SIZE>; ++b)
+      output_asm_insn ("sbc %0,%0", &all_regs_rtx[regno + b]);
+    return "";
+  }
+  [(set (attr "length")
+        (plus (symbol_ref "<QIPSI:SIZE> * INTVAL (operands[2])")
+              (symbol_ref "<HISI:SIZE> - <QIPSI:SIZE>")))])
+
 ;******************************************************************************
 ; mul HI: $1 = sign-/zero-/one-extend, $2 = reg
 ;******************************************************************************
@@ -3153,10 +3050,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (sign_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulsqihi3"
   [(set (match_operand:HI 0 "register_operand"                        "=&r")
@@ -3178,10 +3073,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (zero_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*muluqihi3"
   [(set (match_operand:HI 0 "register_operand"                        "=&r")
@@ -3205,10 +3098,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (not:HI (zero_extend:HI (not:QI (match_dup 1))))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*muloqihi3"
   [(set (match_operand:HI 0 "register_operand"                                        "=&r")
@@ -3277,10 +3168,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulhi3_enh"
   [(set (match_operand:HI 0 "register_operand" "=&r")
@@ -3319,11 +3208,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (mult:HI (reg:HI 24) (reg:HI 22)))
-              (clobber (reg:HI 22))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulhi3_call"
   [(set (reg:HI 24)
@@ -3719,11 +3605,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (reg:SI 22)
-                            (reg:SI 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn_and_split "*mulsi3_call_pr118012_split"
   [(set (reg:SI 22)
@@ -3737,13 +3620,8 @@
    && ! AVR_TINY"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (reg:SI 22)
-                            (reg:SI 18)))
-              (clobber (reg:SI 18))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulsi3_call"
   [(set (reg:SI 22)
@@ -3779,10 +3657,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (any_extend:SI (reg:HI 18))
-                            (any_extend:SI (reg:HI 26))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>mulhisi3_call"
   [(set (reg:SI 22)
@@ -3804,12 +3680,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (lshiftrt:SI (mult:SI (any_extend:SI (reg:HI 18))
-                                                      (any_extend:SI (reg:HI 26)))
-                                             (const_int 16))))
-              (clobber (reg:HI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_su>mulhi3_highpart_call"
   [(set (reg:HI 24)
@@ -3829,10 +3701,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (zero_extend:SI (reg:HI 18))
-                            (sign_extend:SI (reg:HI 26))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*usmulhisi3_call"
   [(set (reg:SI 22)
@@ -3850,10 +3720,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (any_extend:SI (reg:HI 26))
-                            (reg:SI 18)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mul<extend_su>hisi3_call"
   [(set (reg:SI 22)
@@ -3871,10 +3739,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (not:SI (zero_extend:SI (not:HI (reg:HI 26))))
-                            (reg:SI 18)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulohisi3_call"
   [(set (reg:SI 22)
@@ -3925,11 +3791,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24) (div:QI (reg:QI 24) (reg:QI 22)))
-              (set (reg:QI 25) (mod:QI (reg:QI 24) (reg:QI 22)))
-              (clobber (reg:QI 22))
-              (clobber (reg:QI 23))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodqi4_call"
   [(set (reg:QI 24) (div:QI (reg:QI 24) (reg:QI 22)))
@@ -3969,10 +3832,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24) (udiv:QI (reg:QI 24) (reg:QI 22)))
-              (set (reg:QI 25) (umod:QI (reg:QI 24) (reg:QI 22)))
-              (clobber (reg:QI 23))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodqi4_call"
   [(set (reg:QI 24) (udiv:QI (reg:QI 24) (reg:QI 22)))
@@ -4013,11 +3874,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22) (div:HI (reg:HI 24) (reg:HI 22)))
-              (set (reg:HI 24) (mod:HI (reg:HI 24) (reg:HI 22)))
-              (clobber (reg:HI 26))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodhi4_call"
   [(set (reg:HI 22) (div:HI (reg:HI 24) (reg:HI 22)))
@@ -4059,11 +3917,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22) (udiv:HI (reg:HI 24) (reg:HI 22)))
-              (set (reg:HI 24) (umod:HI (reg:HI 24) (reg:HI 22)))
-              (clobber (reg:HI 26))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodhi4_call"
   [(set (reg:HI 22) (udiv:HI (reg:HI 24) (reg:HI 22)))
@@ -4112,10 +3967,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:PSI (zero_extend:PSI (match_dup 1))
-                             (zero_extend:PSI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*umulqihipsi3"
   [(set (match_operand:PSI 0 "register_operand"                         "=&r")
@@ -4134,31 +3987,17 @@
 
 (define_insn_and_split "*umulhiqipsi3_split"
   [(set (match_operand:PSI 0 "register_operand"                         "=&r")
-        (mult:PSI (zero_extend:PSI (match_operand:HI 2 "register_operand" "r"))
-                  (zero_extend:PSI (match_operand:QI 1 "register_operand" "r"))))]
+        (mult:PSI (zero_extend:PSI (match_operand:HI 1 "register_operand" "r"))
+                  (zero_extend:PSI (match_operand:QI 2 "register_operand" "r"))))]
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
+  ; "*umulqihipsi3"
   [(parallel [(set (match_dup 0)
                    (mult:PSI (zero_extend:PSI (match_dup 2))
                              (zero_extend:PSI (match_dup 1))))
               (clobber (reg:CC REG_CC))])])
 
-(define_insn "*umulhiqipsi3"
-  [(set (match_operand:PSI 0 "register_operand"                         "=&r")
-        (mult:PSI (zero_extend:PSI (match_operand:HI 2 "register_operand" "r"))
-                  (zero_extend:PSI (match_operand:QI 1 "register_operand" "r"))))
-   (clobber (reg:CC REG_CC))]
-  "AVR_HAVE_MUL && reload_completed"
-  "mul %1,%A2
-	movw %A0,r0
-	mul %1,%B2
-	add %B0,r0
-	mov %C0,r1
-	clr __zero_reg__
-	adc %C0,__zero_reg__"
-  [(set_attr "length" "7")])
-
 (define_expand "mulsqipsi3"
   [(parallel [(set (match_operand:PSI 0 "pseudo_register_operand" "")
                    (mult:PSI (sign_extend:PSI (match_operand:QI 1 "pseudo_register_operand" ""))
@@ -4229,10 +4068,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 18)
-                   (mult:PSI (sign_extend:PSI (reg:QI 25))
-                             (reg:PSI 22)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulsqipsi3.libgcc"
   [(set (reg:PSI 18)
@@ -4253,13 +4090,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 22)
-                   (mult:PSI (reg:PSI 22)
-                             (reg:PSI 18)))
-              (clobber (reg:QI 21))
-              (clobber (reg:QI 25))
-              (clobber (reg:HI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulpsi3.libgcc"
   [(set (reg:PSI 22)
@@ -4311,12 +4143,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 22) (div:PSI (reg:PSI 22) (reg:PSI 18)))
-              (set (reg:PSI 18) (mod:PSI (reg:PSI 22) (reg:PSI 18)))
-              (clobber (reg:QI 21))
-              (clobber (reg:QI 25))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodpsi4_call"
   [(set (reg:PSI 22) (div:PSI (reg:PSI 22) (reg:PSI 18)))
@@ -4360,12 +4188,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 22) (udiv:PSI (reg:PSI 22) (reg:PSI 18)))
-              (set (reg:PSI 18) (umod:PSI (reg:PSI 22) (reg:PSI 18)))
-              (clobber (reg:QI 21))
-              (clobber (reg:QI 25))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodpsi4_call"
   [(set (reg:PSI 22) (udiv:PSI (reg:PSI 22) (reg:PSI 18)))
@@ -4411,11 +4235,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 18) (div:SI (reg:SI 22) (reg:SI 18)))
-              (set (reg:SI 22) (mod:SI (reg:SI 22) (reg:SI 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodsi4_call"
   [(set (reg:SI 18) (div:SI (reg:SI 22) (reg:SI 18)))
@@ -4458,11 +4279,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 18) (udiv:SI (reg:SI 22) (reg:SI 18)))
-              (set (reg:SI 22) (umod:SI (reg:SI 22) (reg:SI 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodsi4_call"
   [(set (reg:SI 18) (udiv:SI (reg:SI 22) (reg:SI 18)))
@@ -4484,10 +4302,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:QI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andqi3"
   [(set (match_operand:QI 0 "register_operand"       "=??r,d,*l ,r")
@@ -4511,11 +4327,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:HI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andhi3"
   [(set (match_operand:HI 0 "register_operand"       "=??r,d,d,r  ,r  ,r")
@@ -4545,11 +4358,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:PSI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andpsi3"
   [(set (match_operand:PSI 0 "register_operand"        "=??r,d,r  ,r  ,r")
@@ -4580,11 +4390,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:SI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andsi3"
   [(set (match_operand:SI 0 "register_operand"       "=??r,d,r  ,r  ,r")
@@ -4634,10 +4441,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:QI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorqi3"
   [(set (match_operand:QI 0 "register_operand"       "=??r,d,*l")
@@ -4659,11 +4464,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:HI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorhi3"
   [(set (match_operand:HI 0 "register_operand"       "=??r,d,d,r  ,r")
@@ -4691,11 +4493,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:PSI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorpsi3"
   [(set (match_operand:PSI 0 "register_operand"        "=??r,d,r  ,r")
@@ -4723,11 +4522,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:SI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorsi3"
   [(set (match_operand:SI 0 "register_operand"       "=??r,d,r  ,r")
@@ -4758,10 +4554,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:QI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorqi3"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -4780,11 +4574,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:HI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorhi3"
   [(set (match_operand:HI 0 "register_operand"       "=??r,r  ,d  ,r")
@@ -4810,11 +4601,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:PSI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorpsi3"
   [(set (match_operand:PSI 0 "register_operand"        "=??r,r  ,d  ,r")
@@ -4842,11 +4630,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:SI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorsi3"
   [(set (match_operand:SI 0 "register_operand"       "=??r,r  ,d  ,r")
@@ -4918,7 +4703,7 @@
               (clobber (reg:CC REG_CC))])]
   "optimize
    && reload_completed"
-  [(const_int 1)]
+  [(scratch)]
   {
     for (int i = 0; i < <SIZE>; i++)
       {
@@ -5026,10 +4811,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:QI (match_dup 1)
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlqi3"
   [(set (match_operand:QI 0 "register_operand"               "=r,r,r  ,r  ,r  ,r  ,r  ,r")
@@ -5099,10 +4882,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:HI (match_dup 1)
-                              (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlhi2.1"
   [(set (match_operand:HI 0 "register_operand"           "=r")
@@ -5120,10 +4901,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:HI (match_dup 1)
-                              (const_int 15)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlhi2.15"
   [(set (match_operand:HI 0 "register_operand"           "=r")
@@ -5141,10 +4920,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:PSI (match_dup 1)
-                               (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlpsi2.1"
   [(set (match_operand:PSI 0 "register_operand"            "=r")
@@ -5162,10 +4939,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:PSI (match_dup 1)
-                               (const_int 23)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlpsi2.23"
   [(set (match_operand:PSI 0 "register_operand"            "=r")
@@ -5183,10 +4958,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:SI (match_dup 1)
-                              (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlsi2.1"
   [(set (match_operand:SI 0 "register_operand"           "=r")
@@ -5204,10 +4977,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:SI (match_dup 1)
-                              (const_int 31)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlsi2.31"
   [(set (match_operand:SI 0 "register_operand"           "=r")
@@ -5239,7 +5010,7 @@
    && 0 == INTVAL (operands[2]) % 16"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     avr_rotate_bytes (operands);
     DONE;
@@ -5263,7 +5034,7 @@
            && 0 == INTVAL (operands[2]) % 16))"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     avr_rotate_bytes (operands);
     DONE;
@@ -5273,41 +5044,6 @@
 ;;<< << << << << << << << << << << << << << << << << << << << << << << << << <<
 ;; arithmetic shift left
 
-;; Work around PR120423: Transform left shift of a paradoxical subreg
-;; into left shift of the zero-extended entity.
-(define_split ; PR120423
-  [(set (match_operand:HISI 0 "register_operand")
-        (ashift:HISI (subreg:HISI (match_operand:QIPSI 1 "nonimmediate_operand")
-                                  0)
-                     (match_operand:QI 2 "const_int_operand")))]
-  "!reload_completed
-   && !avropt_lra_p
-   && <HISI:SIZE> > <QIPSI:SIZE>"
-  [(set (match_dup 4)
-        (zero_extend:HISI (match_dup 5)))
-   (set (match_dup 0)
-        (ashift:HISI (match_dup 4)
-                     (match_dup 2)))]
-  {
-    operands[4] = gen_reg_rtx (<HISI:MODE>mode);
-    operands[5] = force_reg (<QIPSI:MODE>mode, operands[1]);
-  })
-
-;; Similar happens for PR116389.
-(define_split ; PR116389
-  [(set (match_operand:HISI 0 "register_operand")
-        (subreg:HISI (match_operand:QIPSI 1 "nonimmediate_operand")
-                     0))]
-  "!reload_completed
-   && !avropt_lra_p
-   && <HISI:SIZE> > <QIPSI:SIZE>"
-  [(set (match_dup 0)
-        (zero_extend:HISI (match_dup 2)))]
-  {
-    operands[2] = force_reg (<QIPSI:MODE>mode, operands[1]);
-  })
-
-
 ;; "ashlqi3"
 ;; "ashlqq3"  "ashluqq3"
 (define_expand "ashl<mode>3"
@@ -5363,10 +5099,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:ALL1 (match_dup 1)
-                                (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ashl<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"              "=r,r  ,r      ,r,r")
@@ -5390,11 +5124,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:ALL2 (match_dup 1)
-                                (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 ;; "*ashlhi3"
@@ -5506,11 +5237,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:ALL4 (match_dup 1)
-                                (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*ashl<mode>3"
@@ -5749,12 +5477,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:PSI (match_dup 1)
-                               (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*")])
 
 (define_insn "*ashlpsi3"
@@ -5808,10 +5532,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:ALL1 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ashr<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"                  "=r,r              ,r      ,r")
@@ -5835,11 +5557,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:ALL2 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-   ""
+  [(scratch)]
+  { DONE_ADD_CCC }
    [(set_attr "isa" "*,*,*,3op,*,*")])
 
 ;; "*ashrhi3"
@@ -5866,12 +5585,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:PSI (match_dup 1)
-                                 (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*")])
 
 (define_insn "*ashrpsi3"
@@ -5898,11 +5613,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:ALL4 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*ashr<mode>3"
@@ -6013,10 +5725,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:ALL1 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*lshr<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"                  "=r,r  ,r      ,r,r")
@@ -6039,11 +5749,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:ALL2 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*lshr<mode>3"
@@ -6066,12 +5773,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:PSI (match_dup 1)
-                                 (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*")])
 
 (define_insn "*lshrpsi3"
@@ -6098,11 +5801,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:ALL4 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*lshr<mode>3"
@@ -6217,9 +5917,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (abs:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*absqi2"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -6237,9 +5936,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (abs:SF (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*abssf2"
   [(set (match_operand:SF 0 "register_operand" "=d,r")
@@ -6260,9 +5958,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negqi2"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -6278,9 +5975,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:HI (sign_extend:HI (match_dup 1))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negqihi2"
   [(set (match_operand:HI 0 "register_operand"                        "=r")
@@ -6296,9 +5992,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:HI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*neghi2"
   [(set (match_operand:HI 0 "register_operand"        "=r,&r")
@@ -6316,9 +6011,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negpsi2"
   [(set (match_operand:PSI 0 "register_operand"        "=!d,r,&r")
@@ -6337,10 +6031,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,mov,movw")])
 
 (define_insn "*negsi2.libgcc"
@@ -6371,9 +6063,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:SF (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negsf2"
   [(set (match_operand:SF 0 "register_operand" "=d,r")
@@ -6394,9 +6085,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplqi2"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -6412,9 +6102,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:HI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplhi2"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -6431,9 +6120,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplpsi2"
   [(set (match_operand:PSI 0 "register_operand" "=r")
@@ -6449,9 +6137,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplsi2"
   [(set (match_operand:SI 0 "register_operand" "=r")
@@ -6480,9 +6167,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:HI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendqihi2"
   [(set (match_operand:HI 0 "register_operand" "=r,r")
@@ -6501,9 +6187,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendqipsi2"
   [(set (match_operand:PSI 0 "register_operand" "=r,r")
@@ -6522,9 +6207,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendqisi2"
   [(set (match_operand:SI 0 "register_operand" "=r,r")
@@ -6543,9 +6227,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendhipsi2"
   [(set (match_operand:PSI 0 "register_operand"                               "=r,r")
@@ -6564,9 +6247,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendhisi2"
   [(set (match_operand:SI 0 "register_operand"                               "=r,r")
@@ -6585,9 +6267,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendpsisi2"
   [(set (match_operand:SI 0 "register_operand"                                "=r")
@@ -7032,10 +6713,11 @@
    "#"
    "reload_completed"
    [(set (reg:CC REG_CC)
-         (compare:CC (match_dup 1) (match_dup 2)))
+         (compare:CC (match_dup 1)
+                     (match_dup 2)))
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))])
 
@@ -7054,11 +6736,12 @@
    "#"
    "reload_completed"
    [(parallel [(set (reg:CC REG_CC)
-                    (compare:CC (match_dup 1) (match_dup 2)))
+                    (compare:CC (match_dup 1)
+                                (match_dup 2)))
                (clobber (match_dup 4))])
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))]
    {
@@ -7081,11 +6764,12 @@
    "#"
    "reload_completed"
    [(parallel [(set (reg:CC REG_CC)
-                    (compare:CC (match_dup 1) (match_dup 2)))
+                    (compare:CC (match_dup 1)
+                                (match_dup 2)))
                (clobber (match_dup 4))])
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))]
    {
@@ -7109,11 +6793,12 @@
    "#"
    "reload_completed"
    [(parallel [(set (reg:CC REG_CC)
-                    (compare:CC (match_dup 1) (match_dup 2)))
+                    (compare:CC (match_dup 1)
+                                (match_dup 2)))
                (clobber (match_dup 4))])
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))]
    {
@@ -7668,17 +7353,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_op_dup 0
-                                  [(zero_extract:QIDI
-                                    (match_dup 1)
-                                    (const_int 1)
-                                    (match_dup 2))
-                                   (const_int 0)])
-                    (label_ref (match_dup 3))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbrx_branch<mode>"
   [(set (pc)
@@ -7721,13 +7397,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else (match_op_dup 0 [(and:QISI (match_dup 1)
-                                                            (match_dup 2))
-                                                  (const_int 0)])
-                                 (label_ref (match_dup 3))
-                                 (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbrx_and_branch<mode>"
   [(set (pc)
@@ -7968,14 +7639,8 @@
   "!AVR_HAVE_EIJMP_EICALL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (unspec:HI [(match_dup 0)]
-                              UNSPEC_INDEX_JMP))
-              (use (label_ref (match_dup 1)))
-              (clobber (match_dup 2))
-              (clobber (const_int 0))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "rjmp,rjmp,jmp")])
 
 (define_insn "*tablejump"
@@ -8004,14 +7669,8 @@
   "AVR_HAVE_EIJMP_EICALL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (unspec:HI [(reg:HI REG_Z)]
-                              UNSPEC_INDEX_JMP))
-              (use (label_ref (match_dup 0)))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:QI 24))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "eijmp")])
 
 
@@ -8182,17 +7841,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "eqne_operator"
-                                    [(zero_extract:QIHI
-                                      (mem:QI (match_dup 1))
-                                      (const_int 1)
-                                      (match_dup 2))
-                                     (const_int 0)])
-                    (label_ref (match_dup 3))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch"
   [(set (pc)
@@ -8230,14 +7880,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "gelt_operator"
-                                    [(mem:QI (match_dup 1))
-                                     (const_int 0)])
-                    (label_ref (match_dup 2))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch_bit7"
   [(set (pc)
@@ -8277,17 +7921,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "eqne_operator"
-                                    [(zero_extract:QIHI
-                                      (mem:QI (match_dup 1))
-                                      (const_int 1)
-                                      (match_dup 2))
-                                     (const_int 0)])
-                    (label_ref (match_dup 3))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch_tmp"
   [(set (pc)
@@ -8324,14 +7959,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "gelt_operator"
-                                    [(mem:QI (match_dup 1))
-                                     (const_int 0)])
-                    (label_ref (match_dup 2))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch_tmp_bit7"
   [(set (pc)
@@ -8784,13 +8413,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 1)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*delay_cycles_1"
   [(unspec_volatile [(match_operand:QI 0 "const_int_operand" "n")
@@ -8816,14 +8440,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 2)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "adiw,no_adiw")])
 
 (define_insn "*delay_cycles_2"
@@ -8853,15 +8471,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 3)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*delay_cycles_3"
   [(unspec_volatile [(match_operand:SI 0 "const_int_operand" "n")
@@ -8896,16 +8507,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 4)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (match_dup 5))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*delay_cycles_4"
   [(unspec_volatile [(match_operand:SI 0 "const_int_operand" "n")
@@ -8942,12 +8545,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:QI [(match_dup 1)
-                               (match_dup 2)
-                               (match_dup 3)]
-                              UNSPEC_INSERT_BITS))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*insert_bits"
   [(set (match_operand:QI 0 "register_operand"              "=r  ,d  ,r")
@@ -9127,12 +8726,13 @@
    "#"
    "reload_completed"
    [(set (reg:CC REG_CC)
-         (compare:CC (match_dup 0) (const_int 0)))
+         (compare:CC (match_dup 0)
+                     (const_int 0)))
     (set (pc)
-         (if_then_else (ge (reg:CC REG_CC) (const_int 0))
+         (if_then_else (ge (reg:CC REG_CC)
+                           (const_int 0))
                        (label_ref (match_dup 1))
-                       (pc)))]
-   "")
+                       (pc)))])
 
 (define_expand "flash_segment"
   [(parallel [(match_operand:QI 0 "register_operand" "")
@@ -9235,9 +8835,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (parity:HI (reg:HI 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*parityhi2.libgcc"
   [(set (reg:HI 24)
@@ -9253,9 +8852,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (zero_extend:HI (parity:QI (reg:QI 24))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*parityqihi2.libgcc"
   [(set (reg:HI 24)
@@ -9271,9 +8869,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (parity:SI (reg:SI 22))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*paritysihi2.libgcc"
   [(set (reg:HI 24)
@@ -9329,9 +8926,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (popcount:HI (reg:HI 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*popcounthi2.libgcc"
   [(set (reg:HI 24)
@@ -9347,9 +8943,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (popcount:SI (reg:SI 22))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*popcountsi2.libgcc"
   [(set (reg:HI 24)
@@ -9365,9 +8960,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24)
-                   (popcount:QI (reg:QI 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*popcountqi2.libgcc"
   [(set (reg:QI 24)
@@ -9421,10 +9015,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (clz:HI (reg:HI 24)))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*clzhi2.libgcc"
   [(set (reg:HI 24)
@@ -9442,10 +9034,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (clz:SI (reg:SI 22))))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*clzsihi2.libgcc"
   [(set (reg:HI 24)
@@ -9490,10 +9080,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (ctz:HI (reg:HI 24)))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ctzhi2.libgcc"
   [(set (reg:HI 24)
@@ -9512,11 +9100,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (ctz:SI (reg:SI 22))))
-              (clobber (reg:QI 22))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ctzsihi2.libgcc"
   [(set (reg:HI 24)
@@ -9562,10 +9147,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (ffs:HI (reg:HI 24)))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ffshi2.libgcc"
   [(set (reg:HI 24)
@@ -9584,11 +9167,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (ffs:SI (reg:SI 22))))
-              (clobber (reg:QI 22))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ffssihi2.libgcc"
   [(set (reg:HI 24)
@@ -9633,9 +9213,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (bswap:SI (reg:SI 22)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*bswapsi2.libgcc"
   [(set (reg:SI 22)
@@ -9742,11 +9321,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:HI [(match_dup 1)
-                               (match_dup 2)]
-                              UNSPEC_FMUL))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmul_insn"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -9768,11 +9344,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22)
-                   (unspec:HI [(reg:QI 24)
-                               (reg:QI 25)] UNSPEC_FMUL))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmul.call"
   [(set (reg:HI 22)
@@ -9814,11 +9387,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:HI [(match_dup 1)
-                               (match_dup 2)]
-                              UNSPEC_FMULS))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmuls_insn"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -9840,11 +9410,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22)
-                   (unspec:HI [(reg:QI 24)
-                               (reg:QI 25)] UNSPEC_FMULS))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmuls.call"
   [(set (reg:HI 22)
@@ -9886,11 +9453,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:HI [(match_dup 1)
-                               (match_dup 2)]
-                              UNSPEC_FMULSU))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmulsu_insn"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -9912,11 +9476,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22)
-                   (unspec:HI [(reg:QI 24)
-                               (reg:QI 25)] UNSPEC_FMULSU))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmulsu.call"
   [(set (reg:HI 22)
@@ -10037,11 +9598,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (zero_extract:QI (match_dup 0)
-                                    (const_int 1)
-                                    (match_dup 1))
-                   (match_dup 2))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*insv.reg"
   [(set (zero_extract:QI (match_operand:QI 0 "register_operand"    "+r,d,d,l,l")
@@ -10478,11 +10036,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (zero_extract:QI (not:QI (match_dup 1))
-                                    (const_int 1)
-                                    (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extzv.not"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -10619,11 +10174,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:QISI (any_shift:QISI (match_dup 1)
-                                             (match_dup 2))
-                             (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*insv.any_shift.<mode>"
   [(set (match_operand:QISI 0 "register_operand" "=r")
@@ -10686,11 +10238,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extract:QISI (match_dup 1)
-                                      (const_int 1)
-                                      (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sextr.<QISI:mode>.<QISI2:mode>"
   [(set (match_operand:QISI 0 "register_operand"                    "=r")
diff --git a/gcc/config/avr/avr.opt b/gcc/config/avr/avr.opt
index 9883119..2bed8ea 100644
--- a/gcc/config/avr/avr.opt
+++ b/gcc/config/avr/avr.opt
@@ -18,10 +18,6 @@
 ; along with GCC; see the file COPYING3.  If not see
 ; <http://www.gnu.org/licenses/>.
 
-mlra
-Target Var(avropt_lra_p) UInteger Init(1) Optimization Undocumented
-Usa LRA for reload instead of the old reload framework.  This option is experimental, on per default, and it may be removed in future versions of the compiler.
-
 mcall-prologues
 Target Mask(CALL_PROLOGUES) Optimization
 Optimization. Use subroutines for function prologues and epilogues.
@@ -164,6 +160,10 @@ mfuse-move=
 Target Joined RejectNegative UInteger Var(avropt_fuse_move) Init(0) Optimization IntegerRange(0, 23)
 -mfuse-move=<0,23>	Optimization. Run a post-reload pass that tweaks move instructions.
 
+mfuse-move2
+Target Var(avropt_fuse_move2) Init(0) Optimization
+Optimization. Fuse some move insns after insn combine.
+
 mabsdata
 Target Mask(ABSDATA)
 Assume that all data in static storage can be accessed by LDS / STS instructions.  This option is only useful for reduced Tiny devices like ATtiny40.
diff --git a/gcc/config/avr/avr.opt.urls b/gcc/config/avr/avr.opt.urls
index 662fdee..fa560bc 100644
--- a/gcc/config/avr/avr.opt.urls
+++ b/gcc/config/avr/avr.opt.urls
@@ -1,7 +1,5 @@
 ; Autogenerated by regenerate-opt-urls.py from gcc/config/avr/avr.opt and generated HTML
 
-; skipping UrlSuffix for 'mlra' due to finding no URLs
-
 mcall-prologues
 UrlSuffix(gcc/AVR-Options.html#index-mcall-prologues)
 
@@ -92,6 +90,9 @@ UrlSuffix(gcc/AVR-Options.html#index-mfuse-move)
 mfuse-move=
 UrlSuffix(gcc/AVR-Options.html#index-mfuse-move)
 
+mfuse-move2
+UrlSuffix(gcc/AVR-Options.html#index-mfuse-move2)
+
 mabsdata
 UrlSuffix(gcc/AVR-Options.html#index-mabsdata)
 
diff --git a/gcc/config/cris/cris.cc b/gcc/config/cris/cris.cc
index a34c9e9..4acdd1d 100644
--- a/gcc/config/cris/cris.cc
+++ b/gcc/config/cris/cris.cc
@@ -3711,9 +3711,11 @@ cris_md_asm_adjust (vec<rtx> &outputs, vec<rtx> &inputs,
   /* Determine if the source using MOF.  If it is, automatically
      clobbering MOF would cause it to have impossible constraints.  */
 
-  /* Look for a use of the MOF constraint letter: h.  */
+  /* Look for a use of the MOF constraint letter h or a hard register
+     constraint.  */
   for (unsigned i = 0, n = constraints.length(); i < n; ++i)
-    if (strchr (constraints[i], 'h') != NULL)
+    if (strchr (constraints[i], 'h') != NULL
+	|| strstr (constraints[i], "{mof}") != NULL)
       return NULL;
 
   /* Look for an output or an input that touches MOF.  */
diff --git a/gcc/config/darwin-driver.cc b/gcc/config/darwin-driver.cc
index 224e0a0..e83b7cd 100644
--- a/gcc/config/darwin-driver.cc
+++ b/gcc/config/darwin-driver.cc
@@ -64,7 +64,8 @@ validate_macosx_version_min (const char *version_str)
 
   major = strtoul (version_str, &end, 10);
 
-  /* macOS 10, 11, and 12 are known. clang accepts up to 99.  */
+  /* macOS 10, 11, 12, 13, 14, 15 and 26 are known.
+     clang accepts up to 99.  */
   if (major < 10 || major > 99)
     return NULL;
 
@@ -159,15 +160,16 @@ darwin_find_version_from_kernel (void)
   if (*version_p++ != '.')
     goto parse_failed;
 
-  /* Darwin20 sees a transition to macOS 11.  In this, it seems that the
-     mapping to macOS minor version and patch level is now always 0, 0
-     (at least for macOS 11 and 12).  */
-  if (major_vers >= 20)
-    {
-      /* Apple clang doesn't include the minor version or the patch level
-	 in the object file, nor does it pass it to ld  */
-      asprintf (&new_flag, "%d.00.00", major_vers - 9);
-    }
+  /* Darwin25 saw a transition to macOS 26.  */
+  if (major_vers >= 25)
+    /* Apple clang doesn't include the minor version or the patch level
+       in the object file, nor does it pass it to ld  */
+    asprintf (&new_flag, "%d.00.00", major_vers + 1);
+  /* Darwin20 saw a transition to macOS 11.  */
+  else if (major_vers >= 20)
+    /* Apple clang doesn't include the minor version or the patch level
+       in the object file, nor does it pass it to ld  */
+    asprintf (&new_flag, "%d.00.00", major_vers - 9);
   else if (major_vers - 4 <= 4)
     /* On 10.4 and earlier, the old linker is used which does not
        support three-component system versions.
diff --git a/gcc/config/epiphany/epiphany.cc b/gcc/config/epiphany/epiphany.cc
index 16626f8..f53a643 100644
--- a/gcc/config/epiphany/epiphany.cc
+++ b/gcc/config/epiphany/epiphany.cc
@@ -2816,12 +2816,16 @@ epiphany_vector_alignment_reachable (const_tree type, bool is_packed)
 
 static bool
 epiphany_support_vector_misalignment (machine_mode mode, const_tree type,
-				      int misalignment, bool is_packed)
+				      int misalignment, bool is_packed,
+				      bool is_gather_scatter)
 {
+  if (is_gather_scatter)
+    return true;
   if (GET_MODE_SIZE (mode) == 8 && misalignment % 4 == 0)
     return true;
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* STRUCTURE_SIZE_BOUNDARY seems a bit crude in how it enlarges small
diff --git a/gcc/config/gcn/gcn-opts.h b/gcc/config/gcn/gcn-opts.h
index 0bfc786..0287400 100644
--- a/gcc/config/gcn/gcn-opts.h
+++ b/gcc/config/gcn/gcn-opts.h
@@ -82,11 +82,18 @@ enum hsaco_attr_type
 #define TARGET_DPP_FULL !TARGET_RDNA2_PLUS
 #define TARGET_DPP16 TARGET_RDNA2_PLUS
 #define TARGET_DPP8 TARGET_RDNA2_PLUS
+/* Device requires no manually inserted wait states; that's the
+   case for RDNA 2, 3 and 3.5 (but not for RNDA 4).  */
+#define TARGET_NO_MANUAL_NOPS TARGET_RDNA2_PLUS
 /* Device requires CDNA1-style manually inserted wait states for AVGPRs.  */
 #define TARGET_AVGPR_CDNA1_NOPS TARGET_CDNA1
+/* Device requires CDNA3-style manually inserted wait states.  */
+#define TARGET_CDNA3_NOPS TARGET_CDNA3
 /* Whether to use the 'globally coherent' (glc) or the 'scope' (sc0) flag
    for non-scalar memory operations. The string starts on purpose with a space.
    Note: for scalar memory operations (i.e. 's_...'), 'glc' is still used.
+   Note: on atomics, glc/sc0 denotes whether the pre-op operation should
+   be used.
    CDNA3 also uses 'nt' instead of 'slc' and 'sc1' instead of 'scc'; however,
    there is no non-scalar user so far.  */
 #define TARGET_GLC_NAME (TARGET_CDNA3 ? " sc0" : " glc")
diff --git a/gcc/config/gcn/gcn-valu.md b/gcc/config/gcn/gcn-valu.md
index 4b21302..a34d2e3 100644
--- a/gcc/config/gcn/gcn-valu.md
+++ b/gcc/config/gcn/gcn-valu.md
@@ -811,7 +811,7 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 ; FIXME: 64bit operations really should be splitters, but I am not sure how
 ; to represent vertical subregs.
@@ -828,7 +828,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "16")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 (define_expand "vec_set<mode>"
   [(set (match_operand:V_MOV 0 "register_operand")
@@ -854,7 +854,7 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 (define_insn "*vec_set<mode>_1"
   [(set (match_operand:V_2REG 0 "register_operand"		   "=v")
@@ -871,7 +871,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "16")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 (define_insn "vec_duplicate<mode><exec>"
   [(set (match_operand:V_1REG 0 "register_operand"	   "=v")
@@ -910,7 +910,7 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "read")])
 
 (define_insn "vec_extract<mode><scalar_mode>"
   [(set (match_operand:<SCALAR_MODE> 0 "register_operand"  "=&Sg")
@@ -922,7 +922,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "16")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "read")])
 
 (define_insn "vec_extract<mode><scalar_mode>"
   [(set (match_operand:<SCALAR_MODE> 0 "register_operand"  "=&Sg")
@@ -934,7 +934,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "32")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "read")])
 
 (define_insn "vec_extract<V_1REG:mode><V_1REG_ALT:mode>_nop"
   [(set (match_operand:V_1REG_ALT 0 "register_operand" "=v,v")
@@ -1133,6 +1133,23 @@
     DONE;
   })
 
+(define_expand "gather_load<mode><vndi>"
+  [(match_operand:V_MOV 0 "register_operand")
+   (match_operand:DI 1 "register_operand")
+   (match_operand:<VnDI> 2 "register_operand")
+   (match_operand 3 "immediate_operand")
+   (match_operand:SI 4 "gcn_alu_operand")]
+  ""
+  {
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
+					  operands[2], operands[4],
+					  INTVAL (operands[3]), NULL);
+
+    emit_insn (gen_gather<mode>_insn_1offset (operands[0], addr, const0_rtx,
+					      const0_rtx, const0_rtx));
+    DONE;
+  })
+
 ; Allow any address expression
 (define_expand "gather<mode>_expr<exec>"
   [(set (match_operand:V_MOV 0 "register_operand")
@@ -1175,6 +1192,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "load")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2,*,cdna2")
    (set_attr "xnack" "off,off,on,on")])
@@ -1233,6 +1251,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "load")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2,*,cdna2")
    (set_attr "xnack" "off,off,on,on")])
@@ -1259,6 +1278,23 @@
     DONE;
   })
 
+(define_expand "scatter_store<mode><vndi>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:<VnDI> 1 "register_operand")
+   (match_operand 2 "immediate_operand")
+   (match_operand:SI 3 "gcn_alu_operand")
+   (match_operand:V_MOV 4 "register_operand")]
+  ""
+  {
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
+					  operands[1], operands[3],
+					  INTVAL (operands[2]), NULL);
+
+    emit_insn (gen_scatter<mode>_insn_1offset (addr, const0_rtx, operands[4],
+					       const0_rtx, const0_rtx));
+    DONE;
+  })
+
 ; Allow any address expression
 (define_expand "scatter<mode>_expr<exec_scatter>"
   [(set (mem:BLK (scratch))
@@ -1301,6 +1337,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "store")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2")])
 
@@ -1356,6 +1393,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "store")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2")])
 
@@ -1455,28 +1493,26 @@
 ;; }}}
 ;; {{{ ALU special case: add/sub
 
-(define_insn "add<mode>3<exec_clobber>"
+(define_insn "add<mode>3<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand")
 	(plus:V_INT_1REG
 	  (match_operand:V_INT_1REG 1 "register_operand")
-	  (match_operand:V_INT_1REG 2 "gcn_alu_operand")))
-   (clobber (reg:DI VCC_REG))]
+	  (match_operand:V_INT_1REG 2 "gcn_alu_operand")))]
   ""
   {@ [cons: =0, %1, 2; attrs: type, length]
-  [v,v,vSvA;vop2,4] v_add_co_u32\t%0, vcc, %2, %1
+  [v,v,vSvA;vop2,4] {v_add_u32|v_add_nc_u32}\t%0, %2, %1
   [v,v,vSvB;vop2,8] ^
   })
 
-(define_insn "add<mode>3_dup<exec_clobber>"
+(define_insn "add<mode>3_dup<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand")
 	(plus:V_INT_1REG
 	  (vec_duplicate:V_INT_1REG
 	    (match_operand:<SCALAR_MODE> 2 "gcn_alu_operand"))
-	  (match_operand:V_INT_1REG 1 "register_operand")))
-   (clobber (reg:DI VCC_REG))]
+	  (match_operand:V_INT_1REG 1 "register_operand")))]
   ""
   {@ [cons: =0, 1, 2; attrs: type, length]
-  [v,v,SvA;vop2,4] v_add_co_u32\t%0, vcc, %2, %1
+  [v,v,SvA;vop2,4] {v_add_u32|v_add_nc_u32}\t%0, %2, %1
   [v,v,SvB;vop2,8] ^
   })
 
@@ -1503,16 +1539,16 @@
 	(plus:V_SI
 	  (vec_duplicate:V_SI
 	    (match_operand:SI 1 "gcn_alu_operand"))
-	  (match_operand:V_SI 2 "register_operand")))
+	  (match_operand:V_SI 2 "gcn_alu_operand")))
    (set (match_operand:DI 3 "register_operand")
-	(ltu:DI (plus:V_SI (vec_duplicate:V_SI (match_dup 2))
-			   (match_dup 1))
-		(vec_duplicate:V_SI (match_dup 2))))]
+	(ltu:DI (plus:V_SI (vec_duplicate:V_SI (match_dup 1))
+			   (match_dup 2))
+		(match_dup 2)))]
   ""
   {@ [cons: =0, 1, 2, =3; attrs: type, length]
-  [v,SvA,v,cV;vop2 ,4] v_add_co_u32\t%0, %3, %1, %2
-  [v,SvB,v,cV;vop2 ,8] ^
-  [v,SvA,v,Sg;vop3b,8] ^
+  [v,SvA,vA,cV;vop2 ,4] v_add_co_u32\t%0, %3, %1, %2
+  [v,SvB,vA,cV;vop2 ,8] ^
+  [v,SvA,vA,Sg;vop3b,8] ^
   })
 
 ; v_addc does not accept an SGPR because the VCC read already counts as an
@@ -1551,16 +1587,15 @@
   [(set_attr "type" "vop2,vop3b")
    (set_attr "length" "4,8")])
 
-(define_insn "sub<mode>3<exec_clobber>"
+(define_insn "sub<mode>3<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand"  "=  v,   v")
 	(minus:V_INT_1REG
 	  (match_operand:V_INT_1REG 1 "gcn_alu_operand" "vSvB,   v")
-	  (match_operand:V_INT_1REG 2 "gcn_alu_operand" "   v,vSvB")))
-   (clobber (reg:DI VCC_REG))]
+	  (match_operand:V_INT_1REG 2 "gcn_alu_operand" "   v,vSvB")))]
   ""
   "@
-   v_sub_co_u32\t%0, vcc, %1, %2
-   v_subrev_co_u32\t%0, vcc, %2, %1"
+   {v_sub_u32|v_sub_nc_u32}\t%0, %1, %2
+   {v_subrev_u32|v_subrev_nc_u32}\t%0, %2, %1"
   [(set_attr "type" "vop2")
    (set_attr "length" "8,8")])
 
@@ -1648,6 +1683,39 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "8")])
 
+(define_insn_and_split "add<mode>3_dup"
+  [(set (match_operand:V_DI 0 "register_operand" "= v")
+	(plus:V_DI
+	  (vec_duplicate:V_DI
+	    (match_operand:DI 1 "register_operand"   "SvB"))
+	  (match_operand:V_DI 2 "gcn_alu_operand"    "vDb")))
+   (clobber (reg:DI VCC_REG))
+   (clobber (match_scratch:<VnSI> 3 "=&v"))]
+  ""
+  "#"
+  "gcn_can_split_p  (<MODE>mode, operands[0])
+   && gcn_can_split_p (<MODE>mode, operands[1])
+   && gcn_can_split_p (<MODE>mode, operands[2])"
+  [(const_int 0)]
+  {
+    rtx vcc = gen_rtx_REG (DImode, VCC_REG);
+    emit_insn (gen_add<vnsi>3_vcc_dup
+		(gcn_operand_part (<MODE>mode, operands[0], 0),
+		 gcn_operand_part (DImode, operands[1], 0),
+		 gcn_operand_part (<MODE>mode, operands[2], 0),
+		 vcc));
+    emit_insn (gen_vec_duplicate<vnsi> (operands[3],
+				  gcn_operand_part (DImode, operands[1], 1)));
+    emit_insn (gen_addc<vnsi>3
+		(gcn_operand_part (<MODE>mode, operands[0], 1),
+		 operands[3],
+		 gcn_operand_part (<MODE>mode, operands[2], 1),
+		 vcc, vcc));
+    DONE;
+  }
+  [(set_attr "type" "vmult")
+   (set_attr "length" "8")])
+
 (define_insn_and_split "add<mode>3_exec"
   [(set (match_operand:V_DI 0 "register_operand"		 "=  v")
 	(vec_merge:V_DI
@@ -1685,6 +1753,49 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "8")])
 
+(define_insn_and_split "add<mode>3_dup_exec"
+  [(set (match_operand:V_DI 0 "register_operand"		     "= v")
+	(vec_merge:V_DI
+	  (plus:V_DI
+	    (vec_duplicate:V_DI
+	      (match_operand:DI 1 "register_operand"		     "SvB"))
+	    (match_operand:V_DI 2 "gcn_alu_operand"		         "vDb"))
+	  (match_operand:V_DI 3 "gcn_register_or_unspec_operand" " U0")
+	  (match_operand:DI 4 "gcn_exec_reg_operand"		     "  e")))
+   (clobber (reg:DI VCC_REG))
+   (clobber (match_scratch:<VnSI> 5 "=&v"))]
+  ""
+  "#"
+  "gcn_can_split_p  (<MODE>mode, operands[0])
+   && gcn_can_split_p (<MODE>mode, operands[1])
+   && gcn_can_split_p (<MODE>mode, operands[2])
+   && gcn_can_split_p (<MODE>mode, operands[4])"
+  [(const_int 0)]
+  {
+    rtx vcc = gen_rtx_REG (DImode, VCC_REG);
+    emit_insn (gen_add<vnsi>3_vcc_dup_exec
+		(gcn_operand_part (<MODE>mode, operands[0], 0),
+		 gcn_operand_part (DImode, operands[1], 0),
+		 gcn_operand_part (<MODE>mode, operands[2], 0),
+		 vcc,
+		 gcn_operand_part (<MODE>mode, operands[3], 0),
+		 operands[4]));
+    emit_insn (gen_vec_duplicate<vnsi>_exec (operands[5],
+				gcn_operand_part (DImode, operands[1], 1),
+				gcn_gen_undef (<VnSI>mode),
+				operands[4]));
+    emit_insn (gen_addc<vnsi>3_exec
+		(gcn_operand_part (<MODE>mode, operands[0], 1),
+		 operands[5],
+		 gcn_operand_part (<MODE>mode, operands[2], 1),
+		 vcc, vcc,
+		 gcn_operand_part (<MODE>mode, operands[3], 1),
+		 operands[4]));
+    DONE;
+  }
+  [(set_attr "type" "vmult")
+   (set_attr "length" "8")])
+
 (define_insn_and_split "sub<mode>3"
   [(set (match_operand:V_DI 0 "register_operand"  "= v,  v")
 	(minus:V_DI                                        
@@ -1827,7 +1938,7 @@
 	(ltu:DI (plus:V_DI 
 		  (zero_extend:V_DI (vec_duplicate:<VnSI> (match_dup 1)))
 		  (match_dup 2))
-		(match_dup 1)))]
+		(match_dup 2)))]
   ""
   {@ [cons: =0, 1, 2, =3]
   [v,ASv,v,&Sg] #
@@ -1878,7 +1989,7 @@
 	  (ltu:DI (plus:V_DI 
 		    (zero_extend:V_DI (vec_duplicate:<VnSI> (match_dup 1)))
 		    (match_dup 2))
-		  (match_dup 1))
+		  (match_dup 2))
 	  (match_dup 5)))]
   ""
   {@ [cons: =0, 1, 2, =3, 4, 5]
@@ -1932,7 +2043,7 @@
 	(ltu:DI (plus:V_DI 
 		  (zero_extend:V_DI (match_dup 1))
 		  (vec_duplicate:V_DI (match_dup 2)))
-		(match_dup 1)))]
+		(vec_duplicate:V_DI (match_dup 2))))]
   ""
   {@ [cons: =0, 1, 2, =3]
   [v,v,DbSv,&cV] #
@@ -1981,7 +2092,7 @@
 	  (ltu:DI (plus:V_DI 
 		    (zero_extend:V_DI (match_dup 1))
 		    (vec_duplicate:V_DI (match_dup 2)))
-		  (match_dup 1))
+		  (vec_duplicate:V_DI (match_dup 2)))
 	  (match_dup 5)))]
   ""
   {@ [cons: =0, 1, 2, =3, 4, 5]
@@ -2190,6 +2301,22 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")])
 
+(define_insn "<su>mul<mode>3_highpart_dup<exec>"
+  [(set (match_operand:V_SI 0 "register_operand" "= v")
+	(truncate:V_SI
+	  (lshiftrt:<VnDI>
+	    (mult:<VnDI>
+	      (any_extend:<VnDI>
+		(vec_duplicate:V_SI
+		  (match_operand:SI 1 "gcn_alu_operand"  "SvA")))
+	      (any_extend:<VnDI>
+		(match_operand:V_SI 2 "gcn_alu_operand"  " vA")))
+	    (const_int 32))))]
+  ""
+  "v_mul_hi<sgnsuffix>0\t%0, %2, %1"
+  [(set_attr "type" "vop3a")
+   (set_attr "length" "8")])
+
 (define_insn "mul<mode>3<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand"  "=   v")
 	(mult:V_INT_1REG
@@ -2201,11 +2328,11 @@
    (set_attr "length" "8")])
 
 (define_insn "mul<mode>3_dup<exec>"
-  [(set (match_operand:V_INT_1REG 0 "register_operand"	     "=   v")
+  [(set (match_operand:V_INT_1REG 0 "register_operand"	 "= v")
 	(mult:V_INT_1REG
-	  (match_operand:V_INT_1REG 1 "gcn_alu_operand"	     "%vSvA")
 	  (vec_duplicate:V_INT_1REG
-	    (match_operand:<SCALAR_MODE> 2 "gcn_alu_operand" "  SvA"))))]
+	    (match_operand:<SCALAR_MODE> 1 "gcn_alu_operand" "SvA"))
+	  (match_operand:V_INT_1REG 2 "gcn_alu_operand"	     " vA")))]
   ""
   "v_mul_lo_u32\t%0, %1, %2"
   [(set_attr "type" "vop3a")
@@ -2241,6 +2368,37 @@
     DONE;
   })
 
+(define_insn_and_split "mul<mode>3_dup"
+  [(set (match_operand:V_DI 0 "register_operand" "=&v")
+	(mult:V_DI
+	  (vec_duplicate:V_DI
+	    (match_operand:DI 1 "gcn_alu_operand"    " Sv"))
+	  (match_operand:V_DI 2 "gcn_alu_operand"    "vDA")))
+   (clobber (match_scratch:<VnSI> 3 "=&v"))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+  {
+    rtx out_lo = gcn_operand_part (<MODE>mode, operands[0], 0);
+    rtx out_hi = gcn_operand_part (<MODE>mode, operands[0], 1);
+    rtx left_lo = gcn_operand_part (DImode, operands[1], 0);
+    rtx left_hi = gcn_operand_part (DImode, operands[1], 1);
+    rtx right_lo = gcn_operand_part (<MODE>mode, operands[2], 0);
+    rtx right_hi = gcn_operand_part (<MODE>mode, operands[2], 1);
+    rtx tmp = operands[3];
+
+    emit_insn (gen_mul<vnsi>3_dup (out_lo, left_lo, right_lo));
+    emit_insn (gen_umul<vnsi>3_highpart_dup (out_hi, left_lo, right_lo));
+    emit_insn (gen_mul<vnsi>3_dup (tmp, left_hi, right_lo));
+    emit_insn (gen_add<vnsi>3 (out_hi, out_hi, tmp));
+    emit_insn (gen_mul<vnsi>3_dup (tmp, left_lo, right_hi));
+    emit_insn (gen_add<vnsi>3 (out_hi, out_hi, tmp));
+    emit_insn (gen_mul<vnsi>3_dup (tmp, left_hi, right_hi));
+    emit_insn (gen_add<vnsi>3 (out_hi, out_hi, tmp));
+    DONE;
+  })
+
 (define_insn_and_split "mul<mode>3_exec"
   [(set (match_operand:V_DI 0 "register_operand"		 "=&v")
 	(vec_merge:V_DI
@@ -2289,6 +2447,56 @@
     DONE;
   })
 
+(define_insn_and_split "mul<mode>3_dup_exec"
+  [(set (match_operand:V_DI 0 "register_operand"		 "=&v")
+	(vec_merge:V_DI
+	  (mult:V_DI
+	    (vec_duplicate:V_DI
+	      (match_operand:DI 1 "gcn_alu_operand"		 " Sv"))
+	    (match_operand:V_DI 2 "gcn_alu_operand"		 "vDA"))
+	  (match_operand:V_DI 3 "gcn_register_or_unspec_operand" " U0")
+	  (match_operand:DI 4 "gcn_exec_reg_operand"		 "  e")))
+   (clobber (match_scratch:<VnSI> 5 "=&v"))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+  {
+    rtx out_lo = gcn_operand_part (<MODE>mode, operands[0], 0);
+    rtx out_hi = gcn_operand_part (<MODE>mode, operands[0], 1);
+    rtx left_lo = gcn_operand_part (DImode, operands[1], 0);
+    rtx left_hi = gcn_operand_part (DImode, operands[1], 1);
+    rtx right_lo = gcn_operand_part (<MODE>mode, operands[2], 0);
+    rtx right_hi = gcn_operand_part (<MODE>mode, operands[2], 1);
+    rtx exec = operands[4];
+    rtx tmp = operands[5];
+
+    rtx old_lo, old_hi;
+    if (GET_CODE (operands[3]) == UNSPEC)
+      {
+	old_lo = old_hi = gcn_gen_undef (<VnSI>mode);
+      }
+    else
+      {
+	old_lo = gcn_operand_part (<MODE>mode, operands[3], 0);
+	old_hi = gcn_operand_part (<MODE>mode, operands[3], 1);
+      }
+
+    rtx undef = gcn_gen_undef (<VnSI>mode);
+
+    emit_insn (gen_mul<vnsi>3_dup_exec (out_lo, left_lo, right_lo, old_lo,
+					exec));
+    emit_insn (gen_umul<vnsi>3_highpart_dup_exec (out_hi, left_lo, right_lo,
+						  old_hi, exec));
+    emit_insn (gen_mul<vnsi>3_dup_exec (tmp, left_hi, right_lo, undef, exec));
+    emit_insn (gen_add<vnsi>3_exec (out_hi, out_hi, tmp, out_hi, exec));
+    emit_insn (gen_mul<vnsi>3_dup_exec (tmp, left_lo, right_hi, undef, exec));
+    emit_insn (gen_add<vnsi>3_exec (out_hi, out_hi, tmp, out_hi, exec));
+    emit_insn (gen_mul<vnsi>3_dup_exec (tmp, left_hi, right_hi, undef, exec));
+    emit_insn (gen_add<vnsi>3_exec (out_hi, out_hi, tmp, out_hi, exec));
+    DONE;
+  })
+
 (define_insn_and_split "mul<mode>3_zext"
   [(set (match_operand:V_DI 0 "register_operand"      "=&v")
 	(mult:V_DI
@@ -3056,7 +3264,8 @@
   "flag_unsafe_math_optimizations"
   "v_sqrt%i0\t%0, %1"
   [(set_attr "type" "vop1")
-   (set_attr "length" "8")])
+   (set_attr "length" "8")
+   (set_attr "transop" "yes")])
 
 (define_insn "sqrt<mode>2"
   [(set (match_operand:FP 0 "register_operand"  "=  v")
@@ -3065,7 +3274,8 @@
   "flag_unsafe_math_optimizations"
   "v_sqrt%i0\t%0, %1"
   [(set_attr "type" "vop1")
-   (set_attr "length" "8")])
+   (set_attr "length" "8")
+   (set_attr "transop" "yes")])
 
 ; These FP unops have f64, f32 and f16 versions.
 (define_int_iterator MATH_UNOP_1OR2REG
@@ -3355,7 +3565,8 @@
   ""
   "v_rcp%i0\t%0, %1"
   [(set_attr "type" "vop1")
-   (set_attr "length" "8")])
+   (set_attr "length" "8")
+   (set_attr "transop" "yes")])
 
 ;; v_div_scale takes a numerator (op2) and denominator (op1) and returns the
 ;; one that matches op3 adjusted for best results in reciprocal division.
@@ -3727,6 +3938,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,*,yes,yes")])
 
@@ -3781,6 +3993,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,*,yes,yes")])
 
@@ -3795,9 +4008,9 @@
     /* Unsigned comparisons use the same patterns as signed comparisons,
        except that they use unsigned operators (e.g. LTU vs LT).
        The '%E1' directive then does the Right Thing.  */
-    emit_insn (gen_vec_cmpu<mode>di_exec (operands[0], operands[1],
-					  operands[2], operands[3],
-					  operands[4]));
+    emit_insn (gen_vec_cmp<mode>di_exec (operands[0], operands[1],
+					 operands[2], operands[3],
+					 operands[4]));
     DONE;
   })
 
@@ -3839,6 +4052,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,yes,yes")])
 
@@ -3862,6 +4076,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,yes,yes")])
 
@@ -4052,6 +4267,32 @@
     DONE;
   })
 
+(define_expand "mask_gather_load<mode><vndi>"
+  [(set:V_MOV (match_operand:V_MOV 0 "register_operand")
+	      (unspec:V_MOV
+		[(match_operand:DI 1 "register_operand")
+		 (match_operand:<VnDI> 2 "register_operand")
+		 (match_operand 3 "immediate_operand")
+		 (match_operand:SI 4 "gcn_alu_operand")
+		 (match_operand:DI 5 "")
+		 (match_operand:V_MOV 6 "maskload_else_operand")]
+		UNSPEC_GATHER))]
+  ""
+  {
+    rtx exec = force_reg (DImode, operands[5]);
+
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
+					  operands[2], operands[4],
+					  INTVAL (operands[3]), exec);
+
+    emit_insn (gen_gather<mode>_insn_1offset_exec (operands[0], addr,
+						   const0_rtx, const0_rtx,
+						   const0_rtx,
+						   gcn_gen_undef (<MODE>mode),
+						   exec));
+    DONE;
+  })
+
 (define_expand "mask_scatter_store<mode><vnsi>"
   [(match_operand:DI 0 "register_operand")
    (match_operand:<VnSI> 1 "register_operand")
@@ -4080,6 +4321,27 @@
     DONE;
   })
 
+(define_expand "mask_scatter_store<mode><vndi>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:<VnDI> 1 "register_operand")
+   (match_operand 2 "immediate_operand")
+   (match_operand:SI 3 "gcn_alu_operand")
+   (match_operand:V_MOV 4 "register_operand")
+   (match_operand:DI 5 "")]
+  ""
+  {
+    rtx exec = force_reg (DImode, operands[5]);
+
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
+					  operands[1], operands[3],
+					  INTVAL (operands[2]), exec);
+
+    emit_insn (gen_scatter<mode>_insn_1offset_exec (addr, const0_rtx,
+						    operands[4], const0_rtx,
+						    const0_rtx, exec));
+    DONE;
+  })
+
 (define_code_iterator cond_op [plus minus mult])
 
 (define_expand "cond_<expander><mode>"
@@ -4400,7 +4662,7 @@
     rtx tmp = gen_reg_rtx (<MODE>mode);
     rtx v1 = gen_rtx_REG (<MODE>mode, VGPR_REGNO (1));
 
-    emit_insn (gen_mul<mode>3_dup (tmp, v1, operands[2]));
+    emit_insn (gen_mul<mode>3_dup (tmp, operands[2], v1));
     emit_insn (gen_add<mode>3_dup (operands[0], tmp, operands[1]));
     DONE;
   })
diff --git a/gcc/config/gcn/gcn.cc b/gcc/config/gcn/gcn.cc
index 0ce5a29..5ffeb23 100644
--- a/gcc/config/gcn/gcn.cc
+++ b/gcc/config/gcn/gcn.cc
@@ -54,6 +54,7 @@
 #include "gimple.h"
 #include "cgraph.h"
 #include "case-cfn-macros.h"
+#include "opts.h"
 
 /* This file should be included last.  */
 #include "target-def.h"
@@ -183,6 +184,11 @@ gcn_option_override (void)
 
   if (flag_sram_ecc == HSACO_ATTR_DEFAULT)
     flag_sram_ecc = gcn_devices[gcn_arch].sramecc_default;
+
+  /* TODO: This seems to produce tighter loops, but the testsuites expects it
+     to be set to '2', so I'll leave it default for now.
+  SET_OPTION_IF_UNSET (&global_options, &global_options_set,
+		       param_vect_partial_vector_usage, 1);  */
 }
 
 /* }}}  */
@@ -1275,13 +1281,13 @@ gen_##PREFIX##vN##SUFFIX (PARAMS) \
 }
 
 #define GEN_VNM_NOEXEC(PREFIX, SUFFIX, PARAMS, ARGS) \
-GEN_VN_NOEXEC (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN_NOEXEC (PREFIX, si##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN_NOEXEC (PREFIX, di##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS))) \
 static rtx \
 gen_##PREFIX##vNm##SUFFIX (PARAMS) \
 { \
@@ -1289,13 +1295,13 @@ gen_##PREFIX##vNm##SUFFIX (PARAMS) \
   \
   switch (mode) \
     { \
-    case E_QImode: return gen_##PREFIX##vNqi##SUFFIX (ARGS); \
-    case E_HImode: return gen_##PREFIX##vNhi##SUFFIX (ARGS); \
-    case E_HFmode: return gen_##PREFIX##vNhf##SUFFIX (ARGS); \
+    USE_QHF (case E_QImode: return gen_##PREFIX##vNqi##SUFFIX (ARGS);) \
+    USE_QHF (case E_HImode: return gen_##PREFIX##vNhi##SUFFIX (ARGS);) \
+    USE_QHF (case E_HFmode: return gen_##PREFIX##vNhf##SUFFIX (ARGS);) \
     case E_SImode: return gen_##PREFIX##vNsi##SUFFIX (ARGS); \
-    case E_SFmode: return gen_##PREFIX##vNsf##SUFFIX (ARGS); \
+    USE_QHF (case E_SFmode: return gen_##PREFIX##vNsf##SUFFIX (ARGS);) \
     case E_DImode: return gen_##PREFIX##vNdi##SUFFIX (ARGS); \
-    case E_DFmode: return gen_##PREFIX##vNdf##SUFFIX (ARGS); \
+    USE_QHF (case E_DFmode: return gen_##PREFIX##vNdf##SUFFIX (ARGS);) \
     default: \
       break; \
     } \
@@ -1340,13 +1346,13 @@ gen_##PREFIX##vN##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
 }
 
 #define GEN_VNM(PREFIX, SUFFIX, PARAMS, ARGS) \
-GEN_VN (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN (PREFIX, si##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN (PREFIX, di##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS))) \
 USE_TI (GEN_VN (PREFIX, ti##SUFFIX, A(PARAMS), A(ARGS))) \
 static rtx \
 gen_##PREFIX##vNm##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
@@ -1355,15 +1361,22 @@ gen_##PREFIX##vNm##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
   \
   switch (mode) \
     { \
-    case E_QImode: return gen_##PREFIX##vNqi##SUFFIX (ARGS, merge_src, exec); \
-    case E_HImode: return gen_##PREFIX##vNhi##SUFFIX (ARGS, merge_src, exec); \
-    case E_HFmode: return gen_##PREFIX##vNhf##SUFFIX (ARGS, merge_src, exec); \
-    case E_SImode: return gen_##PREFIX##vNsi##SUFFIX (ARGS, merge_src, exec); \
-    case E_SFmode: return gen_##PREFIX##vNsf##SUFFIX (ARGS, merge_src, exec); \
-    case E_DImode: return gen_##PREFIX##vNdi##SUFFIX (ARGS, merge_src, exec); \
-    case E_DFmode: return gen_##PREFIX##vNdf##SUFFIX (ARGS, merge_src, exec); \
-    case E_TImode: \
-	USE_TI (return gen_##PREFIX##vNti##SUFFIX (ARGS, merge_src, exec);) \
+    USE_QHF (case E_QImode: \
+	return gen_##PREFIX##vNqi##SUFFIX (ARGS, merge_src, exec);) \
+    USE_QHF (case E_HImode: \
+	return gen_##PREFIX##vNhi##SUFFIX (ARGS, merge_src, exec);) \
+    USE_QHF (case E_HFmode: \
+	return gen_##PREFIX##vNhf##SUFFIX (ARGS, merge_src, exec);) \
+    case E_SImode: \
+	return gen_##PREFIX##vNsi##SUFFIX (ARGS, merge_src, exec); \
+    USE_QHF (case E_SFmode: \
+	return gen_##PREFIX##vNsf##SUFFIX (ARGS, merge_src, exec);) \
+    case E_DImode: \
+	return gen_##PREFIX##vNdi##SUFFIX (ARGS, merge_src, exec); \
+    USE_QHF (case E_DFmode: \
+	return gen_##PREFIX##vNdf##SUFFIX (ARGS, merge_src, exec);) \
+    USE_TI (case E_TImode: \
+	return gen_##PREFIX##vNti##SUFFIX (ARGS, merge_src, exec);) \
     default: \
       break; \
     } \
@@ -1372,7 +1385,8 @@ gen_##PREFIX##vNm##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
   return NULL_RTX; \
 }
 
-/* These have TImode support.  */
+/* These support everything.  */
+#define USE_QHF(ARGS) ARGS
 #define USE_TI(ARGS) ARGS
 GEN_VNM (mov,, A(rtx dest, rtx src), A(dest, src))
 GEN_VNM (vec_duplicate,, A(rtx dest, rtx src), A(dest, src))
@@ -1382,6 +1396,7 @@ GEN_VNM (vec_duplicate,, A(rtx dest, rtx src), A(dest, src))
 #define USE_TI(ARGS)
 GEN_VNM (add,3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN (add,si3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
+GEN_VN (add,di3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN (add,si3_vcc_dup, A(rtx dest, rtx src1, rtx src2, rtx vcc),
 	A(dest, src1, src2, vcc))
 GEN_VN (add,di3_sext_dup2, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
@@ -1393,15 +1408,20 @@ GEN_VN (add,di3_vcc_zext_dup2, A(rtx dest, rtx src1, rtx src2, rtx vcc),
 GEN_VN (addc,si3, A(rtx dest, rtx src1, rtx src2, rtx vccout, rtx vccin),
 	A(dest, src1, src2, vccout, vccin))
 GEN_VN (and,si3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
-GEN_VN (ashl,si3, A(rtx dest, rtx src, rtx shift), A(dest, src, shift))
 GEN_VNM_NOEXEC (ds_bpermute,, A(rtx dest, rtx addr, rtx src, rtx exec),
 		A(dest, addr, src, exec))
 GEN_VNM (gather,_expr, A(rtx dest, rtx addr, rtx as, rtx vol),
 	 A(dest, addr, as, vol))
-GEN_VN (mul,si3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN (sub,si3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN_NOEXEC (vec_series,si, A(rtx dest, rtx x, rtx c), A(dest, x, c))
 
+/* These do not have QI, HI, or any FP support.  */
+#undef USE_QHF
+#define USE_QHF(ARGS)
+GEN_VNM (ashl,3, A(rtx dest, rtx src, rtx shift), A(dest, src, shift))
+GEN_VNM (mul,3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
+
+#undef USE_QHF
 #undef USE_TI
 #undef GEN_VNM
 #undef GEN_VN
@@ -1995,8 +2015,8 @@ gcn_expand_vector_init (rtx op0, rtx vec)
   rtx addr = gen_reg_rtx (addrmode);
 
   int unit_size = GET_MODE_SIZE (GET_MODE_INNER (GET_MODE (op0)));
-  emit_insn (gen_mulvNsi3_dup (ramp, gen_rtx_REG (offsetmode, VGPR_REGNO (1)),
-			       GEN_INT (unit_size)));
+  emit_insn (gen_mulvNsi3_dup (ramp, GEN_INT (unit_size),
+			       gen_rtx_REG (offsetmode, VGPR_REGNO (1))));
 
   bool simple_repeat = true;
 
@@ -2293,36 +2313,46 @@ gcn_expand_scalar_to_vector_address (machine_mode mode, rtx exec, rtx mem,
 
    Return values.
      ADDR_SPACE_FLAT   - return VnDImode vector of absolute addresses.
-     ADDR_SPACE_GLOBAL - return VnSImode vector of offsets.  */
+     ADDR_SPACE_GLOBAL - return VnSImode vector of offsets.
+     64-bit offsets    - return VnDImode vector of absolute addresses. */
 
 rtx
 gcn_expand_scaled_offsets (addr_space_t as, rtx base, rtx offsets, rtx scale,
 			   bool unsigned_p, rtx exec)
 {
   int vf = GET_MODE_NUNITS (GET_MODE (offsets));
-  rtx tmpsi = gen_reg_rtx (VnMODE (vf, SImode));
-  rtx tmpdi = gen_reg_rtx (VnMODE (vf, DImode));
+  rtx scaled_offsets = gen_reg_rtx (GET_MODE (offsets));
+  rtx abs_addr = gen_reg_rtx (VnMODE (vf, DImode));
+  bool use_di = GET_MODE_INNER (GET_MODE (scaled_offsets)) == DImode;
 
   if (CONST_INT_P (scale)
       && INTVAL (scale) > 0
       && exact_log2 (INTVAL (scale)) >= 0)
-    emit_insn (gen_ashlvNsi3 (tmpsi, offsets,
-			      GEN_INT (exact_log2 (INTVAL (scale))),
-			      NULL, exec));
+    emit_insn (gen_ashlvNm3 (scaled_offsets, offsets,
+			     GEN_INT (exact_log2 (INTVAL (scale))),
+			     NULL, exec));
   else
-     emit_insn (gen_mulvNsi3_dup (tmpsi, offsets, scale, NULL, exec));
+     emit_insn (gen_mulvNm3_dup (scaled_offsets, scale, offsets, NULL, exec));
 
+  /* No instructions support DImode offsets.  */
+  if (use_di)
+    {
+      emit_insn (gen_addvNdi3_dup (abs_addr, base, scaled_offsets, NULL, exec));
+      return abs_addr;
+    }
   /* "Global" instructions do not support negative register offsets.  */
-  if (as == ADDR_SPACE_FLAT || !unsigned_p)
+  else if (as == ADDR_SPACE_FLAT || !unsigned_p)
     {
       if (unsigned_p)
-	 emit_insn (gen_addvNdi3_zext_dup2 (tmpdi, tmpsi, base, NULL, exec));
+	emit_insn (gen_addvNdi3_zext_dup2 (abs_addr, scaled_offsets, base,
+					   NULL, exec));
       else
-	 emit_insn (gen_addvNdi3_sext_dup2 (tmpdi, tmpsi, base, NULL, exec));
-      return tmpdi;
+	emit_insn (gen_addvNdi3_sext_dup2 (abs_addr, scaled_offsets, base,
+					   NULL, exec));
+      return abs_addr;
     }
   else if (as == ADDR_SPACE_GLOBAL)
-    return tmpsi;
+    return scaled_offsets;
 
   gcc_unreachable ();
 }
@@ -5315,8 +5345,12 @@ gcn_preferred_vector_alignment (const_tree type)
 static bool
 gcn_vectorize_support_vector_misalignment (machine_mode ARG_UNUSED (mode),
 					   const_tree type, int misalignment,
-					   bool is_packed)
+					   bool is_packed,
+					   bool is_gather_scatter)
 {
+  if (is_gather_scatter)
+    return true;
+
   if (is_packed)
     return false;
 
@@ -5761,6 +5795,16 @@ gcn_libc_has_function (enum function_class fn_class,
   return bsd_libc_has_function (fn_class, type);
 }
 
+/* Implement TARGET_VECTORIZE_PREFER_GATHER_SCATTER. */
+
+static bool
+gcn_prefer_gather_scatter (machine_mode ARG_UNUSED (mode),
+			   int ARG_UNUSED (scale),
+			   unsigned int ARG_UNUSED (group_size))
+{
+  return true;
+}
+
 /* }}}  */
 /* {{{ md_reorg pass.  */
 
@@ -6124,12 +6168,22 @@ gcn_md_reorg (void)
      detects the missed cases, and inserts the documented number of NOPs
      required for correct execution.  */
 
+  /* RDNA4 (not yet implemented) differs from RNDA 2/3/3.5 and requires some
+     s_nop, see 5.7 and esp. 5.7.2. in its ISA manual.
+     The assert here is a reminder to add those.  */
+  STATIC_ASSERT (ISA_CDNA1 - ISA_RDNA3 == 1);
+
+  if (TARGET_NO_MANUAL_NOPS)
+    return;
+
   const int max_waits = 5;
   struct ilist
   {
     rtx_insn *insn;
     attr_unit unit;
-    attr_delayeduse delayeduse;
+    attr_type type;
+    attr_flatmemaccess flatmemaccess;
+    bool delayeduse;
     HARD_REG_SET writes;
     HARD_REG_SET reads;
     int age;
@@ -6150,7 +6204,29 @@ gcn_md_reorg (void)
 
       attr_type itype = get_attr_type (insn);
       attr_unit iunit = get_attr_unit (insn);
-      attr_delayeduse idelayeduse = get_attr_delayeduse (insn);
+      attr_flatmemaccess iflatmemaccess = get_attr_flatmemaccess (insn);
+      bool delayeduse;
+      if (TARGET_CDNA3_NOPS)
+	switch (iflatmemaccess)
+	  {
+	  case FLATMEMACCESS_STORE:
+	  case FLATMEMACCESS_STOREX34:
+	  case FLATMEMACCESS_ATOMIC:
+	  case FLATMEMACCESS_CMPSWAPX2:
+	    delayeduse = true;
+	    break;
+	  case FLATMEMACCESS_LOAD:
+	  case FLATMEMACCESS_ATOMICWAIT:
+	  case FLATMEMACCESS_NO:
+	    delayeduse = false;
+	    break;
+	  default:
+	    gcc_unreachable ();
+	  }
+      else
+	delayeduse = (iflatmemaccess == FLATMEMACCESS_CMPSWAPX2
+		      || iflatmemaccess == FLATMEMACCESS_STOREX34);
+
       int ivccwait = get_attr_vccwait (insn);
       HARD_REG_SET ireads, iwrites;
       CLEAR_HARD_REG_SET (ireads);
@@ -6195,16 +6271,26 @@ gcn_md_reorg (void)
 		   && TEST_HARD_REG_BIT (ireads, VCCZ_REG))))
 	    nops_rqd = 5 - prev_insn->age;
 
-	  /* VALU writes SGPR/VCC followed by v_{read,write}lane using
-	     SGPR/VCC as lane select requires 4 wait states.  */
+	  /* VALU writes SGPR/VCC followed by
+	     - v_{read,write}lane using SGPR/VCC as lane select requires
+	       4 wait states
+	     - [CDNA3] VALU reads SGPR as constant requires 1 wait state
+	     - [CDNA3] VALU reads SGPR as carry-in requires no wait states  */
 	  if ((prev_insn->age + nops_rqd) < 4
 	      && prev_insn->unit == UNIT_VECTOR
-	      && get_attr_laneselect (insn) == LANESELECT_YES
+	      && get_attr_laneselect (insn) != LANESELECT_NO
 	      && (hard_reg_set_intersect_p
 		    (depregs, reg_class_contents[(int) SGPR_REGS])
 		  || hard_reg_set_intersect_p
 		       (depregs, reg_class_contents[(int) VCC_CONDITIONAL_REG])))
 	    nops_rqd = 4 - prev_insn->age;
+	  else if (TARGET_CDNA3_NOPS
+		   && (prev_insn->age + nops_rqd) < 1
+		   && prev_insn->unit == UNIT_VECTOR
+		   && iunit == UNIT_VECTOR
+		   && hard_reg_set_intersect_p
+			(depregs, reg_class_contents[(int) SGPR_REGS]))
+	    nops_rqd = 1 - prev_insn->age;
 
 	  /* VALU writes VGPR followed by VALU_DPP reading that VGPR
 	     requires 2 wait states.  */
@@ -6217,22 +6303,128 @@ gcn_md_reorg (void)
 		nops_rqd = 2 - prev_insn->age;
 	    }
 
+	  /* VALU writes EXEC followed by VALU DPP op requires 5 nop.  */
+	  if ((prev_insn->age + nops_rqd) < 5
+	      && itype == TYPE_VOP_DPP
+	      && prev_insn->unit == UNIT_VECTOR
+	      && TEST_HARD_REG_BIT (prev_insn->writes, EXECZ_REG))
+	    nops_rqd = 5 - prev_insn->age;
+
 	  /* Store that requires input registers are not overwritten by
-	     following instruction.  */
-	  if ((prev_insn->age + nops_rqd) < 1
-	      && prev_insn->delayeduse == DELAYEDUSE_YES
+	     following instruction.
+	     For CDNA3, only, VALU writes require 2 not 1 nop.
+	     CDNA3 additionally requires that 1 or 2 nop for global & scatch
+	     store/atomic.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 2
+	      && prev_insn->delayeduse
+	      && iunit == UNIT_VECTOR
+	      && ((hard_reg_set_intersect_p
+		   (prev_insn->reads, iwrites))))
+	    nops_rqd = 2 - prev_insn->age;
+	  else if ((prev_insn->age + nops_rqd) < 1
+	      && prev_insn->delayeduse
 	      && ((hard_reg_set_intersect_p
 		   (prev_insn->reads, iwrites))))
 	    nops_rqd = 1 - prev_insn->age;
 
-	  /* Instruction that requires VCC is not written too close before
-	     using it.  */
+	  /* Instruction (such as v_div_fmas) that requires VCC is not written
+	     too close before using it  */
 	  if (prev_insn->age < ivccwait
 	      && (hard_reg_set_intersect_p
 		  (prev_insn->writes,
 		   reg_class_contents[(int)VCC_CONDITIONAL_REG])))
 	    nops_rqd = ivccwait - prev_insn->age;
 
+	  /* NOTE: The following condition for adding wait state exists, but
+	     GCC does not access the special registers using their SGPR#.
+	     Thus, no action is required here.  The following wait-state
+	     condition exists at least for VEGA/gfx900+ to CDNA3:
+		Mixed use of VCC: alias vs. SGPR# - v_readlane,
+		v_readfirstlane, v_cmp, v_add_*i/u, v_sub_*i/u, v_div_*scale
+		followed by VALU reads VCC as constant requires 1 wait state.
+		(As carry-in, it requires none.)
+		[VCC can be accessed by name or logical SGPR that holds it.]  */
+
+	  /* Testing indicates that CDNA3 requires an s_nop between
+	     e.g. 'v_cmp_eq_u64 vcc, v[4:5], v[8:9]' and 'v_mov_b32 v0, vcc_lo'.
+	     Thus: add it between v_cmp writing VCC and VALU read of VCC.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && iunit == UNIT_VECTOR
+	      && (hard_reg_set_intersect_p
+		  (depregs, reg_class_contents[(int)VCC_CONDITIONAL_REG]))
+	      && get_attr_vcmp (prev_insn->insn) == VCMP_VCMP)
+	    nops_rqd = 1 - prev_insn->age;
+
+	  /* CDNA3: VALU writes SGPR/VCC: v_readlane, v_readfirstlane, v_cmp,
+	     v_add_*i/u, v_sub_*i/u, v_div_*scale - followed by:
+	     - VALU reads SGPR as constant requires 1 waite state
+	     - VALU reads SGPR as carry-in requires no waite state
+	     - v_readlane/v_writelane reads SGPR as lane select requires 4 wait
+	       states.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 4
+	      && iunit == UNIT_VECTOR
+	      && prev_insn->unit == UNIT_VECTOR
+	      && hard_reg_set_intersect_p
+		   (depregs, reg_class_contents[(int) SGPR_SRC_REGS]))
+	    {
+	      if (get_attr_laneselect (insn) != LANESELECT_NO)
+		nops_rqd = 4 - prev_insn->age;
+	      else if ((prev_insn->age + nops_rqd) < 1)
+		nops_rqd = 1 - prev_insn->age;
+	    }
+
+	  /* CDNA3: v_cmpx followed by
+	     - V_readlane, v_readfirstlane, v_writelane requires 4 wait states
+	     - VALU reads EXEC as constant requires 2 wait states
+	     - other VALU requires no wait state  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 4
+	      && get_attr_vcmp (prev_insn->insn) == VCMP_VCMPX
+	      && get_attr_laneselect (insn) != LANESELECT_NO)
+	    nops_rqd = 4 - prev_insn->age;
+	  else if (TARGET_CDNA3_NOPS
+		   && (prev_insn->age + nops_rqd) < 2
+		   && iunit == UNIT_VECTOR
+		   && get_attr_vcmp (prev_insn->insn) == VCMP_VCMPX
+		   && TEST_HARD_REG_BIT (ireads, EXECZ_REG))
+	    nops_rqd = 2 - prev_insn->age;
+
+	  /* CDNA3: VALU writes VGPR followed by v_readlane vsrc0 reads VGPRn
+	     requires 1 wait state. */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && prev_insn->unit == UNIT_VECTOR
+	      && prev_insn->flatmemaccess != FLATMEMACCESS_LOAD
+	      && get_attr_laneselect (insn) == LANESELECT_READ
+	      && hard_reg_set_intersect_p
+		    (depregs, reg_class_contents[(int) VGPR_REGS]))
+	    nops_rqd = 1 - prev_insn->age;
+
+	  /* CDNA3: VALU op which uses OPSEL or SDWA with changes the result's
+	     bit position followed by VALU op consumes result of that op
+	     requires 1 wait state.
+	     FIXME: Handle OPSEL, once used.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && prev_insn->unit == UNIT_VECTOR
+	      && prev_insn->type == TYPE_VOP_SDWA
+	      && !hard_reg_set_empty_p (depregs))
+	    nops_rqd = 1 - prev_insn->age;
+
+	  /* CNDA3: VALU Trans Op (such as v_rcp_f64) followed by non-trans VALU
+	     op consumes result of that op requires 1 wait state.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && prev_insn->unit == UNIT_VECTOR
+	      && iunit == UNIT_VECTOR
+	      && get_attr_transop (prev_insn->insn) == TRANSOP_YES
+	      && get_attr_transop (insn) == TRANSOP_NO
+	      && !hard_reg_set_empty_p (depregs))
+	    nops_rqd = 1 - prev_insn->age;
+
 	  /* CDNA1: write VGPR before v_accvgpr_write reads it.  */
 	  if (TARGET_AVGPR_CDNA1_NOPS
 	      && (prev_insn->age + nops_rqd) < 2
@@ -6264,8 +6456,8 @@ gcn_md_reorg (void)
 	}
 
       /* Insert the required number of NOPs.  */
-      for (int i = nops_rqd; i > 0; i--)
-	emit_insn_after (gen_nop (), last_insn);
+      if (nops_rqd > 0)
+	emit_insn_after (gen_nops (GEN_INT (nops_rqd-1)), last_insn);
 
       /* Age the previous instructions.  We can also ignore writes to
          registers subsequently overwritten.  */
@@ -6288,7 +6480,9 @@ gcn_md_reorg (void)
       /* Track the current instruction as a previous instruction.  */
       back[oldest].insn = insn;
       back[oldest].unit = iunit;
-      back[oldest].delayeduse = idelayeduse;
+      back[oldest].type = itype;
+      back[oldest].flatmemaccess = iflatmemaccess;
+      back[oldest].delayeduse = delayeduse;
       back[oldest].writes = iwrites;
       back[oldest].reads = ireads;
       back[oldest].age = 0;
@@ -7109,6 +7303,11 @@ print_operand_address (FILE *file, rtx mem)
    H - print second part of a multi-reg value (high-part of 2-reg value)
    J - print third part of a multi-reg value
    K - print fourth part of a multi-reg value
+   R   Print a scalar register number as an integer.  Temporary hack.
+   V - Print a vector register number as an integer.  Temporary hack.
+
+   Additionally, the standard builtin c, n, a, and l exist; see gccint's
+   "Output Templates and Operand Substitution" for details.
  */
 
 void
@@ -7957,6 +8156,8 @@ gcn_dwarf_register_span (rtx rtl)
   gcn_vectorize_builtin_vectorized_function
 #undef  TARGET_VECTORIZE_GET_MASK_MODE
 #define TARGET_VECTORIZE_GET_MASK_MODE gcn_vectorize_get_mask_mode
+#undef  TARGET_VECTORIZE_PREFER_GATHER_SCATTER
+#define TARGET_VECTORIZE_PREFER_GATHER_SCATTER gcn_prefer_gather_scatter
 #undef  TARGET_VECTORIZE_PREFERRED_SIMD_MODE
 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE gcn_vectorize_preferred_simd_mode
 #undef  TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
diff --git a/gcc/config/gcn/gcn.md b/gcc/config/gcn/gcn.md
index 2ce2e05..4130cf6 100644
--- a/gcc/config/gcn/gcn.md
+++ b/gcc/config/gcn/gcn.md
@@ -312,18 +312,33 @@
 ; We need to be able to identify v_readlane and v_writelane with
 ; SGPR lane selection in order to handle "Manually Inserted Wait States".
 
-(define_attr "laneselect" "yes,no" (const_string "no"))
+(define_attr "laneselect" "write,read,no" (const_string "no"))
 
-; Identify instructions that require a "Manually Inserted Wait State" if
-; their inputs are overwritten by subsequent instructions.
+; Global or flat memory access using store or load followed by waitcnt
+; and using flat/global atomic access, possibly followed by a waitcnt.
+; 'storex34' denotes FLAT_STORE_X{3,4}.
+; 'cmpswapx2' denotes FLAT_ATOMIC_{F}CMPSWAP_X2
+; Used to handle "Manually Inserted Wait State".
 
-(define_attr "delayeduse" "yes,no" (const_string "no"))
+(define_attr "flatmemaccess"
+             "store,storex34,load,atomic,atomicwait,cmpswapx2,no"
+             (const_string "no"))
+
+; Identify v_cmp and v_cmpx instructions for "Manually Inserted Wait State"
+; handling.
+
+(define_attr "vcmp" "vcmp,vcmpx,no" (const_string "no"))
 
 ; Identify instructions that require "Manually Inserted Wait State" if
 ; a previous instruction writes to VCC.  The number gives the number of NOPs.
 
 (define_attr "vccwait" "" (const_int 0))
 
+; Mark trans ops such as v_{exp,rsq,sqrt,sin,cos,log,...}_F{16,32,64}
+; for later conditional s_nop insertion.
+
+(define_attr "transop" "yes,no" (const_string "no"))
+
 ;; }}}
 ;; {{{ Iterators useful across the wole machine description
 
@@ -414,6 +429,15 @@
   "s_nop\t0x0"
   [(set_attr "type" "sopp")])
 
+; Variant of 'nop' that accepts a count argument.
+; s_nop accepts 0x0 to 0xf for 1 to 16 nops; however,
+; as %0 prints decimals, only 0 to 9 (= 1 to 10 nops) can be used.
+(define_insn "nops"
+  [(match_operand 0 "const_int_operand")]
+  ""
+  "s_nop\t0x%0"
+  [(set_attr "type" "sopp")])
+
 ; FIXME: What should the value of the immediate be? Zero is disallowed, so
 ; pick 1 for now.
 (define_insn "trap"
@@ -555,9 +579,12 @@
   }
   [(set_attr "type" "sop1,vop1,vop3a,sopk,vopc,mult,smem,smem,smem,flat,flat,
 		     flat,flat,flat,flat")
+   (set_attr "flatmemaccess" "*,*,*,*,*,*,*,*,*,load,load,store,load,load,store")
+   (set_attr "vcmp" "*,*,*,*,vcmp,*,*,*,*,*,*,*,*,*,*")
    (set_attr "exec" "*,*,none,*,*,*,*,*,*,*,*,*,*,*,*")
    (set_attr "length" "4,4,4,4,4,8,12,12,12,12,12,12,12,12,12")
-   (set_attr "xnack" "*,*,*,*,*,*,off,on,*,off,on,*,off,on,*")])
+   (set_attr "xnack" "*,*,*,*,*,*,off,on,*,off,on,*,off,on,*")
+   (set_attr "laneselect" "*,*,read,*,*,*,*,*,*,*,*,*,*,*,*")])
 
 ; 32bit move pattern
 
@@ -565,38 +592,38 @@
   [(set (match_operand:SISF 0 "nonimmediate_operand")
 	(match_operand:SISF 1 "gcn_load_operand"))]
   ""
-  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack]
-   [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ] s_mov_b32\t%0, %1
-   [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ] s_movk_i32\t%0, %1
-   [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
-   [SD  ,RB  ;smem ,*   ,12,*    ,off] s_buffer_load%s0\t%0, s[0:3], %1\;s_waitcnt\tlgkmcnt(0)
-   [&SD ,RB  ;smem ,*   ,12,*    ,on ] ^
-   [RB  ,Sm  ;smem ,*   ,12,*    ,*  ] s_buffer_store%s1\t%1, s[0:3], %0
-   [Sm  ,RS  ;smem ,*   ,12,*    ,off] s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-   [&Sm ,RS  ;smem ,*   ,12,*    ,on ] ^
-   [RS  ,Sm  ;smem ,*   ,12,*    ,*  ] s_store_dword\t%1, %A0
-   [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ] v_mov_b32\t%0, %1
-   [Sg  ,v   ;vop3a,none,8 ,*    ,*  ] v_readlane_b32\t%0, %1, 0
-   [v   ,Sv  ;vop3a,none,8 ,*    ,*  ] v_writelane_b32\t%0, %1, 0
-   [v   ,^a  ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_read_b32\t%0, %1
-   [a   ,v   ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_write_b32\t%0, %1
-   [a   ,a   ;vop1 ,*    ,4,cdna2,*  ] v_accvgpr_mov_b32\t%0, %1
-   [v   ,RF  ;flat ,*   ,12,*    ,off] flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0
-   [&v  ,RF  ;flat ,*   ,12,*    ,on ] ^
-   [^a  ,RF  ;flat ,*   ,12,cdna2,off] ^
-   [&^a ,RF  ;flat ,*   ,12,cdna2,on ] ^
-   [RF  ,v   ;flat ,*   ,12,*    ,*  ] flat_store_dword\t%A0, %1%O0%g0
-   [RF  ,a   ;flat ,*   ,12,cdna2,*  ] ^
-   [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ] v_mov_b32\t%0, %1
-   [RLRG,v   ;ds   ,*   ,12,*    ,*  ] ds_write_b32\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-   [v   ,RLRG;ds   ,*   ,12,*    ,*  ] ds_read_b32\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-   [SD  ,Y   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
-   [v   ,RM  ;flat ,*   ,12,*    ,off] global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-   [&v  ,RM  ;flat ,*   ,12,*    ,on ] ^
-   [^a  ,RM  ;flat ,*   ,12,cdna2,off] ^
-   [&^a ,RM  ;flat ,*   ,12,cdna2,on ] ^
-   [RM  ,v   ;flat ,*   ,12,*    ,*  ] global_store_dword\t%A0, %1%O0%g0
-   [RM  ,a   ;flat ,*   ,12,cdna2,*  ] ^
+  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack, laneselect, flatmemaccess]
+   [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+   [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ,*    ,*    ] s_movk_i32\t%0, %1
+   [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+   [SD  ,RB  ;smem ,*   ,12,*    ,off,*    ,*    ] s_buffer_load%s0\t%0, s[0:3], %1\;s_waitcnt\tlgkmcnt(0)
+   [&SD ,RB  ;smem ,*   ,12,*    ,on ,*    ,*    ] ^
+   [RB  ,Sm  ;smem ,*   ,12,*    ,*  ,*    ,*    ] s_buffer_store%s1\t%1, s[0:3], %0
+   [Sm  ,RS  ;smem ,*   ,12,*    ,off,*    ,*    ] s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+   [&Sm ,RS  ;smem ,*   ,12,*    ,on ,*    ,*    ] ^
+   [RS  ,Sm  ;smem ,*   ,12,*    ,*  ,*    ,*    ] s_store_dword\t%1, %A0
+   [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+   [Sg  ,v   ;vop3a,none,8 ,*    ,*  ,read ,*    ] v_readlane_b32\t%0, %1, 0
+   [v   ,Sv  ;vop3a,none,8 ,*    ,*  ,write,*    ] v_writelane_b32\t%0, %1, 0
+   [v   ,^a  ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_read_b32\t%0, %1
+   [a   ,v   ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_write_b32\t%0, %1
+   [a   ,a   ;vop1 ,*    ,4,cdna2,*  ,*    ,*    ] v_accvgpr_mov_b32\t%0, %1
+   [v   ,RF  ;flat ,*   ,12,*    ,off,*    ,load ] flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0
+   [&v  ,RF  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+   [^a  ,RF  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+   [&^a ,RF  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+   [RF  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] flat_store_dword\t%A0, %1%O0%g0
+   [RF  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
+   [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+   [RLRG,v   ;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_write_b32\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+   [v   ,RLRG;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_read_b32\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+   [SD  ,Y   ;sop1 ,*   ,8 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+   [v   ,RM  ;flat ,*   ,12,*    ,off,*    ,load ] global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+   [&v  ,RM  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+   [^a  ,RM  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+   [&^a ,RM  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+   [RM  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] global_store_dword\t%A0, %1%O0%g0
+   [RM  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
   })
 
 ; 8/16bit move pattern
@@ -606,31 +633,31 @@
   [(set (match_operand:QIHI 0 "nonimmediate_operand")
 	(match_operand:QIHI 1 "gcn_load_operand"))]
   "gcn_valid_move_p (<MODE>mode, operands[0], operands[1])"
-  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack]
-  [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ] s_mov_b32\t%0, %1
-  [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ] s_movk_i32\t%0, %1
-  [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
-  [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ] v_mov_b32\t%0, %1
-  [Sg  ,v   ;vop3a,none,4 ,*    ,*  ] v_readlane_b32\t%0, %1, 0
-  [v   ,Sv  ;vop3a,none,4 ,*    ,*  ] v_writelane_b32\t%0, %1, 0
-  [v   ,^a  ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_read_b32\t%0, %1
-  [a   ,v   ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_write_b32\t%0, %1
-  [a   ,a   ;vop1 ,*    ,8,cdna2,*  ] v_accvgpr_mov_b32\t%0, %1
-  [v   ,RF  ;flat ,*   ,12,*    ,off] flat_load%o1\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [&v  ,RF  ;flat ,*   ,12,*    ,on ] ^
-  [^a  ,RF  ;flat ,*   ,12,cdna2,off] ^
-  [&^a ,RF  ;flat ,*   ,12,cdna2,on ] ^
-  [RF  ,v   ;flat ,*   ,12,*    ,*  ] flat_store%s0\t%A0, %1%O0%g0
-  [RF  ,a   ;flat ,*   ,12,cdna2,*  ] ^
-  [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ] v_mov_b32\t%0, %1
-  [RLRG,v   ;ds   ,*   ,12,*    ,*  ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RLRG;ds   ,*   ,12,*    ,*  ] ds_read%u1\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RM  ;flat ,*   ,12,*    ,off] global_load%o1\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [&v  ,RM  ;flat ,*   ,12,*    ,on ] ^
-  [^a  ,RM  ;flat ,*   ,12,cdna2,off] ^
-  [&^a ,RM  ;flat ,*   ,12,cdna2,on ] ^
-  [RM  ,v   ;flat ,*   ,12,*    ,*  ] global_store%s0\t%A0, %1%O0%g0
-  [RM  ,a   ;flat ,*   ,12,cdna2,*  ] ^
+  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack, laneselect, flatmemaccess]
+  [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+  [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ,*    ,*    ] s_movk_i32\t%0, %1
+  [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+  [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+  [Sg  ,v   ;vop3a,none,4 ,*    ,*  ,read ,*    ] v_readlane_b32\t%0, %1, 0
+  [v   ,Sv  ;vop3a,none,4 ,*    ,*  ,write,*    ] v_writelane_b32\t%0, %1, 0
+  [v   ,^a  ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_read_b32\t%0, %1
+  [a   ,v   ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_write_b32\t%0, %1
+  [a   ,a   ;vop1 ,*    ,8,cdna2,*  ,*    ,*    ] v_accvgpr_mov_b32\t%0, %1
+  [v   ,RF  ;flat ,*   ,12,*    ,off,*    ,load ] flat_load%o1\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v  ,RF  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+  [^a  ,RF  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+  [&^a ,RF  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+  [RF  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] flat_store%s0\t%A0, %1%O0%g0
+  [RF  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
+  [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+  [RLRG,v   ;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RLRG;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_read%u1\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RM  ;flat ,*   ,12,*    ,off,*    ,load ] global_load%o1\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v  ,RM  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+  [^a  ,RM  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+  [&^a ,RM  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+  [RM  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] global_store%s0\t%A0, %1%O0%g0
+  [RM  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
   })
 
 ; 64bit move pattern
@@ -639,34 +666,34 @@
   [(set (match_operand:DIDF 0 "nonimmediate_operand")
 	(match_operand:DIDF 1 "general_operand"))]
   "GET_CODE(operands[1]) != SYMBOL_REF"
-  {@ [cons: =0, 1; attrs: type, length, cdna, xnack]
-  [SD  ,SSA ;sop1 ,4 ,*    ,*  ] s_mov_b64\t%0, %1
-  [SD  ,C   ;sop1 ,8 ,*    ,*  ] ^
-  [SD  ,DB  ;mult ,* ,*    ,*  ] #
-  [RS  ,Sm  ;smem ,12,*    ,*  ] s_store_dwordx2\t%1, %A0
-  [Sm  ,RS  ;smem ,12,*    ,off] s_load_dwordx2\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-  [&Sm ,RS  ;smem ,12,*    ,on ] ^
-  [v   ,v   ;vmult,* ,*    ,*  ] #
-  [v   ,DB  ;vmult,* ,*    ,*  ] #
-  [Sg  ,v   ;vmult,* ,*    ,*  ] #
-  [v   ,Sv  ;vmult,* ,*    ,*  ] #
-  [v   ,^a  ;vmult,* ,*    ,*  ] #
-  [a   ,v   ;vmult,* ,*    ,*  ] #
-  [a   ,a   ;vmult,* ,cdna2,*  ] #
-  [v   ,RF  ;flat ,12,*    ,off] flat_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [&v  ,RF  ;flat ,12,*    ,on ] ^
-  [^a  ,RF  ;flat ,12,cdna2,off] ^
-  [&^a ,RF  ;flat ,12,cdna2,on ] ^
-  [RF  ,v   ;flat ,12,*    ,*  ] flat_store_dwordx2\t%A0, %1%O0%g0
-  [RF  ,a   ;flat ,12,cdna2,*  ] ^
-  [RLRG,v   ;ds   ,12,*    ,*  ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RLRG;ds   ,12,*    ,*  ] ds_read_b64\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RM  ;flat ,12,*    ,off] global_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [&v  ,RM  ;flat ,12,*    ,on ] ^
-  [^a  ,RM  ;flat ,12,cdna2,off] ^
-  [&^a ,RM  ;flat ,12,cdna2,on ] ^
-  [RM  ,v   ;flat ,12,*    ,*  ] global_store_dwordx2\t%A0, %1%O0%g0
-  [RM  ,a   ;flat ,12,cdna2,*  ] ^
+  {@ [cons: =0, 1; attrs: type, length, cdna, xnack, flatmemaccess]
+  [SD  ,SSA ;sop1 ,4 ,*    ,*  ,*    ] s_mov_b64\t%0, %1
+  [SD  ,C   ;sop1 ,8 ,*    ,*  ,*    ] ^
+  [SD  ,DB  ;mult ,* ,*    ,*  ,*    ] #
+  [RS  ,Sm  ;smem ,12,*    ,*  ,*    ] s_store_dwordx2\t%1, %A0
+  [Sm  ,RS  ;smem ,12,*    ,off,*    ] s_load_dwordx2\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+  [&Sm ,RS  ;smem ,12,*    ,on ,*    ] ^
+  [v   ,v   ;vmult,* ,*    ,*  ,*    ] #
+  [v   ,DB  ;vmult,* ,*    ,*  ,*    ] #
+  [Sg  ,v   ;vmult,* ,*    ,*  ,*    ] #
+  [v   ,Sv  ;vmult,* ,*    ,*  ,*    ] #
+  [v   ,^a  ;vmult,* ,*    ,*  ,*    ] #
+  [a   ,v   ;vmult,* ,*    ,*  ,*    ] #
+  [a   ,a   ;vmult,* ,cdna2,*  ,*    ] #
+  [v   ,RF  ;flat ,12,*    ,off,load ] flat_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v  ,RF  ;flat ,12,*    ,on ,load ] ^
+  [^a  ,RF  ;flat ,12,cdna2,off,load ] ^
+  [&^a ,RF  ;flat ,12,cdna2,on ,load ] ^
+  [RF  ,v   ;flat ,12,*    ,*  ,store] flat_store_dwordx2\t%A0, %1%O0%g0
+  [RF  ,a   ;flat ,12,cdna2,*  ,store] ^
+  [RLRG,v   ;ds   ,12,*    ,*  ,*    ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RLRG;ds   ,12,*    ,*  ,*    ] ds_read_b64\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RM  ;flat ,12,*    ,off,load ] global_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v  ,RM  ;flat ,12,*    ,on ,load ] ^
+  [^a  ,RM  ;flat ,12,cdna2,off,load ] ^
+  [&^a ,RM  ;flat ,12,cdna2,on ,load ] ^
+  [RM  ,v   ;flat ,12,*    ,*  ,store] global_store_dwordx2\t%A0, %1%O0%g0
+  [RM  ,a   ;flat ,12,cdna2,*  ,store] ^
   }
   "reload_completed
    && ((!MEM_P (operands[0]) && !MEM_P (operands[1])
@@ -704,31 +731,31 @@
   [(set (match_operand:TI 0 "nonimmediate_operand")
 	(match_operand:TI 1 "general_operand"  ))]
   ""
-  {@ [cons: =0, 1; attrs: type, delayeduse, length, cdna, xnack]
-  [SD ,SSB;mult ,*  ,* ,*    ,*  ] #
-  [RS ,Sm ;smem ,*  ,12,*    ,*  ] s_store_dwordx4\t%1, %A0
-  [Sm ,RS ;smem ,yes,12,*    ,off] s_load_dwordx4\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-  [&Sm,RS ;smem ,yes,12,*    ,on ] ^
-  [RF ,v  ;flat ,*  ,12,*    ,*  ] flat_store_dwordx4\t%A0, %1%O0%g0
-  [RF ,a  ;flat ,*  ,12,cdna2,*  ] ^
-  [v  ,RF ;flat ,*  ,12,*    ,off] flat_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [&v ,RF ;flat ,*  ,12,*    ,on ] ^
-  [^a ,RF ;flat ,*  ,12,cdna2,off] ^
-  [&^a,RF ;flat ,*  ,12,cdna2,on ] ^
-  [v  ,v  ;vmult,*  ,* ,*    ,*  ] #
-  [v  ,Sv ;vmult,*  ,* ,*    ,*  ] #
-  [SD ,v  ;vmult,*  ,* ,*    ,*  ] #
-  [RM ,v  ;flat ,yes,12,*    ,*  ] global_store_dwordx4\t%A0, %1%O0%g0
-  [RM ,a  ;flat ,yes,12,cdna2,*  ] ^
-  [v  ,RM ;flat ,*  ,12,*    ,off] global_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [&v ,RM ;flat ,*  ,12,*    ,on ] ^
-  [^a ,RM ;flat ,*  ,12,cdna2,off] ^
-  [&^a,RM ;flat ,*  ,12,cdna2,on ] ^
-  [RL ,v  ;ds   ,*  ,12,*    ,*  ] ds_write_b128\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v  ,RL ;ds   ,*  ,12,*    ,*  ] ds_read_b128\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v  ,^a ;vmult,*  ,* ,*    ,*  ] #
-  [a  ,v  ;vmult,*  ,* ,*    ,*  ] #
-  [a  ,a  ;vmult,*  ,* ,cdna2,*  ] #
+  {@ [cons: =0, 1; attrs: type, length, cdna, xnack, flatmemaccess]
+  [SD ,SSB;mult ,* ,*    ,*  ,*       ] #
+  [RS ,Sm ;smem ,12,*    ,*  ,*       ] s_store_dwordx4\t%1, %A0
+  [Sm ,RS ;smem ,12,*    ,off,*       ] s_load_dwordx4\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+  [&Sm,RS ;smem ,12,*    ,on ,*       ] ^
+  [RF ,v  ;flat ,12,*    ,*  ,storex34] flat_store_dwordx4\t%A0, %1%O0%g0
+  [RF ,a  ;flat ,12,cdna2,*  ,storex34] ^
+  [v  ,RF ;flat ,12,*    ,off,load    ] flat_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v ,RF ;flat ,12,*    ,on ,load    ] ^
+  [^a ,RF ;flat ,12,cdna2,off,load    ] ^
+  [&^a,RF ;flat ,12,cdna2,on ,load    ] ^
+  [v  ,v  ;vmult,* ,*    ,*  ,*       ] #
+  [v  ,Sv ;vmult,* ,*    ,*  ,*       ] #
+  [SD ,v  ;vmult,* ,*    ,*  ,*       ] #
+  [RM ,v  ;flat ,12,*    ,*  ,storex34] global_store_dwordx4\t%A0, %1%O0%g0
+  [RM ,a  ;flat ,12,cdna2,*  ,storex34] ^
+  [v  ,RM ;flat ,12,*    ,off,load    ] global_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v ,RM ;flat ,12,*    ,on ,load    ] ^
+  [^a ,RM ;flat ,12,cdna2,off,load    ] ^
+  [&^a,RM ;flat ,12,cdna2,on ,load    ] ^
+  [RL ,v  ;ds   ,12,*    ,*  ,*       ] ds_write_b128\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v  ,RL ;ds   ,12,*    ,*  ,*       ] ds_read_b128\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v  ,^a ;vmult,* ,*    ,*  ,*       ] #
+  [a  ,v  ;vmult,* ,*    ,*  ,*       ] #
+  [a  ,a  ;vmult,* ,cdna2,*  ,*       ] #
   }
   "reload_completed
    && REG_P (operands[0])
@@ -1077,6 +1104,7 @@
    s_cmp%D1\t%2, %3
    v_cmp%E1\tvcc, %2, %3"
   [(set_attr "type" "sopc,vopc")
+   (set_attr "vcmp" "vcmp")
    (set_attr "length" "8")])
 
 (define_insn "cstoredi4_vector"
@@ -1087,6 +1115,7 @@
   ""
   "v_cmp%E1\tvcc, %2, %3"
   [(set_attr "type" "vopc")
+   (set_attr "vcmp" "vcmp")
    (set_attr "length" "8")])
 
 (define_expand "cbranchdi4"
@@ -1113,6 +1142,7 @@
   ""
   "v_cmp%E1\tvcc, %2, %3"
   [(set_attr "type" "vopc")
+   (set_attr "vcmp" "vcmp")
    (set_attr "length" "8")])
 
 (define_expand "cbranch<mode>4"
@@ -1136,14 +1166,13 @@
   [(set (match_operand:SI 0 "register_operand"         "= Sg, Sg, Sg,   v")
         (plus:SI (match_operand:SI 1 "gcn_alu_operand" "%SgA,  0,SgA,   v")
 		 (match_operand:SI 2 "gcn_alu_operand" " SgA,SgJ,  B,vBSv")))
-   (clobber (match_scratch:BI 3			       "= cs, cs, cs,   X"))
-   (clobber (match_scratch:DI 4			       "=  X,  X,  X,  cV"))]
+   (clobber (match_scratch:BI 3			       "= cs, cs, cs,   X"))]
   ""
   "@
    s_add_i32\t%0, %1, %2
    s_addk_i32\t%0, %2
    s_add_i32\t%0, %1, %2
-   v_add_co_u32\t%0, vcc, %2, %1"
+   {v_add_u32|v_add_nc_u32}\t%0, %2, %1"
   [(set_attr "type" "sop2,sopk,sop2,vop2")
    (set_attr "length" "4,4,8,8")])
 
@@ -1151,8 +1180,7 @@
   [(parallel [(set (match_operand:SI 0 "register_operand")
 		   (plus:SI (match_operand:SI 1 "gcn_alu_operand")
 			    (match_operand:SI 2 "gcn_alu_operand")))
-	      (clobber (reg:BI SCC_REG))
-	      (clobber (scratch:DI))])]
+	      (clobber (reg:BI SCC_REG))])]
   ""
   {})
 
@@ -1332,14 +1360,13 @@
   [(set (match_operand:SI 0 "register_operand"          "=Sg, Sg,    v,   v")
 	(minus:SI (match_operand:SI 1 "gcn_alu_operand" "SgA,SgA,    v,vBSv")
 		  (match_operand:SI 2 "gcn_alu_operand" "SgA,  B, vBSv,   v")))
-   (clobber (match_scratch:BI 3				"=cs, cs,    X,   X"))
-   (clobber (match_scratch:DI 4				"= X,  X,   cV,  cV"))]
+   (clobber (match_scratch:BI 3				"=cs, cs,    X,   X"))]
   ""
   "@
    s_sub_i32\t%0, %1, %2
    s_sub_i32\t%0, %1, %2
-   v_subrev_co_u32\t%0, vcc, %2, %1
-   v_sub_co_u32\t%0, vcc, %1, %2"
+   {v_subrev_u32|v_subrev_nc_u32}\t%0, %2, %1
+   {v_sub_u32|v_sub_nc_u32}\t%0, %1, %2"
   [(set_attr "type" "sop2,sop2,vop2,vop2")
    (set_attr "length" "4,8,8,8")])
 
@@ -1569,8 +1596,7 @@
 	(mult:DI (match_operand:DI 1 "register_operand" "%Sg, Sg,  v, v")
 		 (match_operand:DI 2 "nonmemory_operand" "Sg,  i,vSv, A")))
    (clobber (match_scratch:SI 3 "=&Sg,&Sg,&v,&v"))
-   (clobber (match_scratch:BI 4  "=cs, cs, X, X"))
-   (clobber (match_scratch:DI 5   "=X,  X,cV,cV"))]
+   (clobber (match_scratch:BI 4  "=cs, cs, X, X"))]
   ""
   "#"
   "reload_completed"
@@ -1585,15 +1611,13 @@
     emit_insn (gen_umulsidi3 (operands[0], op1lo, op2lo));
     emit_insn (gen_mulsi3 (tmp, op1lo, op2hi));
     rtx add = gen_rtx_SET (dsthi, gen_rtx_PLUS (SImode, dsthi, tmp));
-    rtx clob1 = gen_rtx_CLOBBER (VOIDmode, operands[4]);
-    rtx clob2 = gen_rtx_CLOBBER (VOIDmode, operands[5]);
-    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (3, add, clob1, clob2));
+    rtx clob = gen_rtx_CLOBBER (VOIDmode, operands[4]);
+    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, add, clob));
     emit_insn (add);
     emit_insn (gen_mulsi3 (tmp, op1hi, op2lo));
     add = gen_rtx_SET (dsthi, gen_rtx_PLUS (SImode, dsthi, tmp));
-    clob1 = gen_rtx_CLOBBER (VOIDmode, operands[4]);
-    clob2 = gen_rtx_CLOBBER (VOIDmode, operands[5]);
-    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (3, add, clob1, clob2));
+    clob = gen_rtx_CLOBBER (VOIDmode, operands[4]);
+    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, add, clob));
     emit_insn (add);
     DONE;
   })
@@ -1991,6 +2015,7 @@
    flat_atomic_<bare_mnemonic><X>\t%0, %1, %2 %G2\;s_waitcnt\t0
    global_atomic_<bare_mnemonic><X>\t%0, %A1, %2%O1 %G2\;s_waitcnt\tvmcnt(0)"
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,atomicwait,atomicwait")
    (set_attr "length" "12")])
 
 ; FIXME: These patterns are disabled because the instructions don't
@@ -2012,6 +2037,7 @@
    flat_atomic_<bare_mnemonic><X>\t%0, %1\;s_waitcnt\t0
    global_atomic_<bare_mnemonic><X>\t%A0, %1%O0\;s_waitcnt\tvmcnt(0)"
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,atomicwait,atomicwait")
    (set_attr "length" "12")])
 
 (define_mode_attr x2 [(SI "DI") (DI "TI")])
@@ -2059,7 +2085,7 @@
    global_atomic_cmpswap<X>\t%0, %A1, %2%O1 %G2\;s_waitcnt\tvmcnt(0)"
   [(set_attr "type" "smem,flat,flat")
    (set_attr "length" "12")
-   (set_attr "delayeduse" "*,yes,yes")])
+   (set_attr "flatmemaccess" "*,cmpswapx2,cmpswapx2")])
 
 (define_insn "sync_compare_and_swap<mode>_lds_insn"
   [(set (match_operand:SIDI 0 "register_operand"    "= v")
@@ -2157,7 +2183,7 @@
 		    ? "buffer_gl1_inv\;buffer_gl0_inv\;flat_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\t0\;buffer_gl1_inv\;buffer_gl0_inv"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;flat_load%o0\t%0, %A1%O1 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\t0\;buffer_inv sc1"
 		    : "buffer_wbinvl1_vol\;flat_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\t0\;buffer_wbinvl1_vol");
@@ -2169,7 +2195,7 @@
 		    ? "buffer_gl1_inv\;buffer_gl0_inv\;global_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_gl1_inv\;buffer_gl0_inv"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;global_load%o0\t%0, %A1%O1 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;global_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
 		    : "buffer_wbinvl1_vol\;global_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol");
@@ -2179,6 +2205,7 @@
     gcc_unreachable ();
   }
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,load,load")
    (set_attr "length" "28")
    (set_attr "rdna" "no,*,*")])
 
@@ -2215,7 +2242,7 @@
 		    : TARGET_WBINVL1_CACHE
 		    ? "buffer_wbinvl1_vol\;flat_store%o1\t%A0, %1%O0 %G1"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;flat_store%o1\t%A0, %1%O0 %G1"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_store%o1\t%A0, %1%O0 %G1"
 		    : "error: cache architectire unspecified");
 	  case 2:
 	    return (TARGET_GLn_CACHE
@@ -2223,7 +2250,7 @@
 		    : TARGET_WBINVL1_CACHE
 		    ? "buffer_wbinvl1_vol\;global_store%o1\t%A0, %1%O0 %G1"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;global_store%o1\t%A0, %1%O0 %G1"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;global_store%o1\t%A0, %1%O0 %G1"
 		    : "error: cache architecture unspecified");
 	  }
 	break;
@@ -2243,7 +2270,8 @@
 		    ? "buffer_wbinvl1_vol\;flat_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\t0\;buffer_wbinvl1_vol"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;flat_store%o1\t%A0, %1%O0 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;"
+		      "flat_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\t0\;buffer_inv sc1"
 		    : "error: cache architecture unspecified");
 	  case 2:
@@ -2254,7 +2282,8 @@
 		    ? "buffer_wbinvl1_vol\;global_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;global_store%o1\t%A0, %1%O0 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
+		      "global_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
 		    : "error: cache architecture unspecified");
 	  }
@@ -2263,6 +2292,7 @@
     gcc_unreachable ();
   }
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,store,store")
    (set_attr "length" "28")
    (set_attr "rdna" "no,*,*")])
 
@@ -2337,7 +2367,7 @@
             ? "buffer_wbinvl1_vol\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0"
             : "error: cache architecture unspecified");
 	  case 2:
@@ -2350,7 +2380,7 @@
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)"
             : "error: cache architecture unspecified");
@@ -2372,7 +2402,7 @@
             ? "buffer_wbinvl1_vol\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0\;buffer_wbinvl1_vol"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0\;buffer_inv sc1"
             : "error: cache architecture unspecified");
 	  case 2:
@@ -2385,7 +2415,7 @@
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
             : "error: cache architecture unspecified");
@@ -2395,6 +2425,7 @@
     gcc_unreachable ();
   }
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,atomicwait,atomicwait")
    (set_attr "length" "28")
    (set_attr "rdna" "no,*,*")])
 
diff --git a/gcc/config/h8300/h8300.h b/gcc/config/h8300/h8300.h
index 6b2dd65..8018b63 100644
--- a/gcc/config/h8300/h8300.h
+++ b/gcc/config/h8300/h8300.h
@@ -610,6 +610,12 @@ struct cum_arg
 #define DATA_SECTION_ASM_OP "\t.section .data"
 #define BSS_SECTION_ASM_OP "\t.section .bss"
 
+/* Override default definitions from elfos.h. */
+#undef INIT_SECTION_ASM_OP
+#define INIT_SECTION_ASM_OP "\t.section\t.init,\"ax\""
+#undef FINI_SECTION_ASM_OP
+#define FINI_SECTION_ASM_OP "\t.section\t.fini,\"ax\""
+
 #undef DO_GLOBAL_CTORS_BODY
 #define DO_GLOBAL_CTORS_BODY			\
 {						\
@@ -647,19 +653,11 @@ struct cum_arg
 /* Globalizing directive for a label.  */
 #define GLOBAL_ASM_OP "\t.global "
 
+/* Override default definition from elfos.h. */
+#undef ASM_DECLARE_FUNCTION_NAME
 #define ASM_DECLARE_FUNCTION_NAME(FILE, NAME, DECL) \
    ASM_OUTPUT_FUNCTION_LABEL (FILE, NAME, DECL)
 
-/* This is how to store into the string LABEL
-   the symbol_ref name of an internal numbered label where
-   PREFIX is the class of label and NUM is the number within the class.
-   This is suitable for output with `assemble_name'.
-
-   N.B.: The h8300.md branch_true and branch_false patterns also know
-   how to generate internal labels.  */
-#define ASM_GENERATE_INTERNAL_LABEL(LABEL, PREFIX, NUM)	\
-  sprintf (LABEL, "*.%s%lu", PREFIX, (unsigned long)(NUM))
-
 /* This is how to output an insn to push a register on the stack.
    It need not be very fast code.  */
 
@@ -690,9 +688,6 @@ struct cum_arg
   if ((LOG) != 0)				\
     fprintf (FILE, "\t.align %d\n", (LOG))
 
-#define ASM_OUTPUT_SKIP(FILE, SIZE) \
-  fprintf (FILE, "\t.space %d\n", (int)(SIZE))
-
 /* This says how to output an assembler line
    to define a global common symbol.  */
 
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 83076ad..09aa9b1 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -387,7 +387,7 @@ ix86_expand_move (machine_mode mode, rtx operands[])
       tmp = XEXP (op1, 0);
 
       if (GET_CODE (tmp) != PLUS
-	  || GET_CODE (XEXP (tmp, 0)) != SYMBOL_REF)
+	  || !SYMBOL_REF_P (XEXP (tmp, 0)))
 	break;
 
       op1 = XEXP (tmp, 0);
@@ -487,7 +487,7 @@ ix86_expand_move (machine_mode mode, rtx operands[])
 		op1 = machopic_legitimize_pic_address (op1, mode,
 						       tmp == op1 ? 0 : tmp);
 	    }
-	  if (op0 != op1 && GET_CODE (op0) != MEM)
+	  if (op0 != op1 && !MEM_P (op0))
 	    {
 	      rtx insn = gen_rtx_SET (op0, op1);
 	      emit_insn (insn);
@@ -1396,11 +1396,11 @@ ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
      to cast them temporarily to integer vectors.  */
   if (op1
       && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
-      && (SUBREG_P (op2) || GET_CODE (op2) == CONST_VECTOR)
+      && (SUBREG_P (op2) || CONST_VECTOR_P (op2))
       && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
       && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
       && SUBREG_BYTE (op1) == 0
-      && (GET_CODE (op2) == CONST_VECTOR
+      && (CONST_VECTOR_P (op2)
 	  || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
 	      && SUBREG_BYTE (op2) == 0))
       && can_create_pseudo_p ())
@@ -1415,7 +1415,7 @@ ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
 	case E_V4DFmode:
 	case E_V8DFmode:
 	  dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
-	  if (GET_CODE (op2) == CONST_VECTOR)
+	  if (CONST_VECTOR_P (op2))
 	    {
 	      op2 = gen_lowpart (GET_MODE (dst), op2);
 	      op2 = force_reg (GET_MODE (dst), op2);
@@ -4918,7 +4918,7 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
 	case LEU:
 	  /* x <= cst can be handled as x < cst + 1 unless there is
 	     wrap around in cst + 1.  */
-	  if (GET_CODE (cop1) == CONST_VECTOR
+	  if (CONST_VECTOR_P (cop1)
 	      && GET_MODE_INNER (mode) != TImode)
 	    {
 	      unsigned int n_elts = GET_MODE_NUNITS (mode), i;
@@ -4962,7 +4962,7 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
 	case GEU:
 	  /* x >= cst can be handled as x > cst - 1 unless there is
 	     wrap around in cst - 1.  */
-	  if (GET_CODE (cop1) == CONST_VECTOR
+	  if (CONST_VECTOR_P (cop1)
 	      && GET_MODE_INNER (mode) != TImode)
 	    {
 	      unsigned int n_elts = GET_MODE_NUNITS (mode), i;
@@ -5033,9 +5033,9 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
 	    }
 	}
 
-      if (GET_CODE (cop0) == CONST_VECTOR)
+      if (CONST_VECTOR_P (cop0))
 	cop0 = force_reg (mode, cop0);
-      else if (GET_CODE (cop1) == CONST_VECTOR)
+      else if (CONST_VECTOR_P (cop1))
 	cop1 = force_reg (mode, cop1);
 
       rtx optrue = op_true ? op_true : CONSTM1_RTX (data_mode);
@@ -5234,7 +5234,7 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
   if (*negate)
     std::swap (op_true, op_false);
 
-  if (GET_CODE (cop1) == CONST_VECTOR)
+  if (CONST_VECTOR_P (cop1))
     cop1 = force_reg (mode, cop1);
 
   /* Allow the comparison to be done in one mode, but the movcc to
@@ -6188,7 +6188,7 @@ ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
 
   rtx constant = get_pool_constant (XEXP (mem, 0));
 
-  if (GET_CODE (constant) != CONST_VECTOR)
+  if (!CONST_VECTOR_P (constant))
     return false;
 
   /* There could be some rtx like
@@ -6198,7 +6198,7 @@ ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
     {
       constant = simplify_subreg (mode, constant, GET_MODE (constant), 0);
 
-      if (constant == nullptr || GET_CODE (constant) != CONST_VECTOR)
+      if (constant == nullptr || !CONST_VECTOR_P (constant))
 	return false;
     }
 
@@ -6244,7 +6244,7 @@ ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
       return size;
     }
 
-  if (GET_CODE (operand) == CONST_VECTOR)
+  if (CONST_VECTOR_P (operand))
     {
       scalar_int_mode imode = int_mode_for_mode (mode).require ();
       /* Caution: if we looked through a constant pool memory above,
@@ -6378,7 +6378,7 @@ ix86_split_long_move (rtx operands[])
 	 fp moves, that force all constants to memory to allow combining.  */
 
       if (MEM_P (operands[1])
-	  && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
+	  && SYMBOL_REF_P (XEXP (operands[1], 0))
 	  && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
 	operands[1] = get_pool_constant (XEXP (operands[1], 0));
       if (push_operand (operands[0], VOIDmode))
@@ -7899,7 +7899,8 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
 			       rtx count, machine_mode mode, int unroll,
 			       int expected_size, bool issetmem)
 {
-  rtx_code_label *out_label, *top_label;
+  rtx_code_label *out_label = nullptr;
+  rtx_code_label *top_label = nullptr;
   rtx iter, tmp;
   machine_mode iter_mode = counter_mode (count);
   int piece_size_n = GET_MODE_SIZE (mode) * unroll;
@@ -7907,9 +7908,19 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
   rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
   rtx size;
   int i;
+  int loop_count;
+
+  if (expected_size != -1 && CONST_INT_P (count))
+    loop_count = INTVAL (count) / GET_MODE_SIZE (mode) / unroll;
+  else
+    loop_count = -1;
 
-  top_label = gen_label_rtx ();
-  out_label = gen_label_rtx ();
+  /* Don't generate the loop if the loop count is 1.  */
+  if (loop_count != 1)
+    {
+      top_label = gen_label_rtx ();
+      out_label = gen_label_rtx ();
+    }
   iter = gen_reg_rtx (iter_mode);
 
   size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
@@ -7923,7 +7934,8 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
     }
   emit_move_insn (iter, const0_rtx);
 
-  emit_label (top_label);
+  if (loop_count != 1)
+    emit_label (top_label);
 
   tmp = convert_modes (Pmode, iter_mode, iter, true);
 
@@ -7991,21 +8003,25 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
   if (tmp != iter)
     emit_move_insn (iter, tmp);
 
-  emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
-			   true, top_label);
-  if (expected_size != -1)
+  if (loop_count != 1)
     {
-      expected_size /= GET_MODE_SIZE (mode) * unroll;
-      if (expected_size == 0)
-	predict_jump (0);
-      else if (expected_size > REG_BR_PROB_BASE)
-	predict_jump (REG_BR_PROB_BASE - 1);
+      emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
+			       true, top_label);
+      if (expected_size != -1)
+	{
+	  expected_size /= GET_MODE_SIZE (mode) * unroll;
+	  if (expected_size == 0)
+	    predict_jump (0);
+	  else if (expected_size > REG_BR_PROB_BASE)
+	    predict_jump (REG_BR_PROB_BASE - 1);
+	  else
+	    predict_jump (REG_BR_PROB_BASE
+			  - (REG_BR_PROB_BASE + expected_size / 2)
+			    / expected_size);
+	}
       else
-        predict_jump (REG_BR_PROB_BASE - (REG_BR_PROB_BASE + expected_size / 2)
-		      / expected_size);
+	predict_jump (REG_BR_PROB_BASE * 80 / 100);
     }
-  else
-    predict_jump (REG_BR_PROB_BASE * 80 / 100);
   iter = ix86_zero_extend_to_Pmode (iter);
   tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
 			     true, OPTAB_LIB_WIDEN);
@@ -8018,7 +8034,8 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
       if (tmp != srcptr)
 	emit_move_insn (srcptr, tmp);
     }
-  emit_label (out_label);
+  if (loop_count != 1)
+    emit_label (out_label);
 }
 
 /* Divide COUNTREG by SCALE.  */
@@ -8221,19 +8238,11 @@ expand_cpymem_epilogue (rtx destmem, rtx srcmem,
   rtx src, dest;
   if (CONST_INT_P (count))
     {
-      HOST_WIDE_INT countval = INTVAL (count);
-      HOST_WIDE_INT epilogue_size = countval % max_size;
-      int i;
-
-      /* For now MAX_SIZE should be a power of 2.  This assert could be
-	 relaxed, but it'll require a bit more complicated epilogue
-	 expanding.  */
-      gcc_assert ((max_size & (max_size - 1)) == 0);
-      for (i = max_size; i >= 1; i >>= 1)
-	{
-	  if (epilogue_size & i)
-	    destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
-	}
+      unsigned HOST_WIDE_INT countval = UINTVAL (count);
+      unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
+      unsigned int destalign = MEM_ALIGN (destmem);
+      move_by_pieces (destmem, srcmem, epilogue_size, destalign,
+		      RETURN_BEGIN);
       return;
     }
   if (max_size > 8)
@@ -8394,6 +8403,81 @@ expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
 				 1, max_size / 2, true);
 }
 
+/* Callback routine for store_by_pieces.  Return the RTL of a register
+   containing GET_MODE_SIZE (MODE) bytes in the RTL register op_p which
+   is a word or a word vector register.  If PREV_P isn't nullptr, it
+   has the RTL info from the previous iteration.  */
+
+static rtx
+setmem_epilogue_gen_val (void *op_p, void *prev_p, HOST_WIDE_INT,
+			 fixed_size_mode mode)
+{
+  rtx target;
+  by_pieces_prev *prev = (by_pieces_prev *) prev_p;
+  if (prev)
+    {
+      rtx prev_op = prev->data;
+      if (prev_op)
+	{
+	  machine_mode prev_mode = GET_MODE (prev_op);
+	  if (prev_mode == mode)
+	    return prev_op;
+	  if (VECTOR_MODE_P (prev_mode)
+	      && VECTOR_MODE_P (mode)
+	      && GET_MODE_INNER (prev_mode) == GET_MODE_INNER (mode))
+	    {
+	      target = gen_rtx_SUBREG (mode, prev_op, 0);
+	      return target;
+	    }
+	}
+    }
+
+  rtx op = (rtx) op_p;
+  machine_mode op_mode = GET_MODE (op);
+
+  gcc_assert (op_mode == word_mode
+	      || (VECTOR_MODE_P (op_mode)
+		  && GET_MODE_INNER (op_mode) == word_mode));
+
+  if (VECTOR_MODE_P (mode))
+    {
+      gcc_assert (GET_MODE_INNER (mode) == QImode);
+
+      unsigned int op_size = GET_MODE_SIZE (op_mode);
+      unsigned int size = GET_MODE_SIZE (mode);
+      unsigned int nunits = op_size / GET_MODE_SIZE (QImode);
+      machine_mode vec_mode
+	= mode_for_vector (QImode, nunits).require ();
+      target = gen_reg_rtx (vec_mode);
+      op = gen_rtx_SUBREG (vec_mode, op, 0);
+      emit_move_insn (target, op);
+      if (op_size == size)
+	return target;
+
+      rtx tmp = gen_reg_rtx (mode);
+      target = gen_rtx_SUBREG (mode, target, 0);
+      emit_move_insn (tmp, target);
+      return tmp;
+    }
+
+  target = gen_reg_rtx (word_mode);
+  if (VECTOR_MODE_P (op_mode))
+    {
+      op = gen_rtx_SUBREG (word_mode, op, 0);
+      emit_move_insn (target, op);
+    }
+  else
+    target = op;
+
+  if (mode == word_mode)
+    return target;
+
+  rtx tmp = gen_reg_rtx (mode);
+  target = gen_rtx_SUBREG (mode, target, 0);
+  emit_move_insn (tmp, target);
+  return tmp;
+}
+
 /* Output code to set at most count & (max_size - 1) bytes starting by DEST.  */
 static void
 expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
@@ -8403,24 +8487,12 @@ expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
 
   if (CONST_INT_P (count))
     {
-      HOST_WIDE_INT countval = INTVAL (count);
-      HOST_WIDE_INT epilogue_size = countval % max_size;
-      int i;
-
-      /* For now MAX_SIZE should be a power of 2.  This assert could be
-	 relaxed, but it'll require a bit more complicated epilogue
-	 expanding.  */
-      gcc_assert ((max_size & (max_size - 1)) == 0);
-      for (i = max_size; i >= 1; i >>= 1)
-	{
-	  if (epilogue_size & i)
-	    {
-	      if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
-		destmem = emit_memset (destmem, destptr, vec_value, i);
-	      else
-		destmem = emit_memset (destmem, destptr, value, i);
-	    }
-	}
+      unsigned HOST_WIDE_INT countval = UINTVAL (count);
+      unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
+      unsigned int destalign = MEM_ALIGN (destmem);
+      store_by_pieces (destmem, epilogue_size, setmem_epilogue_gen_val,
+		       vec_value ? vec_value : value, destalign, true,
+		       RETURN_BEGIN);
       return;
     }
   if (max_size > 32)
@@ -8552,6 +8624,7 @@ expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
   rtx_code_label *label = ix86_expand_aligntest (count, size, false);
   machine_mode mode = int_mode_for_size (size * BITS_PER_UNIT, 1).else_blk ();
   rtx modesize;
+  rtx scalar_value = value;
   int n;
 
   /* If we do not have vector value to copy, we must reduce size.  */
@@ -8571,11 +8644,57 @@ expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
     {
       /* Choose appropriate vector mode.  */
       if (size >= 32)
-	mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode;
+	switch (MOVE_MAX)
+	  {
+	  case 64:
+	    if (size >= 64)
+	      {
+		mode = V64QImode;
+		break;
+	      }
+	    /* FALLTHRU */
+	  case 32:
+	    mode = V32QImode;
+	    break;
+	  case 16:
+	    mode = V16QImode;
+	    break;
+	  case 8:
+	    mode = DImode;
+	    break;
+	  default:
+	    gcc_unreachable ();
+	  }
       else if (size >= 16)
 	mode = TARGET_SSE ? V16QImode : DImode;
       srcmem = change_address (srcmem, mode, srcptr);
     }
+  if (issetmem && vec_value && GET_MODE_SIZE (mode) > size)
+    {
+      /* For memset with vector and the size is smaller than the vector
+	 size, first try the narrower vector, otherwise, use the
+	 original value. */
+      machine_mode inner_mode = GET_MODE_INNER (mode);
+      unsigned int nunits = size / GET_MODE_SIZE (inner_mode);
+      if (nunits > 1)
+	{
+	  mode = mode_for_vector (GET_MODE_INNER (mode),
+				  nunits).require ();
+	  value = gen_rtx_SUBREG (mode, value, 0);
+	}
+      else
+	{
+	  scalar_int_mode smode
+	    = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
+	  gcc_assert (GET_MODE_SIZE (GET_MODE (scalar_value))
+		      >= GET_MODE_SIZE (smode));
+	  mode = smode;
+	  if (GET_MODE (scalar_value) == mode)
+	    value = scalar_value;
+	  else
+	    value = gen_rtx_SUBREG (mode, scalar_value, 0);
+	}
+    }
   destmem = change_address (destmem, mode, destptr);
   modesize = GEN_INT (GET_MODE_SIZE (mode));
   gcc_assert (GET_MODE_SIZE (mode) <= size);
@@ -9179,13 +9298,26 @@ decide_alignment (int align,
 static rtx
 promote_duplicated_reg (machine_mode mode, rtx val)
 {
+  if (val == const0_rtx)
+    return copy_to_mode_reg (mode, CONST0_RTX (mode));
+
   machine_mode valmode = GET_MODE (val);
+  if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
+    {
+      /* Duplicate the scalar value for integer vector.  */
+      gcc_assert ((val == const0_rtx || val == constm1_rtx)
+		  || GET_MODE_INNER (mode) == valmode);
+      rtx dup = gen_reg_rtx (mode);
+      bool ok = ix86_expand_vector_init_duplicate (false, mode, dup,
+						   val);
+      gcc_assert (ok);
+      return dup;
+    }
+
   rtx tmp;
   int nops = mode == DImode ? 3 : 2;
 
-  gcc_assert (mode == SImode || mode == DImode || val == const0_rtx);
-  if (val == const0_rtx)
-    return copy_to_mode_reg (mode, CONST0_RTX (mode));
+  gcc_assert (mode == SImode || mode == DImode);
   if (CONST_INT_P (val))
     {
       HOST_WIDE_INT v = INTVAL (val) & 255;
@@ -9413,11 +9545,6 @@ ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
     return false;
   gcc_assert (alg != no_stringop);
 
-  /* For now vector-version of memset is generated only for memory zeroing, as
-     creating of promoted vector value is very cheap in this case.  */
-  if (issetmem && alg == vector_loop && val_exp != const0_rtx)
-    alg = unrolled_loop;
-
   if (!count)
     count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
   destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
@@ -9510,20 +9637,41 @@ ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
        && ((desired_align > align && !align_bytes)
 	   || (!count && epilogue_size_needed > 1)));
 
+  /* Destination is aligned after the misaligned prologue.  */
+  bool aligned_dstmem = misaligned_prologue_used;
+
+  if (noalign && !misaligned_prologue_used)
+    {
+      /* Also use misaligned prologue if alignment isn't needed and
+	 destination isn't aligned.   Since alignment isn't needed,
+	 the destination after prologue won't be aligned.  */
+      aligned_dstmem = (GET_MODE_ALIGNMENT (move_mode)
+			<= MEM_ALIGN (dst));
+      if (!aligned_dstmem)
+	misaligned_prologue_used = true;
+    }
+
   /* Do the cheap promotion to allow better CSE across the
      main loop and epilogue (ie one load of the big constant in the
      front of all code.
      For now the misaligned move sequences do not have fast path
      without broadcasting.  */
-  if (issetmem && ((CONST_INT_P (val_exp) || misaligned_prologue_used)))
+  if (issetmem
+      && (alg == vector_loop
+	  || CONST_INT_P (val_exp)
+	  || misaligned_prologue_used))
     {
       if (alg == vector_loop)
 	{
-	  gcc_assert (val_exp == const0_rtx);
-	  vec_promoted_val = promote_duplicated_reg (move_mode, val_exp);
 	  promoted_val = promote_duplicated_reg_to_size (val_exp,
 							 GET_MODE_SIZE (word_mode),
 							 desired_align, align);
+	  /* Duplicate the promoted scalar value if not 0 nor -1.  */
+	  vec_promoted_val
+	    = promote_duplicated_reg (move_mode,
+				      (val_exp == const0_rtx
+				       || val_exp == constm1_rtx)
+				      ? val_exp : promoted_val);
 	}
       else
 	{
@@ -9548,7 +9696,8 @@ ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
       if (!issetmem)
         src = change_address (src, BLKmode, srcreg);
       dst = change_address (dst, BLKmode, destreg);
-      set_mem_align (dst, desired_align * BITS_PER_UNIT);
+      if (aligned_dstmem)
+	set_mem_align (dst, desired_align * BITS_PER_UNIT);
       epilogue_size_needed = 0;
       if (need_zero_guard
 	  && min_size < (unsigned HOST_WIDE_INT) size_needed)
@@ -10096,7 +10245,7 @@ construct_plt_address (rtx symbol)
 {
   rtx tmp, unspec;
 
-  gcc_assert (GET_CODE (symbol) == SYMBOL_REF);
+  gcc_assert (SYMBOL_REF_P (symbol));
   gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
   gcc_assert (Pmode == DImode);
 
@@ -10130,7 +10279,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
   tree fndecl;
   bool call_no_callee_saved_registers = false;
 
-  if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+  if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
     {
       fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
       if (fndecl)
@@ -10167,7 +10316,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
   if (TARGET_MACHO && !TARGET_64BIT)
     {
 #if TARGET_MACHO
-      if (flag_pic && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+      if (flag_pic && SYMBOL_REF_P (XEXP (fnaddr, 0)))
 	fnaddr = machopic_indirect_call_target (fnaddr);
 #endif
     }
@@ -10177,7 +10326,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
 	 check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
 	 it an indirect call.  */
       if (flag_pic
-	  && GET_CODE (addr) == SYMBOL_REF
+	  && SYMBOL_REF_P (addr)
 	  && ix86_call_use_plt_p (addr))
 	{
 	  if (flag_plt
@@ -10251,7 +10400,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
   if (ix86_cmodel == CM_LARGE_PIC
       && !TARGET_PECOFF
       && MEM_P (fnaddr)
-      && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
+      && SYMBOL_REF_P (XEXP (fnaddr, 0))
       && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
     fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
   /* Since x32 GOT slot is 64 bit with zero upper 32 bits, indirect
@@ -10354,7 +10503,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
     }
 
   if (TARGET_MACHO && TARGET_64BIT && !sibcall
-      && ((GET_CODE (addr) == SYMBOL_REF && !SYMBOL_REF_LOCAL_P (addr))
+      && ((SYMBOL_REF_P (addr) && !SYMBOL_REF_LOCAL_P (addr))
 	  || !fndecl || TREE_PUBLIC (fndecl)))
     {
       /* We allow public functions defined in a TU to bind locally for PIC
@@ -12463,7 +12612,7 @@ ix86_expand_args_builtin (const struct builtin_description *d,
 static rtx
 ix86_erase_embedded_rounding (rtx pat)
 {
-  if (GET_CODE (pat) == INSN)
+  if (NONJUMP_INSN_P (pat))
     pat = PATTERN (pat);
 
   gcc_assert (GET_CODE (pat) == SET);
@@ -25178,7 +25327,7 @@ const_vector_equal_evenodd_p (rtx op)
 {
   machine_mode mode = GET_MODE (op);
   int i, nunits = GET_MODE_NUNITS (mode);
-  if (GET_CODE (op) != CONST_VECTOR
+  if (!CONST_VECTOR_P (op)
       || nunits != CONST_VECTOR_NUNITS (op))
     return false;
   for (i = 0; i < nunits; i += 2)
@@ -25521,7 +25670,7 @@ ix86_notrack_prefixed_insn_p (rtx_insn *insn)
 
       /* Do not emit 'notrack' if it's not an indirect call.  */
       if (MEM_P (addr)
-	  && GET_CODE (XEXP (addr, 0)) == SYMBOL_REF)
+	  && SYMBOL_REF_P (XEXP (addr, 0)))
 	return false;
       else
 	return find_reg_note (insn, REG_CALL_NOCF_CHECK, 0);
@@ -26338,8 +26487,8 @@ do_mem_operand:
       if (rtx_equal_p (op, args[2]))
 	return 0xaa;
       /* Check if CONST_VECTOR is the ones-complement of args[2].  */
-      if (GET_CODE (op) == CONST_VECTOR
-	  && GET_CODE (args[2]) == CONST_VECTOR
+      if (CONST_VECTOR_P (op)
+	  && CONST_VECTOR_P (args[2])
 	  && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
 							  op, GET_MODE (op)),
 			  args[2]))
@@ -26352,8 +26501,8 @@ do_mem_operand:
       if (rtx_equal_p (op, args[0]))
 	return 0xf0;
       /* Check if CONST_VECTOR is the ones-complement of args[0].  */
-      if (GET_CODE (op) == CONST_VECTOR
-	  && GET_CODE (args[0]) == CONST_VECTOR
+      if (CONST_VECTOR_P (op)
+	  && CONST_VECTOR_P (args[0])
 	  && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
 							  op, GET_MODE (op)),
 			  args[0]))
@@ -26366,8 +26515,8 @@ do_mem_operand:
       if (rtx_equal_p (op, args[1]))
 	return 0xcc;
       /* Check if CONST_VECTOR is the ones-complement of args[1].  */
-      if (GET_CODE (op) == CONST_VECTOR
-	  && GET_CODE (args[1]) == CONST_VECTOR
+      if (CONST_VECTOR_P (op)
+	  && CONST_VECTOR_P (args[1])
 	  && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
 							  op, GET_MODE (op)),
 			  args[1]))
@@ -26597,15 +26746,6 @@ ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
 	  && (!op2 || !side_effects_p (op2))
 	  && op0)
 	{
-	  if (GET_MODE (op0) != mode)
-	    op0 = gen_lowpart (mode, op0);
-	  if (!TARGET_64BIT && !register_operand (op0, mode))
-	    {
-	      /* Avoid force_reg (mode, op0).  */
-	      rtx reg = gen_reg_rtx (mode);
-	      emit_move_insn (reg, op0);
-	      op0 = reg;
-	    }
 	  emit_move_insn (target, gen_rtx_XOR (mode, op0, CONSTM1_RTX (mode)));
 	  return target;
 	}
@@ -26630,15 +26770,6 @@ ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
 	  && (!op2 || !side_effects_p (op2))
 	  && op1)
 	{
-	  if (GET_MODE (op1) != mode)
-	    op1 = gen_lowpart (mode, op1);
-	  if (!TARGET_64BIT && !register_operand (op1, mode))
-	    {
-	      /* Avoid force_reg (mode, op1).  */
-	      rtx reg = gen_reg_rtx (mode);
-	      emit_move_insn (reg, op1);
-	      op1 = reg;
-	    }
 	  emit_move_insn (target, gen_rtx_XOR (mode, op1, CONSTM1_RTX (mode)));
 	  return target;
 	}
@@ -26670,15 +26801,6 @@ ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
 	  && (!op1 || !side_effects_p (op1))
 	  && op2)
 	{
-	  if (GET_MODE (op2) != mode)
-	    op2 = gen_lowpart (mode, op2);
-	  if (!TARGET_64BIT && !register_operand (op2, mode))
-	    {
-	      /* Avoid force_reg (mode, op2).  */
-	      rtx reg = gen_reg_rtx (mode);
-	      emit_move_insn (reg, op2);
-	      op2 = reg;
-	    }
 	  emit_move_insn (target, gen_rtx_XOR (mode, op2, CONSTM1_RTX (mode)));
 	  return target;
 	}
diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
index 054f8d5..9941e61 100644
--- a/gcc/config/i386/i386-features.cc
+++ b/gcc/config/i386/i386-features.cc
@@ -2141,7 +2141,7 @@ convertible_comparison_p (rtx_insn *insn, enum machine_mode mode)
 
   gcc_assert (GET_CODE (src) == COMPARE);
 
-  if (GET_CODE (dst) != REG
+  if (!REG_P (dst)
       || REGNO (dst) != FLAGS_REG
       || GET_MODE (dst) != CCZmode)
     return false;
@@ -2953,7 +2953,7 @@ rest_of_insert_endbr_and_patchable_area (bool need_endbr,
 
 		  /* Also generate ENDBRANCH for non-tail call which
 		     may return via indirect branch.  */
-		  if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+		  if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
 		    fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
 		  if (fndecl == NULL_TREE)
 		    fndecl = MEM_EXPR (fnaddr);
@@ -3085,21 +3085,63 @@ ix86_rpad_gate ()
 	  && optimize_function_for_speed_p (cfun));
 }
 
+enum x86_cse_kind
+{
+  X86_CSE_CONST0_VECTOR,
+  X86_CSE_CONSTM1_VECTOR,
+  X86_CSE_VEC_DUP
+};
+
+struct redundant_load
+{
+  /* Bitmap of basic blocks with broadcast instructions.  */
+  auto_bitmap bbs;
+  /* Bitmap of broadcast instructions.  */
+  auto_bitmap insns;
+  /* The broadcast inner scalar.  */
+  rtx val;
+  /* The inner scalar mode.  */
+  machine_mode mode;
+  /* The instruction which sets the inner scalar.  Nullptr if the inner
+     scalar is applied to the whole function, instead of within the same
+     block.  */
+  rtx_insn *def_insn;
+  /* The widest broadcast source.  */
+  rtx broadcast_source;
+  /* The widest broadcast register.  */
+  rtx broadcast_reg;
+  /* The basic block of the broadcast instruction.  */
+  basic_block bb;
+  /* The number of broadcast instructions with the same inner scalar.  */
+  unsigned HOST_WIDE_INT count;
+  /* The threshold of broadcast instructions with the same inner
+     scalar.  */
+  unsigned int threshold;
+  /* The widest broadcast size in bytes.  */
+  unsigned int size;
+  /* Load kind.  */
+  x86_cse_kind kind;
+};
+
 /* Generate a vector set, DEST = SRC, at entry of the nearest dominator
    for basic block map BBS, which is in the fake loop that contains the
    whole function, so that there is only a single vector set in the
-   whole function.  If not nullptr, INNER_SCALAR is the inner scalar of
-   SRC, as (reg:SI 99) in (vec_duplicate:V4SI (reg:SI 99)).  */
+   whole function.  If not nullptr, LOAD is a pointer to the load.  */
 
 static void
 ix86_place_single_vector_set (rtx dest, rtx src, bitmap bbs,
-			      rtx inner_scalar = nullptr)
+			      redundant_load *load = nullptr)
 {
   basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
-  while (bb->loop_father->latch
-	 != EXIT_BLOCK_PTR_FOR_FN (cfun))
-    bb = get_immediate_dominator (CDI_DOMINATORS,
-				  bb->loop_father->header);
+  /* For X86_CSE_VEC_DUP, don't place the vector set outside of the loop
+     to avoid extra spills.  */
+  if (!load || load->kind != X86_CSE_VEC_DUP)
+    {
+      while (bb->loop_father->latch
+	     != EXIT_BLOCK_PTR_FOR_FN (cfun))
+	bb = get_immediate_dominator (CDI_DOMINATORS,
+				      bb->loop_father->header);
+    }
 
   rtx set = gen_rtx_SET (dest, src);
 
@@ -3141,8 +3183,14 @@ ix86_place_single_vector_set (rtx dest, rtx src, bitmap bbs,
 	}
     }
 
-  if (inner_scalar)
+  if (load && load->kind == X86_CSE_VEC_DUP)
     {
+      /* Get the source from LOAD as (reg:SI 99) in
+
+	 (vec_duplicate:V4SI (reg:SI 99))
+
+       */
+      rtx inner_scalar = load->val;
       /* Set the source in (vec_duplicate:V4SI (reg:SI 99)).  */
       rtx reg = XEXP (src, 0);
       if ((REG_P (inner_scalar) || MEM_P (inner_scalar))
@@ -3226,7 +3274,7 @@ remove_partial_avx_dependency (void)
 	      break;
 	    }
 
-	  /* Only hanlde conversion here.  */
+	  /* Only handle conversion here.  */
 	  machine_mode src_mode
 	    = convert_p ? GET_MODE (XEXP (src, 0)) : VOIDmode;
 	  switch (src_mode)
@@ -3489,44 +3537,6 @@ replace_vector_const (machine_mode vector_mode, rtx vector_const,
     }
 }
 
-enum x86_cse_kind
-{
-  X86_CSE_CONST0_VECTOR,
-  X86_CSE_CONSTM1_VECTOR,
-  X86_CSE_VEC_DUP
-};
-
-struct redundant_load
-{
-  /* Bitmap of basic blocks with broadcast instructions.  */
-  auto_bitmap bbs;
-  /* Bitmap of broadcast instructions.  */
-  auto_bitmap insns;
-  /* The broadcast inner scalar.  */
-  rtx val;
-  /* The inner scalar mode.  */
-  machine_mode mode;
-  /* The instruction which sets the inner scalar.  Nullptr if the inner
-     scalar is applied to the whole function, instead of within the same
-     block.  */
-  rtx_insn *def_insn;
-  /* The widest broadcast source.  */
-  rtx broadcast_source;
-  /* The widest broadcast register.  */
-  rtx broadcast_reg;
-  /* The basic block of the broadcast instruction.  */
-  basic_block bb;
-  /* The number of broadcast instructions with the same inner scalar.  */
-  unsigned HOST_WIDE_INT count;
-  /* The threshold of broadcast instructions with the same inner
-     scalar.  */
-  unsigned int threshold;
-  /* The widest broadcast size in bytes.  */
-  unsigned int size;
-  /* Load kind.  */
-  x86_cse_kind kind;
-};
-
 /* Return the inner scalar if OP is a broadcast, else return nullptr.  */
 
 static rtx
@@ -3534,22 +3544,20 @@ ix86_broadcast_inner (rtx op, machine_mode mode,
 		      machine_mode *scalar_mode_p,
 		      x86_cse_kind *kind_p, rtx_insn **insn_p)
 {
-  if (op == const0_rtx || op == CONST0_RTX (mode))
+  switch (standard_sse_constant_p (op, mode))
     {
+    case 1:
       *scalar_mode_p = QImode;
       *kind_p = X86_CSE_CONST0_VECTOR;
       *insn_p = nullptr;
       return const0_rtx;
-    }
-  else if ((GET_MODE_CLASS (mode) == MODE_VECTOR_INT
-	    && (op == constm1_rtx || op == CONSTM1_RTX (mode)))
-	    || (GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
-		&& float_vector_all_ones_operand (op, mode)))
-    {
+    case 2:
       *scalar_mode_p = QImode;
       *kind_p = X86_CSE_CONSTM1_VECTOR;
       *insn_p = nullptr;
       return constm1_rtx;
+    default:
+      break;
     }
 
   mode = GET_MODE (op);
@@ -3874,10 +3882,7 @@ remove_redundant_vector_load (void)
 	    else
 	      ix86_place_single_vector_set (load->broadcast_reg,
 					    load->broadcast_source,
-					    load->bbs,
-					    (load->kind == X86_CSE_VEC_DUP
-					     ? load->val
-					     : nullptr));
+					    load->bbs, load);
 	  }
 
       loop_optimizer_finalize ();
diff --git a/gcc/config/i386/i386-modes.def b/gcc/config/i386/i386-modes.def
index 2fedbeb..c2db305 100644
--- a/gcc/config/i386/i386-modes.def
+++ b/gcc/config/i386/i386-modes.def
@@ -91,7 +91,6 @@ VECTOR_MODES (FLOAT, 16);     /*         V8HF V4SF V2DF */
 VECTOR_MODES (FLOAT, 32);     /*   V16HF V8SF V4DF V2TF */
 VECTOR_MODES (FLOAT, 64);     /*  V32HF V16SF V8DF V4TF */
 VECTOR_MODES (FLOAT, 128);    /* V64HF V32SF V16DF V8TF */
-VECTOR_MODES (FLOAT, 256);    /* V128HF V64SF V32DF V16TF */
 VECTOR_MODE (FLOAT, HF, 2);   /* 	      	   V2HF */
 VECTOR_MODE (FLOAT, BF, 2);   /* 	      	   V2BF */
 VECTOR_MODE (FLOAT, HF, 6);   /*		   V6HF */
@@ -102,7 +101,6 @@ VECTOR_MODE (INT, QI, 2);     /*                   V2QI */
 VECTOR_MODE (INT, QI, 12);    /*                  V12QI */
 VECTOR_MODE (INT, QI, 14);    /*                  V14QI */
 VECTOR_MODE (INT, HI, 6);     /*                   V6HI */
-VECTOR_MODE (INT, SI, 64);    /* 		  V64SI */
 
 INT_MODE (OI, 32);
 INT_MODE (XI, 64);
diff --git a/gcc/config/i386/i386-options.cc b/gcc/config/i386/i386-options.cc
index 09cb133..09a35ef 100644
--- a/gcc/config/i386/i386-options.cc
+++ b/gcc/config/i386/i386-options.cc
@@ -1362,7 +1362,9 @@ ix86_valid_target_attribute_inner_p (tree fndecl, tree args, char *p_strings[],
 	  arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
 	  if (arg_ok)
 	    set_option (opts, enum_opts_set, opt, value,
-			p + opt_len, DK_UNSPECIFIED, input_location,
+			p + opt_len,
+			static_cast<int> (diagnostics::kind::unspecified),
+			input_location,
 			global_dc);
 	  else
 	    {
@@ -2839,7 +2841,9 @@ ix86_option_override_internal (bool main_args_p,
 
   /* Set the default value for -mfentry.  */
   if (!opts_set->x_flag_fentry)
-    opts->x_flag_fentry = TARGET_SEH;
+    opts->x_flag_fentry = (TARGET_SEH
+			   || (TARGET_64BIT_P (opts->x_ix86_isa_flags)
+			       && ENABLE_X86_64_MFENTRY));
   else
     {
       if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && opts->x_flag_pic
@@ -2850,6 +2854,17 @@ ix86_option_override_internal (bool main_args_p,
 	sorry ("%<-mno-fentry%> isn%'t compatible with SEH");
     }
 
+#ifdef OPTION_GLIBC_P
+  /* -mfentry is supported only on glibc targets.  */
+  if (!opts->x_flag_fentry
+      && OPTION_GLIBC_P (opts)
+      && (TARGET_64BIT_P (opts->x_ix86_isa_flags) || !opts->x_flag_pic)
+      && opts->x_flag_shrink_wrap
+      && opts->x_profile_flag)
+    warning (0, "%<-pg%> without %<-mfentry%> may be unreliable with "
+	     "shrink wrapping");
+#endif
+
   if (TARGET_SEH && TARGET_CALL_MS2SYSV_XLOGUES)
     sorry ("%<-mcall-ms2sysv-xlogues%> isn%'t currently supported with SEH");
 
@@ -3600,6 +3615,18 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
       return NULL_TREE;
     }
 
+  if (TARGET_64BIT)
+    {
+      /* Do not warn when emulating the MS ABI.  */
+      if ((TREE_CODE (*node) != FUNCTION_TYPE
+	   && TREE_CODE (*node) != METHOD_TYPE)
+	  || ix86_function_type_abi (*node) != MS_ABI)
+	warning (OPT_Wattributes, "%qE attribute ignored",
+		 name);
+      *no_add_attrs = true;
+      return NULL_TREE;
+    }
+
   /* Can combine regparm with all attributes but fastcall, and thiscall.  */
   if (is_attribute_p ("regparm", name))
     {
@@ -3612,7 +3639,7 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
 
       if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
 	{
-	  error ("regparam and thiscall attributes are not compatible");
+	  error ("regparm and thiscall attributes are not compatible");
 	}
 
       cst = TREE_VALUE (args);
@@ -3633,19 +3660,7 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
       return NULL_TREE;
     }
 
-  if (TARGET_64BIT)
-    {
-      /* Do not warn when emulating the MS ABI.  */
-      if ((TREE_CODE (*node) != FUNCTION_TYPE
-	   && TREE_CODE (*node) != METHOD_TYPE)
-	  || ix86_function_type_abi (*node) != MS_ABI)
-	warning (OPT_Wattributes, "%qE attribute ignored",
-	         name);
-      *no_add_attrs = true;
-      return NULL_TREE;
-    }
-
-  /* Can combine fastcall with stdcall (redundant) and sseregparm.  */
+  /* Can combine fastcall with sseregparm.  */
   if (is_attribute_p ("fastcall", name))
     {
       if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
@@ -3666,8 +3681,7 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
 	}
     }
 
-  /* Can combine stdcall with fastcall (redundant), regparm and
-     sseregparm.  */
+  /* Can combine stdcall with regparm and sseregparm.  */
   else if (is_attribute_p ("stdcall", name))
     {
       if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
@@ -3717,6 +3731,10 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
 	{
 	  error ("cdecl and thiscall attributes are not compatible");
 	}
+      if (lookup_attribute ("regparm", TYPE_ATTRIBUTES (*node)))
+	{
+	  error ("regparm and thiscall attributes are not compatible");
+	}
     }
 
   /* Can combine sseregparm with all attributes.  */
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index b64175d..65e04d3 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -5979,7 +5979,7 @@ symbolic_reference_mentioned_p (rtx op)
   const char *fmt;
   int i;
 
-  if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
+  if (SYMBOL_REF_P (op) || LABEL_REF_P (op))
     return true;
 
   fmt = GET_RTX_FORMAT (GET_CODE (op));
@@ -6526,7 +6526,7 @@ output_set_got (rtx dest, rtx label)
 
   xops[0] = dest;
 
-  if (TARGET_VXWORKS_RTP && flag_pic)
+  if (TARGET_VXWORKS_GOTTPIC && TARGET_VXWORKS_RTP && flag_pic)
     {
       /* Load (*VXWORKS_GOTT_BASE) into the PIC register.  */
       xops[2] = gen_rtx_MEM (Pmode,
@@ -10724,8 +10724,7 @@ split_stack_prologue_scratch_regno (void)
 
 static GTY(()) rtx split_stack_fn;
 
-/* A SYMBOL_REF for the more stack function when using the large
-   model.  */
+/* A SYMBOL_REF for the more stack function when using the large model.  */
 
 static GTY(()) rtx split_stack_fn_large;
 
@@ -11413,7 +11412,7 @@ ix86_force_load_from_GOT_p (rtx x, bool call_p)
 	  && (!flag_pic || this_is_asm_operands)
 	  && ix86_cmodel != CM_LARGE
 	  && ix86_cmodel != CM_LARGE_PIC
-	  && GET_CODE (x) == SYMBOL_REF
+	  && SYMBOL_REF_P (x)
 	  && ((!call_p
 	       && (!ix86_direct_extern_access
 		   || (SYMBOL_REF_DECL (x)
@@ -11459,23 +11458,23 @@ ix86_legitimate_constant_p (machine_mode mode, rtx x)
 	  case UNSPEC_TPOFF:
 	  case UNSPEC_NTPOFF:
 	    x = XVECEXP (x, 0, 0);
-	    return (GET_CODE (x) == SYMBOL_REF
+	    return (SYMBOL_REF_P (x)
 		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
 	  case UNSPEC_DTPOFF:
 	    x = XVECEXP (x, 0, 0);
-	    return (GET_CODE (x) == SYMBOL_REF
+	    return (SYMBOL_REF_P (x)
 		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
 	  case UNSPEC_SECREL32:
 	    x = XVECEXP (x, 0, 0);
-	    return GET_CODE (x) == SYMBOL_REF;
+	    return SYMBOL_REF_P (x);
 	  default:
 	    return false;
 	  }
 
       /* We must have drilled down to a symbol.  */
-      if (GET_CODE (x) == LABEL_REF)
+      if (LABEL_REF_P (x))
 	return true;
-      if (GET_CODE (x) != SYMBOL_REF)
+      if (!SYMBOL_REF_P (x))
 	return false;
       /* FALLTHRU */
 
@@ -11602,11 +11601,11 @@ legitimate_pic_operand_p (rtx x)
 	    return TARGET_64BIT;
 	  case UNSPEC_TPOFF:
 	    x = XVECEXP (inner, 0, 0);
-	    return (GET_CODE (x) == SYMBOL_REF
+	    return (SYMBOL_REF_P (x)
 		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
 	  case UNSPEC_SECREL32:
 	    x = XVECEXP (inner, 0, 0);
-	    return GET_CODE (x) == SYMBOL_REF;
+	    return SYMBOL_REF_P (x);
 	  case UNSPEC_MACHOPIC_OFFSET:
 	    return legitimate_pic_address_disp_p (x);
 	  default:
@@ -11657,7 +11656,7 @@ legitimate_pic_address_disp_p (rtx disp)
 	  if (INTVAL (op1) >= 16*1024*1024
 	      || INTVAL (op1) < -16*1024*1024)
 	    break;
-	  if (GET_CODE (op0) == LABEL_REF)
+	  if (LABEL_REF_P (op0))
 	    return true;
 	  if (GET_CODE (op0) == CONST
 	      && GET_CODE (XEXP (op0, 0)) == UNSPEC
@@ -11666,7 +11665,7 @@ legitimate_pic_address_disp_p (rtx disp)
 	  if (GET_CODE (op0) == UNSPEC
 	      && XINT (op0, 1) == UNSPEC_PCREL)
 	    return true;
-	  if (GET_CODE (op0) != SYMBOL_REF)
+	  if (!SYMBOL_REF_P (op0))
 	    break;
 	  /* FALLTHRU */
 
@@ -11731,8 +11730,8 @@ legitimate_pic_address_disp_p (rtx disp)
 	      && XINT (disp, 1) != UNSPEC_PLTOFF))
 	return false;
 
-      if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
-	  && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
+      if (!SYMBOL_REF_P (XVECEXP (disp, 0, 0))
+	  && !LABEL_REF_P (XVECEXP (disp, 0, 0)))
 	return false;
       return true;
     }
@@ -11760,14 +11759,14 @@ legitimate_pic_address_disp_p (rtx disp)
       /* We need to check for both symbols and labels because VxWorks loads
 	 text labels with @GOT rather than @GOTOFF.  See gotoff_operand for
 	 details.  */
-      return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
-	      || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
+      return (SYMBOL_REF_P (XVECEXP (disp, 0, 0))
+	      || LABEL_REF_P (XVECEXP (disp, 0, 0)));
     case UNSPEC_GOTOFF:
       /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
 	 While ABI specify also 32bit relocation but we don't produce it in
 	 small PIC model at all.  */
-      if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
-	   || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
+      if ((SYMBOL_REF_P (XVECEXP (disp, 0, 0))
+	   || LABEL_REF_P (XVECEXP (disp, 0, 0)))
 	  && !TARGET_64BIT)
         return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
       return false;
@@ -11777,19 +11776,19 @@ legitimate_pic_address_disp_p (rtx disp)
       if (saw_plus)
 	return false;
       disp = XVECEXP (disp, 0, 0);
-      return (GET_CODE (disp) == SYMBOL_REF
+      return (SYMBOL_REF_P (disp)
 	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
     case UNSPEC_NTPOFF:
       disp = XVECEXP (disp, 0, 0);
-      return (GET_CODE (disp) == SYMBOL_REF
+      return (SYMBOL_REF_P (disp)
 	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
     case UNSPEC_DTPOFF:
       disp = XVECEXP (disp, 0, 0);
-      return (GET_CODE (disp) == SYMBOL_REF
+      return (SYMBOL_REF_P (disp)
 	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
     case UNSPEC_SECREL32:
       disp = XVECEXP (disp, 0, 0);
-      return GET_CODE (disp) == SYMBOL_REF;
+      return SYMBOL_REF_P (disp);
     }
 
   return false;
@@ -12131,11 +12130,11 @@ ix86_legitimate_address_p (machine_mode, rtx addr, bool strict,
 	     that never results in lea, this seems to be easier and
 	     correct fix for crash to disable this test.  */
 	}
-      else if (GET_CODE (disp) != LABEL_REF
+      else if (!LABEL_REF_P (disp)
 	       && !CONST_INT_P (disp)
 	       && (GET_CODE (disp) != CONST
 		   || !ix86_legitimate_constant_p (Pmode, disp))
-	       && (GET_CODE (disp) != SYMBOL_REF
+	       && (!SYMBOL_REF_P (disp)
 		   || !ix86_legitimate_constant_p (Pmode, disp)))
 	/* Displacement is not constant.  */
 	return false;
@@ -12242,10 +12241,10 @@ legitimize_pic_address (rtx orig, rtx reg)
       else
 	new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
     }
-  else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
+  else if ((SYMBOL_REF_P (addr) && SYMBOL_REF_TLS_MODEL (addr) == 0)
 	   /* We can't always use @GOTOFF for text labels
 	      on VxWorks, see gotoff_operand.  */
-	   || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
+	   || (TARGET_VXWORKS_VAROFF && LABEL_REF_P (addr)))
     {
 #if TARGET_PECOFF
       rtx tmp = legitimize_pe_coff_symbol (addr, true);
@@ -12380,8 +12379,8 @@ legitimize_pic_address (rtx orig, rtx reg)
 		  /* For %rip addressing, we have to use
 		     just disp32, not base nor index.  */
 		  if (TARGET_64BIT
-		      && (GET_CODE (base) == SYMBOL_REF
-			  || GET_CODE (base) == LABEL_REF))
+		      && (SYMBOL_REF_P (base)
+			  || LABEL_REF_P (base)))
 		    base = force_reg (mode, base);
 		  if (GET_CODE (new_rtx) == PLUS
 		      && CONSTANT_P (XEXP (new_rtx, 1)))
@@ -12443,6 +12442,28 @@ static GTY(()) rtx ix86_tls_symbol;
 static rtx
 ix86_tls_get_addr (void)
 {
+  if (cfun->machine->call_saved_registers
+      == TYPE_NO_CALLER_SAVED_REGISTERS)
+    {
+      /* __tls_get_addr doesn't preserve vector registers.  When a
+	 function with no_caller_saved_registers attribute calls
+	 __tls_get_addr, YMM and ZMM registers will be clobbered.
+	 Issue an error and suggest -mtls-dialect=gnu2 in this case.  */
+      if (cfun->machine->func_type == TYPE_NORMAL)
+	error (G_("%<-mtls-dialect=gnu2%> must be used with a function"
+		  " with the %<no_caller_saved_registers%> attribute"));
+      else
+	error (cfun->machine->func_type == TYPE_EXCEPTION
+	       ? G_("%<-mtls-dialect=gnu2%> must be used with an"
+		    " exception service routine")
+	       : G_("%<-mtls-dialect=gnu2%> must be used with an"
+		    " interrupt service routine"));
+      /* Don't issue the same error twice.  */
+      cfun->machine->func_type = TYPE_NORMAL;
+      cfun->machine->call_saved_registers
+	= TYPE_DEFAULT_CALL_SAVED_REGISTERS;
+    }
+
   if (!ix86_tls_symbol)
     {
       const char *sym
@@ -12883,12 +12904,12 @@ ix86_legitimize_address (rtx x, rtx, machine_mode mode)
   bool changed = false;
   unsigned log;
 
-  log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
+  log = SYMBOL_REF_P (x) ? SYMBOL_REF_TLS_MODEL (x) : 0;
   if (log)
     return legitimize_tls_address (x, (enum tls_model) log, false);
   if (GET_CODE (x) == CONST
       && GET_CODE (XEXP (x, 0)) == PLUS
-      && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
+      && SYMBOL_REF_P (XEXP (XEXP (x, 0), 0))
       && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
     {
       rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
@@ -13305,7 +13326,7 @@ ix86_delegitimize_tls_address (rtx orig_x)
   if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
     return orig_x;
   x = XVECEXP (unspec, 0, 0);
-  gcc_assert (GET_CODE (x) == SYMBOL_REF);
+  gcc_assert (SYMBOL_REF_P (x));
   if (unspec != XEXP (addr.disp, 0))
     x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
   if (addr.index)
@@ -13472,7 +13493,7 @@ ix86_delegitimize_address_1 (rtx x, bool base_term_p)
       else if (base_term_p
 	       && pic_offset_table_rtx
 	       && !TARGET_MACHO
-	       && !TARGET_VXWORKS_RTP)
+	       && !TARGET_VXWORKS_VAROFF)
 	{
 	  rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
 	  tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
@@ -14669,7 +14690,7 @@ ix86_print_operand (FILE *file, rtx x, int code)
       /* We have patterns that allow zero sets of memory, for instance.
 	 In 64-bit mode, we should probably support all 8-byte vectors,
 	 since we can in fact encode that into an immediate.  */
-      if (GET_CODE (x) == CONST_VECTOR)
+      if (CONST_VECTOR_P (x))
 	{
 	  if (x != CONST0_RTX (GET_MODE (x)))
 	    output_operand_lossage ("invalid vector immediate");
@@ -14699,8 +14720,8 @@ ix86_print_operand (FILE *file, rtx x, int code)
 	      if (ASSEMBLER_DIALECT == ASM_ATT)
 		putc ('$', file);
 	    }
-	  else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
-		   || GET_CODE (x) == LABEL_REF)
+	  else if (GET_CODE (x) == CONST || SYMBOL_REF_P (x)
+		   || LABEL_REF_P (x))
 	    {
 	      if (ASSEMBLER_DIALECT == ASM_ATT)
 		putc ('$', file);
@@ -14795,8 +14816,8 @@ ix86_print_operand_address_as (FILE *file, rtx addr,
 	  && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
 	symbol = XEXP (XEXP (disp, 0), 0);
 
-      if (GET_CODE (symbol) == LABEL_REF
-	  || (GET_CODE (symbol) == SYMBOL_REF
+      if (LABEL_REF_P (symbol)
+	  || (SYMBOL_REF_P (symbol)
 	      && SYMBOL_REF_TLS_MODEL (symbol) == 0))
 	base = pc_rtx;
     }
@@ -14884,7 +14905,7 @@ ix86_print_operand_address_as (FILE *file, rtx addr,
 	    {
 	      if (flag_pic)
 		output_pic_addr_const (file, disp, 0);
-	      else if (GET_CODE (disp) == LABEL_REF)
+	      else if (LABEL_REF_P (disp))
 		output_asm_label (disp);
 	      else
 		output_addr_const (file, disp);
@@ -14920,7 +14941,7 @@ ix86_print_operand_address_as (FILE *file, rtx addr,
 
 	      if (flag_pic)
 		output_pic_addr_const (file, disp, 0);
-	      else if (GET_CODE (disp) == LABEL_REF)
+	      else if (LABEL_REF_P (disp))
 		output_asm_label (disp);
 	      else if (CONST_INT_P (disp))
 		offset = disp;
@@ -15872,7 +15893,7 @@ ix86_output_addr_diff_elt (FILE *file, int value, int rel)
   gcc_assert (!TARGET_64BIT);
 #endif
   /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand.  */
-  if (TARGET_64BIT || TARGET_VXWORKS_RTP)
+  if (TARGET_64BIT || TARGET_VXWORKS_VAROFF)
     fprintf (file, "%s%s%d-%s%d\n",
 	     directive, LPREFIX, value, LPREFIX, rel);
 #if TARGET_MACHO
@@ -16704,6 +16725,10 @@ ix86_convert_const_vector_to_integer (rtx op, machine_mode mode)
 	  val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
 	}
       break;
+    case E_V1SImode:
+    case E_V1DImode:
+      op = CONST_VECTOR_ELT (op, 0);
+      return INTVAL (op);
     case E_V2HFmode:
     case E_V2BFmode:
     case E_V4HFmode:
@@ -17679,8 +17704,8 @@ ix86_rip_relative_addr_p (struct ix86_address *parts)
 	      && CONST_INT_P (XEXP (symbol, 1)))
 	    symbol = XEXP (symbol, 0);
 
-	  if (GET_CODE (symbol) == LABEL_REF
-	      || (GET_CODE (symbol) == SYMBOL_REF
+	  if (LABEL_REF_P (symbol)
+	      || (SYMBOL_REF_P (symbol)
 		  && SYMBOL_REF_TLS_MODEL (symbol) == 0)
 	      || (GET_CODE (symbol) == UNSPEC
 		  && (XINT (symbol, 1) == UNSPEC_GOTPCREL
@@ -20004,7 +20029,7 @@ ix86_gimple_fold_builtin (gimple_stmt_iterator *gsi)
 	tree utype, ures, vce;
 	utype = unsigned_type_for (TREE_TYPE (arg0));
 	/* PABSB/W/D/Q store the unsigned result in dst, use ABSU_EXPR
-	   instead of ABS_EXPR to hanlde overflow case(TYPE_MIN).  */
+	   instead of ABS_EXPR to handle overflow case(TYPE_MIN).  */
 	ures = gimple_build (&stmts, ABSU_EXPR, utype, arg0);
 	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
 	loc = gimple_location (stmt);
@@ -21488,8 +21513,7 @@ ix86_hard_regno_nregs (unsigned int regno, machine_mode mode)
   /* Register pair for mask registers.  */
   if (mode == P2QImode || mode == P2HImode)
     return 2;
-  if (mode == V64SFmode || mode == V64SImode)
-    return 4;
+
   return 1;
 }
 
@@ -23087,7 +23111,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 	  /* Make (subreg:V4SI (not:V16QI (reg:V16QI ..)) 0)
 	     cost the same as register.
 	     This is used by avx_cmp<mode>3_ltint_not.  */
-	  if (GET_CODE (unsop0) == SUBREG)
+	  if (SUBREG_P (unsop0))
 	    unsop0 = XEXP (unsop0, 0);
 	  if (GET_CODE (unsop0) == NOT)
 	    unsop0 = XEXP (unsop0, 0);
@@ -23129,7 +23153,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 	     So current solution is make constant disp as cheap as possible.  */
 	  if (GET_CODE (addr) == PLUS
 	      && x86_64_immediate_operand (XEXP (addr, 1), Pmode)
-	      /* Only hanlde (reg + disp) since other forms of addr are mostly LEA,
+	      /* Only handle (reg + disp) since other forms of addr are mostly LEA,
 		 there's no additional cost for the plus of disp.  */
 	      && register_operand (XEXP (addr, 0), Pmode))
 	    {
@@ -24785,6 +24809,12 @@ static void map_egpr_constraints (vec<const char *> &constraints)
 	      buf.safe_push (cur[j + 1]);
 	      j++;
 	      break;
+	    case '{':
+	      do
+		{
+		  buf.safe_push (cur[j]);
+		} while (cur[j++] != '}');
+	      break;
 	    default:
 	      buf.safe_push (cur[j]);
 	      break;
@@ -25202,20 +25232,14 @@ asm_preferred_eh_data_format (int code, int global)
   return DW_EH_PE_absptr;
 }
 
-/* Implement targetm.vectorize.builtin_vectorization_cost.  */
+/* Worker for ix86_builtin_vectorization_cost and the fallback calls
+   from ix86_vector_costs::add_stmt_cost.  */
 static int
-ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
-                                 tree vectype, int)
+ix86_default_vector_cost (enum vect_cost_for_stmt type_of_cost,
+			  machine_mode mode)
 {
-  bool fp = false;
-  machine_mode mode = TImode;
+  bool fp = FLOAT_MODE_P (mode);
   int index;
-  if (vectype != NULL)
-    {
-      fp = FLOAT_TYPE_P (vectype);
-      mode = TYPE_MODE (vectype);
-    }
-
   switch (type_of_cost)
     {
       case scalar_stmt:
@@ -25274,14 +25298,14 @@ ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 			      COSTS_N_INSNS
 				 (ix86_cost->gather_static
 				  + ix86_cost->gather_per_elt
-				    * TYPE_VECTOR_SUBPARTS (vectype)) / 2);
+				    * GET_MODE_NUNITS (mode)) / 2);
 
       case vector_scatter_store:
         return ix86_vec_cost (mode,
 			      COSTS_N_INSNS
 				 (ix86_cost->scatter_static
 				  + ix86_cost->scatter_per_elt
-				    * TYPE_VECTOR_SUBPARTS (vectype)) / 2);
+				    * GET_MODE_NUNITS (mode)) / 2);
 
       case cond_branch_taken:
         return ix86_cost->cond_taken_branch_cost;
@@ -25299,7 +25323,7 @@ ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 
       case vec_construct:
 	{
-	  int n = TYPE_VECTOR_SUBPARTS (vectype);
+	  int n = GET_MODE_NUNITS (mode);
 	  /* N - 1 element inserts into an SSE vector, the possible
 	     GPR -> XMM move is accounted for in add_stmt_cost.  */
 	  if (GET_MODE_BITSIZE (mode) <= 128)
@@ -25327,6 +25351,17 @@ ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
     }
 }
 
+/* Implement targetm.vectorize.builtin_vectorization_cost.  */
+static int
+ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
+				 tree vectype, int)
+{
+  machine_mode mode = TImode;
+  if (vectype != NULL)
+    mode = TYPE_MODE (vectype);
+  return ix86_default_vector_cost (type_of_cost, mode);
+}
+
 
 /* This function returns the calling abi specific va_list type node.
    It returns  the FNDECL specific va_list type.  */
@@ -25780,7 +25815,7 @@ ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
 unsigned
 ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 				  stmt_vec_info stmt_info, slp_tree node,
-				  tree vectype, int misalign,
+				  tree vectype, int,
 				  vect_cost_model_location where)
 {
   unsigned retval = 0;
@@ -25798,6 +25833,14 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
       if (scalar_p)
 	mode = TYPE_MODE (TREE_TYPE (vectype));
     }
+  /* When we are costing a scalar stmt use the scalar stmt to get at the
+     type of the operation.  */
+  else if (scalar_p && stmt_info)
+    if (tree lhs = gimple_get_lhs (stmt_info->stmt))
+      {
+	fp = FLOAT_TYPE_P (TREE_TYPE (lhs));
+	mode = TYPE_MODE (TREE_TYPE (lhs));
+      }
 
   if ((kind == vector_stmt || kind == scalar_stmt)
       && stmt_info
@@ -26111,32 +26154,24 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
      (AGU and load ports).  Try to account for this by scaling the
      construction cost by the number of elements involved.  */
   if ((kind == vec_construct || kind == vec_to_scalar)
-      && ((stmt_info
-	   && (STMT_VINFO_TYPE (stmt_info) == load_vec_info_type
-	       || STMT_VINFO_TYPE (stmt_info) == store_vec_info_type)
-	   && ((STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_ELEMENTWISE
-		&& (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
+      && ((node
+	   && (((SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
+		 || (SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP
+		     && SLP_TREE_LANES (node) == 1))
+		&& (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF
+					(SLP_TREE_REPRESENTATIVE (node))))
 		    != INTEGER_CST))
-	       || (STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info)
-		   == VMAT_GATHER_SCATTER)))
-	  || (node
-	      && (((SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
-		    || (SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP
-			&& SLP_TREE_LANES (node) == 1))
-		   && (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF
-					     (SLP_TREE_REPRESENTATIVE (node))))
-		      != INTEGER_CST))
-		  || (SLP_TREE_MEMORY_ACCESS_TYPE (node)
-		      == VMAT_GATHER_SCATTER)))))
-    {
-      stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+	       || (SLP_TREE_MEMORY_ACCESS_TYPE (node)
+		   == VMAT_GATHER_SCATTER)))))
+    {
+      stmt_cost = ix86_default_vector_cost (kind, mode);
       stmt_cost *= (TYPE_VECTOR_SUBPARTS (vectype) + 1);
     }
   else if ((kind == vec_construct || kind == scalar_to_vec)
 	   && node
 	   && SLP_TREE_DEF_TYPE (node) == vect_external_def)
     {
-      stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+      stmt_cost = ix86_default_vector_cost (kind, mode);
       unsigned i;
       tree op;
       FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
@@ -26200,7 +26235,7 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 	  TREE_VISITED (op) = 0;
     }
   if (stmt_cost == -1)
-    stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+    stmt_cost = ix86_default_vector_cost (kind, mode);
 
   if (kind == vec_perm && vectype
       && GET_MODE_SIZE (TYPE_MODE (vectype)) == 32)
@@ -26295,6 +26330,65 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
       && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () >= 16)
     m_suggested_epilogue_mode = V8QImode;
 
+  /* When X86_TUNE_AVX512_MASKED_EPILOGUES is enabled try to use
+     a masked epilogue if that doesn't seem detrimental.  */
+  if (loop_vinfo
+      && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
+      && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () > 2
+      && ix86_tune_features[X86_TUNE_AVX512_MASKED_EPILOGUES]
+      && !OPTION_SET_P (param_vect_partial_vector_usage))
+    {
+      bool avoid = false;
+      if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+	  && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) >= 0)
+	{
+	  unsigned int peel_niter
+	    = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
+	  if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
+	    peel_niter += 1;
+	  /* When we know the number of scalar iterations of the epilogue,
+	     avoid masking when a single vector epilog iteration handles
+	     it in full.  */
+	  if (pow2p_hwi ((LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter)
+			 % LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ()))
+	    avoid = true;
+	}
+      if (!avoid && loop_outer (loop_outer (LOOP_VINFO_LOOP (loop_vinfo))))
+	for (auto ddr : LOOP_VINFO_DDRS (loop_vinfo))
+	  {
+	    if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
+	      ;
+	    else if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
+	      ;
+	    else
+	      {
+		int loop_depth
+		    = index_in_loop_nest (LOOP_VINFO_LOOP (loop_vinfo)->num,
+					  DDR_LOOP_NEST (ddr));
+		if (DDR_NUM_DIST_VECTS (ddr) == 1
+		    && DDR_DIST_VECTS (ddr)[0][loop_depth] == 0)
+		  {
+		    /* Avoid the case when there's an outer loop that might
+		       traverse a multi-dimensional array with the inner
+		       loop just executing the masked epilogue with a
+		       read-write where the next outer iteration might
+		       read from the masked part of the previous write,
+		       'n' filling half a vector.
+			 for (j = 0; j < m; ++j)
+			   for (i = 0; i < n; ++i)
+			     a[j][i] = c * a[j][i];  */
+		    avoid = true;
+		    break;
+		  }
+	      }
+	  }
+      if (!avoid)
+	{
+	  m_suggested_epilogue_mode = loop_vinfo->vector_mode;
+	  m_masked_epilogue = 1;
+	}
+    }
+
   vector_costs::finish_cost (scalar_costs);
 }
 
@@ -26731,7 +26825,7 @@ ix86_reloc_rw_mask (void)
 static bool
 symbolic_base_address_p (rtx addr)
 {
-  if (GET_CODE (addr) == SYMBOL_REF)
+  if (SYMBOL_REF_P (addr))
     return true;
 
   if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_GOTOFF)
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index 3f7ad68..791f3b9 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -1842,8 +1842,8 @@ typedef struct ix86_args {
 #define STRIP_UNARY(X) (UNARY_P (X) ? XEXP (X, 0) : X)
 
 #define SYMBOLIC_CONST(X)	\
-  (GET_CODE (X) == SYMBOL_REF						\
-   || GET_CODE (X) == LABEL_REF						\
+  (SYMBOL_REF_P (X)							\
+   || LABEL_REF_P (X)							\
    || (GET_CODE (X) == CONST && symbolic_reference_mentioned_p (X)))
 
 /* Max number of args passed in registers.  If this is more than 3, we will
@@ -2465,10 +2465,11 @@ constexpr wide_int_bitmask PTA_ARROWLAKE = PTA_ALDERLAKE | PTA_AVXIFMA
   | PTA_AVXVNNIINT8 | PTA_AVXNECONVERT | PTA_CMPCCXADD | PTA_UINTR;
 constexpr wide_int_bitmask PTA_ARROWLAKE_S = PTA_ARROWLAKE | PTA_AVXVNNIINT16
   | PTA_SHA512 | PTA_SM3 | PTA_SM4;
-constexpr wide_int_bitmask PTA_CLEARWATERFOREST = PTA_SIERRAFOREST
-  | PTA_AVXVNNIINT16 | PTA_SHA512 | PTA_SM3 | PTA_SM4 | PTA_USER_MSR
-  | PTA_PREFETCHI;
-constexpr wide_int_bitmask PTA_PANTHERLAKE = PTA_ARROWLAKE_S | PTA_PREFETCHI;
+constexpr wide_int_bitmask PTA_CLEARWATERFOREST =
+  (PTA_SIERRAFOREST & (~(PTA_KL | PTA_WIDEKL))) | PTA_AVXVNNIINT16 | PTA_SHA512
+  | PTA_SM3 | PTA_SM4 | PTA_USER_MSR | PTA_PREFETCHI;
+constexpr wide_int_bitmask PTA_PANTHERLAKE =
+  (PTA_ARROWLAKE_S & (~(PTA_KL | PTA_WIDEKL))) | PTA_PREFETCHI;
 constexpr wide_int_bitmask PTA_DIAMONDRAPIDS = PTA_GRANITERAPIDS_D
   | PTA_AVXIFMA | PTA_AVXNECONVERT | PTA_AVXVNNIINT16 | PTA_AVXVNNIINT8
   | PTA_CMPCCXADD | PTA_SHA512 | PTA_SM3 | PTA_SM4 | PTA_AVX10_2
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 21b9f5c..2b0dd66 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -1618,10 +1618,8 @@
 	(compare
 	  (match_operand:QI 0 "nonimmediate_operand" "QBn")
 	  (subreg:QI
-	    (match_operator:SWI248 2 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))]
+	    (match_operator:SWI248 2 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)))]
   "ix86_match_ccmode (insn, CCmode)"
   "cmp{b}\t{%h1, %0|%0, %h1}"
   [(set_attr "addr" "gpr8")
@@ -1632,10 +1630,8 @@
   [(set (reg FLAGS_REG)
 	(compare
 	  (subreg:QI
-	    (match_operator:SWI248 2 "extract_operator"
-	      [(match_operand 0 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 2 "extract_high_operator"
+	      [(match_operand 0 "int248_register_operand" "Q")]) 0)
 	  (match_operand:QI 1 "const0_operand")))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "test{b}\t%h0, %h0"
@@ -1657,10 +1653,8 @@
   [(set (reg FLAGS_REG)
 	(compare
 	  (subreg:QI
-	    (match_operator:SWI248 2 "extract_operator"
-	      [(match_operand 0 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 2 "extract_high_operator"
+	      [(match_operand 0 "int248_register_operand" "Q")]) 0)
 	  (match_operand:QI 1 "general_operand" "QnBn")))]
   "ix86_match_ccmode (insn, CCmode)"
   "cmp{b}\t{%1, %h0|%h0, %1}"
@@ -1672,15 +1666,11 @@
   [(set (reg FLAGS_REG)
 	(compare
 	  (subreg:QI
-	    (match_operator:SWI248 2 "extract_operator"
-	      [(match_operand 0 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 2 "extract_high_operator"
+	      [(match_operand 0 "int248_register_operand" "Q")]) 0)
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))]
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)))]
   "ix86_match_ccmode (insn, CCmode)"
   "cmp{b}\t{%h1, %h0|%h0, %h1}"
   [(set_attr "type" "icmp")
@@ -2968,7 +2958,8 @@
 	(match_operand:SWI248 1 "const_int_operand"))]
   "optimize_insn_for_size_p () && optimize_size > 1
    && operands[1] != const0_rtx
-   && operands[1] != constm1_rtx
+   && (operands[1] != constm1_rtx
+       || (<MODE>mode == DImode && LEGACY_INT_REG_P (operands[0])))
    && IN_RANGE (INTVAL (operands[1]), -128, 127)
    && !ix86_red_zone_used
    && REGNO (operands[0]) != SP_REG"
@@ -3479,10 +3470,8 @@
   [(set (strict_low_part
 	  (match_operand:QI 0 "register_operand" "+Q"))
 	  (subreg:QI
-	    (match_operator:SWI248 2 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0))]
+	    (match_operator:SWI248 2 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0))]
   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
   "mov{b}\t{%h1, %0|%0, %h1}"
   [(set_attr "type" "imov")
@@ -3565,10 +3554,8 @@
 (define_insn "*extzvqi"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=QBn,?R")
 	(subreg:QI
-	  (match_operator:SWI248 2 "extract_operator"
-	    [(match_operand 1 "int248_register_operand" "Q,Q")
-	     (const_int 8)
-	     (const_int 8)]) 0))]
+	  (match_operator:SWI248 2 "extract_high_operator"
+	    [(match_operand 1 "int248_register_operand" "Q,Q")]) 0))]
   ""
 {
   switch (get_attr_type (insn))
@@ -3689,10 +3676,8 @@
 	  (match_operand 0 "int248_register_operand" "+Q")
 	  (const_int 8)
 	  (const_int 8))
-	(match_operator:SWI248 2 "extract_operator"
-	  [(match_operand 1 "int248_register_operand" "Q")
-	   (const_int 8)
-	   (const_int 8)]))]
+	(match_operator:SWI248 2 "extract_high_operator"
+	  [(match_operand 1 "int248_register_operand" "Q")]))]
   ""
   "mov{b}\t{%h1, %h0|%h0, %h1}"
   [(set_attr "type" "imov")
@@ -5259,10 +5244,8 @@
   [(set (match_operand:SWI24 0 "register_operand" "=R")
 	(sign_extend:SWI24
 	  (subreg:QI
-	    (match_operator:SWI248 2 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))]
+	    (match_operator:SWI248 2 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)))]
   ""
   "movs{b<SWI24:imodesuffix>|x}\t{%h1, %0|%0, %h1}"
    [(set_attr "type" "imovx")
@@ -7008,10 +6991,8 @@
   [(set (strict_low_part (match_operand:QI 0 "register_operand" "+Q,&Q"))
 	(plus:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q,Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q,Q")]) 0)
 	  (match_operand:QI 1 "nonimmediate_operand" "0,!qm")))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
@@ -7025,8 +7006,8 @@
      [(set (strict_low_part (match_dup 0))
 	   (plus:QI
 	     (subreg:QI
-	       (match_op_dup 3
-		 [(match_dup 2) (const_int 8) (const_int 8)]) 0)
+	       (zero_extract:SWI248
+		 (match_dup 2) (const_int 8) (const_int 8)) 0)
 	     (match_dup 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -7037,29 +7018,25 @@
   [(set (strict_low_part (match_operand:QI 0 "register_operand" "+&Q"))
 	(plus:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)
 	  (subreg:QI
-	    (match_operator:SWI248 4 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 4 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
   "#"
   "&& reload_completed"
   [(set (strict_low_part (match_dup 0))
 	(subreg:QI
-	  (match_op_dup 4
-	    [(match_dup 2) (const_int 8) (const_int 8)]) 0))
+	  (zero_extract:SWI248
+	    (match_dup 2) (const_int 8) (const_int 8)) 0))
    (parallel
      [(set (strict_low_part (match_dup 0))
 	   (plus:QI
 	     (subreg:QI
-	       (match_op_dup 3
-		 [(match_dup 1) (const_int 8) (const_int 8)]) 0)
+	       (zero_extract:SWI248
+		 (match_dup 1) (const_int 8) (const_int 8)) 0)
 	     (match_dup 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -7474,10 +7451,8 @@
   [(set (match_operand:QI 0 "nonimmediate_operand" "=QBn")
 	(plus:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)
 	  (match_operand:QI 1 "nonimmediate_operand" "0")))
    (clobber (reg:CC FLAGS_REG))]
   ""
@@ -7490,29 +7465,25 @@
   [(set (match_operand:QI 0 "register_operand" "=&Q")
 	(plus:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)
 	  (subreg:QI
-	    (match_operator:SWI248 4 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 4 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "#"
   "&& reload_completed"
   [(set (match_dup 0)
 	(subreg:QI
-	  (match_op_dup 4
-	    [(match_dup 2) (const_int 8) (const_int 8)]) 0))
+	  (zero_extract:SWI248
+	    (match_dup 2) (const_int 8) (const_int 8)) 0))
    (parallel
      [(set (match_dup 0)
 	   (plus:QI
 	     (subreg:QI
-	       (match_op_dup 3
-		 [(match_dup 1) (const_int 8) (const_int 8)]) 0)
+	       (zero_extract:SWI248
+		 (match_dup 1) (const_int 8) (const_int 8)) 0)
 	   (match_dup 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -7542,10 +7513,8 @@
 	(subreg:SWI248
 	  (plus:QI
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "0,!Q")]) 0)
 	    (match_operand:QI 2 "general_operand" "QnBn,QnBn")) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
@@ -7580,8 +7549,8 @@
 	   (subreg:SWI248
 	     (plus:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)
 	       (match_dup 2)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -7601,15 +7570,11 @@
 	(subreg:SWI248
 	  (plusminus:QI
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "<comm>0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "<comm>0,!Q")]) 0)
 	    (subreg:QI
-	      (match_operator:SWI248 4 "extract_operator"
-		[(match_operand 2 "int248_register_operand" "Q,Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)) 0))
+	      (match_operator:SWI248 4 "extract_high_operator"
+		[(match_operand 2 "int248_register_operand" "Q,Q")]) 0)) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "@
@@ -7628,11 +7593,11 @@
 	   (subreg:SWI248
 	     (plusminus:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)
 	       (subreg:QI
-		 (match_op_dup 4
-		   [(match_dup 2) (const_int 8) (const_int 8)]) 0)) 0))
+		 (zero_extract:SWI248
+		   (match_dup 2) (const_int 8) (const_int 8)) 0)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
   [(set_attr "type" "alu")
@@ -8229,10 +8194,8 @@
 	(minus:QI
 	  (match_operand:QI 1 "nonimmediate_operand" "0,!qm")
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q,Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q,Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
   "@
@@ -8246,8 +8209,8 @@
 	   (minus:QI
 	     (match_dup 0)
 	     (subreg:QI
-	       (match_op_dup 3
-		 [(match_dup 2) (const_int 8) (const_int 8)]) 0)))
+	       (zero_extract:SWI248
+		 (match_dup 2) (const_int 8) (const_int 8)) 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
   [(set_attr "type" "alu")
@@ -8257,30 +8220,26 @@
   [(set (strict_low_part (match_operand:QI 0 "register_operand" "+&Q"))
 	(minus:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)
 	  (subreg:QI
-	    (match_operator:SWI248 4 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 4 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
   "#"
   "&& reload_completed"
   [(set (strict_low_part (match_dup 0))
 	(subreg:QI
-	  (match_op_dup 3
-	    [(match_dup 1) (const_int 8) (const_int 8)]) 0))
+	  (zero_extract:SWI248
+	    (match_dup 1) (const_int 8) (const_int 8)) 0))
    (parallel
      [(set (strict_low_part (match_dup 0))
 	   (minus:QI
 	   (match_dup 0)
 	     (subreg:QI
-	       (match_op_dup 4
-		 [(match_dup 2) (const_int 8) (const_int 8)]) 0)))
+	       (zero_extract:SWI248
+		 (match_dup 2) (const_int 8) (const_int 8)) 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
   [(set_attr "type" "alu")
@@ -8331,10 +8290,8 @@
 	(minus:QI
 	  (match_operand:QI 1 "nonimmediate_operand" "0")
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "sub{b}\t{%h2, %0|%0, %h2}"
@@ -8346,30 +8303,26 @@
   [(set (match_operand:QI 0 "register_operand" "=&Q")
 	(minus:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)
 	  (subreg:QI
-	    (match_operator:SWI248 4 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 4 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "#"
   "&& reload_completed"
   [(set (match_dup 0)
 	(subreg:QI
-	  (match_op_dup 3
-	    [(match_dup 1) (const_int 8) (const_int 8)]) 0))
+	  (zero_extract:SWI248
+	    (match_dup 1) (const_int 8) (const_int 8)) 0))
    (parallel
      [(set (match_dup 0)
 	   (minus:QI
 	     (match_dup 0)
 	     (subreg:QI
-	       (match_op_dup 4
-		 [(match_dup 2) (const_int 8) (const_int 8)]) 0)))
+	       (zero_extract:SWI248
+		 (match_dup 2) (const_int 8) (const_int 8)) 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
   [(set_attr "type" "alu")
@@ -8384,10 +8337,8 @@
 	(subreg:SWI248
 	  (minus:QI
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "0,!Q")]) 0)
 	    (match_operand:QI 2 "general_operand" "QnBn,QnBn")) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
@@ -8406,8 +8357,8 @@
 	   (subreg:SWI248
 	     (minus:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)
 	       (match_dup 2)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -12355,10 +12306,8 @@
 	(compare
 	  (and:QI
 	    (subreg:QI
-	      (match_operator:SWI248 2 "extract_operator"
-		[(match_operand 0 "int248_register_operand" "Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 2 "extract_high_operator"
+		[(match_operand 0 "int248_register_operand" "Q")]) 0)
 	    (match_operand:QI 1 "general_operand" "QnBn"))
 	  (const_int 0)))]
   "ix86_match_ccmode (insn, CCNOmode)"
@@ -12372,15 +12321,11 @@
 	(compare
 	  (and:QI
 	    (subreg:QI
-	      (match_operator:SWI248 2 "extract_operator"
-		[(match_operand 0 "int248_register_operand" "Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 2 "extract_high_operator"
+		[(match_operand 0 "int248_register_operand" "Q")]) 0)
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "Q")
-		 (const_int 8)
-		 (const_int 8)]) 0))
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "Q")]) 0))
 	  (const_int 0)))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "test{b}\t{%h1, %h0|%h0, %h1}"
@@ -12810,8 +12755,8 @@
 		   (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))
 	      (clobber (reg:CC FLAGS_REG))])]
 {
-  if (GET_CODE (operands[2]) == SYMBOL_REF
-      || GET_CODE (operands[2]) == LABEL_REF)
+  if (SYMBOL_REF_P (operands[2])
+      || LABEL_REF_P (operands[2]))
     {
       operands[2] = shallow_copy_rtx (operands[2]);
       PUT_MODE (operands[2], SImode);
@@ -12969,10 +12914,8 @@
   [(set (strict_low_part (match_operand:QI 0 "register_operand" "+Q,&Q"))
 	(any_logic:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q,Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q,Q")]) 0)
 	  (match_operand:QI 1 "nonimmediate_operand" "0,!qm")))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
@@ -12986,8 +12929,8 @@
      [(set (strict_low_part (match_dup 0))
 	   (any_logic:QI
 	     (subreg:QI
-	       (match_op_dup 3
-		 [(match_dup 2) (const_int 8) (const_int 8)]) 0)
+	       (zero_extract:SWI248
+		 (match_dup 2) (const_int 8) (const_int 8)) 0)
 	     (match_dup 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -12998,29 +12941,25 @@
   [(set (strict_low_part (match_operand:QI 0 "register_operand" "+&Q"))
 	(any_logic:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)
 	  (subreg:QI
-	    (match_operator:SWI248 4 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 4 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
   "#"
   "&& reload_completed"
   [(set (strict_low_part (match_dup 0))
 	(subreg:QI
-	  (match_op_dup 4
-	    [(match_dup 2) (const_int 8) (const_int 8)]) 0))
+	  (zero_extract:SWI248
+	    (match_dup 2) (const_int 8) (const_int 8)) 0))
    (parallel
      [(set (strict_low_part (match_dup 0))
 	   (any_logic:QI
 	     (subreg:QI
-	       (match_op_dup 3
-		 [(match_dup 1) (const_int 8) (const_int 8)]) 0)
+	       (zero_extract:SWI248
+		 (match_dup 1) (const_int 8) (const_int 8)) 0)
 	     (match_dup 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -13223,10 +13162,8 @@
   [(set (match_operand:QI 0 "nonimmediate_operand" "=QBn")
 	(any_logic:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)
 	  (match_operand:QI 1 "nonimmediate_operand" "0")))
    (clobber (reg:CC FLAGS_REG))]
   ""
@@ -13239,29 +13176,25 @@
   [(set (match_operand:QI 0 "register_operand" "=&Q")
 	(any_logic:QI
 	  (subreg:QI
-	    (match_operator:SWI248 3 "extract_operator"
-	      [(match_operand 1 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)
+	    (match_operator:SWI248 3 "extract_high_operator"
+	      [(match_operand 1 "int248_register_operand" "Q")]) 0)
 	  (subreg:QI
-	    (match_operator:SWI248 4 "extract_operator"
-	      [(match_operand 2 "int248_register_operand" "Q")
-	       (const_int 8)
-	       (const_int 8)]) 0)))
+	    (match_operator:SWI248 4 "extract_high_operator"
+	      [(match_operand 2 "int248_register_operand" "Q")]) 0)))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "#"
   "&& reload_completed"
   [(set (match_dup 0)
 	(subreg:QI
-	  (match_op_dup 4
-	    [(match_dup 2) (const_int 8) (const_int 8)]) 0))
+	  (zero_extract:SWI248
+	    (match_dup 2) (const_int 8) (const_int 8)) 0))
    (parallel
      [(set (match_dup 0)
 	   (any_logic:QI
 	     (subreg:QI
-	       (match_op_dup 3
-		 [(match_dup 1) (const_int 8) (const_int 8)]) 0)
+	       (zero_extract:SWI248
+		 (match_dup 1) (const_int 8) (const_int 8)) 0)
 	   (match_dup 0)))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -13291,10 +13224,8 @@
 	(subreg:SWI248
 	  (any_logic:QI
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "0,!Q")]) 0)
 	    (match_operand:QI 2 "general_operand" "QnBn,QnBn")) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
@@ -13313,8 +13244,8 @@
 	   (subreg:SWI248
 	     (any_logic:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)
 	       (match_dup 2)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -13328,10 +13259,8 @@
 	(match_operator 5 "compare_operator"
 	  [(any_logic:QI
 	     (subreg:QI
-	       (match_operator:SWI248 3 "extract_operator"
-		 [(match_operand 1 "int248_register_operand" "0,!Q")
-		  (const_int 8)
-		  (const_int 8)]) 0)
+	       (match_operator:SWI248 3 "extract_high_operator"
+		 [(match_operand 1 "int248_register_operand" "0,!Q")]) 0)
 	     (match_operand:QI 2 "general_operand" "QnBn,QnBn"))
 	  (const_int 0)]))
    (set (zero_extract:SWI248
@@ -13341,8 +13270,8 @@
 	(subreg:SWI248
 	  (any_logic:QI
 	    (subreg:QI
-	      (match_op_dup 3
-		[(match_dup 0) (const_int 8) (const_int 8)]) 0)
+	      (zero_extract:SWI248
+		(match_dup 0) (const_int 8) (const_int 8)) 0)
 	    (match_dup 2)) 0))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "@
@@ -13358,9 +13287,9 @@
      [(set (match_dup 4)
 	   (match_op_dup 5
 	     [(any_logic:QI
-	        (subreg:QI
-		  (match_op_dup 3
-		    [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		(subreg:QI
+		  (zero_extract:SWI248
+		    (match_dup 0) (const_int 8) (const_int 8)) 0)
 		(match_dup 2))
 	      (const_int 0)]))
       (set (zero_extract:SWI248
@@ -13368,8 +13297,8 @@
 	   (subreg:SWI248
 	     (any_logic:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 1) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 1) (const_int 8) (const_int 8)) 0)
 	       (match_dup 2)) 0))])]
   ""
   [(set_attr "addr" "gpr8")
@@ -13385,15 +13314,11 @@
 	(subreg:SWI248
 	  (any_logic:QI
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "%0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "%0,!Q")]) 0)
 	    (subreg:QI
-	      (match_operator:SWI248 4 "extract_operator"
-		[(match_operand 2 "int248_register_operand" "Q,Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)) 0))
+	      (match_operator:SWI248 4 "extract_high_operator"
+		[(match_operand 2 "int248_register_operand" "Q,Q")]) 0)) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "@
@@ -13412,11 +13337,11 @@
 	   (subreg:SWI248
 	     (any_logic:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)
 	       (subreg:QI
-		 (match_op_dup 4
-		   [(match_dup 2) (const_int 8) (const_int 8)]) 0)) 0))
+		 (zero_extract:SWI248
+		   (match_dup 2) (const_int 8) (const_int 8)) 0)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
   [(set_attr "type" "alu")
@@ -13428,12 +13353,10 @@
 	  (match_operand 0 "int248_register_operand" "+Q,&Q")
 	  (const_int 8)
 	  (const_int 8))
-	(match_operator:SWI248 3 "extract_operator"
+	(match_operator:SWI248 3 "extract_high_operator"
 	  [(any_logic
 	     (match_operand 1 "int248_register_operand" "%0,!Q")
-	     (match_operand 2 "int248_register_operand" "Q,Q"))
-	   (const_int 8)
-	   (const_int 8)]))
+	     (match_operand 2 "int248_register_operand" "Q,Q"))]))
    (clobber (reg:CC FLAGS_REG))]
   "GET_MODE (operands[1]) == GET_MODE (operands[2])"
   "@
@@ -13449,9 +13372,9 @@
    (parallel
      [(set (zero_extract:SWI248
 	     (match_dup 0) (const_int 8) (const_int 8))
-	   (match_op_dup 3
-	     [(any_logic (match_dup 4) (match_dup 2))
-	      (const_int 8) (const_int 8)]))
+	   (zero_extract:SWI248
+	     (any_logic (match_dup 4) (match_dup 2))
+	     (const_int 8) (const_int 8)))
       (clobber (reg:CC FLAGS_REG))])]
   "operands[4] = gen_lowpart (GET_MODE (operands[1]), operands[0]);"
   [(set_attr "type" "alu")
@@ -14696,10 +14619,8 @@
 	(subreg:SWI248
 	  (neg:QI
 	    (subreg:QI
-	      (match_operator:SWI248 2 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)) 0))
+	      (match_operator:SWI248 2 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "0,!Q")]) 0)) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "@
@@ -14717,8 +14638,8 @@
 	   (subreg:SWI248
 	     (neg:QI
 	       (subreg:QI
-		 (match_op_dup 2
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)) 0))
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
   [(set_attr "type" "negnot")
@@ -14769,6 +14690,17 @@
 		   (compare:CCZ (neg:SWI (match_dup 0)) (const_int 0)))
 	      (set (match_dup 0) (neg:SWI (match_dup 0)))])])
 
+;; Optimize *negsi_1 followed by *cmpsi_ccno_1 (PR target/91384) with APX_F
+(define_peephole2
+  [(parallel [(set (match_operand:SWI 0 "general_reg_operand")
+		   (neg:SWI (match_operand:SWI 1 "general_reg_operand")))
+	      (clobber (reg:CC FLAGS_REG))])
+   (set (reg:CCZ FLAGS_REG) (compare:CCZ (match_dup 1) (const_int 0)))]
+  "TARGET_APX_NDD"
+  [(parallel [(set (reg:CCZ FLAGS_REG)
+		   (compare:CCZ (neg:SWI (match_dup 1)) (const_int 0)))
+	      (set (match_dup 0) (neg:SWI (match_dup 1)))])])
+
 ;; Special expand pattern to handle integer mode abs
 
 (define_expand "abs<mode>2"
@@ -15339,13 +15271,9 @@
 	  (match_operand 0 "int248_register_operand" "+Q,&Q")
 	  (const_int 8)
 	  (const_int 8))
-	(subreg:SWI248
-	  (not:QI
-	    (subreg:QI
-	      (match_operator:SWI248 2 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)) 0))]
+	(not:SWI248
+	  (match_operator:SWI248 2 "extract_high_operator"
+	    [(match_operand 1 "int248_register_operand" "0,!Q")])))]
   ""
   "@
    not{b}\t%h0
@@ -15358,11 +15286,8 @@
 	  (match_dup 1) (const_int 8) (const_int 8)))
    (set (zero_extract:SWI248
 	  (match_dup 0) (const_int 8) (const_int 8))
-	(subreg:SWI248
-	  (not:QI
-	    (subreg:QI
-	      (match_op_dup 2
-		[(match_dup 0) (const_int 8) (const_int 8)]) 0)) 0))]
+	(not:SWI248
+	  (zero_extract:SWI248 (match_dup 0) (const_int 8) (const_int 8))))]
   ""
   [(set_attr "type" "negnot")
    (set_attr "mode" "QI")])
@@ -16709,10 +16634,8 @@
 	(subreg:SWI248
 	  (ashift:QI
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "0,!Q")]) 0)
 	    (match_operand:QI 2 "nonmemory_operand" "cI,cI")) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
@@ -16746,8 +16669,8 @@
 	   (subreg:SWI248
 	     (ashift:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)
 	       (match_dup 2)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -17993,10 +17916,8 @@
 	(subreg:SWI248
 	  (any_shiftrt:QI
 	    (subreg:QI
-	      (match_operator:SWI248 3 "extract_operator"
-		[(match_operand 1 "int248_register_operand" "0,!Q")
-		 (const_int 8)
-		 (const_int 8)]) 0)
+	      (match_operator:SWI248 3 "extract_high_operator"
+		[(match_operand 1 "int248_register_operand" "0,!Q")]) 0)
 	    (match_operand:QI 2 "nonmemory_operand" "cI,cI")) 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
@@ -18022,8 +17943,8 @@
 	   (subreg:SWI248
 	     (any_shiftrt:QI
 	       (subreg:QI
-		 (match_op_dup 3
-		   [(match_dup 0) (const_int 8) (const_int 8)]) 0)
+		 (zero_extract:SWI248
+		   (match_dup 0) (const_int 8) (const_int 8)) 0)
 	       (match_dup 2)) 0))
       (clobber (reg:CC FLAGS_REG))])]
   ""
@@ -20102,7 +20023,7 @@
 
       /* We can't use @GOTOFF for text labels on VxWorks;
 	 see gotoff_operand.  */
-      if (TARGET_64BIT || TARGET_VXWORKS_RTP)
+      if (TARGET_64BIT || TARGET_VXWORKS_VAROFF)
 	{
 	  code = PLUS;
 	  op0 = operands[0];
@@ -20970,7 +20891,7 @@
       (clobber (reg:CC FLAGS_REG))])]
   "!TARGET_64BIT"
 {
-  if (flag_pic && !TARGET_VXWORKS_RTP)
+  if (flag_pic && !TARGET_VXWORKS_GOTTPIC)
     ix86_pc_thunk_call_expanded = true;
 })
 
@@ -20991,7 +20912,7 @@
       (clobber (reg:CC FLAGS_REG))])]
   "!TARGET_64BIT"
 {
-  if (flag_pic && !TARGET_VXWORKS_RTP)
+  if (flag_pic && !TARGET_VXWORKS_GOTTPIC)
     ix86_pc_thunk_call_expanded = true;
 })
 
@@ -28240,10 +28161,8 @@
 	(match_operator 1 "compare_operator"
 	  [(and:QI
 	     (subreg:QI
-	       (match_operator:SWI248 4 "extract_operator"
-		 [(match_operand 2 "int248_register_operand")
-		  (const_int 8)
-		  (const_int 8)]) 0)
+	       (match_operator:SWI248 4 "extract_high_operator"
+		 [(match_operand 2 "int248_register_operand")]) 0)
 	     (match_operand 3 "const_int_operand"))
 	   (const_int 0)]))]
   "! TARGET_PARTIAL_REG_STALL
@@ -28255,9 +28174,9 @@
 	   (match_op_dup 1
 	     [(and:QI
 		(subreg:QI
-		  (match_op_dup 4 [(match_dup 2)
-				   (const_int 8)
-				   (const_int 8)]) 0)
+		  (zero_extract:SWI248 (match_dup 2)
+				       (const_int 8)
+				       (const_int 8)) 0)
 		(match_dup 3))
 	      (const_int 0)]))
       (set (zero_extract:SWI248 (match_dup 2)
@@ -28266,9 +28185,9 @@
 	   (subreg:SWI248
 	     (and:QI
 	       (subreg:QI
-		 (match_op_dup 4 [(match_dup 2)
-				  (const_int 8)
-				  (const_int 8)]) 0)
+		 (zero_extract:SWI248 (match_dup 2)
+				      (const_int 8)
+				      (const_int 8)) 0)
 	       (match_dup 3)) 0))])])
 
 ;; Don't do logical operations with memory inputs.
diff --git a/gcc/config/i386/mmx.md b/gcc/config/i386/mmx.md
index 7920232..1f97993 100644
--- a/gcc/config/i386/mmx.md
+++ b/gcc/config/i386/mmx.md
@@ -81,12 +81,13 @@
 ;; 4-byte and 2-byte QImode vector modes
 (define_mode_iterator VI1_16_32 [V4QI V2QI])
 
-;; All 2-byte, 4-byte and 8-byte vector modes with more than 1 element
+;; All 2-byte, 4-byte and 8-byte vector modes.
 (define_mode_iterator V_16_32_64
-   [V2QI V4QI V2HI V2HF
+   [V2QI V4QI V2HI V1SI V2HF V2BF
     (V8QI "TARGET_64BIT") (V4HI "TARGET_64BIT")
     (V4HF "TARGET_64BIT") (V4BF "TARGET_64BIT")
-    (V2SI "TARGET_64BIT") (V2SF "TARGET_64BIT")])
+    (V2SI "TARGET_64BIT") (V2SF "TARGET_64BIT")
+    (V1DI "TARGET_64BIT")])
 
 ;; V2S* modes
 (define_mode_iterator V2FI [V2SF V2SI])
@@ -107,6 +108,7 @@
   [(V8QI "DI") (V4QI "SI") (V2QI "HI")
    (V4HI "DI") (V2HI "SI")
    (V2SI "DI")
+   (V1DI "DI") (V1SI "SI")
    (V4HF "DI") (V2HF "SI")
    (V4BF "DI") (V2BF "SI")
    (V2SF "DI")])
@@ -329,7 +331,7 @@
 
 (define_expand "mov<mode>"
   [(set (match_operand:V_32 0 "nonimmediate_operand")
-	(match_operand:V_32 1 "nonimmediate_operand"))]
+	(match_operand:V_32 1 "nonimm_or_0_operand"))]
   ""
 {
   ix86_expand_vector_move (<MODE>mode, operands);
@@ -339,7 +341,7 @@
 (define_insn "*mov<mode>_internal"
   [(set (match_operand:V_32 0 "nonimmediate_operand"
     "=r ,m ,v,v,v,m,r,v")
-	(match_operand:V_32 1 "general_operand"
+	(match_operand:V_32 1 "nonimm_or_0_operand"
     "rmC,rC,C,v,m,v,v,r"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && ix86_hardreg_mov_ok (operands[0], operands[1])"
@@ -407,22 +409,6 @@
 	   ]
 	   (symbol_ref "true")))])
 
-;; 16-bit, 32-bit and 64-bit constant vector stores.  After reload,
-;; convert them to immediate integer stores.
-(define_insn_and_split "*mov<mode>_imm"
-  [(set (match_operand:V_16_32_64 0 "memory_operand" "=m")
-	(match_operand:V_16_32_64 1 "x86_64_const_vector_operand" "i"))]
-  ""
-  "#"
-  "&& reload_completed"
-  [(set (match_dup 0) (match_dup 1))]
-{
-  HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (operands[1],
-							    <MODE>mode);
-  operands[1] = GEN_INT (val);
-  operands[0] = lowpart_subreg (<mmxinsnmode>mode, operands[0], <MODE>mode);
-})
-
 ;; For TARGET_64BIT we always round up to 8 bytes.
 (define_insn "*push<mode>2_rex64"
   [(set (match_operand:V_32 0 "push_operand" "=X,X")
@@ -457,7 +443,7 @@
 
 (define_expand "movv2qi"
   [(set (match_operand:V2QI 0 "nonimmediate_operand")
-	(match_operand:V2QI 1 "nonimmediate_operand"))]
+	(match_operand:V2QI 1 "nonimm_or_0_operand"))]
   ""
 {
   ix86_expand_vector_move (V2QImode, operands);
@@ -467,9 +453,10 @@
 (define_insn "*movv2qi_internal"
   [(set (match_operand:V2QI 0 "nonimmediate_operand"
     "=r,r,r,m ,v,v,v,jm,m,r,v")
-	(match_operand:V2QI 1 "general_operand"
+	(match_operand:V2QI 1 "nonimm_or_0_operand"
     "r ,C,m,rC,C,v,m,x,v,v,r"))]
-  "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
+  "!(MEM_P (operands[0]) && MEM_P (operands[1]))
+   && ix86_hardreg_mov_ok (operands[0], operands[1])"
 {
   switch (get_attr_type (insn))
     {
@@ -587,6 +574,42 @@
 	   ]
 	   (symbol_ref "true")))])
 
+(define_split
+  [(set (match_operand:V_16_32_64 0 "general_reg_operand")
+	(match_operand:V_16_32_64 1 "memory_operand"))]
+  "reload_completed
+   && SYMBOL_REF_P (XEXP (operands[1], 0))
+   && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0))"
+  [(set (match_dup 0) (match_dup 1))]
+{
+  rtx op1 = avoid_constant_pool_reference (operands[1]);
+
+  if (!CONST_VECTOR_P (op1))
+    FAIL;
+
+  HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (op1, <MODE>mode);
+
+  operands[0] = lowpart_subreg (<mmxinsnmode>mode, operands[0], <MODE>mode);
+  operands[1] = GEN_INT (val);
+})
+
+;; 16-bit, 32-bit and 64-bit constant vector stores.  After reload,
+;; convert them to immediate integer stores.
+(define_insn_and_split "*mov<mode>_imm"
+  [(set (match_operand:V_16_32_64 0 "memory_operand" "=m")
+	(match_operand:V_16_32_64 1 "x86_64_const_vector_operand" "i"))]
+  ""
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (match_dup 1))]
+{
+  rtx op1 = operands[1];
+  HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (op1, <MODE>mode);
+
+  operands[0] = adjust_address (operands[0], <mmxinsnmode>mode, 0);
+  operands[1] = GEN_INT (val);
+})
+
 ;; We always round up to UNITS_PER_WORD bytes.
 (define_insn "*pushv2qi2"
   [(set (match_operand:V2QI 0 "push_operand" "=X,X")
diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index 1bd63b2..0f31090 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -573,8 +573,8 @@
 
     case CONST:
       op = XEXP (op, 0);
-      if (GET_CODE (op) == SYMBOL_REF
-	  || GET_CODE (op) == LABEL_REF
+      if (SYMBOL_REF_P (op)
+	  || LABEL_REF_P (op)
 	  || (GET_CODE (op) == UNSPEC
 	      && (XINT (op, 1) == UNSPEC_GOT
 		  || XINT (op, 1) == UNSPEC_GOTOFF
@@ -586,8 +586,8 @@
 	return false;
 
       op = XEXP (op, 0);
-      if (GET_CODE (op) == SYMBOL_REF
-	  || GET_CODE (op) == LABEL_REF)
+      if (SYMBOL_REF_P (op)
+	  || LABEL_REF_P (op))
 	return true;
       /* Only @GOTOFF gets offsets.  */
       if (GET_CODE (op) != UNSPEC
@@ -595,8 +595,8 @@
 	return false;
 
       op = XVECEXP (op, 0, 0);
-      if (GET_CODE (op) == SYMBOL_REF
-	  || GET_CODE (op) == LABEL_REF)
+      if (SYMBOL_REF_P (op)
+	  || LABEL_REF_P (op))
 	return true;
       return false;
 
@@ -614,10 +614,10 @@
       && CONST_INT_P (XEXP (XEXP (op, 0), 1)))
     op = XEXP (XEXP (op, 0), 0);
 
-  if (GET_CODE (op) == LABEL_REF)
+  if (LABEL_REF_P (op))
     return true;
 
-  if (GET_CODE (op) != SYMBOL_REF)
+  if (!SYMBOL_REF_P (op))
     return false;
 
   if (SYMBOL_REF_TLS_MODEL (op))
@@ -649,7 +649,7 @@
       && CONST_INT_P (XEXP (XEXP (op, 0), 1)))
     op = XEXP (XEXP (op, 0), 0);
 
-  if (GET_CODE (op) == SYMBOL_REF
+  if (SYMBOL_REF_P (op)
       && !SYMBOL_REF_FUNCTION_P (op))
     return false;
 
@@ -664,8 +664,9 @@
 ;; same segment as the GOT.  Unfortunately, the flexibility of linker
 ;; scripts means that we can't be sure of that in general, so assume
 ;; @GOTOFF is not valid on VxWorks, except with the large code model.
+;; The comments above seem to apply only to VxWorks releases before 7.
 (define_predicate "gotoff_operand"
-  (and (ior (not (match_test "TARGET_VXWORKS_RTP"))
+  (and (ior (not (match_test "TARGET_VXWORKS_VAROFF"))
             (match_test "ix86_cmodel == CM_LARGE")
             (match_test "ix86_cmodel == CM_LARGE_PIC"))
        (match_operand 0 "local_symbolic_operand")))
@@ -1144,7 +1145,7 @@
   unsigned n_elts;
   op = avoid_constant_pool_reference (op);
 
-  if (GET_CODE (op) != CONST_VECTOR)
+  if (!CONST_VECTOR_P (op))
     return false;
 
   n_elts = CONST_VECTOR_NUNITS (op);
@@ -1172,7 +1173,7 @@
   if (MEM_P (op))
     {
       op = get_pool_constant (XEXP (op, 0));
-      if (GET_CODE (op) != CONST_VECTOR)
+      if (!CONST_VECTOR_P (op))
 	return false;
 
       if (GET_MODE (op) != mode
@@ -1421,8 +1422,8 @@
 	}
       if (TARGET_64BIT
 	  && flag_pic
-	  && (GET_CODE (disp) == SYMBOL_REF
-	      || GET_CODE (disp) == LABEL_REF))
+	  && (SYMBOL_REF_P (disp)
+	      || LABEL_REF_P (disp)))
 	return false;
     }
 
@@ -1739,8 +1740,12 @@
 (define_predicate "compare_operator"
   (match_code "compare"))
 
-(define_predicate "extract_operator"
-  (match_code "zero_extract,sign_extract"))
+(define_predicate "extract_high_operator"
+  (match_code "zero_extract,sign_extract,ashiftrt,lshiftrt")
+{
+  return (const8_operand (XEXP (op, 1), VOIDmode)
+	  && (BINARY_P (op) || const8_operand (XEXP (op, 2), VOIDmode)));
+})
 
 ;; Return true if OP is a memory operand, aligned to
 ;; less than its natural alignment.
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 252ba07..ec74f93 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -12756,7 +12756,7 @@
 	     lo insns have =m and 0C constraints.  */
 	  : (operands[2] != const0_rtx
 	     || (!rtx_equal_p (dest, operands[3])
-		 && GET_CODE (operands[3]) != CONST_VECTOR))))
+		 && !CONST_VECTOR_P (operands[3])))))
     dest = gen_reg_rtx (<ssehalfvecmode>mode);
   switch (INTVAL (operands[2]))
     {
@@ -21729,6 +21729,19 @@
 	   (const_string "orig")))
    (set_attr "mode" "TI,TI,TI,TI,TI,TI,V4SF,V2SF,V2SF")])
 
+;; Eliminate redundancy caused by
+;; /* Special case TImode to 128-bit vector conversions via V2DI.  */
+;; in ix86_expand_vector_move
+
+(define_split
+  [(set (match_operand:V2DI 0 "register_operand")
+	(vec_concat:V2DI
+	  (subreg:DI (match_operand:TI 1 "register_operand") 0)
+	  (subreg:DI (match_dup 1) 8)))]
+  "TARGET_SSE2 && ix86_pre_reload_split ()"
+  [(set (match_dup 0)
+	(subreg:V2DI (match_dup 1) 0))])
+
 (define_insn "*vec_concatv2di_0"
   [(set (match_operand:V2DI 0 "register_operand"     "=v,v ,x")
 	(vec_concat:V2DI
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 91cdca7..a86cbad 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -31,7 +31,7 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 	- Updating ix86_issue_rate and ix86_adjust_cost in i386.md
 	- possibly updating ia32_multipass_dfa_lookahead, ix86_sched_reorder
 	  and ix86_sched_init_global if those tricks are needed.
-    - Tunning the flags bellow. Those are split into sections and each
+    - tuning flags below; those are split into sections and each
       section is very roughly ordered by importance.  */
 
 /*****************************************************************************/
@@ -639,6 +639,11 @@ DEF_TUNE (X86_TUNE_AVX512_STORE_BY_PIECES, "avx512_store_by_pieces",
 DEF_TUNE (X86_TUNE_AVX512_TWO_EPILOGUES, "avx512_two_epilogues",
 	  m_ZNVER4 | m_ZNVER5)
 
+/* X86_TUNE_AVX512_MAKED_EPILOGUES: Use two masked vector epilogues
+   when fit.  */
+DEF_TUNE (X86_TUNE_AVX512_MASKED_EPILOGUES, "avx512_masked_epilogues",
+	  m_ZNVER4 | m_ZNVER5)
+
 /*****************************************************************************/
 /*****************************************************************************/
 /* Historical relics: tuning flags that helps a specific old CPU designs     */
diff --git a/gcc/config/loongarch/lasx.md b/gcc/config/loongarch/lasx.md
index 43e3ab0..3d71f30 100644
--- a/gcc/config/loongarch/lasx.md
+++ b/gcc/config/loongarch/lasx.md
@@ -2060,9 +2060,9 @@
   [(set_attr "type" "simd_int_arith")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "lasx_xvshuf_<lasxfmt_f>"
+(define_insn "@lasx_xvshuf_<lasxfmt_f>"
   [(set (match_operand:LASX_DWH 0 "register_operand" "=f")
-	(unspec:LASX_DWH [(match_operand:LASX_DWH 1 "register_operand" "0")
+	(unspec:LASX_DWH [(match_operand:<VIMODE> 1 "register_operand" "0")
 			  (match_operand:LASX_DWH 2 "register_operand" "f")
 			  (match_operand:LASX_DWH 3 "register_operand" "f")]
 			UNSPEC_LASX_XVSHUF))]
diff --git a/gcc/config/loongarch/loongarch.cc b/gcc/config/loongarch/loongarch.cc
index f62e416..493f95e 100644
--- a/gcc/config/loongarch/loongarch.cc
+++ b/gcc/config/loongarch/loongarch.cc
@@ -8380,7 +8380,7 @@ static bool
 loongarch_try_expand_lsx_vshuf_const (struct expand_vec_perm_d *d)
 {
   int i;
-  rtx target, op0, op1, sel, tmp;
+  rtx target, op0, op1;
   rtx rperm[MAX_VECT_LEN];
 
   if (GET_MODE_SIZE (d->vmode) == 16)
@@ -8399,47 +8399,23 @@ loongarch_try_expand_lsx_vshuf_const (struct expand_vec_perm_d *d)
       for (i = 0; i < d->nelt; i += 1)
 	  rperm[i] = GEN_INT (d->perm[i]);
 
-      if (d->vmode == E_V2DFmode)
-	{
-	  sel = gen_rtx_CONST_VECTOR (E_V2DImode, gen_rtvec_v (d->nelt, rperm));
-	  tmp = simplify_gen_subreg (E_V2DImode, d->target, d->vmode, 0);
-	  emit_move_insn (tmp, sel);
-	}
-      else if (d->vmode == E_V4SFmode)
-	{
-	  sel = gen_rtx_CONST_VECTOR (E_V4SImode, gen_rtvec_v (d->nelt, rperm));
-	  tmp = simplify_gen_subreg (E_V4SImode, d->target, d->vmode, 0);
-	  emit_move_insn (tmp, sel);
-	}
+      machine_mode sel_mode = related_int_vector_mode (d->vmode)
+	.require ();
+      rtvec sel_v = gen_rtvec_v (d->nelt, rperm);
+
+      /* Despite vshuf.* (except vshuf.b) needs sel == target, we cannot
+	 load sel into target right now: here we are dealing with
+	 pseudo regs, and target may be the same pseudo as one of op0
+	 or op1.  Then we'd clobber the input.  Instead, we use a new
+	 pseudo reg here.  The reload pass will look at the constraint
+	 of vshuf.* and move sel into target first if needed.  */
+      rtx sel = force_reg (sel_mode,
+			   gen_rtx_CONST_VECTOR (sel_mode, sel_v));
+
+      if (d->vmode == E_V16QImode)
+	emit_insn (gen_lsx_vshuf_b (target, op1, op0, sel));
       else
-	{
-	  sel = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (d->nelt, rperm));
-	  emit_move_insn (d->target, sel);
-	}
-
-      switch (d->vmode)
-	{
-	case E_V2DFmode:
-	  emit_insn (gen_lsx_vshuf_d_f (target, target, op1, op0));
-	  break;
-	case E_V2DImode:
-	  emit_insn (gen_lsx_vshuf_d (target, target, op1, op0));
-	  break;
-	case E_V4SFmode:
-	  emit_insn (gen_lsx_vshuf_w_f (target, target, op1, op0));
-	  break;
-	case E_V4SImode:
-	  emit_insn (gen_lsx_vshuf_w (target, target, op1, op0));
-	  break;
-	case E_V8HImode:
-	  emit_insn (gen_lsx_vshuf_h (target, target, op1, op0));
-	  break;
-	case E_V16QImode:
-	  emit_insn (gen_lsx_vshuf_b (target, op1, op0, target));
-	  break;
-	default:
-	  break;
-	}
+	emit_insn (gen_lsx_vshuf (d->vmode, target, sel, op1, op0));
 
       return true;
     }
@@ -9435,7 +9411,7 @@ loongarch_expand_vec_perm_const (struct expand_vec_perm_d *d)
   bool flag = false;
   unsigned int i;
   unsigned char idx;
-  rtx target, op0, op1, sel, tmp;
+  rtx target, op0, op1;
   rtx rperm[MAX_VECT_LEN];
   unsigned int remapped[MAX_VECT_LEN];
   unsigned char perm2[MAX_VECT_LEN];
@@ -9615,63 +9591,23 @@ loongarch_expand_vec_perm_const (struct expand_vec_perm_d *d)
 expand_perm_const_end:
       if (flag)
 	{
-	  /* Copy selector vector from memory to vector register for later insn
-	     gen function.
-	     If vector's element in floating point value, we cannot fit
-	     selector argument into insn gen function directly, because of the
-	     insn template definition.  As a solution, generate a integral mode
-	     subreg of target, then copy selector vector (that is in integral
-	     mode) to this subreg.  */
-	  switch (d->vmode)
-	    {
-	    case E_V4DFmode:
-	      sel = gen_rtx_CONST_VECTOR (E_V4DImode, gen_rtvec_v (d->nelt,
-								   rperm));
-	      tmp = simplify_gen_subreg (E_V4DImode, d->target, d->vmode, 0);
-	      emit_move_insn (tmp, sel);
-	      break;
-	    case E_V8SFmode:
-	      sel = gen_rtx_CONST_VECTOR (E_V8SImode, gen_rtvec_v (d->nelt,
-								   rperm));
-	      tmp = simplify_gen_subreg (E_V8SImode, d->target, d->vmode, 0);
-	      emit_move_insn (tmp, sel);
-	      break;
-	    default:
-	      sel = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (d->nelt,
-								 rperm));
-	      emit_move_insn (d->target, sel);
-	      break;
-	    }
-
 	  target = d->target;
 	  op0 = d->op0;
 	  op1 = d->one_vector_p ? d->op0 : d->op1;
 
-	  /* We FINALLY can generate xvshuf.* insn.  */
-	  switch (d->vmode)
-	    {
-	    case E_V4DFmode:
-	      emit_insn (gen_lasx_xvshuf_d_f (target, target, op1, op0));
-	      break;
-	    case E_V4DImode:
-	      emit_insn (gen_lasx_xvshuf_d (target, target, op1, op0));
-	      break;
-	    case E_V8SFmode:
-	      emit_insn (gen_lasx_xvshuf_w_f (target, target, op1, op0));
-	      break;
-	    case E_V8SImode:
-	      emit_insn (gen_lasx_xvshuf_w (target, target, op1, op0));
-	      break;
-	    case E_V16HImode:
-	      emit_insn (gen_lasx_xvshuf_h (target, target, op1, op0));
-	      break;
-	    case E_V32QImode:
-	      emit_insn (gen_lasx_xvshuf_b (target, op1, op0, target));
-	      break;
-	    default:
-	      gcc_unreachable ();
-	      break;
-	    }
+	  machine_mode sel_mode = related_int_vector_mode (d->vmode)
+	    .require ();
+	  rtvec sel_v = gen_rtvec_v (d->nelt, rperm);
+
+	  /* See the comment in loongarch_expand_lsx_shuffle for why
+	     we don't simply use a SUBREG to pun target.  */
+	  rtx sel = force_reg (sel_mode,
+			       gen_rtx_CONST_VECTOR (sel_mode, sel_v));
+
+	  if (d->vmode == E_V32QImode)
+	    emit_insn (gen_lasx_xvshuf_b (target, op1, op0, sel));
+	  else
+	    emit_insn (gen_lasx_xvshuf (d->vmode, target, sel, op1, op0));
 
 	  return true;
 	}
@@ -11116,17 +11052,21 @@ static bool
 loongarch_builtin_support_vector_misalignment (machine_mode mode,
 					       const_tree type,
 					       int misalignment,
-					       bool is_packed)
+					       bool is_packed,
+					       bool is_gather_scatter)
 {
   if ((ISA_HAS_LSX || ISA_HAS_LASX) && STRICT_ALIGNMENT)
     {
+      if (is_gather_scatter)
+	return true;
       if (optab_handler (movmisalign_optab, mode) == CODE_FOR_nothing)
 	return false;
       if (misalignment == -1)
 	return false;
     }
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* Return a PARALLEL containing NELTS elements, with element I equal
diff --git a/gcc/config/loongarch/loongarch.h b/gcc/config/loongarch/loongarch.h
index d897763..5fc8665 100644
--- a/gcc/config/loongarch/loongarch.h
+++ b/gcc/config/loongarch/loongarch.h
@@ -823,8 +823,6 @@ typedef struct {
 
 #define CASE_VECTOR_MODE Pmode
 
-#define CASE_VECTOR_SHORTEN_MODE(MIN, MAX, BODY) Pmode
-
 /* Define this as 1 if `char' should by default be signed; else as 0.  */
 #ifndef DEFAULT_SIGNED_CHAR
 #define DEFAULT_SIGNED_CHAR 1
diff --git a/gcc/config/loongarch/loongarch.md b/gcc/config/loongarch/loongarch.md
index f7005de..32ef980 100644
--- a/gcc/config/loongarch/loongarch.md
+++ b/gcc/config/loongarch/loongarch.md
@@ -4573,9 +4573,10 @@
   "&& true"
   [(set (match_dup 3) (match_dup 2))
    (set (match_dup 0)
-	(unspec:SI [(match_dup 3) (subreg:SI (match_dup 1) 0)] CRC))]
+	(unspec:SI [(match_dup 3) (match_dup 1)] CRC))]
   {
     operands[3] = gen_reg_rtx (<MODE>mode);
+    operands[1] = lowpart_subreg (SImode, operands[1], DImode);
   })
 
 ;; With normal or medium code models, if the only use of a pc-relative
diff --git a/gcc/config/loongarch/lsx.md b/gcc/config/loongarch/lsx.md
index 407c868..fb0236b 100644
--- a/gcc/config/loongarch/lsx.md
+++ b/gcc/config/loongarch/lsx.md
@@ -535,9 +535,9 @@
   DONE;
 })
 
-(define_insn "lsx_vshuf_<lsxfmt_f>"
+(define_insn "@lsx_vshuf_<lsxfmt_f>"
   [(set (match_operand:LSX_DWH 0 "register_operand" "=f")
-	(unspec:LSX_DWH [(match_operand:LSX_DWH 1 "register_operand" "0")
+	(unspec:LSX_DWH [(match_operand:<VIMODE> 1 "register_operand" "0")
 			 (match_operand:LSX_DWH 2 "register_operand" "f")
 			 (match_operand:LSX_DWH 3 "register_operand" "f")]
 			UNSPEC_LSX_VSHUF))]
diff --git a/gcc/config/nvptx/nvptx.opt b/gcc/config/nvptx/nvptx.opt
index d326ca4..9796839 100644
--- a/gcc/config/nvptx/nvptx.opt
+++ b/gcc/config/nvptx/nvptx.opt
@@ -120,6 +120,51 @@ Target RejectNegative Alias(misa=,sm_89)
 march-map=sm_90a
 Target RejectNegative Alias(misa=,sm_89)
 
+march-map=sm_100
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_100f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_100a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_101
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_101f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_101a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_103
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_103f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_103a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_120
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_120f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_120a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_121
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_121f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_121a
+Target RejectNegative Alias(misa=,sm_89)
+
 Enum
 Name(ptx_version) Type(enum ptx_version)
 Known PTX ISA versions (for use with the -mptx= option):
diff --git a/gcc/config/pru/pru-pragma.cc b/gcc/config/pru/pru-pragma.cc
index c3f3d33..9338780 100644
--- a/gcc/config/pru/pru-pragma.cc
+++ b/gcc/config/pru/pru-pragma.cc
@@ -46,21 +46,24 @@ pru_pragma_ctable_entry (cpp_reader *)
   enum cpp_ttype type;
 
   type = pragma_lex (&ctable_index);
-  if (type == CPP_NUMBER && tree_fits_uhwi_p (ctable_index))
+  if (type == CPP_NUMBER && tree_fits_shwi_p (ctable_index))
     {
       type = pragma_lex (&base_addr);
-      if (type == CPP_NUMBER  && tree_fits_uhwi_p (base_addr))
+      if (type == CPP_NUMBER && tree_fits_shwi_p (base_addr))
 	{
-	  unsigned HOST_WIDE_INT i = tree_to_uhwi (ctable_index);
-	  unsigned HOST_WIDE_INT base = tree_to_uhwi (base_addr);
+	  HOST_WIDE_INT i = tree_to_shwi (ctable_index);
+	  HOST_WIDE_INT base = sext_hwi (tree_to_shwi (base_addr),
+					 POINTER_SIZE);
 
 	  type = pragma_lex (&base_addr);
 	  if (type != CPP_EOF)
 	    error ("junk at end of %<#pragma CTABLE_ENTRY%>");
-	  else if (i >= ARRAY_SIZE (pru_ctable))
+	  else if (!IN_RANGE (i, 0, ARRAY_SIZE (pru_ctable) - 1))
 	    error ("%<CTABLE_ENTRY%> index %wd is not valid", i);
 	  else if (pru_ctable[i].valid && pru_ctable[i].base != base)
 	    error ("redefinition of %<CTABLE_ENTRY %wd%>", i);
+	  else if (!IN_RANGE (base, INT32_MIN, INT32_MAX))
+	    error ("%<CTABLE_ENTRY%> base address does not fit in 32-bits");
 	  else
 	    {
 	      if (base & 0xff)
diff --git a/gcc/config/pru/pru-protos.h b/gcc/config/pru/pru-protos.h
index c73fad8..4750f0e 100644
--- a/gcc/config/pru/pru-protos.h
+++ b/gcc/config/pru/pru-protos.h
@@ -23,7 +23,7 @@
 
 struct pru_ctable_entry {
     bool valid;
-    unsigned HOST_WIDE_INT base;
+    HOST_WIDE_INT base;
 };
 
 extern struct pru_ctable_entry pru_ctable[32];
@@ -66,9 +66,9 @@ pru_regno_ok_for_index_p (int regno, bool strict_p)
   return pru_regno_ok_for_base_p (regno, strict_p);
 }
 
-extern int pru_get_ctable_exact_base_index (unsigned HOST_WIDE_INT caddr);
-extern int pru_get_ctable_base_index (unsigned HOST_WIDE_INT caddr);
-extern int pru_get_ctable_base_offset (unsigned HOST_WIDE_INT caddr);
+extern int pru_get_ctable_exact_base_index (HOST_WIDE_INT caddr);
+extern int pru_get_ctable_base_index (HOST_WIDE_INT caddr);
+extern int pru_get_ctable_base_offset (HOST_WIDE_INT caddr);
 
 extern int pru_symref2ioregno (rtx op);
 
diff --git a/gcc/config/pru/pru.cc b/gcc/config/pru/pru.cc
index 47e5f24..322e319 100644
--- a/gcc/config/pru/pru.cc
+++ b/gcc/config/pru/pru.cc
@@ -1428,7 +1428,7 @@ pru_valid_const_ubyte_offset (machine_mode mode, HOST_WIDE_INT offset)
 /* Recognize a CTABLE base address.  Return CTABLE entry index, or -1 if
    base was not found in the pragma-filled pru_ctable.  */
 int
-pru_get_ctable_exact_base_index (unsigned HOST_WIDE_INT caddr)
+pru_get_ctable_exact_base_index (HOST_WIDE_INT caddr)
 {
   unsigned int i;
 
@@ -1444,7 +1444,7 @@ pru_get_ctable_exact_base_index (unsigned HOST_WIDE_INT caddr)
 /* Check if the given address can be addressed via CTABLE_BASE + UBYTE_OFFS,
    and return the base CTABLE index if possible.  */
 int
-pru_get_ctable_base_index (unsigned HOST_WIDE_INT caddr)
+pru_get_ctable_base_index (HOST_WIDE_INT caddr)
 {
   unsigned int i;
 
@@ -1461,7 +1461,7 @@ pru_get_ctable_base_index (unsigned HOST_WIDE_INT caddr)
 
 /* Return the offset from some CTABLE base for this address.  */
 int
-pru_get_ctable_base_offset (unsigned HOST_WIDE_INT caddr)
+pru_get_ctable_base_offset (HOST_WIDE_INT caddr)
 {
   int i;
 
@@ -2004,7 +2004,7 @@ pru_print_operand_address (FILE *file, machine_mode mode, rtx op)
 
     case CONST_INT:
       {
-	unsigned HOST_WIDE_INT caddr = INTVAL (op);
+	HOST_WIDE_INT caddr = INTVAL (op);
 	int base = pru_get_ctable_base_index (caddr);
 	int offs = pru_get_ctable_base_offset (caddr);
 	if (base < 0)
diff --git a/gcc/config/riscv/arch-canonicalize b/gcc/config/riscv/arch-canonicalize
index fd55255..34dad45 100755
--- a/gcc/config/riscv/arch-canonicalize
+++ b/gcc/config/riscv/arch-canonicalize
@@ -32,7 +32,7 @@ import itertools
 from functools import reduce
 
 SUPPORTED_ISA_SPEC = ["2.2", "20190608", "20191213"]
-CANONICAL_ORDER = "imafdgqlcbkjtpvn"
+CANONICAL_ORDER = "imafdqlcbkjtpvnh"
 LONG_EXT_PREFIXES = ['z', 's', 'h', 'x']
 
 #
diff --git a/gcc/config/riscv/autovec-opt.md b/gcc/config/riscv/autovec-opt.md
index 8df7f64..6531996 100644
--- a/gcc/config/riscv/autovec-opt.md
+++ b/gcc/config/riscv/autovec-opt.md
@@ -1714,6 +1714,74 @@
   }
   [(set_attr "type" "vialu")])
 
+(define_insn_and_split "*<sat_op_v_vdup>_vx_<mode>"
+ [(set (match_operand:V_VLSI   0 "register_operand")
+   (if_then_else:V_VLSI
+    (unspec:<VM>
+     [(match_operand:<VM>      1 "vector_mask_operand")
+      (match_operand           5 "vector_length_operand")
+      (match_operand           6 "const_int_operand")
+      (match_operand           7 "const_int_operand")
+      (match_operand           8 "const_int_operand")
+      (match_operand           9 "const_int_operand")
+      (reg:SI VL_REGNUM)
+      (reg:SI VTYPE_REGNUM)
+      (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
+    (unspec:V_VLSI
+     [(match_operand:V_VLSI    3 "register_operand")
+      (vec_duplicate:V_VLSI
+       (match_operand:<VEL>    4 "reg_or_int_operand"))] VSAT_VX_OP_V_VDUP)
+    (unspec:V_VLSI
+     [(match_operand:DI        2 "register_operand")] UNSPEC_VUNDEF)))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    int vxrm_val = INTVAL (operands[9]);
+    riscv_vector::expand_vx_binary_vxrm_vec_vec_dup (operands[0], operands[3],
+						     operands[4],
+						     <VSAT_VX_OP_V_VDUP>,
+						     vxrm_val, <MODE>mode);
+
+    DONE;
+  }
+  [(set_attr "type" "vaalu")])
+
+(define_insn_and_split "*<sat_op_vdup_v>_vx_<mode>"
+ [(set (match_operand:V_VLSI   0 "register_operand")
+   (if_then_else:V_VLSI
+    (unspec:<VM>
+     [(match_operand:<VM>      1 "vector_mask_operand")
+      (match_operand           5 "vector_length_operand")
+      (match_operand           6 "const_int_operand")
+      (match_operand           7 "const_int_operand")
+      (match_operand           8 "const_int_operand")
+      (match_operand           9 "const_int_operand")
+      (reg:SI VL_REGNUM)
+      (reg:SI VTYPE_REGNUM)
+      (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
+    (unspec:V_VLSI
+     [(vec_duplicate:V_VLSI
+       (match_operand:<VEL>    4 "reg_or_int_operand"))
+      (match_operand:V_VLSI    3 "register_operand")] VSAT_VX_OP_VDUP_V)
+    (unspec:V_VLSI
+     [(match_operand:DI        2 "register_operand")] UNSPEC_VUNDEF)))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    int vxrm_val = INTVAL (operands[9]);
+    riscv_vector::expand_vx_binary_vxrm_vec_dup_vec (operands[0], operands[3],
+						     operands[4],
+						     <VSAT_VX_OP_VDUP_V>,
+						     vxrm_val, <MODE>mode);
+
+    DONE;
+  }
+  [(set_attr "type" "vaalu")])
+
 ;; =============================================================================
 ;; Combine vec_duplicate + op.vv to op.vf
 ;; Include
@@ -1725,6 +1793,8 @@
 ;; - vfmsac.vf
 ;; - vfnmacc.vf
 ;; - vfnmsac.vf
+;; - vfwmacc.vf
+;; - vfwmsac.vf
 ;; =============================================================================
 
 ;; vfmadd.vf, vfmsub.vf, vfmacc.vf, vfmsac.vf
@@ -1796,3 +1866,99 @@
   }
   [(set_attr "type" "vfmuladd")]
 )
+
+;; vfwmacc.vf, vfwmsac.vf
+(define_insn_and_split "*vfwmacc_vf_<mode>"
+  [(set (match_operand:VWEXTF 0 "register_operand")
+    (plus_minus:VWEXTF
+	    (mult:VWEXTF
+	      (float_extend:VWEXTF
+	        (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand"))
+	      (vec_duplicate:VWEXTF
+	        (float_extend:<VEL>
+		  (match_operand:<VSUBEL> 2 "register_operand"))))
+	    (match_operand:VWEXTF 1 "register_operand")))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
+    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul_scalar (<CODE>, <MODE>mode),
+				   riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
+
+;; Intermediate pattern for vfwmacc.vf and vfwmsac.vf used by combine
+(define_insn_and_split "*extend_vf_<mode>"
+ [(set (match_operand:VWEXTF 0 "register_operand")
+    (vec_duplicate:VWEXTF
+      (float_extend:<VEL>
+        (match_operand:<VSUBEL> 1 "register_operand"))))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx tmp = gen_reg_rtx (<VEL>mode);
+    emit_insn (gen_extend<vsubel><vel>2(tmp, operands[1]));
+
+    rtx ops[] = {operands[0], tmp};
+    riscv_vector::expand_broadcast (<MODE>mode, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
+
+;; vfwnmacc.vf
+(define_insn_and_split "*vfwnmacc_vf_<mode>"
+  [(set (match_operand:VWEXTF 0 "register_operand")
+    (minus:VWEXTF
+      (mult:VWEXTF
+	(neg:VWEXTF
+	  (vec_duplicate:VWEXTF
+	    (float_extend:<VEL>
+	      (match_operand:<VSUBEL> 2 "register_operand"))))
+	(float_extend:VWEXTF
+	  (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))
+      (match_operand:VWEXTF 1 "register_operand")))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
+    riscv_vector::emit_vlmax_insn(
+	code_for_pred_widen_mul_neg_scalar(MINUS, <MODE>mode),
+	riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
+
+;; vfwnmsac.vf
+(define_insn_and_split "*vfwnmsac_vf_<mode>"
+  [(set (match_operand:VWEXTF 0 "register_operand")
+    (minus:VWEXTF
+      (match_operand:VWEXTF 1 "register_operand")
+      (mult:VWEXTF
+	(float_extend:VWEXTF
+	  (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand"))
+	(vec_duplicate:VWEXTF
+	  (float_extend:<VEL>
+	    (match_operand:<VSUBEL> 2 "register_operand"))))))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
+    riscv_vector::emit_vlmax_insn(
+	code_for_pred_widen_mul_neg_scalar (PLUS, <MODE>mode),
+	riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
diff --git a/gcc/config/riscv/autovec.md b/gcc/config/riscv/autovec.md
index 94a61bd..48de5ef 100644
--- a/gcc/config/riscv/autovec.md
+++ b/gcc/config/riscv/autovec.md
@@ -1359,9 +1359,7 @@
   if (operands[2] == const0_rtx)
     {
       rtx ops[] = {operands[0], operands[0], operands[1]};
-      riscv_vector::emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
-					riscv_vector::SCALAR_MOVE_MERGED_OP_TU,
-					ops, CONST1_RTX (Pmode));
+      riscv_vector::expand_set_first_tu (<MODE>mode, ops);
     }
   else
     {
@@ -1385,8 +1383,7 @@
 	 VL we need for the slide.  */
       rtx tmp = gen_reg_rtx (<MODE>mode);
       rtx ops1[] = {tmp, operands[1]};
-      emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
-                           riscv_vector::UNARY_OP, ops1, length);
+      riscv_vector::expand_broadcast (<MODE>mode, ops1, length);
 
       /* Slide exactly one element up leaving the tail elements
 	 unchanged.  */
@@ -2489,7 +2486,8 @@
       (sign_extend:VWEXTI
        (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
       (sign_extend:VWEXTI
-       (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))))]
+       (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
+     (const_int 1))))]
   "TARGET_VECTOR"
   {
     insn_code icode = code_for_pred (UNSPEC_VAADD, <V_DOUBLE_TRUNC>mode);
@@ -2499,6 +2497,19 @@
   }
 )
 
+(define_expand "avg<mode>3_floor"
+ [(match_operand:V_VLSI_D 0 "register_operand")
+  (match_operand:V_VLSI_D 1 "register_operand")
+  (match_operand:V_VLSI_D 2 "register_operand")]
+  "TARGET_VECTOR"
+  {
+    insn_code icode = code_for_pred (UNSPEC_VAADD, <MODE>mode);
+    riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_VXRM_RDN,
+				   operands);
+    DONE;
+  }
+)
+
 (define_expand "avg<v_double_trunc>3_ceil"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
    (truncate:<V_DOUBLE_TRUNC>
@@ -2509,7 +2520,8 @@
 	(match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
        (sign_extend:VWEXTI
 	(match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
-      (const_int 1)))))]
+      (const_int 1))
+     (const_int 1))))]
   "TARGET_VECTOR"
   {
     insn_code icode = code_for_pred (UNSPEC_VAADD, <V_DOUBLE_TRUNC>mode);
@@ -2519,6 +2531,19 @@
   }
 )
 
+(define_expand "avg<mode>3_ceil"
+ [(match_operand:V_VLSI_D 0 "register_operand")
+  (match_operand:V_VLSI_D 1 "register_operand")
+  (match_operand:V_VLSI_D 2 "register_operand")]
+  "TARGET_VECTOR"
+  {
+    insn_code icode = code_for_pred (UNSPEC_VAADD, <MODE>mode);
+    riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_VXRM_RNU,
+				   operands);
+    DONE;
+  }
+)
+
 ;; csrwi vxrm, 2
 ;; vaaddu.vv vd, vs2, vs1
 (define_expand "uavg<mode>3_floor"
diff --git a/gcc/config/riscv/constraints.md b/gcc/config/riscv/constraints.md
index ccab1a2..5ecaa19 100644
--- a/gcc/config/riscv/constraints.md
+++ b/gcc/config/riscv/constraints.md
@@ -237,10 +237,11 @@
  (and (match_code "const_vector")
       (match_test "rtx_equal_p (op, riscv_vector::gen_scalar_move_mask (GET_MODE (op)))")))
 
-(define_memory_constraint "Wdm"
+(define_constraint "Wdm"
   "Vector duplicate memory operand"
-  (and (match_code "mem")
-       (match_code "reg" "0")))
+  (and (match_test "strided_load_broadcast_p ()")
+       (and (match_code "mem")
+	    (match_code "reg" "0"))))
 
 ;; Vendor ISA extension constraints.
 
diff --git a/gcc/config/riscv/gen-riscv-mcpu-texi.cc b/gcc/config/riscv/gen-riscv-mcpu-texi.cc
new file mode 100644
index 0000000..9681438
--- /dev/null
+++ b/gcc/config/riscv/gen-riscv-mcpu-texi.cc
@@ -0,0 +1,43 @@
+#include <string>
+#include <vector>
+#include <stdio.h>
+
+int
+main ()
+{
+  puts ("@c Copyright (C) 2025 Free Software Foundation, Inc.");
+  puts ("@c This is part of the GCC manual.");
+  puts ("@c For copying conditions, see the file gcc/doc/include/fdl.texi.");
+  puts ("");
+  puts ("@c This file is generated automatically using");
+  puts ("@c  gcc/config/riscv/gen-riscv-mcpu-texi.cc from:");
+  puts ("@c       gcc/config/riscv/riscv-cores.def");
+  puts ("");
+  puts ("@c Please *DO NOT* edit manually.");
+  puts ("");
+  puts ("@samp{Core Name}");
+  puts ("");
+  puts ("@opindex mcpu");
+  puts ("@item -mcpu=@var{processor-string}");
+  puts ("Use architecture of and optimize the output for the given processor, specified");
+  puts ("by particular CPU name. Permissible values for this option are:");
+  puts ("");
+  puts ("");
+
+  std::vector<std::string> coreNames;
+
+#define RISCV_CORE(CORE_NAME, ARCH, MICRO_ARCH) \
+  coreNames.push_back (CORE_NAME);
+#include "riscv-cores.def"
+#undef RISCV_CORE
+
+  for (size_t i = 0; i < coreNames.size(); ++i) {
+    if (i == coreNames.size() - 1) {
+      printf("@samp{%s}.\n", coreNames[i].c_str());
+    } else {
+      printf("@samp{%s},\n\n", coreNames[i].c_str());
+    }
+  }
+
+  return 0;
+}
diff --git a/gcc/config/riscv/gen-riscv-mtune-texi.cc b/gcc/config/riscv/gen-riscv-mtune-texi.cc
new file mode 100644
index 0000000..1bdfe2a
--- /dev/null
+++ b/gcc/config/riscv/gen-riscv-mtune-texi.cc
@@ -0,0 +1,41 @@
+#include <string>
+#include <vector>
+#include <stdio.h>
+
+int
+main ()
+{
+  puts ("@c Copyright (C) 2025 Free Software Foundation, Inc.");
+  puts ("@c This is part of the GCC manual.");
+  puts ("@c For copying conditions, see the file gcc/doc/include/fdl.texi.");
+  puts ("");
+  puts ("@c This file is generated automatically using");
+  puts ("@c  gcc/config/riscv/gen-riscv-mtune-texi.cc from:");
+  puts ("@c       gcc/config/riscv/riscv-cores.def");
+  puts ("");
+  puts ("@c Please *DO NOT* edit manually.");
+  puts ("");
+  puts ("@samp{Tune Name}");
+  puts ("");
+  puts ("@opindex mtune");
+  puts ("@item -mtune=@var{processor-string}");
+  puts ("Optimize the output for the given processor, specified by microarchitecture or");
+  puts ("particular CPU name.  Permissible values for this option are:");
+  puts ("");
+  puts ("");
+
+  std::vector<std::string> tuneNames;
+
+#define RISCV_TUNE(TUNE_NAME, PIPELINE_MODEL, TUNE_INFO) \
+  tuneNames.push_back (TUNE_NAME);
+#include "riscv-cores.def"
+#undef RISCV_TUNE
+
+  for (size_t i = 0; i < tuneNames.size(); ++i) {
+    printf("@samp{%s},\n\n", tuneNames[i].c_str());
+  }
+
+  puts ("and all valid options for @option{-mcpu=}.");
+
+  return 0;
+}
diff --git a/gcc/config/riscv/generic-vector-ooo.md b/gcc/config/riscv/generic-vector-ooo.md
index ab9e57f..773003b 100644
--- a/gcc/config/riscv/generic-vector-ooo.md
+++ b/gcc/config/riscv/generic-vector-ooo.md
@@ -17,6 +17,9 @@
 ;; <http://www.gnu.org/licenses/>.
 ;; Vector load/store
 
+;; The insn reservations include "generic" as we won't have a in-order
+;; generic definition for vector instructions.
+
 (define_automaton "vector_ooo")
 
 ;; Separate issue queue for vector instructions.
@@ -29,119 +32,141 @@
 (define_cpu_unit "vxu_ooo_multicycle" "vector_ooo")
 
 (define_insn_reservation "vec_load" 6
-  (eq_attr "type" "vlde,vldm,vlds,vldux,vldox,vldff,vldr")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vlde,vldm,vlds,vldux,vldox,vldff,vldr"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 (define_insn_reservation "vec_store" 6
-  (eq_attr "type" "vste,vstm,vsts,vstux,vstox,vstr")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vste,vstm,vsts,vstux,vstox,vstr"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector segment loads/stores.
 (define_insn_reservation "vec_loadstore_seg" 10
-  (eq_attr "type" "vlsegde,vlsegds,vlsegdux,vlsegdox,vlsegdff,\
-		   vssegte,vssegts,vssegtux,vssegtox")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vlsegde,vlsegds,vlsegdux,vlsegdox,vlsegdff,\
+		        vssegte,vssegts,vssegtux,vssegtox"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Regular vector operations and integer comparisons.
 (define_insn_reservation "vec_alu" 3
-  (eq_attr "type" "vialu,viwalu,vext,vicalu,vshift,vnshift,viminmax,vicmp,\
-		   vimov,vsalu,vaalu,vsshift,vnclip,vmov,vfmov,vector,\
-		   vandn,vbrev,vbrev8,vrev8,vclz,vctz,vrol,vror,vwsll")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vialu,viwalu,vext,vicalu,vshift,vnshift,viminmax,vicmp,\
+		        vimov,vsalu,vaalu,vsshift,vnclip,vmov,vfmov,vector,\
+		        vandn,vbrev,vbrev8,vrev8,vclz,vctz,vrol,vror,vwsll"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector float comparison, conversion etc.
 (define_insn_reservation "vec_fcmp" 3
-  (eq_attr "type" "vfrecp,vfminmax,vfcmp,vfsgnj,vfclass,vfcvtitof,\
-                   vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,\
-                   vfncvtftoi,vfncvtftof,vfncvtbf16,vfwcvtbf16")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfrecp,vfminmax,vfcmp,vfsgnj,vfclass,vfcvtitof,\
+                        vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,\
+                        vfncvtftoi,vfncvtftof,vfncvtbf16,vfwcvtbf16"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector integer multiplication.
 (define_insn_reservation "vec_imul" 4
-  (eq_attr "type" "vimul,viwmul,vimuladd,viwmuladd,vsmul,vclmul,vclmulh,\
-                   vghsh,vgmul")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vimul,viwmul,vimuladd,viwmuladd,vsmul,vclmul,vclmulh,\
+                        vghsh,vgmul"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector float addition.
 (define_insn_reservation "vec_fadd" 4
-  (eq_attr "type" "vfalu,vfwalu")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfalu,vfwalu"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector float multiplication and FMA.
 (define_insn_reservation "vec_fmul" 6
-  (eq_attr "type" "vfmul,vfwmul,vfmuladd,vfwmuladd,vfwmaccbf16,sf_vqmacc,sf_vfnrclip")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfmul,vfwmul,vfmuladd,vfwmuladd,vfwmaccbf16,sf_vqmacc,sf_vfnrclip"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, assumed to be a generic operation for now.
 (define_insn_reservation "vec_crypto" 4
-  (eq_attr "type" "crypto,vclz,vctz,vcpop")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "crypto,vclz,vctz,vcpop"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, AES
 (define_insn_reservation "vec_crypto_aes" 4
-  (eq_attr "type" "vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, sha
 (define_insn_reservation "vec_crypto_sha" 4
-  (eq_attr "type" "vsha2ms,vsha2ch,vsha2cl")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vsha2ms,vsha2ch,vsha2cl"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, SM3/4
 (define_insn_reservation "vec_crypto_sm" 4
-  (eq_attr "type" "vsm4k,vsm4r,vsm3me,vsm3c")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vsm4k,vsm4r,vsm3me,vsm3c"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector permute.
 (define_insn_reservation "vec_perm" 3
-  (eq_attr "type" "vimerge,vfmerge,vslideup,vslidedown,vislide1up,\
-                   vislide1down,vfslide1up,vfslide1down,vgather,vcompress")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vimerge,vfmerge,vslideup,vslidedown,vislide1up,\
+                        vislide1down,vfslide1up,vfslide1down,vgather,vcompress"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector reduction.
 (define_insn_reservation "vec_reduction" 8
-  (eq_attr "type" "vired,viwred,vfredu,vfwredu")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vired,viwred,vfredu,vfwredu"))
   "vxu_ooo_issue,vxu_ooo_multicycle")
 
 ;; Vector ordered reduction, assume the latency number is for
 ;; a 128-bit vector.  It is scaled in riscv_sched_adjust_cost
 ;; for larger vectors.
 (define_insn_reservation "vec_ordered_reduction" 10
-  (eq_attr "type" "vfredo,vfwredo")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfredo,vfwredo"))
   "vxu_ooo_issue,vxu_ooo_multicycle*3")
 
 ;; Vector integer division, assume not pipelined.
 (define_insn_reservation "vec_idiv" 16
-  (eq_attr "type" "vidiv")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vidiv"))
   "vxu_ooo_issue,vxu_ooo_multicycle*3")
 
 ;; Vector float divisions and sqrt, assume not pipelined.
 (define_insn_reservation "vec_float_divsqrt" 16
-  (eq_attr "type" "vfdiv,vfsqrt")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfdiv,vfsqrt"))
   "vxu_ooo_issue,vxu_ooo_multicycle*3")
 
 ;; Vector mask operations.
 (define_insn_reservation "vec_mask" 2
-  (eq_attr "type" "vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,\
-                   vfmovvf,vfmovfv")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,\
+                        vfmovvf,vfmovfv"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector vsetvl.
 (define_insn_reservation "vec_vesetvl" 1
-  (eq_attr "type" "vsetvl,vsetvl_pre")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vsetvl,vsetvl_pre"))
   "vxu_ooo_issue")
 
 ;; Vector rounding mode setters, assume pipeline barrier.
 (define_insn_reservation "vec_setrm" 20
-  (eq_attr "type" "wrvxrm,wrfrm")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "wrvxrm,wrfrm"))
   "vxu_ooo_issue,vxu_ooo_issue*3")
 
 ;; Vector read vlen/vlenb.
 (define_insn_reservation "vec_readlen" 4
-  (eq_attr "type" "rdvlenb,rdvl")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "rdvlenb,rdvl"))
   "vxu_ooo_issue,vxu_ooo_issue")
 
 ;; Vector sf_vcp.
 (define_insn_reservation "vec_sf_vcp" 2
-  (eq_attr "type" "sf_vc,sf_vc_se")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "sf_vc,sf_vc_se"))
   "vxu_ooo_issue")
diff --git a/gcc/config/riscv/mips-insn.md b/gcc/config/riscv/mips-insn.md
new file mode 100644
index 0000000..ad46026
--- /dev/null
+++ b/gcc/config/riscv/mips-insn.md
@@ -0,0 +1,35 @@
+;; Machine description for MIPS custom instructions.
+;; Copyright (C) 2025 Free Software Foundation, Inc.
+
+;; This file is part of GCC.
+
+;; GCC is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+
+;; GCC is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GCC; see the file COPYING3.  If not see
+;; <http://www.gnu.org/licenses/>.
+
+(define_insn "*mov<GPR:mode><X:mode>cc_bitmanip"
+  [(set (match_operand:GPR 0 "register_operand" "=r")
+	(if_then_else:GPR (any_eq:X (match_operand:X 1 "register_operand" "r")
+				    (match_operand:X 2 "const_0_operand" "J"))
+			  (match_operand:GPR 3 "reg_or_0_operand" "rJ")
+			  (match_operand:GPR 4 "reg_or_0_operand" "rJ")))]
+  "TARGET_XMIPSCMOV"
+{
+  enum rtx_code code = <CODE>;
+  if (code == NE)
+    return "mips.ccmov\t%0,%1,%z3,%z4";
+  else
+    return "mips.ccmov\t%0,%1,%z4,%z3";
+}
+[(set_attr "type" "condmove")
+ (set_attr "mode" "<GPR:MODE>")])
diff --git a/gcc/config/riscv/mips-p8700.md b/gcc/config/riscv/mips-p8700.md
index ae0ea8d..fac9abb 100644
--- a/gcc/config/riscv/mips-p8700.md
+++ b/gcc/config/riscv/mips-p8700.md
@@ -163,5 +163,5 @@
    vgather,vcompress,vmov,vector,vandn,vbrev,vbrev8,vrev8,vclz,vctz,vcpop,vrol,vror,vwsll,
    vclmul,vclmulh,vghsh,vgmul,vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz,
    vsha2ms,vsha2ch,vsha2cl,vsm4k,vsm4r,vsm3me,vsm3c,vfncvtbf16,vfwcvtbf16,vfwmaccbf16,
-   sf_vc,sf_vc_se"))
+   sf_vc,sf_vc_se,ghost"))
   "mips_p8700_dummies")
diff --git a/gcc/config/riscv/predicates.md b/gcc/config/riscv/predicates.md
index 061904b..381f96c 100644
--- a/gcc/config/riscv/predicates.md
+++ b/gcc/config/riscv/predicates.md
@@ -33,11 +33,11 @@
 (define_predicate "prefetch_operand"
   (ior (match_operand 0 "register_operand")
        (and (match_test "const_arith_operand (op, VOIDmode)")
-	    (match_test "(INTVAL (op) & 0xf) == 0"))
+	    (match_test "(INTVAL (op) & 0x1f) == 0"))
        (and (match_code "plus")
 	    (match_test "register_operand (XEXP (op, 0), word_mode)")
 	    (match_test "const_arith_operand (XEXP (op, 1), VOIDmode)")
-	    (match_test "(INTVAL (XEXP (op, 1)) & 0xf) == 0"))))
+	    (match_test "(INTVAL (XEXP (op, 1)) & 0x1f) == 0"))))
 
 (define_predicate "lui_operand"
   (and (match_code "const_int")
@@ -518,6 +518,10 @@
 
 (define_predicate "vector_broadcast_mask_operand"
   (ior (match_operand 0 "vector_least_significant_set_mask_operand")
+       (match_operand 0 "vector_all_trues_mask_operand")))
+
+(define_predicate "strided_broadcast_mask_operand"
+  (ior (match_operand 0 "vector_least_significant_set_mask_operand")
     (ior (match_operand 0 "register_operand")
          (match_operand 0 "vector_all_trues_mask_operand"))))
 
@@ -617,7 +621,16 @@
 
 ;; The scalar operand can be directly broadcast by RVV instructions.
 (define_predicate "direct_broadcast_operand"
-  (match_test "riscv_vector::can_be_broadcasted_p (op)"))
+  (match_test "riscv_vector::can_be_broadcast_p (op)"))
+
+;; A strided broadcast is just a fallback pattern that loads from
+;; memory.
+(define_predicate "strided_broadcast_operand"
+  (match_test "riscv_vector::strided_broadcast_p (op)"))
+
+(define_predicate "any_broadcast_operand"
+  (ior (match_operand 0 "direct_broadcast_operand")
+       (match_operand 0 "strided_broadcast_operand")))
 
 ;; A CONST_INT operand that has exactly two bits cleared.
 (define_predicate "const_nottwobits_operand"
diff --git a/gcc/config/riscv/riscv-avlprop.cc b/gcc/config/riscv/riscv-avlprop.cc
index bb4aceb..3031c29 100644
--- a/gcc/config/riscv/riscv-avlprop.cc
+++ b/gcc/config/riscv/riscv-avlprop.cc
@@ -508,7 +508,7 @@ pass_avlprop::execute (function *fn)
       simplify_replace_vlmax_avl (rinsn, prop.second);
     }
 
-  if (rvv_vector_bits == RVV_VECTOR_BITS_ZVL)
+  if (rvv_vector_bits == RVV_VECTOR_BITS_ZVL && !TARGET_XTHEADVECTOR)
     {
       /* Simplify VLMAX AVL into immediate AVL.
 	 E.g. Simplify this following case:
diff --git a/gcc/config/riscv/riscv-cores.def b/gcc/config/riscv/riscv-cores.def
index 2096c00..98f3470 100644
--- a/gcc/config/riscv/riscv-cores.def
+++ b/gcc/config/riscv/riscv-cores.def
@@ -169,7 +169,6 @@ RISCV_CORE("xiangshan-kunminghu",   "rv64imafdcbvh_sdtrig_sha_shcounterenw_"
 			      "zvfhmin_zvkt_zvl128b_zvl32b_zvl64b",
 			      "xiangshan-kunminghu")
 
-RISCV_CORE("mips-p8700",     	"rv64imafd_zicsr_zmmul_"
-			      "zaamo_zalrsc_zba_zbb",
+RISCV_CORE("mips-p8700",      "rv64imfd_zicsr_zifencei_zalrsc_zba_zbb",
 			      "mips-p8700")
 #undef RISCV_CORE
diff --git a/gcc/config/riscv/riscv-ext-mips.def b/gcc/config/riscv/riscv-ext-mips.def
new file mode 100644
index 0000000..5d7836d
--- /dev/null
+++ b/gcc/config/riscv/riscv-ext-mips.def
@@ -0,0 +1,35 @@
+/* MIPS extension definition file for RISC-V.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.
+
+Please run `make riscv-regen` in build folder to make sure updated anything.
+
+Format of DEFINE_RISCV_EXT, please refer to riscv-ext.def.  */
+
+DEFINE_RISCV_EXT (
+  /* NAME.  */ xmipscmov,
+  /* UPPERCASE_NAME.  */ XMIPSCMOV,
+  /* FULL_NAME.  */ "Mips conditional move extension",
+  /* DESC.  */ "",
+  /* URL.  */ ,
+  /* DEP_EXTS.  */ ({}),
+  /* SUPPORTED_VERSIONS.  */ ({{1, 0}}),
+  /* FLAG_GROUP.  */ xmips,
+  /* BITMASK_GROUP_ID.  */ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_BIT_POSITION.  */ BITMASK_NOT_YET_ALLOCATED,
+  /* EXTRA_EXTENSION_FLAGS.  */ 0)
diff --git a/gcc/config/riscv/riscv-ext.def b/gcc/config/riscv/riscv-ext.def
index 816acaa..09f18ad 100644
--- a/gcc/config/riscv/riscv-ext.def
+++ b/gcc/config/riscv/riscv-ext.def
@@ -80,8 +80,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{2, 0}}),
   /* FLAG_GROUP */ base,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 0,
+  /* BITMASK_BIT_POSITION*/ 4,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -190,8 +190,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({"zba", "zbb", "zbs"}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ base,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 0,
+  /* BITMASK_BIT_POSITION*/ 1,
   /* EXTRA_EXTENSION_FLAGS */ EXT_FLAG_MACRO)
 
 DEFINE_RISCV_EXT(
@@ -216,8 +216,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ base,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 0,
+  /* BITMASK_BIT_POSITION*/ 7,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -398,8 +398,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{2, 0}}),
   /* FLAG_GROUP */ zi,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 11,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -464,7 +464,7 @@ DEFINE_RISCV_EXT(
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zi,
   /* BITMASK_GROUP_ID */ 1,
-  /* BITMASK_BIT_POSITION*/ 1,
+  /* BITMASK_BIT_POSITION*/ 8,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -476,8 +476,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zm,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 12,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -787,8 +787,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({"zca"}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zc,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 10,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -813,8 +813,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({"zca", "zilsd"}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zc,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 9,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -2082,3 +2082,4 @@ DEFINE_RISCV_EXT(
 #include "riscv-ext-sifive.def"
 #include "riscv-ext-thead.def"
 #include "riscv-ext-ventana.def"
+#include "riscv-ext-mips.def"
diff --git a/gcc/config/riscv/riscv-ext.opt b/gcc/config/riscv/riscv-ext.opt
index 9f8c545..26d6e68 100644
--- a/gcc/config/riscv/riscv-ext.opt
+++ b/gcc/config/riscv/riscv-ext.opt
@@ -47,6 +47,9 @@ TargetVariable
 int riscv_xcv_subext
 
 TargetVariable
+int riscv_xmips_subext
+
+TargetVariable
 int riscv_xsf_subext
 
 TargetVariable
@@ -445,3 +448,4 @@ Mask(XTHEADVECTOR) Var(riscv_xthead_subext)
 
 Mask(XVENTANACONDOPS) Var(riscv_xventana_subext)
 
+Mask(XMIPSCMOV) Var(riscv_xmips_subext)
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index a033120..539321f 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -137,6 +137,7 @@ extern void riscv_expand_usadd (rtx, rtx, rtx);
 extern void riscv_expand_ssadd (rtx, rtx, rtx);
 extern void riscv_expand_ussub (rtx, rtx, rtx);
 extern void riscv_expand_sssub (rtx, rtx, rtx);
+extern void riscv_expand_usmul (rtx, rtx, rtx);
 extern void riscv_expand_ustrunc (rtx, rtx);
 extern void riscv_expand_sstrunc (rtx, rtx);
 extern int riscv_register_move_cost (machine_mode, reg_class_t, reg_class_t);
@@ -413,8 +414,14 @@ enum insn_flags : unsigned int
   /* Means INSN has VXRM operand and the value is VXRM_RNU.  */
   VXRM_RNU_P = 1 << 20,
 
+  /* Means INSN has VXRM operand and the value is VXRM_RNE.  */
+  VXRM_RNE_P = 1 << 21,
+
   /* Means INSN has VXRM operand and the value is VXRM_RDN.  */
-  VXRM_RDN_P = 1 << 21,
+  VXRM_RDN_P = 1 << 22,
+
+  /* Means INSN has VXRM operand and the value is VXRM_ROD.  */
+  VXRM_ROD_P = 1 << 23,
 };
 
 enum insn_type : unsigned int
@@ -476,7 +483,9 @@ enum insn_type : unsigned int
   BINARY_OP_TUMA = __MASK_OP_TUMA | BINARY_OP_P,
   BINARY_OP_FRM_DYN = BINARY_OP | FRM_DYN_P,
   BINARY_OP_VXRM_RNU = BINARY_OP | VXRM_RNU_P,
+  BINARY_OP_VXRM_RNE = BINARY_OP | VXRM_RNE_P,
   BINARY_OP_VXRM_RDN = BINARY_OP | VXRM_RDN_P,
+  BINARY_OP_VXRM_ROD = BINARY_OP | VXRM_ROD_P,
 
   /* Ternary operator. Always have real merge operand.  */
   TERNARY_OP = HAS_DEST_P | HAS_MASK_P | USE_ALL_TRUES_MASK_P | HAS_MERGE_P
@@ -603,6 +612,7 @@ void emit_vlmax_vsetvl (machine_mode, rtx);
 void emit_hard_vlmax_vsetvl (machine_mode, rtx);
 void emit_vlmax_insn (unsigned, unsigned, rtx *);
 void emit_nonvlmax_insn (unsigned, unsigned, rtx *, rtx);
+void emit_avltype_insn (unsigned, unsigned, rtx *, avl_type, rtx = nullptr);
 void emit_vlmax_insn_lra (unsigned, unsigned, rtx *, rtx);
 enum vlmul_type get_vlmul (machine_mode);
 rtx get_vlmax_rtx (machine_mode);
@@ -670,6 +680,8 @@ void expand_vec_oct_sstrunc (rtx, rtx, machine_mode, machine_mode,
 			     machine_mode);
 void expand_vx_binary_vec_dup_vec (rtx, rtx, rtx, rtx_code, machine_mode);
 void expand_vx_binary_vec_vec_dup (rtx, rtx, rtx, rtx_code, machine_mode);
+void expand_vx_binary_vxrm_vec_vec_dup (rtx, rtx, rtx, int, int, machine_mode);
+void expand_vx_binary_vxrm_vec_dup_vec (rtx, rtx, rtx, int, int, machine_mode);
 #endif
 bool sew64_scalar_helper (rtx *, rtx *, rtx, machine_mode,
 			  bool, void (*)(rtx *, rtx), enum avl_type);
@@ -693,6 +705,9 @@ bool expand_block_move (rtx, rtx, rtx, bool);
 machine_mode preferred_simd_mode (scalar_mode);
 machine_mode get_mask_mode (machine_mode);
 void expand_vec_series (rtx, rtx, rtx, rtx = 0);
+void expand_broadcast (machine_mode, rtx *, rtx = 0);
+void expand_set_first (machine_mode, rtx *, rtx = 0);
+void expand_set_first_tu (machine_mode, rtx *, rtx = 0);
 void expand_vec_init (rtx, rtx);
 void expand_vec_perm (rtx, rtx, rtx, rtx);
 void expand_select_vl (rtx *);
@@ -759,7 +774,8 @@ uint8_t get_sew (rtx_insn *);
 enum vlmul_type get_vlmul (rtx_insn *);
 int count_regno_occurrences (rtx_insn *, unsigned int);
 bool imm_avl_p (machine_mode);
-bool can_be_broadcasted_p (rtx);
+bool can_be_broadcast_p (rtx);
+bool strided_broadcast_p (rtx);
 bool gather_scatter_valid_offset_p (machine_mode);
 HOST_WIDE_INT estimated_poly_value (poly_int64, unsigned int);
 bool whole_reg_to_reg_move_p (rtx *, machine_mode, int);
@@ -812,6 +828,7 @@ extern const char *th_output_move (rtx, rtx);
 extern bool th_print_operand_address (FILE *, machine_mode, rtx);
 #endif
 
+extern bool strided_load_broadcast_p (void);
 extern bool riscv_use_divmod_expander (void);
 void riscv_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree, int);
 extern bool
diff --git a/gcc/config/riscv/riscv-selftests.cc b/gcc/config/riscv/riscv-selftests.cc
index 34d01ac..9ca1ffe 100644
--- a/gcc/config/riscv/riscv-selftests.cc
+++ b/gcc/config/riscv/riscv-selftests.cc
@@ -342,9 +342,13 @@ run_broadcast_selftests (void)
 	  expand_vector_broadcast (mode, mem);                                 \
 	  insn = get_last_insn ();                                             \
 	  src = SET_SRC (PATTERN (insn));                                      \
-	  ASSERT_TRUE (MEM_P (XEXP (src, 0)));                                 \
-	  ASSERT_TRUE (                                                        \
-	    rtx_equal_p (src, gen_rtx_VEC_DUPLICATE (mode, XEXP (src, 0))));   \
+	  if (strided_load_broadcast_p ())                                     \
+	    {                                                                  \
+	      ASSERT_TRUE (MEM_P (XEXP (src, 0)));                             \
+	      ASSERT_TRUE (                                                    \
+		rtx_equal_p (src,                                              \
+			     gen_rtx_VEC_DUPLICATE (mode, XEXP (src, 0))));    \
+	    }                                                                  \
 	  end_sequence ();                                                     \
 	  /* Test vmv.v.x or vfmv.v.f.  */                                     \
 	  start_sequence ();                                                   \
diff --git a/gcc/config/riscv/riscv-string.cc b/gcc/config/riscv/riscv-string.cc
index 9080189..61c4a09 100644
--- a/gcc/config/riscv/riscv-string.cc
+++ b/gcc/config/riscv/riscv-string.cc
@@ -1625,16 +1625,14 @@ expand_vec_setmem (rtx dst_in, rtx length_in, rtx fill_value_in)
      Otherwise, use a predicated store.  */
   if (known_eq (GET_MODE_SIZE (info.vmode), INTVAL (info.avl)))
     {
-      emit_vlmax_insn (code_for_pred_broadcast (info.vmode), UNARY_OP,
-		       broadcast_ops);
+      riscv_vector::expand_broadcast (info.vmode, broadcast_ops);
       emit_move_insn (dst, fill_value);
     }
   else
     {
       if (!satisfies_constraint_vl (info.avl))
 	info.avl = force_reg (Pmode, info.avl);
-      emit_nonvlmax_insn (code_for_pred_broadcast (info.vmode),
-			  riscv_vector::UNARY_OP, broadcast_ops, info.avl);
+      riscv_vector::expand_broadcast (info.vmode, broadcast_ops, info.avl);
       machine_mode mask_mode
 	= riscv_vector::get_vector_mode (BImode, GET_MODE_NUNITS (info.vmode))
 	  .require ();
diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index ce1633c..c9c8328 100644
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -351,9 +351,12 @@ public:
       add_rounding_mode_operand (FRM_RNE);
     else if (m_insn_flags & VXRM_RNU_P)
       add_rounding_mode_operand (VXRM_RNU);
+    else if (m_insn_flags & VXRM_RNE_P)
+      add_rounding_mode_operand (VXRM_RNE);
     else if (m_insn_flags & VXRM_RDN_P)
       add_rounding_mode_operand (VXRM_RDN);
-
+    else if (m_insn_flags & VXRM_ROD_P)
+      add_rounding_mode_operand (VXRM_ROD);
 
     if (insn_data[(int) icode].n_operands != m_opno)
       internal_error ("invalid number of operands for insn %s, "
@@ -408,7 +411,7 @@ emit_vlmax_insn_lra (unsigned icode, unsigned insn_flags, rtx *ops, rtx vl)
   gcc_assert (!can_create_pseudo_p ());
   machine_mode mode = GET_MODE (ops[0]);
 
-  if (imm_avl_p (mode))
+  if (imm_avl_p (mode) && !TARGET_XTHEADVECTOR)
     {
       /* Even though VL is a real hardreg already allocated since
 	 it is post-RA now, we still gain benefits that we emit
@@ -437,6 +440,26 @@ emit_nonvlmax_insn (unsigned icode, unsigned insn_flags, rtx *ops, rtx vl)
   e.emit_insn ((enum insn_code) icode, ops);
 }
 
+/* Emit either a VLMAX insn or a non-VLMAX insn depending on TYPE.  For a
+   non-VLMAX insn, the length must be specified in VL.  */
+
+void
+emit_avltype_insn (unsigned icode, unsigned insn_flags, rtx *ops,
+		   avl_type type, rtx vl)
+{
+  if (type != avl_type::VLMAX && vl != NULL_RTX)
+    {
+      insn_expander<RVV_INSN_OPERANDS_MAX> e (insn_flags, false);
+      e.set_vl (vl);
+      e.emit_insn ((enum insn_code) icode, ops);
+    }
+  else
+    {
+      insn_expander<RVV_INSN_OPERANDS_MAX> e (insn_flags, true);
+      e.emit_insn ((enum insn_code) icode, ops);
+    }
+}
+
 /* Return true if the vector duplicated by a super element which is the fusion
    of consecutive elements.
 
@@ -1170,6 +1193,59 @@ expand_vector_init_trailing_same_elem (rtx target,
   return false;
 }
 
+/* Helper function to emit a vmv.vx/vi and float variants.
+   If VL is not given a VLMAX insn will be emitted, otherwise
+   a non-VLMAX insn with length VL.
+   If the value to be broadcast is not suitable for vmv.vx
+   fall back to a vlse with zero stride.  This itself has a
+   fallback if the uarch prefers not to use a strided load
+   for broadcast.  */
+
+void
+expand_broadcast (machine_mode mode, rtx *ops, rtx vl)
+{
+  rtx elt = ops[1];
+  avl_type type = vl ? NONVLMAX : VLMAX;
+  if (can_be_broadcast_p (elt))
+    emit_avltype_insn (code_for_pred_broadcast (mode), UNARY_OP, ops,
+		       type, vl);
+  else
+    emit_avltype_insn (code_for_pred_strided_broadcast (mode),
+		       UNARY_OP, ops, type, vl);
+}
+
+/* Similar to expand_broadcast but emits a vmv.s.x/vfmv.s.f instead.  */
+
+void
+expand_set_first (machine_mode mode, rtx *ops, rtx vl)
+{
+  rtx elt = ops[1];
+  avl_type type = vl ? NONVLMAX : VLMAX;
+  if (can_be_broadcast_p (elt))
+    emit_avltype_insn (code_for_pred_broadcast (mode),
+			SCALAR_MOVE_OP, ops, type, vl);
+  else
+    emit_avltype_insn (code_for_pred_strided_broadcast (mode),
+			SCALAR_MOVE_OP, ops, type, vl);
+}
+
+/* Similar to expand_set_first but keeping the tail elements
+   unchanged (TU) */
+
+void
+expand_set_first_tu (machine_mode mode, rtx *ops, rtx vl)
+{
+  rtx elt = ops[2];
+  if (!vl)
+    vl = const1_rtx;
+  if (can_be_broadcast_p (elt))
+    emit_nonvlmax_insn (code_for_pred_broadcast (mode),
+			SCALAR_MOVE_MERGED_OP_TU, ops, vl);
+  else
+    emit_nonvlmax_insn (code_for_pred_strided_broadcast (mode),
+			SCALAR_MOVE_MERGED_OP_TU, ops, vl);
+}
+
 static void
 expand_const_vec_duplicate (rtx target, rtx src, rtx elt)
 {
@@ -1206,7 +1282,7 @@ expand_const_vec_duplicate (rtx target, rtx src, rtx elt)
       if (lra_in_progress)
 	{
 	  rtx ops[] = {result, elt};
-	  emit_vlmax_insn (code_for_pred_broadcast (mode), UNARY_OP, ops);
+	  expand_broadcast (mode, ops);
 	}
       else
 	{
@@ -1258,8 +1334,7 @@ expand_const_vector_duplicate_repeating (rtx target, rvv_builder *builder)
     {
       dup = gen_reg_rtx (builder->new_mode ());
       rtx ops[] = {dup, ele};
-      emit_vlmax_insn (code_for_pred_broadcast (builder->new_mode ()),
-		       UNARY_OP, ops);
+      expand_broadcast (builder->new_mode (), ops);
     }
   else
     dup = expand_vector_broadcast (builder->new_mode (), ele);
@@ -1302,8 +1377,7 @@ expand_const_vector_duplicate_default (rtx target, rvv_builder *builder)
 
   rtx tmp1 = gen_reg_rtx (builder->mode ());
   rtx dup_ops[] = {tmp1, builder->elt (0)};
-  emit_vlmax_insn (code_for_pred_broadcast (builder->mode ()), UNARY_OP,
-		   dup_ops);
+  expand_broadcast (builder->mode (), dup_ops);
 
   for (unsigned int i = 1; i < builder->npatterns (); i++)
     {
@@ -2116,18 +2190,32 @@ has_vi_variant_p (rtx_code code, rtx x)
     }
 }
 
+/* This is a helper for binary ops with DImode scalar operands that are
+   broadcast (like vadd.vx v1, a1).
+   Instead of having similar code for all the expanders this function
+   unifies the handling.  For 64-bit targets all we do is choose
+   between the vi variant (if available) and the register variant.
+   For 32-bit targets we either create the sign-extending variant
+   of vop.vx (when the immediate fits 32 bits) or emit a vector
+   broadcast of the 64-bit register/immediate and switch to a
+   vop.vv (replacing the scalar op with the broadcast vector.  */
+
 bool
 sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
 		     machine_mode vector_mode, bool has_vi_variant_p,
 		     void (*emit_vector_func) (rtx *, rtx), enum avl_type type)
 {
   machine_mode scalar_mode = GET_MODE_INNER (vector_mode);
+
+  /* If the scalar broadcast op fits an immediate, use the
+     vop.vi variant if there is one.  */
   if (has_vi_variant_p)
     {
       *scalar_op = force_reg (scalar_mode, *scalar_op);
       return false;
     }
 
+  /* On a 64-bit target we can always use the vop.vx variant.  */
   if (TARGET_64BIT)
     {
       if (!rtx_equal_p (*scalar_op, const0_rtx))
@@ -2135,6 +2223,8 @@ sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
       return false;
     }
 
+  /* For 32 bit and if there is no vop.vi variant for a 32-bit immediate
+     we need to use the sign-extending (SI -> DI) vop.vx variants.  */
   if (immediate_operand (*scalar_op, Pmode))
     {
       if (!rtx_equal_p (*scalar_op, const0_rtx))
@@ -2144,6 +2234,17 @@ sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
       return false;
     }
 
+  /* Now we're left with a 64-bit immediate or a register.
+     We cannot use a vop.vx variant but must broadcast the value first
+     and switch to a vop.vv variant.
+     Broadcast can either be done via vlse64.v v1, reg, zero
+     or by loading one 64-bit element (vle64.v) and using a
+     broadcast vrgather.vi.  This is decided when splitting
+     the strided broadcast insn.  */
+  gcc_assert (!TARGET_64BIT
+	      && (CONST_INT_P (*scalar_op)
+		  || register_operand (*scalar_op, scalar_mode)));
+
   if (CONST_INT_P (*scalar_op))
     {
       if (maybe_gt (GET_MODE_SIZE (scalar_mode), GET_MODE_SIZE (Pmode)))
@@ -2154,11 +2255,8 @@ sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
 
   rtx tmp = gen_reg_rtx (vector_mode);
   rtx ops[] = {tmp, *scalar_op};
-  if (type == VLMAX)
-    emit_vlmax_insn (code_for_pred_broadcast (vector_mode), UNARY_OP, ops);
-  else
-    emit_nonvlmax_insn (code_for_pred_broadcast (vector_mode), UNARY_OP, ops,
-			vl);
+  emit_avltype_insn (code_for_pred_strided_broadcast (vector_mode),
+		     UNARY_OP, ops, type, vl);
   emit_vector_func (operands, tmp);
 
   return true;
@@ -2552,8 +2650,7 @@ expand_vector_init_merge_repeating_sequence (rtx target,
 
   /* Step 1: Broadcast the first pattern.  */
   rtx ops[] = {target, force_reg (builder.inner_mode (), builder.elt (0))};
-  emit_vlmax_insn (code_for_pred_broadcast (builder.mode ()),
-		    UNARY_OP, ops);
+  expand_broadcast (builder.mode (), ops);
   /* Step 2: Merge the rest iteration of pattern.  */
   for (unsigned int i = 1; i < builder.npatterns (); i++)
     {
@@ -2566,8 +2663,7 @@ expand_vector_init_merge_repeating_sequence (rtx target,
       if (full_nelts <= builder.inner_bits_size ()) /* vmv.s.x.  */
 	{
 	  rtx ops[] = {dup, merge_mask};
-	  emit_nonvlmax_insn (code_for_pred_broadcast (GET_MODE (dup)),
-			       SCALAR_MOVE_OP, ops, CONST1_RTX (Pmode));
+	  expand_set_first (GET_MODE (dup), ops);
 	}
       else /* vmv.v.x.  */
 	{
@@ -2575,8 +2671,7 @@ expand_vector_init_merge_repeating_sequence (rtx target,
 		       force_reg (GET_MODE_INNER (mask_int_mode), merge_mask)};
 	  rtx vl = gen_int_mode (CEIL (full_nelts, builder.inner_bits_size ()),
 				 Pmode);
-	  emit_nonvlmax_insn (code_for_pred_broadcast (mask_int_mode), UNARY_OP,
-			       ops, vl);
+	  expand_broadcast (mask_int_mode, ops, vl);
 	}
 
       emit_move_insn (mask, gen_lowpart (mask_bit_mode, dup));
@@ -4667,20 +4762,20 @@ expand_reduction (unsigned unspec, unsigned unspec_for_vl0_safe,
 
   rtx m1_tmp = gen_reg_rtx (m1_mode);
   rtx scalar_move_ops[] = {m1_tmp, init};
-  insn_code icode = code_for_pred_broadcast (m1_mode);
   if (need_mask_operand_p (insn_flags))
     {
       if (need_vl0_safe)
-	emit_nonvlmax_insn (icode, SCALAR_MOVE_OP, scalar_move_ops, const1_rtx);
+	expand_set_first (m1_mode, scalar_move_ops, const1_rtx);
       else
-	emit_nonvlmax_insn (icode, SCALAR_MOVE_OP, scalar_move_ops, vl_op);
+	expand_set_first (m1_mode, scalar_move_ops, vl_op);
     }
   else
-    emit_vlmax_insn (icode, SCALAR_MOVE_OP, scalar_move_ops);
+    expand_set_first (m1_mode, scalar_move_ops);
 
   rtx m1_tmp2 = gen_reg_rtx (m1_mode);
   rtx reduc_ops[] = {m1_tmp2, vector_src, m1_tmp};
 
+  insn_code icode;
   if (need_vl0_safe)
     icode = code_for_pred (unspec_for_vl0_safe, vmode);
   else
@@ -5544,6 +5639,7 @@ expand_vx_binary_vec_dup_vec (rtx op_0, rtx op_1, rtx op_2,
     case SMIN:
     case UMIN:
     case US_PLUS:
+    case SS_PLUS:
       icode = code_for_pred_scalar (code, mode);
       break;
     case MINUS:
@@ -5557,6 +5653,82 @@ expand_vx_binary_vec_dup_vec (rtx op_0, rtx op_1, rtx op_2,
   emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops);
 }
 
+static enum insn_type
+get_insn_type_by_vxrm_val (int vxrm_val)
+{
+  enum insn_type itype;
+
+  switch (vxrm_val)
+    {
+    case VXRM_RNU:
+      itype = BINARY_OP_VXRM_RNU;
+      break;
+    case VXRM_RNE:
+      itype = BINARY_OP_VXRM_RNE;
+      break;
+    case VXRM_RDN:
+      itype = BINARY_OP_VXRM_RDN;
+      break;
+    case VXRM_ROD:
+      itype = BINARY_OP_VXRM_ROD;
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  return itype;
+}
+
+/* Expand the binary vx combine with the format like v2 = vop(v1, vec_dup(x))
+   and its' vxrm value.  Aka the second op comes from the vec_duplicate,
+   and the first op is the vector reg.  */
+
+void
+expand_vx_binary_vxrm_vec_vec_dup (rtx op_0, rtx op_1, rtx op_2, int unspec,
+				   int vxrm_val, machine_mode mode)
+{
+  enum insn_code icode;
+  enum insn_type itype = get_insn_type_by_vxrm_val (vxrm_val);
+  rtx ops[] = {op_0, op_1, op_2};
+
+  switch (unspec)
+    {
+    case UNSPEC_VAADD:
+    case UNSPEC_VAADDU:
+      icode = code_for_pred_scalar (unspec, mode);
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  emit_vlmax_insn (icode, itype, ops);
+}
+
+/* Expand the binary vx combine with the format like v2 = vop(vec_dup(x), v1)
+   and its' vxrm value.  Aka the second op comes from the vec_duplicate,
+   and the first op is the vector reg.  */
+
+void
+expand_vx_binary_vxrm_vec_dup_vec (rtx op_0, rtx op_1, rtx op_2, int unspec,
+				   int vxrm_val, machine_mode mode)
+{
+  enum insn_code icode;
+  enum insn_type itype = get_insn_type_by_vxrm_val (vxrm_val);
+  rtx ops[] = {op_0, op_1, op_2};
+
+  switch (unspec)
+    {
+    case UNSPEC_VAADD:
+    case UNSPEC_VAADDU:
+      icode = code_for_pred_scalar (unspec, mode);
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  emit_vlmax_insn (icode, itype, ops);
+}
+
 /* Expand the binary vx combine with the format like v2 = vop(v1, vec_dup(x)).
    Aka the second op comes from the vec_duplicate, and the first op is
    the vector reg.  */
@@ -5584,6 +5756,8 @@ expand_vx_binary_vec_vec_dup (rtx op_0, rtx op_1, rtx op_2,
     case UMIN:
     case US_PLUS:
     case US_MINUS:
+    case SS_PLUS:
+    case SS_MINUS:
       icode = code_for_pred_scalar (code, mode);
       break;
     default:
@@ -5766,24 +5940,84 @@ count_regno_occurrences (rtx_insn *rinsn, unsigned int regno)
   return count;
 }
 
-/* Return true if the OP can be directly broadcasted.  */
+/* Return true if the OP can be broadcast with a
+   v[f]mv.v.[xif] instruction.  */
+
+bool
+can_be_broadcast_p (rtx op)
+{
+  machine_mode mode = GET_MODE (op);
+
+  /* Zero always works and we can always put an immediate into a
+     register.
+     What's tricky is that for an immediate we don't know the
+     register's mode it will end up in, i.e. what element size
+     we want to broadcast.  So even if the immediate is small it might
+     still end up in a DImode register that we cannot broadcast.
+     vmv.s.x, i.e. a single-element set can handle this, though,
+     because it implicitly sign-extends to SEW.  */
+  if (rtx_equal_p (op, CONST0_RTX (mode))
+      || const_int_operand (op, Xmode))
+    return true;
+
+  /* Do not accept DImode broadcasts on !TARGET_64BIT.  Those
+     are handled by strided broadcast.  */
+  if (INTEGRAL_MODE_P (mode)
+      && maybe_gt (GET_MODE_SIZE (mode), UNITS_PER_WORD))
+    return false;
+
+  /* Non-register operands that can be forced into a register we can
+     handle.  These don't need to use strided broadcast. */
+  if (INTEGRAL_MODE_P (mode)
+      && (memory_operand (op, mode) || CONST_POLY_INT_P (op))
+      && can_create_pseudo_p ())
+    return true;
+
+  /* Likewise, do not accept HFmode broadcast if we don't have
+     vfmv.v.f for 16-bit registers available.  */
+  if (mode == HFmode && !TARGET_ZVFH)
+    return false;
+
+  /* Same for float, just that we can always handle 64-bit doubles
+     even on !TARGET_64BIT.  We have ruled out 16-bit HF already
+     above.  */
+  if (FLOAT_MODE_P (mode)
+      && (memory_operand (op, mode) || CONSTANT_P (op))
+      && can_create_pseudo_p ())
+    return true;
+
+  /* After excluding all the cases we cannot handle the register types
+     that remain can always be broadcast.  */
+  if (register_operand (op, mode))
+    return true;
+
+  return false;
+}
+
+/* Returns true for all operands that cannot use vmv.vx, vfmv.vf,
+   vmv.s.x, or vfmv.s.f but rather need to go via memory.  */
+
 bool
-can_be_broadcasted_p (rtx op)
+strided_broadcast_p (rtx op)
 {
   machine_mode mode = GET_MODE (op);
-  /* We don't allow RA (register allocation) reload generate
-    (vec_duplicate:DI reg) in RV32 system wheras we allow
-    (vec_duplicate:DI mem) in RV32 system.  */
-  if (!can_create_pseudo_p () && !FLOAT_MODE_P (mode)
-      && maybe_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (Pmode))
-      && !satisfies_constraint_Wdm (op))
+  if (!memory_operand (op, mode)
+      && !register_operand (op, mode)
+      && !rtx_equal_p (op, CONST0_RTX (mode))
+      && !const_int_operand (op, mode))
     return false;
 
-  if (satisfies_constraint_K (op) || register_operand (op, mode)
-      || satisfies_constraint_Wdm (op) || rtx_equal_p (op, CONST0_RTX (mode)))
+  /* !TARGET64_BIT does not have a vmv.v.x/vmv.s.x for 64-bit
+     DImode elements.  */
+  if (INTEGRAL_MODE_P (mode)
+      && maybe_gt (GET_MODE_SIZE (mode), UNITS_PER_WORD))
+    return true;
+
+  /* Zvfhmin does not have a vfmv.v.f/vfmv.s.f.  for 16-bit elements.  */
+  if (!TARGET_ZVFH && mode == HFmode)
     return true;
 
-  return can_create_pseudo_p () && nonmemory_operand (op, mode);
+  return false;
 }
 
 void
@@ -5898,7 +6132,10 @@ whole_reg_to_reg_move_p (rtx *ops, machine_mode mode, int avl_type_index)
   return false;
 }
 
-/* Return true if we can transform vmv.v.x/vfmv.v.f to vmv.s.x/vfmv.s.f.  */
+/* Return true if we can transform vmv.v.x/vfmv.v.f to vmv.s.x/vfmv.s.f.
+   That's the case if we're dealing with a scalar broadcast that
+   has VL = 1.  */
+
 bool
 splat_to_scalar_move_p (rtx *ops)
 {
diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.cc b/gcc/config/riscv/riscv-vector-builtins-bases.cc
index bf5172c..7e4d396 100644
--- a/gcc/config/riscv/riscv-vector-builtins-bases.cc
+++ b/gcc/config/riscv/riscv-vector-builtins-bases.cc
@@ -643,7 +643,8 @@ public:
 	return e.use_exact_insn (code_for_pred_mov (e.vector_mode ()));
       case OP_TYPE_x:
       case OP_TYPE_f:
-	return e.use_exact_insn (code_for_pred_broadcast (e.vector_mode ()));
+	return e.use_scalar_broadcast_insn
+	  (code_for_pred_broadcast (e.vector_mode ()));
       default:
 	gcc_unreachable ();
       }
diff --git a/gcc/config/riscv/riscv-vector-builtins.cc b/gcc/config/riscv/riscv-vector-builtins.cc
index f652a12..0db7549 100644
--- a/gcc/config/riscv/riscv-vector-builtins.cc
+++ b/gcc/config/riscv/riscv-vector-builtins.cc
@@ -4753,7 +4753,10 @@ function_expander::use_ternop_insn (bool vd_accum_p, insn_code icode)
 }
 
 /* Implement the call using instruction ICODE, with a 1:1 mapping between
-   arguments and input operands.  */
+   arguments and input operands.
+   There are operands that cannot be broadcast using v[f]mv.  In that case
+   we switch to a strided broadcast.  */
+
 rtx
 function_expander::use_widen_ternop_insn (insn_code icode)
 {
@@ -4794,7 +4797,10 @@ function_expander::use_widen_ternop_insn (insn_code icode)
 }
 
 /* Implement the call using instruction ICODE, with a 1:1 mapping between
-   arguments and input operands.  */
+   arguments and input operands.
+   There are operands that cannot be broadcast using v[f]mv.  In that case
+   we switch to a strided broadcast.  */
+
 rtx
 function_expander::use_scalar_move_insn (insn_code icode)
 {
@@ -4812,6 +4818,37 @@ function_expander::use_scalar_move_insn (insn_code icode)
   for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++)
     add_input_operand (argno);
 
+  if (!can_be_broadcast_p (m_ops[3].value))
+    icode = code_for_pred_strided_broadcast (vector_mode ());
+
+  add_input_operand (Pmode, get_tail_policy_for_pred (pred));
+  add_input_operand (Pmode, get_mask_policy_for_pred (pred));
+  add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX));
+  return generate_insn (icode);
+}
+
+/* Implement the call using instruction ICODE, with a 1:1 mapping between
+   arguments and input operands.  */
+rtx
+function_expander::use_scalar_broadcast_insn (insn_code icode)
+{
+  machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
+
+  /* Record the offset to get the argument.  */
+  int arg_offset = 0;
+  add_all_one_mask_operand (mask_mode ());
+
+  if (use_real_merge_p (pred))
+    add_input_operand (arg_offset++);
+  else
+    add_vundef_operand (mode);
+
+  for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++)
+    add_input_operand (argno);
+
+  if (!can_be_broadcast_p (m_ops[3].value))
+    icode = code_for_pred_strided_broadcast (vector_mode ());
+
   add_input_operand (Pmode, get_tail_policy_for_pred (pred));
   add_input_operand (Pmode, get_mask_policy_for_pred (pred));
   add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX));
@@ -4977,6 +5014,12 @@ registered_function::overloaded_hash () const
   for (unsigned int i = 0; i < argument_types.length (); i++)
     {
       type = argument_types[i];
+
+      /* If we're passed something entirely unreasonable, just ignore here.
+	 We'll warn later anyway.  */
+      if (TREE_CODE_CLASS (TREE_CODE (type)) != tcc_type)
+	continue;
+
       unsigned_p = POINTER_TYPE_P (type) ? TYPE_UNSIGNED (TREE_TYPE (type))
 					 : TYPE_UNSIGNED (type);
       mode_p = POINTER_TYPE_P (type) ? TYPE_MODE (TREE_TYPE (type))
diff --git a/gcc/config/riscv/riscv-vector-builtins.h b/gcc/config/riscv/riscv-vector-builtins.h
index 1f2587a..86d8115 100644
--- a/gcc/config/riscv/riscv-vector-builtins.h
+++ b/gcc/config/riscv/riscv-vector-builtins.h
@@ -497,6 +497,7 @@ public:
   rtx use_ternop_insn (bool, insn_code);
   rtx use_widen_ternop_insn (insn_code);
   rtx use_scalar_move_insn (insn_code);
+  rtx use_scalar_broadcast_insn (insn_code);
   rtx generate_insn (insn_code);
 
   /* The function call expression.  */
diff --git a/gcc/config/riscv/riscv-vector-costs.cc b/gcc/config/riscv/riscv-vector-costs.cc
index 4d8170d..44ef44a 100644
--- a/gcc/config/riscv/riscv-vector-costs.cc
+++ b/gcc/config/riscv/riscv-vector-costs.cc
@@ -178,8 +178,8 @@ get_live_range (hash_map<tree, pair> *live_ranges, tree arg)
        STMT 5 (be vectorized)      -- point 2
        ...
 */
-static void
-compute_local_program_points (
+void
+costs::compute_local_program_points (
   vec_info *vinfo,
   hash_map<basic_block, vec<stmt_point>> &program_points_per_bb)
 {
@@ -274,14 +274,14 @@ loop_invariant_op_p (class loop *loop,
 
 /* Return true if the variable should be counted into liveness.  */
 static bool
-variable_vectorized_p (class loop *loop, stmt_vec_info stmt_info, tree var,
-		       bool lhs_p)
+variable_vectorized_p (class loop *loop, stmt_vec_info stmt_info,
+		       slp_tree node, tree var, bool lhs_p)
 {
   if (!var)
     return false;
   gimple *stmt = STMT_VINFO_STMT (stmt_info);
-  enum stmt_vec_info_type type
-    = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info));
+  stmt_info = vect_stmt_to_vectorize (stmt_info);
+  enum stmt_vec_info_type type = SLP_TREE_TYPE (node);
   if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
     {
       if (gimple_call_internal_fn (stmt) == IFN_MASK_STORE
@@ -357,8 +357,8 @@ variable_vectorized_p (class loop *loop, stmt_vec_info stmt_info, tree var,
 
    The live range of SSA 1 is [1, 3] in bb 2.
    The live range of SSA 2 is [0, 4] in bb 3.  */
-static machine_mode
-compute_local_live_ranges (
+machine_mode
+costs::compute_local_live_ranges (
   loop_vec_info loop_vinfo,
   const hash_map<basic_block, vec<stmt_point>> &program_points_per_bb,
   hash_map<basic_block, hash_map<tree, pair>> &live_ranges_per_bb)
@@ -388,8 +388,11 @@ compute_local_live_ranges (
 	      unsigned int point = program_point.point;
 	      gimple *stmt = program_point.stmt;
 	      tree lhs = gimple_get_lhs (stmt);
-	      if (variable_vectorized_p (loop, program_point.stmt_info, lhs,
-					 true))
+	      slp_tree *node = vinfo_slp_map.get (program_point.stmt_info);
+	      if (!node)
+		continue;
+	      if (variable_vectorized_p (loop, program_point.stmt_info,
+					 *node, lhs, true))
 		{
 		  biggest_mode = get_biggest_mode (biggest_mode,
 						   TYPE_MODE (TREE_TYPE (lhs)));
@@ -397,7 +400,7 @@ compute_local_live_ranges (
 		  pair &live_range
 		    = live_ranges->get_or_insert (lhs, &existed_p);
 		  gcc_assert (!existed_p);
-		  if (STMT_VINFO_MEMORY_ACCESS_TYPE (program_point.stmt_info)
+		  if (SLP_TREE_MEMORY_ACCESS_TYPE (*node)
 		      == VMAT_LOAD_STORE_LANES)
 		    point = get_first_lane_point (program_points,
 						  program_point.stmt_info);
@@ -406,8 +409,8 @@ compute_local_live_ranges (
 	      for (i = 0; i < gimple_num_args (stmt); i++)
 		{
 		  tree var = gimple_arg (stmt, i);
-		  if (variable_vectorized_p (loop, program_point.stmt_info, var,
-					     false))
+		  if (variable_vectorized_p (loop, program_point.stmt_info,
+					     *node, var, false))
 		    {
 		      biggest_mode
 			= get_biggest_mode (biggest_mode,
@@ -415,8 +418,7 @@ compute_local_live_ranges (
 		      bool existed_p = false;
 		      pair &live_range
 			= live_ranges->get_or_insert (var, &existed_p);
-		      if (STMT_VINFO_MEMORY_ACCESS_TYPE (
-			    program_point.stmt_info)
+		      if (SLP_TREE_MEMORY_ACCESS_TYPE (*node)
 			  == VMAT_LOAD_STORE_LANES)
 			point = get_last_lane_point (program_points,
 						     program_point.stmt_info);
@@ -597,15 +599,15 @@ get_store_value (gimple *stmt)
 }
 
 /* Return true if additional vector vars needed.  */
-static bool
-need_additional_vector_vars_p (stmt_vec_info stmt_info)
+bool
+costs::need_additional_vector_vars_p (stmt_vec_info stmt_info,
+				      slp_tree node)
 {
-  enum stmt_vec_info_type type
-    = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info));
+  enum stmt_vec_info_type type = SLP_TREE_TYPE (node);
   if (type == load_vec_info_type || type == store_vec_info_type)
     {
       if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
-	  && STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_GATHER_SCATTER)
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
 	return true;
 
       machine_mode mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
@@ -657,8 +659,8 @@ compute_estimated_lmul (loop_vec_info loop_vinfo, machine_mode mode)
 
    Then, after this function, we update SSA 1 live range in bb 2
    into [2, 4] since SSA 1 is live out into bb 3.  */
-static void
-update_local_live_ranges (
+void
+costs::update_local_live_ranges (
   vec_info *vinfo,
   hash_map<basic_block, vec<stmt_point>> &program_points_per_bb,
   hash_map<basic_block, hash_map<tree, pair>> &live_ranges_per_bb,
@@ -685,8 +687,13 @@ update_local_live_ranges (
 	{
 	  gphi *phi = psi.phi ();
 	  stmt_vec_info stmt_info = vinfo->lookup_stmt (phi);
-	  if (STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info))
-	      == undef_vec_info_type)
+	  stmt_info = vect_stmt_to_vectorize (stmt_info);
+	  slp_tree *node = vinfo_slp_map.get (stmt_info);
+
+	  if (!node)
+	    continue;
+
+	  if (SLP_TREE_TYPE (*node) == undef_vec_info_type)
 	    continue;
 
 	  for (j = 0; j < gimple_phi_num_args (phi); j++)
@@ -761,9 +768,12 @@ update_local_live_ranges (
 	  if (!is_gimple_assign_or_call (gsi_stmt (si)))
 	    continue;
 	  stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
-	  enum stmt_vec_info_type type
-	    = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info));
-	  if (need_additional_vector_vars_p (stmt_info))
+	  stmt_info = vect_stmt_to_vectorize (stmt_info);
+	  slp_tree *node = vinfo_slp_map.get (stmt_info);
+	  if (!node)
+	    continue;
+	  enum stmt_vec_info_type type = SLP_TREE_TYPE (*node);
+	  if (need_additional_vector_vars_p (stmt_info, *node))
 	    {
 	      /* For non-adjacent load/store STMT, we will potentially
 		 convert it into:
@@ -816,8 +826,8 @@ update_local_live_ranges (
 }
 
 /* Compute the maximum live V_REGS.  */
-static bool
-has_unexpected_spills_p (loop_vec_info loop_vinfo)
+bool
+costs::has_unexpected_spills_p (loop_vec_info loop_vinfo)
 {
   /* Compute local program points.
      It's a fast and effective computation.  */
@@ -899,7 +909,11 @@ costs::analyze_loop_vinfo (loop_vec_info loop_vinfo)
   /* Detect whether we're vectorizing for VLA and should apply the unrolling
      heuristic described above m_unrolled_vls_niters.  */
   record_potential_vls_unrolling (loop_vinfo);
+}
 
+void
+costs::record_lmul_spills (loop_vec_info loop_vinfo)
+{
   /* Detect whether the LOOP has unexpected spills.  */
   record_potential_unexpected_spills (loop_vinfo);
 }
@@ -1071,7 +1085,7 @@ costs::better_main_loop_than_p (const vector_costs *uncast_other) const
    load/store.  */
 static int
 segment_loadstore_group_size (enum vect_cost_for_stmt kind,
-			      stmt_vec_info stmt_info)
+			      stmt_vec_info stmt_info, slp_tree node)
 {
   if (stmt_info
       && (kind == vector_load || kind == vector_store)
@@ -1079,7 +1093,7 @@ segment_loadstore_group_size (enum vect_cost_for_stmt kind,
     {
       stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
       if (stmt_info
-	  && STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_LOAD_STORE_LANES)
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_LOAD_STORE_LANES)
 	return DR_GROUP_SIZE (stmt_info);
     }
   return 0;
@@ -1093,7 +1107,7 @@ segment_loadstore_group_size (enum vect_cost_for_stmt kind,
 unsigned
 costs::adjust_stmt_cost (enum vect_cost_for_stmt kind, loop_vec_info loop,
 			 stmt_vec_info stmt_info,
-			 slp_tree, tree vectype, int stmt_cost)
+			 slp_tree node, tree vectype, int stmt_cost)
 {
   const cpu_vector_cost *costs = get_vector_costs ();
   switch (kind)
@@ -1116,7 +1130,8 @@ costs::adjust_stmt_cost (enum vect_cost_for_stmt kind, loop_vec_info loop,
 		 each vector in the group.  Here we additionally add permute
 		 costs for each.  */
 	      /* TODO: Indexed and ordered/unordered cost.  */
-	      int group_size = segment_loadstore_group_size (kind, stmt_info);
+	      int group_size = segment_loadstore_group_size (kind, stmt_info,
+							     node);
 	      if (group_size > 1)
 		{
 		  switch (group_size)
@@ -1239,8 +1254,12 @@ costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
   int stmt_cost
     = targetm.vectorize.builtin_vectorization_cost (kind, vectype, misalign);
 
+  if (stmt_info && node)
+    vinfo_slp_map.put (stmt_info, node);
+
   /* Do one-time initialization based on the vinfo.  */
   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
+
   if (!m_analyzed_vinfo)
     {
       if (loop_vinfo)
@@ -1326,6 +1345,8 @@ costs::finish_cost (const vector_costs *scalar_costs)
 {
   if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
     {
+      record_lmul_spills (loop_vinfo);
+
       adjust_vect_cost_per_loop (loop_vinfo);
     }
   vector_costs::finish_cost (scalar_costs);
diff --git a/gcc/config/riscv/riscv-vector-costs.h b/gcc/config/riscv/riscv-vector-costs.h
index de546a6..b84ceb1 100644
--- a/gcc/config/riscv/riscv-vector-costs.h
+++ b/gcc/config/riscv/riscv-vector-costs.h
@@ -91,7 +91,10 @@ private:
   typedef pair_hash <tree_operand_hash, tree_operand_hash> tree_pair_hash;
   hash_set <tree_pair_hash> memrefs;
 
+  hash_map <stmt_vec_info, slp_tree> vinfo_slp_map;
+
   void analyze_loop_vinfo (loop_vec_info);
+  void record_lmul_spills (loop_vec_info loop_vinfo);
   void record_potential_vls_unrolling (loop_vec_info);
   bool prefer_unrolled_loop () const;
 
@@ -103,6 +106,19 @@ private:
   bool m_has_unexpected_spills_p = false;
   void record_potential_unexpected_spills (loop_vec_info);
 
+  void compute_local_program_points (vec_info *,
+				     hash_map<basic_block, vec<stmt_point>> &);
+  void update_local_live_ranges (vec_info *,
+				 hash_map<basic_block, vec<stmt_point>> &,
+				 hash_map<basic_block, hash_map<tree, pair>> &,
+				 machine_mode *);
+  machine_mode compute_local_live_ranges
+    (loop_vec_info, const hash_map<basic_block, vec<stmt_point>> &,
+     hash_map<basic_block, hash_map<tree, pair>> &);
+
+  bool has_unexpected_spills_p (loop_vec_info);
+  bool need_additional_vector_vars_p (stmt_vec_info, slp_tree);
+
   void adjust_vect_cost_per_loop (loop_vec_info);
   unsigned adjust_stmt_cost (enum vect_cost_for_stmt kind,
 			     loop_vec_info,
diff --git a/gcc/config/riscv/riscv-vsetvl.def b/gcc/config/riscv/riscv-vsetvl.def
index d7a5ada..0f999d2 100644
--- a/gcc/config/riscv/riscv-vsetvl.def
+++ b/gcc/config/riscv/riscv-vsetvl.def
@@ -79,7 +79,7 @@ DEF_SEW_LMUL_RULE (sew_only, sew_only, sew_only, sew_eq_p, sew_eq_p, nop)
 DEF_SEW_LMUL_RULE (sew_only, ge_sew, sew_only,
 		   sew_ge_and_prev_sew_le_next_max_sew_p, sew_ge_p, nop)
 DEF_SEW_LMUL_RULE (
-  sew_only, ratio_and_ge_sew, sew_lmul,
+  sew_only, ratio_and_ge_sew, ratio_and_ge_sew,
   sew_ge_and_prev_sew_le_next_max_sew_and_next_ratio_valid_for_prev_sew_p,
   always_false, modify_lmul_with_next_ratio)
 
@@ -104,9 +104,9 @@ DEF_SEW_LMUL_RULE (ratio_and_ge_sew, sew_lmul, sew_lmul,
 DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ratio_only, ratio_and_ge_sew, ratio_eq_p,
 		   ratio_eq_p, use_max_sew_and_lmul_with_prev_ratio)
 DEF_SEW_LMUL_RULE (
-  ratio_and_ge_sew, sew_only, sew_only,
+  ratio_and_ge_sew, sew_only, ratio_and_ge_sew,
   sew_le_and_next_sew_le_prev_max_sew_and_prev_ratio_valid_for_next_sew_p,
-  always_false, use_next_sew_with_prev_ratio)
+  sew_eq_p, use_next_sew_with_prev_ratio)
 DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ge_sew, ratio_and_ge_sew,
 		   max_sew_overlap_and_prev_ratio_valid_for_next_sew_p,
 		   sew_ge_p, use_max_sew_and_lmul_with_prev_ratio)
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
index 8fa1082..e0d8904 100644
--- a/gcc/config/riscv/riscv.cc
+++ b/gcc/config/riscv/riscv.cc
@@ -87,6 +87,10 @@ along with GCC; see the file COPYING3.  If not see
 #include "riscv-vector-costs.h"
 #include "riscv-subset.h"
 
+/* Target variants that support full conditional move.  */
+#define	TARGET_COND_MOV						\
+   (TARGET_SFB_ALU || TARGET_XTHEADCONDMOV || TARGET_XMIPSCMOV)
+
 /* True if X is an UNSPEC wrapper around a SYMBOL_REF or LABEL_REF.  */
 #define UNSPEC_ADDRESS_P(X)					\
   (GET_CODE (X) == UNSPEC					\
@@ -166,7 +170,7 @@ struct GTY(())  riscv_frame_info {
 };
 
 enum riscv_privilege_levels {
-  UNKNOWN_MODE, USER_MODE, SUPERVISOR_MODE, MACHINE_MODE
+  UNKNOWN_MODE, SUPERVISOR_MODE, MACHINE_MODE, RNMI_MODE
 };
 
 struct GTY(()) mode_switching_info {
@@ -283,6 +287,10 @@ enum riscv_fusion_pairs
   RISCV_FUSE_AUIPC_LD = (1 << 7),
   RISCV_FUSE_LDPREINCREMENT = (1 << 8),
   RISCV_FUSE_ALIGNED_STD = (1 << 9),
+  RISCV_FUSE_CACHE_ALIGNED_STD = (1 << 10),
+  RISCV_FUSE_BFEXT = (1 << 11),
+  RISCV_FUSE_EXPANDED_LD = (1 << 12),
+  RISCV_FUSE_B_ALUI = (1 << 13),
 };
 
 /* Costs of various operations on the different architectures.  */
@@ -302,6 +310,7 @@ struct riscv_tune_param
   bool vector_unaligned_access;
   bool use_divmod_expansion;
   bool overlap_op_by_pieces;
+  bool use_zero_stride_load;
   bool speculative_sched_vsetvl;
   unsigned int fusible_ops;
   const struct cpu_vector_cost *vec_costs;
@@ -465,6 +474,7 @@ static const struct riscv_tune_param generic_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -488,6 +498,7 @@ static const struct riscv_tune_param rocket_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -511,6 +522,7 @@ static const struct riscv_tune_param sifive_7_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -534,6 +546,7 @@ static const struct riscv_tune_param sifive_p400_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_LUI_ADDI | RISCV_FUSE_AUIPC_ADDI,  /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -557,6 +570,7 @@ static const struct riscv_tune_param sifive_p600_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_LUI_ADDI | RISCV_FUSE_AUIPC_ADDI,  /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -580,6 +594,7 @@ static const struct riscv_tune_param thead_c906_tune_info = {
   false,					/* vector_unaligned_access */
   false,	/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -603,6 +618,7 @@ static const struct riscv_tune_param xiangshan_nanhu_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_ZEXTW | RISCV_FUSE_ZEXTH,          /* fusible_ops */
   NULL,						/* vector cost */
@@ -626,6 +642,7 @@ static const struct riscv_tune_param generic_ooo_tune_info = {
   true,						/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   true,						/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -649,6 +666,7 @@ static const struct riscv_tune_param tt_ascalon_d8_tune_info = {
   true,						/* vector_unaligned_access */
   true,						/* use_divmod_expansion */
   true,						/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -672,6 +690,7 @@ static const struct riscv_tune_param optimize_size_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -695,6 +714,7 @@ static const struct riscv_tune_param mips_p8700_tune_info = {
   false,        /* vector_unaligned_access */
   true,         /* use_divmod_expansion */
   false,        /* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,				/* fusible_ops */
   NULL,         /* vector cost */
@@ -3947,13 +3967,27 @@ get_vector_binary_rtx_cost (rtx x, int scalar2vr_cost)
 {
   gcc_assert (riscv_v_ext_mode_p (GET_MODE (x)));
 
-  rtx op_0 = XEXP (x, 0);
-  rtx op_1 = XEXP (x, 1);
+  rtx neg;
+  rtx op_0;
+  rtx op_1;
+
+  if (GET_CODE (x) == UNSPEC)
+    {
+      op_0 = XVECEXP (x, 0, 0);
+      op_1 = XVECEXP (x, 0, 1);
+    }
+  else
+    {
+      op_0 = XEXP (x, 0);
+      op_1 = XEXP (x, 1);
+    }
 
   if (GET_CODE (op_0) == VEC_DUPLICATE
       || GET_CODE (op_1) == VEC_DUPLICATE)
     return (scalar2vr_cost + 1) * COSTS_N_INSNS (1);
-  else if (GET_CODE (op_0) == NEG && GET_CODE (op_1) == VEC_DUPLICATE)
+  else if (GET_CODE (neg = op_0) == NEG
+	   && (GET_CODE (op_1) == VEC_DUPLICATE
+	       || GET_CODE (XEXP (neg, 0)) == VEC_DUPLICATE))
     return (scalar2vr_cost + 1) * COSTS_N_INSNS (1);
   else
     return COSTS_N_INSNS (1);
@@ -3997,8 +4031,25 @@ riscv_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UN
 		    case UMOD:
 		    case US_PLUS:
 		    case US_MINUS:
+		    case SS_PLUS:
+		    case SS_MINUS:
 		      *total = get_vector_binary_rtx_cost (op, scalar2vr_cost);
 		      break;
+		    case UNSPEC:
+		      {
+			switch (XINT (op, 1))
+			  {
+			  case UNSPEC_VAADDU:
+			  case UNSPEC_VAADD:
+			    *total
+			      = get_vector_binary_rtx_cost (op, scalar2vr_cost);
+			    break;
+			  default:
+			    *total = COSTS_N_INSNS (1);
+			    break;
+			  }
+		      }
+		      break;
 		    default:
 		      *total = COSTS_N_INSNS (1);
 		      break;
@@ -4151,7 +4202,7 @@ riscv_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UN
       return false;
 
     case IF_THEN_ELSE:
-      if ((TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
+      if (TARGET_COND_MOV
 	  && reg_or_0_operand (XEXP (x, 1), mode)
 	  && sfb_alu_operand (XEXP (x, 2), mode)
 	  && comparison_operator (XEXP (x, 0), VOIDmode))
@@ -5469,6 +5520,68 @@ riscv_expand_conditional_branch (rtx label, rtx_code code, rtx op0, rtx op1)
   emit_jump_insn (gen_condjump (condition, label));
 }
 
+/* canonicalization of the comparands.  */
+void
+canonicalize_comparands (rtx_code code, rtx *op0, rtx *op1)
+{
+  /* An integer comparison must be comparing WORD_MODE objects.
+     Extend the comparison arguments as necessary.  */
+  if ((INTEGRAL_MODE_P (GET_MODE (*op0)) && GET_MODE (*op0) != word_mode)
+      || (INTEGRAL_MODE_P (GET_MODE (*op1)) && GET_MODE (*op1) != word_mode))
+    riscv_extend_comparands (code, op0, op1);
+
+  /* We might have been handed back a SUBREG.  Just to make things
+     easy, force it into a REG.  */
+  if (!REG_P (*op0) && !CONST_INT_P (*op0))
+    *op0 = force_reg (word_mode, *op0);
+  if (!REG_P (*op1) && !CONST_INT_P (*op1))
+    *op1 = force_reg (word_mode, *op1);
+}
+
+/* Emit target specific conditional move like TARGET_XMIPSCMOV etc.  */
+bool
+riscv_target_conditional_move (rtx dest, rtx op0, rtx op1, rtx_code code,
+				rtx cons, rtx alt)
+{
+  machine_mode dst_mode = GET_MODE (dest);
+  rtx target;
+
+  /* force the operands to the register.  */
+  cons = force_reg (dst_mode, cons);
+  alt = force_reg (dst_mode, alt);
+
+  if (TARGET_XMIPSCMOV)
+    {
+      if (code == EQ || code == NE)
+	{
+	  op0 = riscv_zero_if_equal (op0, op1);
+	  op1 = const0_rtx;
+	}
+      else
+	{
+	  target = gen_reg_rtx (GET_MODE (op0));
+	  riscv_emit_int_order_test (code, 0, target, op0, op1);
+	  op0 = target;
+	  op1 = const0_rtx;
+	  code = NE;
+	}
+      riscv_emit_int_compare (&code, &op0, &op1);
+      rtx cond = gen_rtx_fmt_ee (code, GET_MODE (op0), op0, op1);
+      emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (dst_mode,
+							   cond, cons, alt)));
+      return true;
+    }
+  /* TARGET_SFB_ALU || TARGET_XTHEADCONDMOV.  */
+  else
+   {
+     riscv_emit_int_compare (&code, &op0, &op1, !TARGET_SFB_ALU);
+     rtx cond = gen_rtx_fmt_ee (code, GET_MODE (op0), op0, op1);
+     emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (dst_mode, cond,
+							 cons, alt)));
+     return true;
+   }
+}
+
 /* Emit a cond move: If OP holds, move CONS to DEST; else move ALT to DEST.
    Return 0 if expansion failed.  */
 
@@ -5521,34 +5634,22 @@ riscv_expand_conditional_move (rtx dest, rtx op, rtx cons, rtx alt)
       /* If we need more special cases, add them here.  */
     }
 
+
   if (((TARGET_ZICOND_LIKE
 	|| (arith_operand (cons, dst_mode) && arith_operand (alt, dst_mode)))
        && GET_MODE_CLASS (dst_mode) == MODE_INT
        && GET_MODE_CLASS (cond_mode) == MODE_INT)
-      || TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
+       || TARGET_COND_MOV)
     {
       machine_mode mode0 = GET_MODE (op0);
-      machine_mode mode1 = GET_MODE (op1);
-
-      /* An integer comparison must be comparing WORD_MODE objects.
-	 Extend the comparison arguments as necessary.  */
-      if ((INTEGRAL_MODE_P (mode0) && mode0 != word_mode)
-	  || (INTEGRAL_MODE_P (mode1) && mode1 != word_mode))
-	riscv_extend_comparands (code, &op0, &op1);
 
-      /* We might have been handed back a SUBREG.  Just to make things
-	 easy, force it into a REG.  */
-      if (!REG_P (op0) && !CONST_INT_P (op0))
-	op0 = force_reg (word_mode, op0);
-      if (!REG_P (op1) && !CONST_INT_P (op1))
-	op1 = force_reg (word_mode, op1);
+      canonicalize_comparands (code,&op0,&op1);
 
       /* In the fallback generic case use DST_MODE rather than WORD_MODE
 	 for the output of the SCC instruction, to match the mode of the NEG
 	 operation below.  The output of SCC is 0 or 1 boolean, so it is
 	 valid for input in any scalar integer mode.  */
-      rtx tmp = gen_reg_rtx ((TARGET_ZICOND_LIKE
-			      || TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
+      rtx tmp = gen_reg_rtx ((TARGET_ZICOND_LIKE || TARGET_COND_MOV)
 			     ? word_mode : dst_mode);
       bool invert = false;
 
@@ -5585,25 +5686,12 @@ riscv_expand_conditional_move (rtx dest, rtx op, rtx cons, rtx alt)
 	  op0 = XEXP (op, 0);
 	  op1 = XEXP (op, 1);
 	}
-      else if (!TARGET_ZICOND_LIKE && !TARGET_SFB_ALU && !TARGET_XTHEADCONDMOV)
+      else if (!TARGET_ZICOND_LIKE && !TARGET_COND_MOV)
 	riscv_expand_int_scc (tmp, code, op0, op1, &invert);
 
-      if (TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
-	{
-	  riscv_emit_int_compare (&code, &op0, &op1, !TARGET_SFB_ALU);
-	  rtx cond = gen_rtx_fmt_ee (code, GET_MODE (op0), op0, op1);
+      if (TARGET_COND_MOV)
+	return riscv_target_conditional_move (dest, op0, op1, code, cons, alt);
 
-	  /* The expander is a bit loose in its specification of the true
-	     arm of the conditional move.  That allows us to support more
-	     cases for extensions which are more general than SFB.  But
-	     does mean we need to force CONS into a register at this point.  */
-	  cons = force_reg (dst_mode, cons);
-	  /* With XTheadCondMov we need to force ALT into a register too.  */
-	  alt = force_reg (dst_mode, alt);
-	  emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (dst_mode, cond,
-							      cons, alt)));
-	  return true;
-	}
       else if (!TARGET_ZICOND_LIKE)
 	{
 	  if (invert)
@@ -6837,12 +6925,18 @@ riscv_handle_type_attribute (tree *node ATTRIBUTE_UNUSED, tree name, tree args,
 	    }
 
 	  string = TREE_STRING_POINTER (cst);
-	  if (strcmp (string, "user") && strcmp (string, "supervisor")
-	      && strcmp (string, "machine"))
+	  if (!strcmp (string, "rnmi") && !TARGET_SMRNMI)
+	    {
+	      error ("attribute 'rnmi' requires the Smrnmi ISA extension");
+	      *no_add_attrs = true;
+	    }
+	  else if (strcmp (string, "supervisor")
+		   && strcmp (string, "machine")
+		   && strcmp (string, "rnmi"))
 	    {
 	      warning (OPT_Wattributes,
-		       "argument to %qE attribute is not %<\"user\"%>, %<\"supervisor\"%>, "
-		       "or %<\"machine\"%>", name);
+		       "argument to %qE attribute is not %<\"supervisor\"%>, "
+		       "%<\"machine\"%>, or %<\"rnmi\"%>", name);
 	      *no_add_attrs = true;
 	    }
 	}
@@ -8990,7 +9084,7 @@ riscv_allocate_and_probe_stack_space (rtx temp1, HOST_WIDE_INT size)
 	  /* We want the CFA independent of the stack pointer for the
 	     duration of the loop.  */
 	  add_reg_note (insn, REG_CFA_DEF_CFA,
-			plus_constant (Pmode, temp1,
+			plus_constant (Pmode, temp2,
 				       initial_cfa_offset + rounded_size));
 	  RTX_FRAME_RELATED_P (insn) = 1;
 	}
@@ -9623,12 +9717,12 @@ riscv_expand_epilogue (int style)
 
       if (th_int_mask && TH_INT_INTERRUPT (cfun))
 	emit_jump_insn (gen_th_int_pop ());
-      else if (mode == MACHINE_MODE)
-	emit_jump_insn (gen_riscv_mret ());
       else if (mode == SUPERVISOR_MODE)
 	emit_jump_insn (gen_riscv_sret ());
-      else
-	emit_jump_insn (gen_riscv_uret ());
+      else if (mode == RNMI_MODE)
+	emit_jump_insn (gen_riscv_mnret ());
+      else /* Must be MACHINE_MODE.  */
+	emit_jump_insn (gen_riscv_mret ());
     }
   else if (style != SIBCALL_RETURN)
     {
@@ -10203,6 +10297,81 @@ riscv_fusion_enabled_p(enum riscv_fusion_pairs op)
   return tune_param->fusible_ops & op;
 }
 
+/* Matches an add:
+   (set (reg:DI rd) (plus:SI (reg:SI rs1) (reg:SI rs2))) */
+
+static bool
+riscv_set_is_add (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && REG_P (XEXP (SET_SRC (set), 0))
+	  && REG_P (XEXP (SET_SRC (set), 1))
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches an addi:
+   (set (reg:DI rd) (plus:SI (reg:SI rs1) (const_int imm))) */
+
+static bool
+riscv_set_is_addi (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && REG_P (XEXP (SET_SRC (set), 0))
+	  && CONST_INT_P (XEXP (SET_SRC (set), 1))
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches an add.uw:
+  (set (reg:DI rd)
+    (plus:DI (zero_extend:DI (reg:SI rs1)) (reg:DI rs2))) */
+
+static bool
+riscv_set_is_adduw (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && GET_CODE (XEXP (SET_SRC (set), 0)) == ZERO_EXTEND
+	  && REG_P (XEXP (XEXP (SET_SRC (set), 0), 0))
+	  && REG_P (XEXP (SET_SRC (set), 1))
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches a shNadd:
+  (set (reg:DI rd)
+       (plus:DI (ashift:DI (reg:DI rs1) (const_int N)) (reg:DI rS2)) */
+
+static bool
+riscv_set_is_shNadd (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && GET_CODE (XEXP (SET_SRC (set), 0)) == ASHIFT
+	  && REG_P (XEXP (XEXP (SET_SRC (set), 0), 0))
+	  && CONST_INT_P (XEXP (XEXP (SET_SRC (set), 0), 1))
+	  && (INTVAL (XEXP (XEXP (SET_SRC (set), 0), 1)) == 1
+	      || INTVAL (XEXP (XEXP (SET_SRC (set), 0), 1)) == 2
+	      || INTVAL (XEXP (XEXP (SET_SRC (set), 0), 1)) == 3)
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches a shNadd.uw:
+  (set (reg:DI rd)
+       (plus:DI (and:DI (ashift:DI (reg:DI rs1) (const_int N))
+			(const_int N))
+		(reg:DI rs2)) */
+
+static bool
+riscv_set_is_shNadduw (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && GET_CODE (XEXP (SET_SRC (set), 0)) == AND
+	  && GET_CODE (XEXP (XEXP (SET_SRC (set), 0), 0)) == ASHIFT
+	  && REG_P (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 0))
+	  && CONST_INT_P (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1))
+	  && (INTVAL (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1)) == 1
+	      || INTVAL (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1)) == 2
+	      || INTVAL (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1)) == 3)
+	  && REG_P (SET_DEST (set)));
+}
+
 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P.  Return true if PREV and CURR
    should be kept together during scheduling.  */
 
@@ -10225,10 +10394,10 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
   bool simple_sets_p = prev_set && curr_set && !any_condjump_p (curr);
   bool sched1 = can_create_pseudo_p ();
 
-  unsigned int prev_dest_regno = (REG_P (SET_DEST (prev_set))
+  unsigned int prev_dest_regno = (prev_set && REG_P (SET_DEST (prev_set))
 				  ? REGNO (SET_DEST (prev_set))
 				  : FIRST_PSEUDO_REGISTER);
-  unsigned int curr_dest_regno = (REG_P (SET_DEST (curr_set))
+  unsigned int curr_dest_regno = (curr_set && REG_P (SET_DEST (curr_set))
 				  ? REGNO (SET_DEST (curr_set))
 				  : FIRST_PSEUDO_REGISTER);
 
@@ -10253,11 +10422,15 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 1))
 	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
 	  && INTVAL (XEXP (SET_SRC (prev_set), 1)) == 32
-	  && (( INTVAL (XEXP (SET_SRC (curr_set), 1)) == 32
-		&& riscv_fusion_enabled_p(RISCV_FUSE_ZEXTW) )
-	      || ( INTVAL (XEXP (SET_SRC (curr_set), 1)) < 32
-		   && riscv_fusion_enabled_p(RISCV_FUSE_ZEXTWS))))
-	return true;
+	  && ((INTVAL (XEXP (SET_SRC (curr_set), 1)) == 32
+	       && riscv_fusion_enabled_p (RISCV_FUSE_ZEXTW) )
+	      || (INTVAL (XEXP (SET_SRC (curr_set), 1)) < 32
+		  && riscv_fusion_enabled_p (RISCV_FUSE_ZEXTWS))))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_ZEXTWS\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_ZEXTH)
@@ -10278,7 +10451,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
 	  && INTVAL (XEXP (SET_SRC (prev_set), 1)) == 48
 	  && INTVAL (XEXP (SET_SRC (curr_set), 1)) == 48)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file,"RISCV_FUSE_ZEXTH\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LDINDEXED)
@@ -10297,7 +10474,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && GET_CODE (SET_SRC (prev_set)) == PLUS
 	  && REG_P (XEXP (SET_SRC (prev_set), 0))
 	  && REG_P (XEXP (SET_SRC (prev_set), 1)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LDINDEXED\n");
+	  return true;
+	}
 
       /* We are trying to match the following:
 	   prev (add) == (set (reg:DI rD)
@@ -10313,7 +10494,144 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && GET_CODE (SET_SRC (prev_set)) == PLUS
 	  && REG_P (XEXP (SET_SRC (prev_set), 0))
 	  && REG_P (XEXP (SET_SRC (prev_set), 1)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LDINDEXED\n");
+	  return true;
+	}
+    }
+
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_EXPANDED_LD)
+      && (sched1 || prev_dest_regno == curr_dest_regno))
+    {
+      /* For the "expanded add/load fusion" family we have 2 main
+	 categories: memory loads with displacement (i.e. with imm offset)
+	 and loads without displacement (i.e. with offset = x0).
+
+	 For loads without displacement we'll need:
+	 - add + ld (done in RISCV_FUSE_LDINDEXED)
+	 - addi + ld (done in RISCV_FUSE_LDPREINCREMENT)
+	 - shNadd + ld
+	 - add.uw + lw
+	 - shNadd.uw + lw
+
+	 For loads with displacement/immediates:
+	 with lw with immediate):
+	 - add + ld with displacement
+	 - addi + ld with displacement
+	 - shNadd + ld with displacement
+	 - add.uw + lw with displacement
+	 - shNadd.uw + lw with displacement */
+
+      /* We're trying to match a curr_set ld with displacement:
+	  prev (add|addi) = (set (reg:DI rd) (...))
+	  curr (ld)  == (set (reg:DI rD)
+		(mem:DI (plus:DI (reg:DI rD) (const_int IMM12)))) */
+      if (MEM_P (SET_SRC (curr_set))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == PLUS
+	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
+	{
+	  if (riscv_set_is_add (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_addi (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadd (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
+
+      /* We're trying to match a ld without displacement:
+	  prev (addi|shNadd) = (reg:DI rD) (...))
+	  curr (ld)  == (set (reg:DI rD)
+			     (mem:DI (reg:DI rD))) */
+      if (MEM_P (SET_SRC (curr_set))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno)
+	{
+	  if (riscv_set_is_addi (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadd (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
+
+      /* We're trying to match a curr_set lw with displacement:
+	  prev (add.uw|shNadd.uw) = (set (reg:DI rd) (...))
+	  curr (lw)  == (set (reg:DI rd)
+		(any_extend:DI (mem:SUBX (plus:DI ((reg:DI rd)
+						   (const_int IMM)))) */
+      if ((GET_CODE (SET_SRC (curr_set)) == SIGN_EXTEND
+	   || (GET_CODE (SET_SRC (curr_set)) == ZERO_EXTEND))
+	  && MEM_P (XEXP (SET_SRC (curr_set), 0))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && GET_CODE (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == PLUS
+	  && REG_P (XEXP (XEXP (XEXP (SET_SRC (curr_set), 0), 0),0))
+	  && (REGNO (XEXP (XEXP (XEXP (SET_SRC (curr_set), 0), 0),0))
+	      == prev_dest_regno))
+	{
+	  if (riscv_set_is_adduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
+
+      /* We're trying to match a curr_set lw without displacement:
+	  prev (add.uw|shNadd.uw) = (set (reg:DI rd) (...))
+	  curr (ld|lh|lw)  == (set (reg:DI rd)
+		(any_extend:DI (mem:SUBX (reg:DI rsd)))) */
+      if ((GET_CODE (SET_SRC (curr_set)) == SIGN_EXTEND
+	   || (GET_CODE (SET_SRC (curr_set)) == ZERO_EXTEND))
+	  && MEM_P (XEXP (SET_SRC (curr_set), 0))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && REG_P (XEXP (XEXP (SET_SRC (curr_set), 0), 0))
+	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
+	{
+	  if (riscv_set_is_adduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LDPREINCREMENT)
@@ -10332,7 +10650,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && GET_CODE (SET_SRC (prev_set)) == PLUS
 	  && REG_P (XEXP (SET_SRC (prev_set), 0))
 	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 1)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LDPREINCREMENT\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LUI_ADDI)
@@ -10350,7 +10672,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && (GET_CODE (SET_SRC (prev_set)) == HIGH
 	      || (CONST_INT_P (SET_SRC (prev_set))
 		  && LUI_OPERAND (INTVAL (SET_SRC (prev_set))))))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_ADDI\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_AUIPC_ADDI)
@@ -10372,7 +10698,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 		  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
 		  && SMALL_OPERAND (INTVAL (XEXP (SET_SRC (curr_set), 1))))))
 
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_AUIPC_ADDI\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LUI_LD)
@@ -10392,14 +10722,22 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
 	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == PLUS
 	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_LD\n");
+	  return true;
+	}
 
       if (GET_CODE (SET_SRC (prev_set)) == HIGH
 	  && MEM_P (SET_SRC (curr_set))
 	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
 	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == LO_SUM
 	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_LD\n");
+	  return true;
+	}
 
       if (GET_CODE (SET_SRC (prev_set)) == HIGH
 	  && (GET_CODE (SET_SRC (curr_set)) == SIGN_EXTEND
@@ -10409,7 +10747,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && (GET_CODE (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == LO_SUM
 	      && (REGNO (XEXP (XEXP (XEXP (SET_SRC (curr_set), 0), 0), 0))
 		  == prev_dest_regno)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_LD\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_AUIPC_LD)
@@ -10425,10 +10767,14 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && MEM_P (SET_SRC (curr_set))
 	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
 	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == PLUS)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_AUIPC_LD\n");
+	  return true;
+	}
     }
 
-  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_ALIGNED_STD))
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_CACHE_ALIGNED_STD))
     {
       /* We are trying to match the following:
 	   prev (sd) == (set (mem (plus (reg sp|fp) (const_int)))
@@ -10473,11 +10819,184 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 		 lower offset.  */
 	      if ((INTVAL (offset_prev) % 16) == 0
 		  && (INTVAL (offset_prev) + 8 == INTVAL (offset_curr)))
-		return true;
+		{
+		  if (dump_file)
+		    fprintf (dump_file, "RISCV_FUSE_ALIGNED_STD\n");
+		  return true;
+		}
 	    }
 	}
     }
 
+  /* More general form of the RISCV_FUSE_CACHE_ALIGNED_STD.  The
+     major difference is the dependency on the stores being opposite
+     halves of a cache line is dropped.  Instead the lowest address
+     needs 2X the alignment of the object and the higher address
+     immediately followed the first object.  */
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_ALIGNED_STD))
+    {
+      /* We are trying to match the following:
+	prev (sd) == (set (mem (plus (reg rS1) (const_int)))
+			  (reg rS2))
+	curr (sd) == (set (mem (plus (reg rS1) (const_int)))
+			  (reg rS3)) */
+
+      if (MEM_P (SET_DEST (prev_set))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && MEM_P (SET_DEST (curr_set))
+	  /* Stores must have the same width */
+	  && GET_MODE (SET_DEST (curr_set)) == GET_MODE (SET_DEST (prev_set)))
+	{
+	  rtx base_prev, base_curr, offset_prev, offset_curr;
+	  unsigned mode_size;
+
+	  extract_base_offset_in_addr (SET_DEST (prev_set),
+				       &base_prev, &offset_prev);
+	  extract_base_offset_in_addr (SET_DEST (curr_set),
+				       &base_curr, &offset_curr);
+
+	  /* Proceed only if we find both bases, both bases
+	     are registers and bases are the same register.  */
+	  if (base_prev != NULL_RTX && base_curr != NULL_RTX
+	      && REG_P (base_prev) && REG_P (base_curr)
+	      && REGNO (base_prev) == REGNO (base_curr))
+	    {
+	      machine_mode mode = GET_MODE (SET_DEST (curr_set));
+	      mode_size = estimated_poly_value (GET_MODE_SIZE (mode));
+
+	      HOST_WIDE_INT offset_prev_int = INTVAL (offset_prev);
+	      HOST_WIDE_INT offset_curr_int = INTVAL (offset_curr);
+
+	      /* Get the smaller offset into OFFSET_PREV_INT.  */
+	      if (offset_prev_int > offset_curr_int)
+		std::swap (offset_prev_int, offset_curr_int);
+
+	      /* We've normalized, so we need to check that the lower
+		 address is aligned to 2X the size of the object.  The
+		 higher address must be the lower address plus the
+		 size of the object.  */
+	      if (((offset_prev_int % (2 * mode_size)) == 0)
+		  && offset_prev_int + mode_size == offset_curr_int)
+		{
+		  if (dump_file)
+		    fprintf (dump_file, "RISCV_FUSE_ALIGNED_STD\n");
+		  return true;
+		}
+	    }
+	}
+    }
+
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_BFEXT)
+      && (sched1 || prev_dest_regno == curr_dest_regno))
+    {
+      /* We are trying to match the following:
+	   prev (slli) == (set (reg:DI rD)
+			       (ashift:DI (reg:DI rS) (const_int)))
+	   curr (srli) == (set (reg:DI rD)
+			       (lshiftrt:DI (reg:DI rD) (const_int))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == ASHIFT
+	  && (GET_CODE (SET_SRC (curr_set)) == LSHIFTRT
+	      || GET_CODE (SET_SRC (curr_set)) == ASHIFTRT)
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno
+	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 1))
+	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1)))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_BFEXT\n");
+	  return true;
+	}
+    }
+
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_B_ALUI)
+      && (sched1 || prev_dest_regno == curr_dest_regno))
+    {
+      /* We are trying to match the following:
+	  prev (orc.b) == (set (reg rD)
+				(unspec (reg rS1)))
+	  curr (not) == (set (reg rD2) (not (reg rD))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == UNSPEC
+	  && GET_CODE (SET_SRC (curr_set)) == NOT
+	  && XINT (SET_SRC (prev_set), 1) == UNSPEC_ORC_B
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno)
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+
+      /* We are trying to match the following:
+	  prev (ctz) == (set (reg rD) (ctz (reg rS1)))
+	  curr (andi) == (set (reg rD)
+				(and (reg rD) (const_int 63))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == CTZ
+	  && GET_CODE (SET_SRC (curr_set)) == AND
+	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
+	  && INTVAL (XEXP (SET_SRC (curr_set), 1)) == 63
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno)
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+
+      /* We are trying to match the following:
+	  prev (sub) == (set (reg rD)
+				(minus (const_int 0) (reg rS2))
+	  curr (max) == (set (reg rD)
+				(smax (reg rD) (reg rS2))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == MINUS
+	  && (XEXP (SET_SRC (prev_set), 0)
+	      == CONST0_RTX (GET_MODE (SET_SRC (prev_set))))
+	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 0))
+	  && GET_CODE (SET_SRC (curr_set)) == SMAX
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno
+	  && REG_P (XEXP (SET_SRC (prev_set), 1))
+	  && REG_P (XEXP (SET_SRC (curr_set), 1))
+	  && (REGNO (XEXP (SET_SRC (prev_set), 1))
+	      == REGNO (XEXP (SET_SRC (curr_set), 1))))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+
+      /* We are trying to match the following:
+	  prev (neg) == (set (reg rD) (neg (reg rS1)))
+	  curr (max) == (set (reg rD)
+				(smax (reg rD) (reg rS1))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == NEG
+	  && GET_CODE (SET_SRC (curr_set)) == SMAX
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno
+	  && REG_P (XEXP (SET_SRC (prev_set), 0))
+	  && REG_P (XEXP (SET_SRC (curr_set), 1))
+	  && (REGNO (XEXP (SET_SRC (prev_set), 0))
+	      == REGNO (XEXP (SET_SRC (curr_set), 1))))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+    }
+
   return false;
 }
 
@@ -11545,10 +12064,10 @@ riscv_get_interrupt_type (tree decl)
     {
       const char *string = TREE_STRING_POINTER (TREE_VALUE (attr_args));
 
-      if (!strcmp (string, "user"))
-	return USER_MODE;
-      else if (!strcmp (string, "supervisor"))
+      if (!strcmp (string, "supervisor"))
 	return SUPERVISOR_MODE;
+      else if (!strcmp (string, "rnmi"))
+	return RNMI_MODE;
       else /* Must be "machine".  */
 	return MACHINE_MODE;
     }
@@ -12165,14 +12684,31 @@ riscv_estimated_poly_value (poly_int64 val,
 /* Return true if the vector misalignment factor is supported by the
    target.  */
 bool
-riscv_support_vector_misalignment (machine_mode mode,
-				   const_tree type ATTRIBUTE_UNUSED,
-				   int misalignment,
-				   bool is_packed ATTRIBUTE_UNUSED)
+riscv_support_vector_misalignment (machine_mode mode, const_tree type,
+				   int misalignment, bool is_packed,
+				   bool is_gather_scatter)
 {
-  /* Depend on movmisalign pattern.  */
+  /* IS_PACKED is true if the corresponding scalar element is not naturally
+     aligned.  If the misalignment is unknown and the the access is packed
+     we defer to the default hook which will check if movmisalign is present.
+     Movmisalign, in turn, depends on TARGET_VECTOR_MISALIGN_SUPPORTED.  */
+  if (misalignment == DR_MISALIGNMENT_UNKNOWN)
+    {
+      if (!is_packed)
+	return true;
+    }
+  else
+    {
+      /* If we know that misalignment is a multiple of the element size, we're
+	 good.  */
+      if (misalignment % TYPE_ALIGN_UNIT (type) == 0)
+	return true;
+    }
+
+  /* Otherwise fall back to movmisalign again.  */
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* Implement TARGET_VECTORIZE_GET_MASK_MODE.  */
@@ -12334,6 +12870,14 @@ riscv_lshift_subword (machine_mode mode ATTRIBUTE_UNUSED, rtx value, rtx shift,
 						  gen_lowpart (QImode, shift)));
 }
 
+/* Return TRUE if we should use the zero stride load, FALSE otherwise. */
+
+bool
+strided_load_broadcast_p ()
+{
+  return tune_param->use_zero_stride_load;
+}
+
 /* Return TRUE if we should use the divmod expander, FALSE otherwise.  This
    allows the behavior to be tuned for specific implementations as well as
    when optimizing for size.  */
@@ -13298,6 +13842,88 @@ riscv_expand_sssub (rtx dest, rtx x, rtx y)
   emit_move_insn (dest, gen_lowpart (mode, xmode_dest));
 }
 
+/* Implement the Xmode usmul.
+
+   b = SAT_MUL (a, b);
+   =>
+   _1 = a * b;
+   _2 = mulhu (a, b);
+   _overflow_p = _2 == 0;
+   _mask = - _overflow_p;
+   b = _1 | _mask;
+ */
+
+static void
+riscv_expand_xmode_usmul (rtx dest, rtx x, rtx y)
+{
+  machine_mode mode = GET_MODE (dest);
+
+  gcc_assert (mode == Xmode);
+
+  rtx mul = gen_reg_rtx (Xmode);
+  rtx mulhu = gen_reg_rtx (Xmode);
+  rtx overflow_p = gen_reg_rtx (Xmode);
+
+  riscv_emit_binary (MULT, mul, x, y);
+
+  if (TARGET_64BIT)
+    emit_insn (gen_umuldi3_highpart (mulhu, x, y));
+  else
+    emit_insn (gen_umulsi3_highpart (mulhu, x, y));
+
+  riscv_emit_binary (NE, overflow_p, mulhu, CONST0_RTX (Xmode));
+  riscv_emit_unary (NEG, overflow_p, overflow_p);
+  riscv_emit_binary (IOR, dest, mul, overflow_p);
+}
+
+/* Implement the non-Xmode usmul.
+
+   b = SAT_MUL (a, b);
+   =>
+   _1 = a * b;
+   _max = (T)-1
+   _overflow_p = _1 > _max;
+   _mask = - _overflow_p;
+   b = _1 | _mask;
+ */
+
+static void
+riscv_expand_non_xmode_usmul (rtx dest, rtx x, rtx y)
+{
+  machine_mode mode = GET_MODE (dest);
+  unsigned bitsize = GET_MODE_BITSIZE (mode).to_constant ();
+
+  gcc_assert (mode != Xmode);
+
+  rtx xmode_x = riscv_extend_to_xmode_reg (x, mode, ZERO_EXTEND);
+  rtx xmode_y = riscv_extend_to_xmode_reg (y, mode, ZERO_EXTEND);
+  rtx xmode_mul = gen_reg_rtx (Xmode);
+  rtx mul_max = gen_reg_rtx (Xmode);
+  rtx overflow_p = gen_reg_rtx (Xmode);
+
+  uint64_t max = ((uint64_t)1 << bitsize) - 1;
+
+  emit_move_insn (mul_max, GEN_INT (max));
+  riscv_emit_binary (MULT, xmode_mul, xmode_x, xmode_y);
+
+  riscv_emit_binary (LTU, overflow_p, mul_max, xmode_mul);
+  riscv_emit_unary (NEG, overflow_p, overflow_p);
+  riscv_emit_binary (IOR, xmode_mul, xmode_mul, overflow_p);
+
+  emit_move_insn (dest, gen_lowpart (mode, xmode_mul));
+}
+
+/* Implements the unsigned saturation mult standard name usmul for int mode.  */
+
+void
+riscv_expand_usmul (rtx dest, rtx x, rtx y)
+{
+  if (GET_MODE (dest) == Xmode)
+    return riscv_expand_xmode_usmul (dest, x, y) ;
+  else
+    return riscv_expand_non_xmode_usmul (dest, x, y);
+}
+
 /* Implement the unsigned saturation truncation for int mode.
 
    b = SAT_TRUNC (a);
diff --git a/gcc/config/riscv/riscv.md b/gcc/config/riscv/riscv.md
index 893c925..578dd43 100644
--- a/gcc/config/riscv/riscv.md
+++ b/gcc/config/riscv/riscv.md
@@ -120,7 +120,7 @@
   ;; Interrupt handler instructions.
   UNSPECV_MRET
   UNSPECV_SRET
-  UNSPECV_URET
+  UNSPECV_MNRET
 
   ;; Blockage and synchronization.
   UNSPECV_BLOCKAGE
@@ -3298,7 +3298,7 @@
 			  (match_operand:GPR 2 "movcc_operand")
 			  (match_operand:GPR 3 "movcc_operand")))]
   "TARGET_SFB_ALU || TARGET_XTHEADCONDMOV || TARGET_ZICOND_LIKE
-   || TARGET_MOVCC"
+   || TARGET_MOVCC || TARGET_XMIPSCMOV"
 {
   if (riscv_expand_conditional_move (operands[0], operands[1],
 				     operands[2], operands[3]))
@@ -4166,11 +4166,11 @@
   "sret"
   [(set_attr "type" "ret")])
 
-(define_insn "riscv_uret"
+(define_insn "riscv_mnret"
   [(return)
-   (unspec_volatile [(const_int 0)] UNSPECV_URET)]
-  ""
-  "uret"
+   (unspec_volatile [(const_int 0)] UNSPECV_MNRET)]
+  "TARGET_SMRNMI"
+  "mnret"
   [(set_attr "type" "ret")])
 
 (define_insn "stack_tie<mode>"
@@ -4402,7 +4402,7 @@
 )
 
 (define_insn "prefetch"
-  [(prefetch (match_operand 0 "prefetch_operand" "Q")
+  [(prefetch (match_operand 0 "prefetch_operand" "Qr")
              (match_operand 1 "imm5_operand" "i")
              (match_operand 2 "const_int_operand" "n"))]
   "TARGET_ZICBOP"
@@ -4634,6 +4634,17 @@
   }
 )
 
+(define_expand "usmul<mode>3"
+  [(match_operand:ANYI 0 "register_operand")
+   (match_operand:ANYI 1 "register_operand")
+   (match_operand:ANYI 2 "register_operand")]
+  ""
+  {
+    riscv_expand_usmul (operands[0], operands[1], operands[2]);
+    DONE;
+  }
+)
+
 (define_expand "ustrunc<mode><anyi_double_truncated>2"
   [(match_operand:<ANYI_DOUBLE_TRUNCATED> 0 "register_operand")
    (match_operand:ANYI_DOUBLE_TRUNC       1 "register_operand")]
@@ -4872,6 +4883,7 @@
 (include "vector-crypto.md")
 (include "vector-bfloat16.md")
 (include "zicond.md")
+(include "mips-insn.md")
 (include "sfb.md")
 (include "zc.md")
 ;; Vendor extensions
diff --git a/gcc/config/riscv/sync.md b/gcc/config/riscv/sync.md
index a75ea68..50ec8b3 100644
--- a/gcc/config/riscv/sync.md
+++ b/gcc/config/riscv/sync.md
@@ -627,7 +627,7 @@
 	(match_operand:SHORT 1 "memory_operand" "+A"))				;; memory
    (set (match_dup 1)
 	(unspec_volatile:SHORT [(match_operand:SHORT 2 "register_operand" "0")  ;; expected_val
-				(match_operand:SHORT 3 "register_operand" "rJ") ;; desired_val
+				(match_operand:SHORT 3 "reg_or_0_operand" "rJ") ;; desired_val
 				(match_operand:SI 4 "const_int_operand")	;; mod_s
 				(match_operand:SI 5 "const_int_operand")]	;; mod_f
 	 UNSPEC_COMPARE_AND_SWAP))]
diff --git a/gcc/config/riscv/t-riscv b/gcc/config/riscv/t-riscv
index 32092d8..a7eaa8b 100644
--- a/gcc/config/riscv/t-riscv
+++ b/gcc/config/riscv/t-riscv
@@ -194,7 +194,8 @@ RISCV_EXT_DEFS = \
   $(srcdir)/config/riscv/riscv-ext.def \
   $(srcdir)/config/riscv/riscv-ext-sifive.def \
   $(srcdir)/config/riscv/riscv-ext-thead.def \
-  $(srcdir)/config/riscv/riscv-ext-ventana.def
+  $(srcdir)/config/riscv/riscv-ext-ventana.def \
+  $(srcdir)/config/riscv/riscv-ext-mips.def
 
 $(srcdir)/config/riscv/riscv-ext.opt: $(RISCV_EXT_DEFS)
 
@@ -228,8 +229,41 @@ s-riscv-ext.texi: build/gen-riscv-ext-texi$(build_exeext)
 	$(SHELL) $(srcdir)/../move-if-change tmp-riscv-ext.texi $(srcdir)/doc/riscv-ext.texi
 	$(STAMP) s-riscv-ext.texi
 
-# Run `riscv-regen' after you changed or added anything from riscv-ext*.def
+RISCV_CORES_DEFS = \
+  $(srcdir)/config/riscv/riscv-cores.def
+
+build/gen-riscv-mtune-texi.o: $(srcdir)/config/riscv/gen-riscv-mtune-texi.cc \
+	$(RISCV_CORES_DEFS)
+	$(CXX_FOR_BUILD) $(CXXFLAGS_FOR_BUILD) -c $< -o $@
+
+build/gen-riscv-mcpu-texi.o: $(srcdir)/config/riscv/gen-riscv-mcpu-texi.cc \
+	$(RISCV_CORES_DEFS)
+	$(CXX_FOR_BUILD) $(CXXFLAGS_FOR_BUILD) -c $< -o $@
+
+build/gen-riscv-mtune-texi$(build_exeext): build/gen-riscv-mtune-texi.o
+	$(LINKER_FOR_BUILD) $(BUILD_LINKERFLAGS) $(BUILD_LDFLAGS) -o $@ $<
+
+build/gen-riscv-mcpu-texi$(build_exeext): build/gen-riscv-mcpu-texi.o
+	$(LINKER_FOR_BUILD) $(BUILD_LINKERFLAGS) $(BUILD_LDFLAGS) -o $@ $<
+
+$(srcdir)/doc/riscv-mtune.texi: $(RISCV_CORES_DEFS)
+$(srcdir)/doc/riscv-mtune.texi: s-riscv-mtune.texi ; @true
+
+$(srcdir)/doc/riscv-mcpu.texi: $(RISCV_CORES_DEFS)
+$(srcdir)/doc/riscv-mcpu.texi: s-riscv-mcpu.texi ; @true
+
+s-riscv-mtune.texi: build/gen-riscv-mtune-texi$(build_exeext)
+	$(RUN_GEN) build/gen-riscv-mtune-texi$(build_exeext) > tmp-riscv-mtune.texi
+	$(SHELL) $(srcdir)/../move-if-change tmp-riscv-mtune.texi $(srcdir)/doc/riscv-mtune.texi
+	$(STAMP) s-riscv-mtune.texi
+
+s-riscv-mcpu.texi: build/gen-riscv-mcpu-texi$(build_exeext)
+	$(RUN_GEN) build/gen-riscv-mcpu-texi$(build_exeext) > tmp-riscv-mcpu.texi
+	$(SHELL) $(srcdir)/../move-if-change tmp-riscv-mcpu.texi $(srcdir)/doc/riscv-mcpu.texi
+	$(STAMP) s-riscv-mcpu.texi
+
+# Run `riscv-regen' after you changed or added anything from riscv-ext*.def and riscv-cores*.def
 
 .PHONY: riscv-regen
 
-riscv-regen: s-riscv-ext.texi s-riscv-ext.opt
+riscv-regen: s-riscv-ext.texi s-riscv-ext.opt s-riscv-mtune.texi s-riscv-mcpu.texi
diff --git a/gcc/config/riscv/vector-iterators.md b/gcc/config/riscv/vector-iterators.md
index 7825444..aa3b6fb 100644
--- a/gcc/config/riscv/vector-iterators.md
+++ b/gcc/config/riscv/vector-iterators.md
@@ -2418,6 +2418,47 @@
   (RVVM1x2DF "rvvm1df")
 ])
 
+(define_mode_attr vsubel [
+  (RVVM8HI "qi") (RVVM4HI "qi") (RVVM2HI "qi") (RVVM1HI "qi") (RVVMF2HI "qi") (RVVMF4HI "qi")
+
+  (RVVM8SI "hi") (RVVM4SI "hi") (RVVM2SI "hi") (RVVM1SI "hi") (RVVMF2SI "hi")
+
+  (RVVM8SF "hf") (RVVM4SF "hf") (RVVM2SF "hf") (RVVM1SF "hf") (RVVMF2SF "hf")
+
+  (RVVM8DI "si") (RVVM4DI "si") (RVVM2DI "si") (RVVM1DI "si")
+
+  (RVVM8DF "sf") (RVVM4DF "sf") (RVVM2DF "sf") (RVVM1DF "sf")
+
+  ;; VLS modes.
+  (V1HI "qi") (V2HI "qi") (V4HI "qi") (V8HI "qi") (V16HI "qi") (V32HI "qi") (V64HI "qi") (V128HI "qi") (V256HI "qi")
+  (V512HI "qi") (V1024HI "qi") (V2048HI "qi")
+  (V1SI "hi") (V2SI "hi") (V4SI "hi") (V8SI "hi") (V16SI "hi") (V32SI "hi") (V64SI "hi") (V128SI "hi") (V256SI "hi")
+  (V512SI "hi") (V1024SI "hi")
+  (V1DI "si") (V2DI "si") (V4DI "si") (V8DI "si") (V16DI "si") (V32DI "si") (V64DI "si") (V128DI "si") (V256DI "si") (V512DI "si")
+
+  (V1SF "hf")
+  (V2SF "hf")
+  (V4SF "hf")
+  (V8SF "hf")
+  (V16SF "hf")
+  (V32SF "hf")
+  (V64SF "hf")
+  (V128SF "hf")
+  (V256SF "hf")
+  (V512SF "hf")
+  (V1024SF "hf")
+  (V1DF "sf")
+  (V2DF "sf")
+  (V4DF "sf")
+  (V8DF "sf")
+  (V16DF "sf")
+  (V32DF "sf")
+  (V64DF "sf")
+  (V128DF "sf")
+  (V256DF "sf")
+  (V512DF "sf")
+])
+
 (define_mode_attr VSUBEL [
   (RVVM8HI "QI") (RVVM4HI "QI") (RVVM2HI "QI") (RVVM1HI "QI") (RVVMF2HI "QI") (RVVMF4HI "QI")
 
@@ -3972,6 +4013,14 @@
 			      UNSPEC_VASUBU UNSPEC_VASUB UNSPEC_VSMUL
 			      UNSPEC_VSSRL UNSPEC_VSSRA])
 
+(define_int_iterator VSAT_VX_OP_V_VDUP [
+  UNSPEC_VAADDU UNSPEC_VAADD
+])
+
+(define_int_iterator VSAT_VX_OP_VDUP_V [
+  UNSPEC_VAADDU UNSPEC_VAADD
+])
+
 (define_int_iterator VSAT_ARITH_OP [UNSPEC_VAADDU UNSPEC_VAADD
 			      	    UNSPEC_VASUBU UNSPEC_VASUB UNSPEC_VSMUL])
 (define_int_iterator VSAT_SHIFT_OP [UNSPEC_VSSRL UNSPEC_VSSRA])
@@ -4006,6 +4055,14 @@
 			 	(UNSPEC_VSSRA "vsshift") (UNSPEC_VNCLIP "vnclip")
 				(UNSPEC_VNCLIPU "vnclip")])
 
+(define_int_attr sat_op_v_vdup [
+  (UNSPEC_VAADDU "aaddu") (UNSPEC_VAADD "aadd")
+])
+
+(define_int_attr sat_op_vdup_v [
+  (UNSPEC_VAADDU "aaddu") (UNSPEC_VAADD "aadd")
+])
+
 (define_int_attr misc_op [(UNSPEC_VMSBF "sbf") (UNSPEC_VMSIF "sif") (UNSPEC_VMSOF "sof")
 			  (UNSPEC_VFRSQRT7 "rsqrt7")])
 
@@ -4042,11 +4099,12 @@
 ])
 
 (define_code_iterator any_int_binop_no_shift_v_vdup [
-  plus minus and ior xor mult div udiv mod umod smax umax smin umin us_plus us_minus
+  plus minus and ior xor mult div udiv mod umod smax umax smin umin us_plus
+  us_minus ss_plus ss_minus
 ])
 
 (define_code_iterator any_int_binop_no_shift_vdup_v [
-  plus minus and ior xor mult smax umax smin umin us_plus
+  plus minus and ior xor mult smax umax smin umin us_plus ss_plus
 ])
 
 (define_code_iterator any_int_unop [neg not])
diff --git a/gcc/config/riscv/vector.md b/gcc/config/riscv/vector.md
index 6753b01..66b7670 100644
--- a/gcc/config/riscv/vector.md
+++ b/gcc/config/riscv/vector.md
@@ -1551,20 +1551,44 @@
 (define_expand "vec_duplicate<mode>"
   [(set (match_operand:V_VLS 0 "register_operand")
         (vec_duplicate:V_VLS
-          (match_operand:<VEL> 1 "direct_broadcast_operand")))]
+          (match_operand:<VEL> 1 "any_broadcast_operand")))]
   "TARGET_VECTOR"
   {
-    /* Early expand DImode broadcast in RV32 system to avoid RA reload
-       generate (set (reg) (vec_duplicate:DI)).  */
+    /* Don't keep a DImode broadcast for RV32 in the vec_duplicate form.
+       Otherwise combine or late combine could end up doing
+	      "64-bit broadcast" (!= vmv.v.x)
+            + vadd.vv
+	    = vadd.vx
+       which would be invalid.  */
     bool gt_p = maybe_gt (GET_MODE_SIZE (<VEL>mode), GET_MODE_SIZE (Pmode));
     if (!FLOAT_MODE_P (<VEL>mode) && gt_p)
       {
-        riscv_vector::emit_vlmax_insn (code_for_pred_broadcast (<MODE>mode),
-				       riscv_vector::UNARY_OP, operands);
-	DONE;
+        riscv_vector::emit_vlmax_insn
+	  (code_for_pred_strided_broadcast
+	    (<MODE>mode), riscv_vector::UNARY_OP, operands);
+       DONE;
       }
-    /* Otherwise, allow it fall into general vec_duplicate pattern
-       which allow us to have vv->vx combine optimization in later pass.  */
+
+    /* Even though we can eventually broadcast any permissible
+       constant by moving it into a register we need to force
+       any non-immediate one into a register here.
+       If we didn't do that we couldn't fwprop/late-combine
+	      vec_duplicate 123.45f
+	    + vfadd.vv
+	    = vfadd.vf
+       because the constant is valid for vec_duplicate but not
+       for vfadd.vf.  Therefore we need to do
+	      fa0 = 123.45f
+	      vec_duplicate fa0
+	    + vfadd.vv
+	    = vfadd.vf  */
+    if (!satisfies_constraint_P (operands[1])
+	&& !satisfies_constraint_J (operands[1])
+	&& !rtx_equal_p (operands[1], CONST0_RTX (<VEL>mode))
+	&& !memory_operand (operands[1], <VEL>mode))
+      operands[1] = force_reg (<VEL>mode, operands[1]);
+
+    /* Otherwise keep the vec_duplicate pattern until split.  */
   })
 
 ;; According to GCC internal:
@@ -1574,14 +1598,20 @@
 (define_insn_and_split "*vec_duplicate<mode>"
   [(set (match_operand:V_VLS 0 "register_operand")
         (vec_duplicate:V_VLS
-          (match_operand:<VEL> 1 "direct_broadcast_operand")))]
+          (match_operand:<VEL> 1 "any_broadcast_operand")))]
   "TARGET_VECTOR && can_create_pseudo_p ()"
   "#"
   "&& 1"
   [(const_int 0)]
   {
-    riscv_vector::emit_vlmax_insn (code_for_pred_broadcast (<MODE>mode),
-                                   riscv_vector::UNARY_OP, operands);
+    if (riscv_vector::can_be_broadcast_p (operands[1]))
+      riscv_vector::emit_vlmax_insn (code_for_pred_broadcast (<MODE>mode),
+				     riscv_vector::UNARY_OP, operands);
+    else
+      riscv_vector::emit_vlmax_insn (code_for_pred_strided_broadcast
+				     (<MODE>mode), riscv_vector::UNARY_OP,
+				     operands);
+
     DONE;
   }
   [(set_attr "type" "vector")]
@@ -1783,7 +1813,7 @@
   [(set_attr "type" "vsetvl")
    (set_attr "mode" "SI")])
 
-;; This pattern use to combine bellow two insns and then further remove
+;; This pattern use to combine below two insns and then further remove
 ;; unnecessary sign_extend operations:
 ;;   (set (reg:DI 134 [ _1 ])
 ;;        (unspec:DI [
@@ -2127,69 +2157,45 @@
 	  (match_operand:V_VLS 2 "vector_merge_operand")))]
   "TARGET_VECTOR"
 {
-  /* Transform vmv.v.x/vfmv.v.f (avl = 1) into vmv.s.x since vmv.s.x/vfmv.s.f
-     has better chances to do vsetvl fusion in vsetvl pass.  */
   bool wrap_vec_dup = true;
   rtx vec_cst = NULL_RTX;
-  if (riscv_vector::splat_to_scalar_move_p (operands))
-    {
-      operands[1] = riscv_vector::gen_scalar_move_mask (<VM>mode);
-      operands[3] = force_reg (<VEL>mode, operands[3]);
-    }
-  else if (immediate_operand (operands[3], <VEL>mode)
-	   && (vec_cst = gen_const_vec_duplicate (<MODE>mode, operands[3]))
-	   && (/* -> pred_broadcast<mode>_zero */
-	       (vector_least_significant_set_mask_operand (operands[1],
-							   <VM>mode)
-		&& vector_const_0_operand (vec_cst, <MODE>mode))
-	       || (/* pred_broadcast<mode>_imm */
-		   vector_all_trues_mask_operand (operands[1], <VM>mode)
-		   && vector_const_int_or_double_0_operand (vec_cst,
-							    <MODE>mode))))
+  if (immediate_operand (operands[3], <VEL>mode)
+      && (vec_cst = gen_const_vec_duplicate (<MODE>mode, operands[3]))
+      && (/* -> pred_broadcast<mode>_zero */
+	  (vector_least_significant_set_mask_operand (operands[1],
+						      <VM>mode)
+	   && vector_const_0_operand (vec_cst, <MODE>mode))
+	  || (/* pred_broadcast<mode>_imm */
+	      vector_all_trues_mask_operand (operands[1], <VM>mode)
+	      && vector_const_int_or_double_0_operand (vec_cst,
+						       <MODE>mode))))
     {
       operands[3] = vec_cst;
       wrap_vec_dup = false;
     }
-  /* Handle vmv.s.x instruction (Wb1 mask) which has memory scalar.  */
-  else if (satisfies_constraint_Wdm (operands[3]))
-    {
-      if (satisfies_constraint_Wb1 (operands[1]))
-	{
-	  /* Case 1: vmv.s.x (TA, x == memory) ==> vlse.v (TA)  */
-	  if (satisfies_constraint_vu (operands[2]))
-	    operands[1] = CONSTM1_RTX (<VM>mode);
-	  else if (GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode))
-	    {
-	      /* Case 2: vmv.s.x (TU, x == memory) ==>
-			   vl = 0 or 1; + vlse.v (TU) in RV32 system  */
-	      operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
-	      operands[1] = CONSTM1_RTX (<VM>mode);
-	    }
-	  else
-	    /* Case 3: load x (memory) to register.  */
-	    operands[3] = force_reg (<VEL>mode, operands[3]);
-	}
-    }
-  else if (GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode)
-           && (immediate_operand (operands[3], Pmode)
+  else if (GET_MODE_SIZE (<VEL>mode) > UNITS_PER_WORD
+	   && satisfies_constraint_Wb1 (operands[1])
+	   && (immediate_operand (operands[3], Xmode)
 	       || (CONST_POLY_INT_P (operands[3])
 	           && known_ge (rtx_to_poly_int64 (operands[3]), 0U)
-		   && known_le (rtx_to_poly_int64 (operands[3]), GET_MODE_SIZE (<MODE>mode)))))
+		   && known_le (rtx_to_poly_int64 (operands[3]),
+				GET_MODE_SIZE (<MODE>mode)))))
     {
       rtx tmp = gen_reg_rtx (Pmode);
       poly_int64 value = rtx_to_poly_int64 (operands[3]);
-      emit_move_insn (tmp, gen_int_mode (value, Pmode));
+      emit_move_insn (tmp, gen_int_mode (value, Xmode));
       operands[3] = gen_rtx_SIGN_EXTEND (<VEL>mode, tmp);
     }
-  /* Never load (const_int 0) into a register, that's silly.  */
-  else if (operands[3] == CONST0_RTX (<VEL>mode))
+
+  /* For a vmv.v.x never load (const_int 0) or valid immediate operands
+     into a register, because we can use vmv.v.i.  */
+  else if (satisfies_constraint_Wc1 (operands[1])
+      && (satisfies_constraint_P (operands[3])
+	  || operands[3] == CONST0_RTX (<VEL>mode)))
     ;
-  /* If we're broadcasting [-16..15] across more than just
-     element 0, then we can use vmv.v.i directly, thus avoiding
-     the load of the constant into a GPR.  */
-  else if (CONST_INT_P (operands[3])
-	   && IN_RANGE (INTVAL (operands[3]), -16, 15)
-	   && !satisfies_constraint_Wb1 (operands[1]))
+  /* For vmv.s.x we have vmv.s.x v1, zero.  */
+  else if (satisfies_constraint_Wb1 (operands[1])
+	   && operands[3] == CONST0_RTX (<VEL>mode))
     ;
   else
     operands[3] = force_reg (<VEL>mode, operands[3]);
@@ -2197,113 +2203,68 @@
     operands[3] = gen_rtx_VEC_DUPLICATE (<MODE>mode, operands[3]);
 })
 
-(define_insn_and_split "*pred_broadcast<mode>"
-  [(set (match_operand:V_VLSI 0 "register_operand"                 "=vr, vr, vd, vd, vr, vr, vr, vr")
+(define_insn_and_rewrite "*pred_broadcast<mode>"
+  [(set (match_operand:V_VLSI 0 "register_operand"                 "=vr, vr, vr, vr")
 	(if_then_else:V_VLSI
 	  (unspec:<VM>
-	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1, vm, vm,Wc1,Wc1,Wb1,Wb1")
-	     (match_operand 4 "vector_length_operand"              "rvl,rvl,rvl,rvl,rvl,rvl,rvl,rvl")
-	     (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
-	     (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
-	     (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
+	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1,Wb1,Wb1")
+	     (match_operand 4 "vector_length_operand"		   "rvl,rvl,rvl,rvl")
+	     (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i")
+	     (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i")
+	     (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i")
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
 	  (vec_duplicate:V_VLSI
-	    (match_operand:<VEL> 3 "direct_broadcast_operand"       "rP,rP,Wdm,Wdm,Wdm,Wdm, rJ, rJ"))
-	  (match_operand:V_VLSI 2 "vector_merge_operand"            "vu, 0, vu,  0, vu,  0, vu,  0")))]
+	    (match_operand:<VEL> 3 "direct_broadcast_operand"      " rP, rP, rJ, rJ"))
+	  (match_operand:V_VLSI 2 "vector_merge_operand"           " vu,  0, vu,  0")))]
   "TARGET_VECTOR"
   "@
    vmv.v.%o3\t%0,%3
    vmv.v.%o3\t%0,%3
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero
-   vlse<sew>.v\t%0,%3,zero
    vmv.s.x\t%0,%z3
    vmv.s.x\t%0,%z3"
-  "(register_operand (operands[3], <VEL>mode)
-  || CONST_POLY_INT_P (operands[3]))
-  && GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode)"
-  [(set (match_dup 0)
-	(if_then_else:V_VLSI (unspec:<VM> [(match_dup 1) (match_dup 4)
-	     (match_dup 5) (match_dup 6) (match_dup 7)
-	     (reg:SI VL_REGNUM) (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
-	  (vec_duplicate:V_VLSI (match_dup 3))
-	  (match_dup 2)))]
-  {
-    gcc_assert (can_create_pseudo_p ());
-    if (CONST_POLY_INT_P (operands[3]))
-      {
-	rtx tmp = gen_reg_rtx (<VEL>mode);
-	emit_move_insn (tmp, operands[3]);
-	operands[3] = tmp;
-      }
-    rtx m = assign_stack_local (<VEL>mode, GET_MODE_SIZE (<VEL>mode),
-				GET_MODE_ALIGNMENT (<VEL>mode));
-    m = validize_mem (m);
-    emit_move_insn (m, operands[3]);
-    m = gen_rtx_MEM (<VEL>mode, force_reg (Pmode, XEXP (m, 0)));
-    operands[3] = m;
-
-    /* For SEW = 64 in RV32 system, we expand vmv.s.x:
-       andi a2,a2,1
-       vsetvl zero,a2,e64
-       vlse64.v  */
-    if (satisfies_constraint_Wb1 (operands[1]))
-      {
-	operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
-	operands[1] = CONSTM1_RTX (<VM>mode);
-      }
-  }
-  [(set_attr "type" "vimov,vimov,vlds,vlds,vlds,vlds,vimovxv,vimovxv")
+  "&& (operands[1] == CONSTM1_RTX (<VM>mode)
+       && operands[4] == CONST1_RTX (Pmode)
+       && (register_operand (operands[3], <VEL>mode)
+           || satisfies_constraint_J (operands[3])))"
+{
+  /* A broadcast of a single element is just a vmv.s.x.  */
+  operands[1] = riscv_vector::gen_scalar_move_mask (<VM>mode);
+}
+  [(set_attr "type" "vimov,vimov,vimovxv,vimovxv")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*pred_broadcast<mode>_zvfh"
-  [(set (match_operand:V_VLSF    0 "register_operand"              "=vr,  vr,  vr,  vr")
+(define_insn_and_rewrite "pred_broadcast<mode>_zvfh"
+  [(set (match_operand:V_VLSF    0 "register_operand"              "=vr, vr, vr, vr")
 	(if_then_else:V_VLSF
 	  (unspec:<VM>
-	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1, Wc1, Wb1, Wb1")
-	     (match_operand      4 "vector_length_operand"         "rvl, rvl, rvl, rvl")
-	     (match_operand      5 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand      6 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand      7 "const_int_operand"             "  i,   i,   i,   i")
+	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1,Wb1,Wb1")
+	     (match_operand      4 "vector_length_operand"	   "rvl,rvl,rvl,rvl")
+	     (match_operand      5 "const_int_operand"             "  i,  i,  i,  i")
+	     (match_operand      6 "const_int_operand"             "  i,  i,  i,  i")
+	     (match_operand      7 "const_int_operand"             "  i,  i,  i,  i")
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
 	  (vec_duplicate:V_VLSF
-	    (match_operand:<VEL> 3 "direct_broadcast_operand"      "  f,   f,   f,   f"))
-	  (match_operand:V_VLSF  2 "vector_merge_operand"          " vu,   0,  vu,   0")))]
+	    (match_operand:<VEL> 3 "direct_broadcast_operand"      "  f,  f,  f,  f"))
+	  (match_operand:V_VLSF  2 "vector_merge_operand"          " vu,  0, vu,  0")))]
   "TARGET_VECTOR"
   "@
    vfmv.v.f\t%0,%3
    vfmv.v.f\t%0,%3
    vfmv.s.f\t%0,%3
    vfmv.s.f\t%0,%3"
+  "&& (operands[1] == CONSTM1_RTX (<VM>mode)
+       && operands[4] == CONST1_RTX (Pmode)
+       && (register_operand (operands[3], <VEL>mode)
+           || satisfies_constraint_J (operands[3])))"
+{
+  /* A broadcast of a single element is just a vfmv.s.f.  */
+  operands[1] = riscv_vector::gen_scalar_move_mask (<VM>mode);
+}
   [(set_attr "type" "vfmov,vfmov,vfmovfv,vfmovfv")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*pred_broadcast<mode>_zvfhmin"
-  [(set (match_operand:V_VLSF_ZVFHMIN   0 "register_operand"              "=vr,  vr,  vr,  vr")
-	(if_then_else:V_VLSF_ZVFHMIN
-	  (unspec:<VM>
-	    [(match_operand:<VM>        1 "vector_broadcast_mask_operand" " vm,  vm, Wc1, Wc1")
-	     (match_operand             4 "vector_length_operand"         "rvl, rvl, rvl, rvl")
-	     (match_operand             5 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand             6 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand             7 "const_int_operand"             "  i,   i,   i,   i")
-	     (reg:SI VL_REGNUM)
-	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
-	  (vec_duplicate:V_VLSF_ZVFHMIN
-	    (match_operand:<VEL>        3 "direct_broadcast_operand"      "Wdm, Wdm, Wdm, Wdm"))
-	  (match_operand:V_VLSF_ZVFHMIN 2 "vector_merge_operand"          " vu,   0,  vu,   0")))]
-  "TARGET_VECTOR"
-  "@
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero
-   vlse<sew>.v\t%0,%3,zero"
-  [(set_attr "type" "vlds,vlds,vlds,vlds")
-   (set_attr "mode" "<MODE>")])
-
 (define_insn "*pred_broadcast<mode>_extended_scalar"
   [(set (match_operand:V_VLSI_D 0 "register_operand"               "=vr, vr, vr, vr")
 	(if_then_else:V_VLSI_D
@@ -2366,6 +2327,117 @@
   [(set_attr "type" "vimov,vimov")
    (set_attr "mode" "<MODE>")])
 
+(define_expand "@pred_strided_broadcast<mode>"
+  [(set (match_operand:V_VLS 0 "register_operand")
+	(if_then_else:V_VLS
+	  (unspec:<VM>
+	    [(match_operand:<VM> 1 "strided_broadcast_mask_operand")
+	     (match_operand 4 "vector_length_operand")
+	     (match_operand 5 "const_int_operand")
+	     (match_operand 6 "const_int_operand")
+	     (match_operand 7 "const_int_operand")
+	     (reg:SI VL_REGNUM)
+	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+	  (vec_duplicate:V_VLS
+	    (match_operand:<VEL> 3 "strided_broadcast_operand"))
+	  (match_operand:V_VLS 2 "vector_merge_operand")))]
+  "TARGET_VECTOR"
+{
+  if (satisfies_constraint_Wb1 (operands[1]))
+    {
+      /* If we're asked to set a single element (like vmv.s.x but we
+	 need to go via memory here) and the tail policy is agnostic
+	 we can overwrite all elements.
+	 Thus, set the mask to broadcast.  */
+      operands[1] = CONSTM1_RTX (<VM>mode);
+      if (!satisfies_constraint_vu (operands[2])
+	  && GET_MODE_SIZE (<VEL>mode) > UNITS_PER_WORD)
+	{
+	  /* Case 2: vmv.s.x (TU, x == memory) ==>
+	     vl = 0 or 1; + vlse.v (TU) in RV32 system  */
+	  /* In this case we must not overwrite the residual elements,
+	     so set the vector length to 0/1.  */
+	  operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
+	}
+    }
+})
+
+(define_insn_and_split "*pred_strided_broadcast<mode>"
+  [(set (match_operand:V_VLSI 0 "register_operand"                  "=vd, vd, vr, vr")
+	(if_then_else:V_VLSI
+	  (unspec:<VM>
+	    [(match_operand:<VM> 1 "strided_broadcast_mask_operand" " vm, vm,Wc1,Wc1")
+	     (match_operand 4 "vector_length_operand"               "rvl,rvl,rvl,rvl")
+	     (match_operand 5 "const_int_operand"                   "  i,  i,  i,  i")
+	     (match_operand 6 "const_int_operand"                   "  i,  i,  i,  i")
+	     (match_operand 7 "const_int_operand"                   "  i,  i,  i,  i")
+	     (reg:SI VL_REGNUM)
+	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+	  (vec_duplicate:V_VLSI
+	    (match_operand:<VEL> 3 "strided_broadcast_operand"	    "  A,  A,  A,  A"))
+	  (match_operand:V_VLSI 2 "vector_merge_operand"            " vu,  0, vu,  0")))]
+  "TARGET_VECTOR"
+  "@
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero
+   vlse<sew>.v\t%0,%3,zero"
+  "&& !strided_load_broadcast_p () && can_create_pseudo_p ()"
+  [(const_int 0)]
+  {
+    rtx tmp = gen_reg_rtx (V1DImode);
+    emit_move_insn (tmp, gen_lowpart (V1DImode, operands[3]));
+    tmp = lowpart_subreg (<MODE>mode, tmp, V1DImode);
+
+    emit_insn
+      (gen_pred_gather<mode>_scalar
+       (operands[0], operands[1], operands[2], tmp, CONST0_RTX (Pmode),
+	operands[4], operands[5], operands[6], operands[7]));
+    DONE;
+  }
+  [(set_attr "type" "vlds,vlds,vlds,vlds")
+   (set_attr "mode" "<MODE>")])
+
+(define_insn_and_split "*pred_strided_broadcast<mode>_zvfhmin"
+  [(set (match_operand:V_VLSF_ZVFHMIN   0 "register_operand"               "=vr,  vr,  vr,  vr")
+	(if_then_else:V_VLSF_ZVFHMIN
+	  (unspec:<VM>
+	    [(match_operand:<VM>        1 "strided_broadcast_mask_operand" " vm,  vm, Wc1, Wc1")
+	     (match_operand             4 "vector_length_operand"          "rvl, rvl, rvl, rvl")
+	     (match_operand             5 "const_int_operand"              "  i,   i,   i,   i")
+	     (match_operand             6 "const_int_operand"              "  i,   i,   i,   i")
+	     (match_operand             7 "const_int_operand"              "  i,   i,   i,   i")
+	     (reg:SI VL_REGNUM)
+	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+	  (vec_duplicate:V_VLSF_ZVFHMIN
+	    (match_operand:<VEL>        3 "strided_broadcast_operand"	   "  A,   A,   A,   A"))
+	  (match_operand:V_VLSF_ZVFHMIN 2 "vector_merge_operand"           " vu,   0,  vu,   0")))]
+  "TARGET_VECTOR"
+  "@
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero
+   vlse<sew>.v\t%0,%3,zero"
+  "&& !strided_load_broadcast_p ()
+   && <VEL>mode == HFmode
+   && can_create_pseudo_p ()"
+  [(const_int 0)]
+  {
+    poly_uint64 nunits = GET_MODE_NUNITS (<MODE>mode);
+    machine_mode vmodehi
+      = riscv_vector::get_vector_mode (HImode, nunits).require ();
+    rtx ops[] = {gen_lowpart (vmodehi, operands[0]),
+		 gen_lowpart (HImode, operands[3])};
+    riscv_vector::emit_avltype_insn (code_for_pred_broadcast (vmodehi),
+				     riscv_vector::UNARY_OP, ops,
+				     (riscv_vector::avl_type) INTVAL (operands[7]),
+				     operands[4]);
+    DONE;
+  }
+  [(set_attr "type" "vlds,vlds,vlds,vlds")
+   (set_attr "mode" "<MODE>")])
+
+
 ;; -------------------------------------------------------------------------------
 ;; ---- Predicated Strided loads/stores
 ;; -------------------------------------------------------------------------------
@@ -4607,8 +4679,8 @@
 ;; Handle GET_MODE_INNER (mode) = DImode. We need to split them since
 ;; we need to deal with SEW = 64 in RV32 system.
 (define_expand "@pred_<sat_op><mode>_scalar"
-  [(set (match_operand:VI_D 0 "register_operand")
-	(if_then_else:VI_D
+  [(set (match_operand:V_VLSI_D 0 "register_operand")
+	(if_then_else:V_VLSI_D
 	  (unspec:<VM>
 	    [(match_operand:<VM> 1 "vector_mask_operand")
 	     (match_operand 5 "vector_length_operand")
@@ -4619,10 +4691,10 @@
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)
 	     (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
-	  (unspec:VI_D
-	    [(match_operand:VI_D 3 "register_operand")
+	  (unspec:V_VLSI_D
+	    [(match_operand:V_VLSI_D 3 "register_operand")
 	     (match_operand:<VEL> 4 "reg_or_int_operand")] VSAT_ARITH_OP)
-	  (match_operand:VI_D 2 "vector_merge_operand")))]
+	  (match_operand:V_VLSI_D 2 "vector_merge_operand")))]
   "TARGET_VECTOR"
 {
   if (riscv_vector::sew64_scalar_helper (
@@ -4641,8 +4713,8 @@
 })
 
 (define_insn "*pred_<sat_op><mode>_scalar"
-  [(set (match_operand:VI_D 0 "register_operand"         "=vd, vr, vd, vr")
-	(if_then_else:VI_D
+  [(set (match_operand:V_VLSI_D 0 "register_operand"         "=vd, vr, vd, vr")
+	(if_then_else:V_VLSI_D
 	  (unspec:<VM>
 	    [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
 	     (match_operand 5 "vector_length_operand"    "rvl,rvl,rvl,rvl")
@@ -4653,18 +4725,18 @@
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)
 	     (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
-	  (unspec:VI_D
-	    [(match_operand:VI_D 3 "register_operand"    " vr, vr, vr, vr")
+	  (unspec:V_VLSI_D
+	    [(match_operand:V_VLSI_D 3 "register_operand"    " vr, vr, vr, vr")
 	     (match_operand:<VEL> 4 "reg_or_0_operand"   " rJ, rJ, rJ, rJ")] VSAT_ARITH_OP)
-	  (match_operand:VI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")))]
+	  (match_operand:V_VLSI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")))]
   "TARGET_VECTOR"
   "v<sat_op>.vx\t%0,%3,%z4%p1"
   [(set_attr "type" "<sat_insn_type>")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*pred_<sat_op><mode>_extended_scalar"
-  [(set (match_operand:VI_D 0 "register_operand"            "=vd, vr, vd, vr")
-	(if_then_else:VI_D
+  [(set (match_operand:V_VLSI_D 0 "register_operand"            "=vd, vr, vd, vr")
+	(if_then_else:V_VLSI_D
 	  (unspec:<VM>
 	    [(match_operand:<VM> 1 "vector_mask_operand"    " vm, vm,Wc1,Wc1")
 	     (match_operand 5 "vector_length_operand"       "rvl,rvl,rvl,rvl")
@@ -4675,11 +4747,11 @@
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)
 	     (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
-	  (unspec:VI_D
-	    [(match_operand:VI_D 3 "register_operand"       " vr, vr, vr, vr")
+	  (unspec:V_VLSI_D
+	    [(match_operand:V_VLSI_D 3 "register_operand"       " vr, vr, vr, vr")
 	     (sign_extend:<VEL>
 	       (match_operand:<VSUBEL> 4 "reg_or_0_operand" " rJ, rJ, rJ, rJ"))] VSAT_ARITH_OP)
-	  (match_operand:VI_D 2 "vector_merge_operand"      " vu,  0, vu,  0")))]
+	  (match_operand:V_VLSI_D 2 "vector_merge_operand"      " vu,  0, vu,  0")))]
   "TARGET_VECTOR && !TARGET_64BIT"
   "v<sat_op>.vx\t%0,%3,%z4%p1"
   [(set_attr "type" "<sat_insn_type>")
@@ -7267,10 +7339,10 @@
 	  (plus_minus:VWEXTF
 	    (mult:VWEXTF
 	      (float_extend:VWEXTF
-	        (vec_duplicate:<V_DOUBLE_TRUNC>
-	          (match_operand:<VSUBEL> 3 "register_operand"       "    f")))
-	      (float_extend:VWEXTF
-	        (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
+	        (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr"))
+	      (vec_duplicate:VWEXTF
+	        (float_extend:<VEL>
+	          (match_operand:<VSUBEL> 3 "register_operand"       "    f"))))
 	    (match_operand:VWEXTF 2 "register_operand"               "    0"))
 	  (match_dup 2)))]
   "TARGET_VECTOR"
diff --git a/gcc/config/riscv/xiangshan.md b/gcc/config/riscv/xiangshan.md
index 5ed6bac..34b4a8f 100644
--- a/gcc/config/riscv/xiangshan.md
+++ b/gcc/config/riscv/xiangshan.md
@@ -107,7 +107,8 @@
 ;; they are just dummies like this one.
 (define_insn_reservation "xiangshan_alu_unknown" 1
   (and (eq_attr "tune" "xiangshan")
-       (eq_attr "type" "zicond,min,max,minu,maxu,clz,ctz,cpop,ghost,rotate,clmul,condmove,crypto,mvpair,rdvlenb,rdvl,wrvxrm,wrfrm,rdfrm,vsetvl,vsetvl_pre,vlde,vste,vldm,vstm,vlds,vsts,vldux,vldox,vstux,vstox,vldff,vldr,vstr,vlsegde,vssegte,vlsegds,vssegts,vlsegdux,vlsegdox,vssegtux,vssegtox,vlsegdff,vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,vimul,vidiv,viwmul,vimuladd,sf_vqmacc,viwmuladd,vimerge,vimov,vsalu,vaalu,vsmul,vsshift,vnclip,sf_vfnrclip,vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,vfmovvf,vfmovfv,vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,vgather,vcompress,vmov,vector,vandn,vbrev,vbrev8,vrev8,vclz,vctz,vcpop,vrol,vror,vwsll,vclmul,vclmulh,vghsh,vgmul,vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz,vsha2ms,vsha2ch,vsha2cl,vsm4k,vsm4r,vsm3me,vsm3c,vfncvtbf16,vfwcvtbf16,vfwmaccbf16"))
+       (eq_attr "type" "zicond,min,max,minu,maxu,clz,ctz,cpop,ghost,rotate,clmul,condmove,crypto,mvpair,rdvlenb,rdvl,wrvxrm,wrfrm,rdfrm,vsetvl,vsetvl_pre,vlde,vste,vldm,vstm,vlds,vsts,vldux,vldox,vstux,vstox,vldff,vldr,vstr,vlsegde,vssegte,vlsegds,vssegts,vlsegdux,vlsegdox,vssegtux,vssegtox,vlsegdff,vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,vimul,vidiv,viwmul,vimuladd,sf_vqmacc,viwmuladd,vimerge,vimov,vsalu,vaalu,vsmul,vsshift,vnclip,sf_vfnrclip,vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,vfmovvf,vfmovfv,vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,vgather,vcompress,vmov,vector,vandn,vbrev,vbrev8,vrev8,vclz,vctz,vcpop,vrol,vror,vwsll,vclmul,vclmulh,vghsh,vgmul,vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz,vsha2ms,vsha2ch,vsha2cl,vsm4k,vsm4r,vsm3me,vsm3c,vfncvtbf16,vfwcvtbf16,vfwmaccbf16,sf_vc,sf_vc_se"))
+
   "xs_alu_rs")
 
 ;; ----------------------------------------------------
diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc
index 7ee26e5..764b499 100644
--- a/gcc/config/rs6000/rs6000.cc
+++ b/gcc/config/rs6000/rs6000.cc
@@ -4951,10 +4951,19 @@ static bool
 rs6000_builtin_support_vector_misalignment (machine_mode mode,
 					    const_tree type,
 					    int misalignment,
-					    bool is_packed)
+					    bool is_packed,
+					    bool is_gather_scatter)
 {
   if (TARGET_VSX)
     {
+      if (is_gather_scatter)
+	{
+	  if (TARGET_ALTIVEC && is_packed)
+	    return false;
+	  else
+	    return true;
+	}
+
       if (TARGET_EFFICIENT_UNALIGNED_VSX)
 	return true;
 
@@ -5165,6 +5174,7 @@ public:
 
 protected:
   void update_target_cost_per_stmt (vect_cost_for_stmt, stmt_vec_info,
+				    slp_tree node,
 				    vect_cost_model_location, unsigned int);
   void density_test (loop_vec_info);
   void adjust_vect_cost_per_loop (loop_vec_info);
@@ -5312,6 +5322,7 @@ rs6000_adjust_vect_cost_per_stmt (enum vect_cost_for_stmt kind,
 void
 rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
 					       stmt_vec_info stmt_info,
+					       slp_tree node,
 					       vect_cost_model_location where,
 					       unsigned int orig_count)
 {
@@ -5372,12 +5383,12 @@ rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
 	 or may not need to apply.  When finalizing the cost of the loop,
 	 the extra penalty is applied when the load density heuristics
 	 are satisfied.  */
-      if (kind == vec_construct && stmt_info
-	  && STMT_VINFO_TYPE (stmt_info) == load_vec_info_type
-	  && (STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_ELEMENTWISE
-	      || STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_STRIDED_SLP))
+      if (kind == vec_construct && node
+	  && SLP_TREE_TYPE (node) == load_vec_info_type
+	  && (SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
+	      || SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP))
 	{
-	  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+	  tree vectype = SLP_TREE_VECTYPE (node);
 	  unsigned int nunits = vect_nunits_for_cost (vectype);
 	  /* As PR103702 shows, it's possible that vectorizer wants to do
 	     costings for only one unit here, it's no need to do any
@@ -5406,7 +5417,7 @@ rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
 
 unsigned
 rs6000_cost_data::add_stmt_cost (int count, vect_cost_for_stmt kind,
-				 stmt_vec_info stmt_info, slp_tree,
+				 stmt_vec_info stmt_info, slp_tree node,
 				 tree vectype, int misalign,
 				 vect_cost_model_location where)
 {
@@ -5424,7 +5435,7 @@ rs6000_cost_data::add_stmt_cost (int count, vect_cost_for_stmt kind,
       retval = adjust_cost_for_freq (stmt_info, where, count * stmt_cost);
       m_costs[where] += retval;
 
-      update_target_cost_per_stmt (kind, stmt_info, where, orig_count);
+      update_target_cost_per_stmt (kind, stmt_info, node, where, orig_count);
     }
 
   return retval;
@@ -10309,15 +10320,18 @@ can_be_rotated_to_negative_lis (HOST_WIDE_INT c, int *rot)
 
   /* case b. xx0..01..1xx: some of 15 x's (and some of 16 0's) are
      rotated over the highest bit.  */
-  int pos_one = clz_hwi ((c << 16) >> 16);
-  middle_zeros = ctz_hwi (c >> (HOST_BITS_PER_WIDE_INT - pos_one));
-  int middle_ones = clz_hwi (~(c << pos_one));
-  if (middle_zeros >= 16 && middle_ones >= 33)
+  unsigned HOST_WIDE_INT uc = c;
+  int pos_one = clz_hwi ((HOST_WIDE_INT) (uc << 16) >> 16);
+  if (pos_one != 0)
     {
-      *rot = pos_one;
-      return true;
+      middle_zeros = ctz_hwi (c >> (HOST_BITS_PER_WIDE_INT - pos_one));
+      int middle_ones = clz_hwi (~(uc << pos_one));
+      if (middle_zeros >= 16 && middle_ones >= 33)
+	{
+	  *rot = pos_one;
+	  return true;
+	}
     }
-
   return false;
 }
 
@@ -10434,7 +10448,8 @@ can_be_built_by_li_and_rldic (HOST_WIDE_INT c, int *shift, HOST_WIDE_INT *mask)
   if (lz >= HOST_BITS_PER_WIDE_INT)
     return false;
 
-  int middle_ones = clz_hwi (~(c << lz));
+  unsigned HOST_WIDE_INT uc = c;
+  int middle_ones = clz_hwi (~(uc << lz));
   if (tz + lz + middle_ones >= ones
       && (tz - lz) < HOST_BITS_PER_WIDE_INT
       && tz < HOST_BITS_PER_WIDE_INT)
@@ -10468,7 +10483,7 @@ can_be_built_by_li_and_rldic (HOST_WIDE_INT c, int *shift, HOST_WIDE_INT *mask)
   if (!IN_RANGE (pos_first_1, 1, HOST_BITS_PER_WIDE_INT-1))
     return false;
 
-  middle_ones = clz_hwi (~c << pos_first_1);
+  middle_ones = clz_hwi ((~(unsigned HOST_WIDE_INT) c) << pos_first_1);
   middle_zeros = ctz_hwi (c >> (HOST_BITS_PER_WIDE_INT - pos_first_1));
   if (pos_first_1 < HOST_BITS_PER_WIDE_INT
       && middle_ones + middle_zeros < HOST_BITS_PER_WIDE_INT
@@ -10570,7 +10585,8 @@ rs6000_emit_set_long_const (rtx dest, HOST_WIDE_INT c, int *num_insns)
     {
       /* li/lis; rldicX */
       unsigned HOST_WIDE_INT imm = (c | ~mask);
-      imm = (imm >> shift) | (imm << (HOST_BITS_PER_WIDE_INT - shift));
+      if (shift != 0)
+	imm = (imm >> shift) | (imm << (HOST_BITS_PER_WIDE_INT - shift));
 
       count_or_emit_insn (temp, GEN_INT (imm));
       if (shift != 0)
diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
index 9c718ca..e31ee40 100644
--- a/gcc/config/rs6000/rs6000.md
+++ b/gcc/config/rs6000/rs6000.md
@@ -1969,7 +1969,7 @@
   [(set (match_dup 0) (plus:GPR (match_dup 1) (match_dup 3)))
    (set (match_dup 0) (plus:GPR (match_dup 0) (match_dup 4)))]
 {
-  HOST_WIDE_INT val = INTVAL (operands[2]);
+  unsigned HOST_WIDE_INT val = UINTVAL (operands[2]);
   HOST_WIDE_INT low = sext_hwi (val, 16);
   HOST_WIDE_INT rest = trunc_int_for_mode (val - low, <MODE>mode);
 
diff --git a/gcc/config/rs6000/vxworks.h b/gcc/config/rs6000/vxworks.h
index fa2c837b..e77247b 100644
--- a/gcc/config/rs6000/vxworks.h
+++ b/gcc/config/rs6000/vxworks.h
@@ -34,6 +34,21 @@ along with GCC; see the file COPYING3.  If not see
 /* Common definitions first.                                   */
 /*-------------------------------------------------------------*/
 
+/* Default to 64 bits when the target is powerpc64*-wrs-vxworks*,
+   and to 32 bits otherwise.  */
+#undef SUBTARGET_DRIVER_SELF_SPECS
+#if TARGET_VXWORKS64
+#define SUBTARGET_DRIVER_SELF_SPECS "%{!m64:%{!m32:-m64}}"
+#else
+#define SUBTARGET_DRIVER_SELF_SPECS "%{!m32:%{!m64:-m32}}"
+#endif
+
+/* Having used the build-time TARGET_VXWORKS64 to choose the default ABI above,
+   redefine it so that it matches whichever ABI is selected for each
+   compilation.  */
+#undef TARGET_VXWORKS64
+#define TARGET_VXWORKS64 TARGET_64BIT
+
 /* CPP predefined macros.  */
 
 #undef TARGET_OS_CPP_BUILTINS
diff --git a/gcc/config/s390/s390-protos.h b/gcc/config/s390/s390-protos.h
index d760a7e..6becad1 100644
--- a/gcc/config/s390/s390-protos.h
+++ b/gcc/config/s390/s390-protos.h
@@ -128,6 +128,8 @@ extern void s390_expand_vcond (rtx, rtx, rtx, enum rtx_code, rtx, rtx);
 extern void s390_expand_vec_init (rtx, rtx);
 extern rtx s390_expand_merge_perm_const (machine_mode, bool);
 extern void s390_expand_merge (rtx, rtx, rtx, bool);
+extern void s390_expand_int_spaceship (rtx, rtx, rtx, rtx);
+extern void s390_expand_fp_spaceship (rtx, rtx, rtx, rtx);
 extern rtx s390_build_signbit_mask (machine_mode);
 extern rtx s390_return_addr_rtx (int, rtx);
 extern rtx s390_back_chain_rtx (void);
diff --git a/gcc/config/s390/s390.cc b/gcc/config/s390/s390.cc
index 3826720..012b6db 100644
--- a/gcc/config/s390/s390.cc
+++ b/gcc/config/s390/s390.cc
@@ -3862,7 +3862,21 @@ s390_register_move_cost (machine_mode mode,
 {
   /* On s390, copy between fprs and gprs is expensive.  */
 
-  /* It becomes somewhat faster having ldgr/lgdr.  */
+  /* With vector extensions any GPR<->VR load up to 8 bytes is supported.  */
+  if (TARGET_VX && GET_MODE_SIZE (mode) <= 8)
+    {
+      /* ldgr/vlvgg take one cycle and vlvg[bhf] take two cycles. */
+      if (reg_classes_intersect_p (from, GENERAL_REGS)
+	  && reg_classes_intersect_p (to, VEC_REGS))
+	return GET_MODE_SIZE (mode) == 8 ? 1 : 2;
+      /* lgdr/vlgv[fg] take three cycles and vlgv[bh] take five cycles. */
+      if (reg_classes_intersect_p (to, GENERAL_REGS)
+	  && reg_classes_intersect_p (from, VEC_REGS))
+	return GET_MODE_SIZE (mode) >= 4 ? 3 : 4;
+    }
+
+  /* Without vector extensions it still becomes somewhat faster having
+     ldgr/lgdr.  */
   if (TARGET_Z10 && GET_MODE_SIZE (mode) == 8)
     {
       /* ldgr is single cycle. */
@@ -8199,6 +8213,167 @@ s390_expand_atomic (machine_mode mode, enum rtx_code code,
 					       NULL_RTX, 1, OPTAB_DIRECT), 1);
 }
 
+/* Expand integer op0 = op1 <=> op2, i.e.,
+   op0 = op1 == op2 ? 0 : op1 < op2 ? -1 : 1.
+
+   Signedness is specified by op3.  If op3 equals 1, then perform an unsigned
+   comparison, and if op3 equals -1, then perform a signed comparison.
+
+   For integer comparisons we strive for a sequence like
+   CR[L] ; LHI ; LOCHIL ; LOCHIH
+   where the first three instructions fit into a group.  */
+
+void
+s390_expand_int_spaceship (rtx op0, rtx op1, rtx op2, rtx op3)
+{
+  gcc_assert (op3 == const1_rtx || op3 == constm1_rtx);
+
+  rtx cc, cond_lt, cond_gt;
+  machine_mode cc_mode;
+  machine_mode mode = GET_MODE (op1);
+
+  /* Prior VXE3 emulate a 128-bit comparison by breaking it up into three
+     comparisons.  First test the high halfs.  In case they equal, then test
+     the low halfs.  Finally, test for equality.  Depending on the results
+     make use of LOCs.  */
+  if (mode == TImode && !TARGET_VXE3)
+    {
+      gcc_assert (TARGET_VX);
+      op1
+	= force_reg (V2DImode, simplify_gen_subreg (V2DImode, op1, TImode, 0));
+      op2
+	= force_reg (V2DImode, simplify_gen_subreg (V2DImode, op2, TImode, 0));
+      rtx lab = gen_label_rtx ();
+      rtx ccz = gen_rtx_REG (CCZmode, CC_REGNUM);
+      /* Compare high halfs for equality.
+	 VEC[L]G op1, op2 sets
+	   CC1 if high(op1) < high(op2)
+	 and
+	   CC2 if high(op1) > high(op2).  */
+      machine_mode cc_mode = op3 == const1_rtx ? CCUmode : CCSmode;
+      rtx lane0 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
+      emit_insn (gen_rtx_SET (
+	gen_rtx_REG (cc_mode, CC_REGNUM),
+	gen_rtx_COMPARE (cc_mode,
+			 gen_rtx_VEC_SELECT (DImode, op1, lane0),
+			 gen_rtx_VEC_SELECT (DImode, op2, lane0))));
+      s390_emit_jump (lab, gen_rtx_NE (CCZmode, ccz, const0_rtx));
+      /* At this point we know that the high halfs equal.
+	 VCHLGS op2, op1 sets CC1 if low(op1) < low(op2)  */
+      emit_insn (gen_rtx_PARALLEL (
+	VOIDmode,
+	gen_rtvec (2,
+		   gen_rtx_SET (gen_rtx_REG (CCVIHUmode, CC_REGNUM),
+				gen_rtx_COMPARE (CCVIHUmode, op2, op1)),
+		   gen_rtx_CLOBBER (VOIDmode, gen_rtx_SCRATCH (V2DImode)))));
+      emit_label (lab);
+      emit_insn (gen_rtx_SET (op0, const1_rtx));
+      emit_insn (
+	gen_movsicc (op0,
+		     gen_rtx_LTU (CCUmode, gen_rtx_REG (CCUmode, CC_REGNUM),
+				  const0_rtx),
+		     constm1_rtx, op0));
+      /* Deal with the case where both halfs equal.  */
+      emit_insn (gen_rtx_PARALLEL (
+	VOIDmode,
+	gen_rtvec (2,
+		   gen_rtx_SET (gen_rtx_REG (CCVEQmode, CC_REGNUM),
+				gen_rtx_COMPARE (CCVEQmode, op1, op2)),
+		   gen_rtx_SET (gen_reg_rtx (V2DImode),
+				gen_rtx_EQ (V2DImode, op1, op2)))));
+      emit_insn (gen_movsicc (op0, gen_rtx_EQ (CCZmode, ccz, const0_rtx),
+			      const0_rtx, op0));
+      return;
+    }
+
+  if (mode == QImode || mode == HImode)
+    {
+      rtx_code extend = op3 == const1_rtx ? ZERO_EXTEND : SIGN_EXTEND;
+      op1 = simplify_gen_unary (extend, SImode, op1, mode);
+      op1 = force_reg (SImode, op1);
+      op2 = simplify_gen_unary (extend, SImode, op2, mode);
+      op2 = force_reg (SImode, op2);
+      mode = SImode;
+    }
+
+  if (op3 == const1_rtx)
+    {
+      cc_mode = CCUmode;
+      cc = gen_rtx_REG (cc_mode, CC_REGNUM);
+      cond_lt = gen_rtx_LTU (mode, cc, const0_rtx);
+      cond_gt = gen_rtx_GTU (mode, cc, const0_rtx);
+    }
+  else
+    {
+      cc_mode = CCSmode;
+      cc = gen_rtx_REG (cc_mode, CC_REGNUM);
+      cond_lt = gen_rtx_LT (mode, cc, const0_rtx);
+      cond_gt = gen_rtx_GT (mode, cc, const0_rtx);
+    }
+
+  emit_insn (gen_rtx_SET (cc, gen_rtx_COMPARE (cc_mode, op1, op2)));
+  emit_move_insn (op0, const0_rtx);
+  emit_insn (gen_movsicc (op0, cond_lt, constm1_rtx, op0));
+  emit_insn (gen_movsicc (op0, cond_gt, const1_rtx, op0));
+}
+
+/* Expand floating-point op0 = op1 <=> op2, i.e.,
+   op0 = op1 == op2 ? 0 : op1 < op2 ? -1 : op1 > op2 ? 1 : 2.
+
+   If op3 equals const0_rtx, then we are interested in the compare only (see
+   test spaceship-fp-4.c).  Otherwise, op3 is a CONST_INT different than
+   const1_rtx and constm1_rtx which is used in order to set op0 for unordered.
+
+   Emit a branch-only solution, i.e., let if-convert fold the branches into
+   LOCs if applicable.  This has the benefit that the solution is also
+   applicable if we are only interested in the compare, i.e., if op3 equals
+   const0_rtx.
+ */
+
+void
+s390_expand_fp_spaceship (rtx op0, rtx op1, rtx op2, rtx op3)
+{
+  gcc_assert (op3 != const1_rtx && op3 != constm1_rtx);
+
+  machine_mode mode = GET_MODE (op1);
+  machine_mode cc_mode = s390_select_ccmode (LTGT, op1, op2);
+  rtx cc_reg = gen_rtx_REG (cc_mode, CC_REGNUM);
+  rtx cond_unordered = gen_rtx_UNORDERED (mode, cc_reg, const0_rtx);
+  rtx cond_eq = gen_rtx_EQ (mode, cc_reg, const0_rtx);
+  rtx cond_gt = gen_rtx_GT (mode, cc_reg, const0_rtx);
+  rtx_insn *insn;
+  rtx l_unordered = gen_label_rtx ();
+  rtx l_eq = gen_label_rtx ();
+  rtx l_gt = gen_label_rtx ();
+  rtx l_end = gen_label_rtx ();
+
+  s390_emit_compare (VOIDmode, LTGT, op1, op2);
+  if (!flag_finite_math_only)
+    {
+      insn = s390_emit_jump (l_unordered, cond_unordered);
+      add_reg_br_prob_note (insn, profile_probability::very_unlikely ());
+    }
+  insn = s390_emit_jump (l_eq, cond_eq);
+  add_reg_br_prob_note (insn, profile_probability::unlikely ());
+  insn = s390_emit_jump (l_gt, cond_gt);
+  add_reg_br_prob_note (insn, profile_probability::even ());
+  emit_move_insn (op0, constm1_rtx);
+  emit_jump (l_end);
+  emit_label (l_eq);
+  emit_move_insn (op0, const0_rtx);
+  emit_jump (l_end);
+  emit_label (l_gt);
+  emit_move_insn (op0, const1_rtx);
+  if (!flag_finite_math_only)
+    {
+      emit_jump (l_end);
+      emit_label (l_unordered);
+      rtx unord_val = op3 == const0_rtx ? const2_rtx : op3;
+      emit_move_insn (op0, unord_val);
+    }
+  emit_label (l_end);
+}
+
 /* This is called from dwarf2out.cc via TARGET_ASM_OUTPUT_DWARF_DTPREL.
    We need to emit DTP-relative relocations.  */
 
@@ -16566,9 +16741,6 @@ s390_option_override_internal (struct gcc_options *opts,
   else
     SET_OPTION_IF_UNSET (opts, opts_set, param_vect_partial_vector_usage, 0);
 
-  /* Do not vectorize loops with a low trip count for now.  */
-  SET_OPTION_IF_UNSET (opts, opts_set, param_min_vect_loop_bound, 2);
-
   /* Set the default alignment.  */
   s390_default_align (opts);
 
@@ -16863,8 +17035,9 @@ s390_valid_target_attribute_inner_p (tree args,
 	      generate_option (opt, NULL, value, CL_TARGET, &decoded);
 	      s390_handle_option (opts, new_opts_set, &decoded, input_location);
 	      set_option (opts, new_opts_set, opt, value,
-			  p + opt_len, DK_UNSPECIFIED, input_location,
-			  global_dc);
+			  p + opt_len,
+			  static_cast<int> (diagnostics::kind::unspecified),
+			  input_location, global_dc);
 	    }
 	  else
 	    {
@@ -16881,8 +17054,9 @@ s390_valid_target_attribute_inner_p (tree args,
 	  arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
 	  if (arg_ok)
 	    set_option (opts, new_opts_set, opt, value,
-			p + opt_len, DK_UNSPECIFIED, input_location,
-			global_dc);
+			p + opt_len,
+			static_cast<int> (diagnostics::kind::unspecified),
+			input_location, global_dc);
 	  else
 	    {
 	      error ("attribute %<target%> argument %qs is unknown", orig_p);
@@ -17334,13 +17508,15 @@ static bool
 s390_support_vector_misalignment (machine_mode mode ATTRIBUTE_UNUSED,
 				  const_tree type ATTRIBUTE_UNUSED,
 				  int misalignment ATTRIBUTE_UNUSED,
-				  bool is_packed ATTRIBUTE_UNUSED)
+				  bool is_packed ATTRIBUTE_UNUSED,
+				  bool is_gather_scatter ATTRIBUTE_UNUSED)
 {
   if (TARGET_VX)
     return true;
 
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* The vector ABI requires vector types to be aligned on an 8 byte
@@ -17832,9 +18008,11 @@ f_constraint_p (const char *constraint)
   for (size_t i = 0, c_len = strlen (constraint); i < c_len;
        i += CONSTRAINT_LEN (constraint[i], constraint + i))
     {
-      if (constraint[i] == 'f')
+      if (constraint[i] == 'f'
+	  || (constraint[i] == '{' && constraint[i + 1] == 'f'))
 	seen_f_p = true;
-      if (constraint[i] == 'v')
+      if (constraint[i] == 'v'
+	  || (constraint[i] == '{' && constraint[i + 1] == 'v'))
 	seen_v_p = true;
     }
 
@@ -17924,7 +18102,8 @@ s390_md_asm_adjust (vec<rtx> &outputs, vec<rtx> &inputs,
 	continue;
       bool allows_mem, allows_reg, is_inout;
       bool ok = parse_output_constraint (&constraint, i, ninputs, noutputs,
-					 &allows_mem, &allows_reg, &is_inout);
+					 &allows_mem, &allows_reg, &is_inout,
+					 nullptr);
       gcc_assert (ok);
       if (!f_constraint_p (constraint))
 	/* Long double with a constraint other than "=f" - nothing to do.  */
@@ -17969,7 +18148,7 @@ s390_md_asm_adjust (vec<rtx> &outputs, vec<rtx> &inputs,
       bool allows_mem, allows_reg;
       bool ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
 					constraints.address (), &allows_mem,
-					&allows_reg);
+					&allows_reg, nullptr);
       gcc_assert (ok);
       if (!f_constraint_p (constraint))
 	/* Long double with a constraint other than "f" (or "=f" for inout
@@ -18041,9 +18220,34 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
   static const unsigned char lo_perm_qi_swap[16]
     = {17, 1, 19, 3, 21, 5, 23, 7, 25, 9, 27, 11, 29, 13, 31, 15};
 
+  static const unsigned char hi_perm_qi_di[16]
+    = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
+  static const unsigned char hi_perm_qi_si[16]
+    = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};
+  static const unsigned char hi_perm_qi_hi[16]
+    = {0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23};
+
+  static const unsigned char lo_perm_qi_di[16]
+    = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
+  static const unsigned char lo_perm_qi_si[16]
+    = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};
+  static const unsigned char lo_perm_qi_hi[16]
+    = {8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31};
+
+  static const unsigned char hi_perm_hi_si[8] = {0, 1, 8, 9, 2, 3, 10, 11};
+  static const unsigned char hi_perm_hi_di[8] = {0, 1, 2, 3, 8, 9, 10, 11};
+
+  static const unsigned char lo_perm_hi_si[8] = {4, 5, 12, 13, 6, 7, 14, 15};
+  static const unsigned char lo_perm_hi_di[8] = {4, 5, 6, 7, 12, 13, 14, 15};
+
+  static const unsigned char hi_perm_si_di[4] = {0, 1, 4, 5};
+
+  static const unsigned char lo_perm_si_di[4] = {2, 3, 6, 7};
+
   bool merge_lo_p = false;
   bool merge_hi_p = false;
   bool swap_operands_p = false;
+  machine_mode mergemode = d.vmode;
 
   if ((d.nelt == 2 && memcmp (d.perm, hi_perm_di, 2) == 0)
       || (d.nelt == 4 && memcmp (d.perm, hi_perm_si, 4) == 0)
@@ -18075,6 +18279,75 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
       merge_lo_p = true;
       swap_operands_p = true;
     }
+  else if (d.nelt == 16)
+    {
+      if (memcmp (d.perm, hi_perm_qi_di, 16) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, hi_perm_qi_si, 16) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V4SImode;
+	}
+      else if (memcmp (d.perm, hi_perm_qi_hi, 16) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V8HImode;
+	}
+      else if (memcmp (d.perm, lo_perm_qi_di, 16) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, lo_perm_qi_si, 16) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V4SImode;
+	}
+      else if (memcmp (d.perm, lo_perm_qi_hi, 16) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V8HImode;
+	}
+    }
+  else if (d.nelt == 8)
+    {
+      if (memcmp (d.perm, hi_perm_hi_di, 8) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, hi_perm_hi_si, 8) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V4SImode;
+	}
+      else if (memcmp (d.perm, lo_perm_hi_di, 8) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, lo_perm_hi_si, 8) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V4SImode;
+	}
+    }
+  else if (d.nelt == 4)
+    {
+      if (memcmp (d.perm, hi_perm_si_di, 4) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, lo_perm_si_di, 4) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V2DImode;
+	}
+    }
 
   if (!merge_lo_p && !merge_hi_p)
     return false;
@@ -18082,7 +18355,7 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
   if (d.testing_p)
     return merge_lo_p || merge_hi_p;
 
-  rtx op0, op1;
+  rtx op0, op1, target = d.target;
   if (swap_operands_p)
     {
       op0 = d.op1;
@@ -18093,12 +18366,80 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
       op0 = d.op0;
       op1 = d.op1;
     }
+  if (mergemode != d.vmode)
+    {
+      target = simplify_gen_subreg (mergemode, target, d.vmode, 0);
+      op0 = simplify_gen_subreg (mergemode, op0, d.vmode, 0);
+      op1 = simplify_gen_subreg (mergemode, op1, d.vmode, 0);
+    }
 
-  s390_expand_merge (d.target, op0, op1, merge_hi_p);
+  s390_expand_merge (target, op0, op1, merge_hi_p);
 
   return true;
 }
 
+/* Try to expand the vector permute operation described by D using the vector
+   pack instruction vpk.  Return true if vector pack could be used.  */
+static bool
+expand_perm_with_pack (const struct expand_vec_perm_d &d)
+{
+  static const unsigned char qi_hi[16]
+    = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31};
+  static const unsigned char qi_si[16]
+    = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31};
+  static const unsigned char qi_di[16]
+    = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
+
+  static const unsigned char hi_si[8]
+    = {1, 3, 5, 7, 9, 11, 13, 15};
+  static const unsigned char hi_di[8]
+    = {2, 3, 6, 7, 10, 11, 14, 15};
+
+  static const unsigned char si_di[4]
+    = {1, 3, 5, 7};
+
+  machine_mode packmode, resmode;
+  enum insn_code code = CODE_FOR_nothing;
+
+  if (d.nelt == 16 && memcmp (d.perm, qi_hi, 16) == 0)
+    {
+      packmode = E_V8HImode;
+      resmode = E_V16QImode;
+      code = CODE_FOR_vec_pack_trunc_v8hi;
+    }
+  else if ((d.nelt == 16 && memcmp (d.perm, qi_si, 16) == 0)
+	   || (d.nelt == 8 && memcmp (d.perm, hi_si, 8) == 0))
+    {
+      packmode = E_V4SImode;
+      resmode = E_V8HImode;
+      code = CODE_FOR_vec_pack_trunc_v4si;
+    }
+  else if ((d.nelt == 16 && memcmp (d.perm, qi_di, 16) == 0)
+	   || (d.nelt == 8 && memcmp (d.perm, hi_di, 8) == 0)
+	   || (d.nelt == 4 && memcmp (d.perm, si_di, 4) == 0))
+    {
+      packmode = E_V2DImode;
+      resmode = E_V4SImode;
+      code = CODE_FOR_vec_pack_trunc_v2di;
+    }
+
+  if (code == CODE_FOR_nothing)
+    return false;
+
+  if (d.testing_p)
+    return true;
+  rtx target = simplify_gen_subreg (resmode, d.target, d.vmode, 0);
+  rtx op0 = simplify_gen_subreg (packmode,
+				 force_reg (GET_MODE (d.op0), d.op0),
+				 d.vmode, 0);
+  rtx op1 = simplify_gen_subreg (packmode,
+				 force_reg (GET_MODE (d.op1), d.op1),
+				 d.vmode, 0);
+  rtx pat = GEN_FCN (code) (target, op0, op1);
+  emit_insn (pat);
+  return true;
+}
+
 /* Try to expand the vector permute operation described by D using the
    vector permute doubleword immediate instruction vpdi.  Return true
    if vpdi could be used.
@@ -18322,6 +18663,9 @@ vectorize_vec_perm_const_1 (const struct expand_vec_perm_d &d)
   if (expand_perm_with_merge (d))
     return true;
 
+  if (expand_perm_with_pack (d))
+    return true;
+
   if (expand_perm_with_vpdi (d))
     return true;
 
diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index 97a4bdf..8cc48b0 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -121,6 +121,7 @@
 
    ; Test Data Class (TDC)
    UNSPEC_TDC_INSN
+   UNSPEC_SIGNBIT
 
    ; Byte-wise Population Count
    UNSPEC_POPCNT
@@ -139,9 +140,6 @@
    UNSPEC_LCBB
 
    ; Vector
-   UNSPEC_VEC_SMULT_HI
-   UNSPEC_VEC_UMULT_HI
-   UNSPEC_VEC_SMULT_LO
    UNSPEC_VEC_SMULT_EVEN
    UNSPEC_VEC_UMULT_EVEN
    UNSPEC_VEC_SMULT_ODD
@@ -241,9 +239,6 @@
 
    UNSPEC_VEC_MSUM
 
-   UNSPEC_VEC_VFMIN
-   UNSPEC_VEC_VFMAX
-
    UNSPEC_VEC_VBLEND
    UNSPEC_VEC_VEVAL
    UNSPEC_VEC_VGEM
@@ -256,6 +251,9 @@
 
    UNSPEC_NNPA_VCFN_V8HI
    UNSPEC_NNPA_VCNF_V8HI
+
+   UNSPEC_FMAX
+   UNSPEC_FMIN
 ])
 
 ;;
@@ -311,6 +309,9 @@
    UNSPECV_SPLIT_STACK_CALL
 
    UNSPECV_OSC_BREAK
+
+   ; Stack Protector
+   UNSPECV_SP_GET_TP
   ])
 
 ;;
@@ -368,6 +369,9 @@
    (VR23_REGNUM                 45)
    (VR24_REGNUM                 46)
    (VR31_REGNUM                 53)
+   ; Access registers
+   (AR0_REGNUM                  36)
+   (AR1_REGNUM                  37)
   ])
 
 ; Rounding modes for binary floating point numbers
@@ -510,7 +514,7 @@
 				    S390_TDC_INFINITY
 				    S390_TDC_NORMAL_BFP])
 
-(define_int_attr tdc_insn [(S390_TDC_SIGNBIT_SET "signbit")
+(define_int_attr tdc_insn [(S390_TDC_SIGNBIT_SET "signbit_tdc")
 			   (S390_TDC_FINITE "isfinite")
 			   (S390_TDC_INFINITY "isinf")
 			   (S390_TDC_NORMAL_BFP "isnormal")
@@ -1523,6 +1527,27 @@
   operands[0] = SET_DEST (PATTERN (curr_insn));
 })
 
+; Restrict spaceship optab to z13 or later since there we have
+; LOAD HALFWORD IMMEDIATE ON CONDITION.
+
+(define_mode_iterator SPACESHIP_INT [(TI "TARGET_VX") DI SI HI QI])
+(define_expand "spaceship<mode>4"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SPACESHIP_INT 1 "register_operand")
+   (match_operand:SPACESHIP_INT 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_Z13 && TARGET_64BIT"
+  "s390_expand_int_spaceship (operands[0], operands[1], operands[2], operands[3]); DONE;")
+
+(define_mode_iterator SPACESHIP_BFP [TF DF SF])
+(define_expand "spaceship<mode>4"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SPACESHIP_BFP 1 "register_operand")
+   (match_operand:SPACESHIP_BFP 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_Z13 && TARGET_64BIT && TARGET_HARD_FLOAT"
+  "s390_expand_fp_spaceship (operands[0], operands[1], operands[2], operands[3]); DONE;")
+
 
 ; (TF|DF|SF|TD|DD|SD) instructions
 
@@ -3779,6 +3804,86 @@
 	(unspec:SI [(reg:CCZ CC_REGNUM)] UNSPEC_CC_TO_INT))]
   "TARGET_HARD_DFP")
 
+(define_mode_iterator SIGNBIT_SINGLE [(SF "TARGET_HARD_FLOAT")
+				      (SD "TARGET_HARD_DFP")])
+(define_expand "signbit<mode>2"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SIGNBIT_SINGLE 1 "nonimmediate_operand")]
+  ""
+{
+  if (TARGET_VX && TARGET_64BIT)
+    {
+      emit_insn (gen_rtx_SET (operands[0], simplify_gen_subreg (SImode, operands[1], <MODE>mode, 0)));
+      emit_insn (gen_rtx_SET (operands[0], gen_rtx_LSHIFTRT (SImode, operands[0], GEN_INT (31))));
+    }
+  else if (TARGET_Z10 && TARGET_64BIT)
+    emit_insn (gen_signbit<mode>2_z10 (operands[0], operands[1]));
+  else
+    emit_insn (gen_signbit_tdc<mode>2 (operands[0], force_reg (<MODE>mode, operands[1])));
+  DONE;
+})
+
+(define_insn "signbit<mode>2_z10"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(unspec:SI [(match_operand:SIGNBIT_SINGLE 1 "nonimmediate_operand" "fRT")]
+		   UNSPEC_SIGNBIT))]
+  "TARGET_Z10 && TARGET_64BIT"
+  "#")
+
+(define_split
+  [(set (match_operand:SI 0 "register_operand")
+	(unspec:SI [(match_operand:SIGNBIT_SINGLE 1 "register_operand")]
+		   UNSPEC_SIGNBIT))]
+  "TARGET_Z10 && TARGET_64BIT && reload_completed"
+  [(set (match_dup 0) (match_dup 1))
+   (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 63)))]
+{
+  operands[0] = gen_rtx_REG (DImode, REGNO (operands[0]));
+  operands[1] = gen_rtx_REG (DImode, REGNO (operands[1]));
+})
+
+(define_split
+  [(set (match_operand:SI 0 "register_operand")
+	(unspec:SI [(match_operand:SIGNBIT_SINGLE 1 "memory_operand")]
+		   UNSPEC_SIGNBIT))]
+  "TARGET_Z10 && TARGET_64BIT && reload_completed"
+  [(set (match_dup 0) (match_dup 1))
+   (set (match_dup 0) (lshiftrt:SI (match_dup 0) (const_int 31)))]
+{
+  operands[1] = change_address (operands[1], SImode, 0);
+})
+
+(define_mode_iterator SIGNBIT_DBL_TETRA [(DF "TARGET_HARD_FLOAT")
+					 (TF "TARGET_HARD_FLOAT")
+					 (DD "TARGET_HARD_DFP")
+					 (TD "TARGET_HARD_DFP")])
+(define_expand "signbit<mode>2"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SIGNBIT_DBL_TETRA 1 "nonimmediate_operand")]
+  ""
+{
+  if (TARGET_Z10 && TARGET_64BIT)
+    {
+      rtx reg_di = gen_reg_rtx (DImode);
+      if (<MODE>mode == TFmode || <MODE>mode == TDmode)
+	{
+	  rtx reg_ti = gen_reg_rtx (TImode);
+	  emit_insn (gen_rtx_SET (reg_ti, simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0)));
+	  emit_insn (gen_rtx_SET (reg_di, simplify_gen_subreg (DImode, reg_ti, TImode, 0)));
+	}
+      else
+	emit_insn (gen_rtx_SET (reg_di, simplify_gen_subreg (DImode, operands[1], <MODE>mode, 0)));
+      emit_insn (gen_rtx_SET (reg_di, gen_rtx_LSHIFTRT (DImode, reg_di, GEN_INT (63))));
+      rtx subreg = gen_rtx_SUBREG (SImode, reg_di, 4);
+      SUBREG_PROMOTED_VAR_P (subreg) = 1;
+      SUBREG_PROMOTED_SET (subreg, SRP_SIGNED_AND_UNSIGNED);
+      emit_insn (gen_rtx_SET (operands[0], subreg));
+    }
+  else
+    emit_insn (gen_signbit_tdc<mode>2 (operands[0], force_reg (<MODE>mode, operands[1])));
+  DONE;
+})
+
 ; This extracts CC into a GPR properly shifted.  The actual IPM
 ; instruction will be issued by reload.  The constraint of operand 1
 ; forces reload to use a GPR.  So reload will issue a movcc insn for
@@ -11927,15 +12032,43 @@
 ; Stack Protector Patterns
 ;
 
+; Insns stack_protect_get_tp{si,di} are similar to *get_tp_{31,64} but still
+; distinct in the sense that they force recomputation of the thread pointer
+; instead of potentially reloading it from stack.
+
+(define_insn_and_split "stack_protect_get_tpsi"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(unspec_volatile:SI [(const_int 0)] UNSPECV_SP_GET_TP))]
+  ""
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (reg:SI AR0_REGNUM))])
+
+(define_insn_and_split "stack_protect_get_tpdi"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(unspec_volatile:DI [(const_int 0)] UNSPECV_SP_GET_TP))]
+  ""
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 1) (reg:SI AR0_REGNUM))
+   (set (match_dup 0) (ashift:DI (match_dup 0) (const_int 32)))
+   (set (strict_low_part (match_dup 1)) (reg:SI AR1_REGNUM))]
+  "operands[1] = gen_rtx_REG (SImode, REGNO (operands[0]));")
+
 (define_expand "stack_protect_set"
   [(set (match_operand 0 "memory_operand" "")
 	(match_operand 1 "memory_operand" ""))]
   ""
 {
 #ifdef TARGET_THREAD_SSP_OFFSET
+  rtx tp = gen_reg_rtx (Pmode);
+  if (TARGET_64BIT)
+    emit_insn (gen_stack_protect_get_tpdi (tp));
+  else
+    emit_insn (gen_stack_protect_get_tpsi (tp));
   operands[1]
-    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
-                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
+    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, tp,
+					GEN_INT (TARGET_THREAD_SSP_OFFSET)));
 #endif
   if (TARGET_64BIT)
     emit_insn (gen_stack_protect_setdi (operands[0], operands[1]));
@@ -11961,9 +12094,14 @@
 {
   rtx cc_reg, test;
 #ifdef TARGET_THREAD_SSP_OFFSET
+  rtx tp = gen_reg_rtx (Pmode);
+  if (TARGET_64BIT)
+    emit_insn (gen_stack_protect_get_tpdi (tp));
+  else
+    emit_insn (gen_stack_protect_get_tpsi (tp));
   operands[1]
-    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
-                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
+    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, tp,
+					GEN_INT (TARGET_THREAD_SSP_OFFSET)));
 #endif
   if (TARGET_64BIT)
     emit_insn (gen_stack_protect_testdi (operands[0], operands[1]));
diff --git a/gcc/config/s390/vector.md b/gcc/config/s390/vector.md
index 7251a76..12bbeb6 100644
--- a/gcc/config/s390/vector.md
+++ b/gcc/config/s390/vector.md
@@ -89,6 +89,13 @@
 (define_mode_iterator VF_HW [(V4SF "TARGET_VXE") V2DF (V1TF "TARGET_VXE")
 			     (TF "TARGET_VXE")])
 
+; FP scalar and vector modes
+(define_mode_iterator VFT_BFP [SF DF
+			      (V1SF "TARGET_VXE") (V2SF "TARGET_VXE") (V4SF "TARGET_VXE")
+			      V1DF V2DF
+			      (V1TF "TARGET_VXE") (TF "TARGET_VXE")])
+
+
 (define_mode_iterator V_8   [V1QI])
 (define_mode_iterator V_16  [V2QI  V1HI])
 (define_mode_iterator V_32  [V4QI  V2HI V1SI V1SF])
@@ -142,13 +149,13 @@
 
 ; The instruction suffix for integer instructions and instructions
 ; which do not care about whether it is floating point or integer.
-(define_mode_attr bhfgq[(V1QI "b") (V2QI "b") (V4QI "b") (V8QI "b") (V16QI "b")
-			(V1HI "h") (V2HI "h") (V4HI "h") (V8HI "h")
-			(V1SI "f") (V2SI "f") (V4SI "f")
-			(V1DI "g") (V2DI "g")
+(define_mode_attr bhfgq[(V1QI "b") (V2QI "b") (V4QI "b") (V8QI "b") (V16QI "b") (QI "b")
+			(V1HI "h") (V2HI "h") (V4HI "h") (V8HI "h") (HI "h")
+			(V1SI "f") (V2SI "f") (V4SI "f") (SI "f")
+			(V1DI "g") (V2DI "g") (DI "g")
 			(V1TI "q") (TI "q")
-			(V1SF "f") (V2SF "f") (V4SF "f")
-			(V1DF "g") (V2DF "g")
+			(V1SF "f") (V2SF "f") (V4SF "f") (SF "f")
+			(V1DF "g") (V2DF "g") (DF "g")
 			(V1TF "q") (TF "q")])
 
 ; This is for vmalhw. It gets an 'w' attached to avoid confusion with
@@ -494,6 +501,54 @@
                         SIL,SIL,RI,RI,RRE,RRE,RIL,RR,RXY,RXY,RIL")])
 
 
+; Instructions vlgvb, vlgvh, vlgvf zero all remaining bits of a GPR, i.e.,
+; an implicit zero extend is done.
+
+(define_insn "*movdi<mode>_zero_extend_A"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(zero_extend:DI (match_operand:SINT 1 "register_operand" "v")))]
+  "TARGET_VX"
+  "vlgv<bhfgq>\t%0,%v1,0"
+  [(set_attr "op_type" "VRS")])
+
+(define_insn "*movsi<mode>_zero_extend_A"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(zero_extend:SI (match_operand:HQI 1 "register_operand" "v")))]
+  "TARGET_VX"
+  "vlgv<bhfgq>\t%0,%v1,0"
+  [(set_attr "op_type" "VRS")])
+
+(define_mode_iterator VLGV_DI [V1QI V2QI V4QI V8QI V16QI
+			       V1HI V2HI V4HI V8HI
+			       V1SI V2SI V4SI])
+(define_insn "*movdi<mode>_zero_extend_B"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(zero_extend:DI (vec_select:<non_vec>
+			  (match_operand:VLGV_DI 1 "register_operand" "v")
+			  (parallel [(match_operand:SI 2 "const_int_operand" "n")]))))]
+  "TARGET_VX"
+{
+  operands[2] = GEN_INT (UINTVAL (operands[2]) & (GET_MODE_NUNITS (<MODE>mode) - 1));
+  return "vlgv<bhfgq>\t%0,%v1,%Y2";
+}
+  [(set_attr "op_type" "VRS")
+   (set_attr "mnemonic" "vlgv<bhfgq>")])
+
+(define_mode_iterator VLGV_SI [V1QI V2QI V4QI V8QI V16QI
+			       V1HI V2HI V4HI V8HI])
+(define_insn "*movsi<mode>_zero_extend_B"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(zero_extend:SI (vec_select:<non_vec>
+			  (match_operand:VLGV_SI 1 "register_operand" "v")
+			  (parallel [(match_operand:SI 2 "const_int_operand" "n")]))))]
+  "TARGET_VX"
+{
+  operands[2] = GEN_INT (UINTVAL (operands[2]) & (GET_MODE_NUNITS (<MODE>mode) - 1));
+  return "vlgv<bhfgq>\t%0,%v1,%Y2";
+}
+  [(set_attr "op_type" "VRS")
+   (set_attr "mnemonic" "vlgv<bhfgq>")])
+
 ; vec_load_lanes?
 
 ; vec_store_lanes?
@@ -3565,14 +3620,341 @@
   "veval\t%v0,%v1,%v2,%v3,%b4"
   [(set_attr "op_type" "VRI")])
 
-; reduc_smin
-; reduc_smax
-; reduc_umin
-; reduc_umax
-
 ; vec_pack_sfix_trunc: convert + pack ?
 ; vec_pack_ufix_trunc
 ; vec_unpacks_float_hi
 ; vec_unpacks_float_lo
 ; vec_unpacku_float_hi
 ; vec_unpacku_float_lo
+
+(define_expand "avg<mode>3_ceil"
+  [(set (match_operand:VIT_HW_VXE3_T                        0 "register_operand")
+	(unspec:VIT_HW_VXE3_T [(match_operand:VIT_HW_VXE3_T 1 "register_operand")
+			       (match_operand:VIT_HW_VXE3_T 2 "register_operand")]
+			      UNSPEC_VEC_AVG))]
+  "TARGET_VX")
+
+(define_expand "uavg<mode>3_ceil"
+  [(set (match_operand:VIT_HW_VXE3_T                        0 "register_operand")
+	(unspec:VIT_HW_VXE3_T [(match_operand:VIT_HW_VXE3_T 1 "register_operand")
+			       (match_operand:VIT_HW_VXE3_T 2 "register_operand")]
+			      UNSPEC_VEC_AVGU))]
+  "TARGET_VX")
+
+(define_expand "smul<mode>3_highpart"
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand")
+	(smul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand")))]
+  "TARGET_VX")
+
+(define_expand "umul<mode>3_highpart"
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand")
+	(umul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand")))]
+  "TARGET_VX")
+
+; fmax
+(define_expand "fmax<mode>3"
+  [(set (match_operand:VFT_BFP                  0 "register_operand")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand")
+	       (match_operand:VFT_BFP           2 "register_operand")
+	       (const_int 4)]
+	      UNSPEC_FMAX))]
+  "TARGET_VXE")
+
+; fmin
+(define_expand "fmin<mode>3"
+  [(set (match_operand:VFT_BFP                  0 "register_operand")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand")
+	       (match_operand:VFT_BFP           2 "register_operand")
+	       (const_int 4)]
+	      UNSPEC_FMIN))]
+  "TARGET_VXE")
+
+; reduc_plus
+(define_expand "reduc_plus_scal_<mode>"
+  [(set (match_dup 4)
+	(unspec:V4SI [(match_operand:VI_HW_QH 1 "register_operand")
+		      (match_dup 2)]
+		      UNSPEC_VEC_VSUM))
+   (set (match_dup 5)
+	(unspec:V2DI [(match_dup 4) (match_dup 3)] UNSPEC_VEC_VSUMQ))
+   (set (match_operand:<non_vec> 0 "register_operand")
+	(vec_select:<non_vec> (match_dup 6)
+			      (parallel [(match_dup 7)])))]
+  "TARGET_VX"
+{
+  operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));
+  operands[3] = simplify_gen_subreg (V4SImode, operands[2], <MODE>mode, 0);
+  operands[4] = gen_reg_rtx (V4SImode);
+  operands[5] = gen_reg_rtx (V2DImode);
+  operands[6] = simplify_gen_subreg (<MODE>mode, operands[5], V2DImode, 0);
+  operands[7] = GEN_INT (16 / GET_MODE_SIZE (<non_vec>mode) - 1);
+})
+
+(define_expand "reduc_plus_scal_<mode>"
+  [(set (match_dup 3)
+	(unspec:V2DI [(match_operand:VI_HW_SD 1 "register_operand")
+		      (match_dup 2)]
+		      UNSPEC_VEC_VSUMQ))
+   (set (match_operand:<non_vec> 0 "register_operand")
+	(vec_select:<non_vec> (match_dup 4)
+			      (parallel [(match_dup 5)])))]
+  "TARGET_VX"
+{
+  operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));
+  operands[3] = gen_reg_rtx (V2DImode);
+  operands[4] = simplify_gen_subreg (<MODE>mode, operands[3], V2DImode, 0);
+  operands[5] = GEN_INT (16 / GET_MODE_SIZE (<non_vec>mode) - 1);
+})
+
+(define_expand "reduc_plus_scal_v2df"
+  [(set (match_dup 2)
+	(unspec:V2DF [(match_operand:V2DF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3) (plus:V2DF (match_dup 1) (match_dup 2)))
+   (set (match_operand:DF 0 "register_operand")
+	(vec_select:DF (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DFmode);
+  operands[3] = gen_reg_rtx (V2DFmode);
+})
+
+(define_expand "reduc_plus_scal_v4sf"
+  [(set (match_dup 2)
+	(unspec:V4SF [(match_operand:V4SF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3) (plus:V4SF (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V4SF [(match_dup 3) (match_dup 3) (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5) (plus:V4SF (match_dup 3) (match_dup 4)))
+   (set (match_operand:SF 0 "register_operand")
+	(vec_select:SF (match_dup 5) (parallel [(const_int 0)])))]
+  "TARGET_VXE"
+{
+  operands[2] = gen_reg_rtx (V4SFmode);
+  operands[3] = gen_reg_rtx (V4SFmode);
+  operands[4] = gen_reg_rtx (V4SFmode);
+  operands[5] = gen_reg_rtx (V4SFmode);
+})
+
+; reduc_fmin, reduc_fmax, reduc_smin, reduc_smax
+
+(define_int_iterator REDUC_FMINMAX [UNSPEC_FMAX UNSPEC_FMIN])
+(define_int_attr reduc_fminmax_name [(UNSPEC_FMAX "fmax") (UNSPEC_FMIN "fmin")])
+(define_code_iterator REDUC_MINMAX [smin smax])
+(define_code_attr reduc_minmax_name [(smin "smin") (smax "smax")])
+
+(define_expand "reduc_<reduc_fminmax_name>_scal_v2df"
+  [(set (match_dup 2)
+	(unspec:V2DF [(match_operand:V2DF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(unspec:V2DF [(match_dup 1) (match_dup 2) (const_int 4)] REDUC_FMINMAX))
+   (set (match_operand:DF 0 "register_operand" "")
+	(vec_select:DF (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DFmode);
+  operands[3] = gen_reg_rtx (V2DFmode);
+})
+
+(define_expand "reduc_<reduc_fminmax_name>_scal_v4sf"
+  [(set (match_dup 2)
+	(unspec:V4SF [(match_operand:V4SF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(unspec:V4SF [(match_dup 1) (match_dup 2) (const_int 4)] REDUC_FMINMAX))
+   (set (match_dup 4)
+	(unspec:V4SF [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(unspec:V4SF [(match_dup 3) (match_dup 4) (const_int 4)] REDUC_FMINMAX))
+   (set (match_operand:SF 0 "register_operand")
+	(vec_select:SF (match_dup 5) (parallel [(const_int 0)])))]
+   "TARGET_VXE"
+{
+  operands[2] = gen_reg_rtx (V4SFmode);
+  operands[3] = gen_reg_rtx (V4SFmode);
+  operands[4] = gen_reg_rtx (V4SFmode);
+  operands[5] = gen_reg_rtx (V4SFmode);
+})
+
+(define_expand "reduc_<reduc_minmax_name>_scal_v2df"
+  [(set (match_dup 2)
+	(unspec:V2DF [(match_operand:V2DF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUC_MINMAX:V2DF (match_dup 1) (match_dup 2)))
+   (set (match_operand:DF 0 "register_operand" "")
+	(vec_select:DF (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DFmode);
+  operands[3] = gen_reg_rtx (V2DFmode);
+})
+
+(define_expand "reduc_<reduc_minmax_name>_scal_v4sf"
+  [(set (match_dup 2)
+	(unspec:V4SF [(match_operand:V4SF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUC_MINMAX:V4SF (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V4SF [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUC_MINMAX:V4SF (match_dup 3) (match_dup 4)))
+   (set (match_operand:SF 0 "register_operand" "")
+	(vec_select:SF (match_dup 5) (parallel [(const_int 0)])))]
+   "TARGET_VXE"
+{
+  operands[2] = gen_reg_rtx (V4SFmode);
+  operands[3] = gen_reg_rtx (V4SFmode);
+  operands[4] = gen_reg_rtx (V4SFmode);
+  operands[5] = gen_reg_rtx (V4SFmode);
+})
+
+; reduce_and, reduc_ior, reduc_xor
+; reduc_smin, reduc_smax, reduc_umin, reduc_umax
+
+(define_code_iterator REDUCBIN [and xor ior smin smax umin umax])
+(define_code_attr reduc_bin_insn [(and "and") (xor "xor") (ior "ior")
+				  (smin "smin") (smax "smax")
+				  (umin "umin") (umax "umax")])
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v2di"
+  [(set (match_dup 2)
+	(unspec:V2DI [(match_operand:V2DI 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V2DI (match_dup 1) (match_dup 2)))
+   (set (match_operand:DI 0 "register_operand" "")
+	(vec_select:DI (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DImode);
+  operands[3] = gen_reg_rtx (V2DImode);
+})
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v4si"
+  [(set (match_dup 2)
+	(unspec:V4SI [(match_operand:V4SI 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V4SI (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V4SI [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUCBIN:V4SI (match_dup 3) (match_dup 4)))
+   (set (match_operand:SI 0 "register_operand" "")
+	(vec_select:SI (match_dup 5) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V4SImode);
+  operands[3] = gen_reg_rtx (V4SImode);
+  operands[4] = gen_reg_rtx (V4SImode);
+  operands[5] = gen_reg_rtx (V4SImode);
+})
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v8hi"
+  [(set (match_dup 2)
+	(unspec:V8HI [(match_operand:V8HI 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 2)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V8HI (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V8HI [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUCBIN:V8HI (match_dup 3) (match_dup 4)))
+   (set (match_dup 6)
+	(unspec:V8HI [(match_dup 5)
+		      (match_dup 5)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 7)
+	(REDUCBIN:V8HI (match_dup 5) (match_dup 6)))
+   (set (match_operand:HI 0 "register_operand" "")
+	(vec_select:HI (match_dup 7) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V8HImode);
+  operands[3] = gen_reg_rtx (V8HImode);
+  operands[4] = gen_reg_rtx (V8HImode);
+  operands[5] = gen_reg_rtx (V8HImode);
+  operands[6] = gen_reg_rtx (V8HImode);
+  operands[7] = gen_reg_rtx (V8HImode);
+})
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v16qi"
+  [(set (match_dup 2)
+	(unspec:V16QI [(match_operand:V16QI 1 "register_operand")
+		       (match_dup 1)
+		       (const_int 1)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V16QI (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V16QI [(match_dup 3)
+		       (match_dup 3)
+		       (const_int 2)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUCBIN:V16QI (match_dup 3) (match_dup 4)))
+   (set (match_dup 6)
+	(unspec:V16QI [(match_dup 5)
+		       (match_dup 5)
+		       (const_int 4)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 7)
+	(REDUCBIN:V16QI (match_dup 5) (match_dup 6)))
+   (set (match_dup 8)
+	(unspec:V16QI [(match_dup 7)
+		       (match_dup 7)
+		       (const_int 8)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 9)
+	(REDUCBIN:V16QI (match_dup 7) (match_dup 8)))
+   (set (match_operand:QI 0 "register_operand" "")
+	(vec_select:QI (match_dup 9) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V16QImode);
+  operands[3] = gen_reg_rtx (V16QImode);
+  operands[4] = gen_reg_rtx (V16QImode);
+  operands[5] = gen_reg_rtx (V16QImode);
+  operands[6] = gen_reg_rtx (V16QImode);
+  operands[7] = gen_reg_rtx (V16QImode);
+  operands[8] = gen_reg_rtx (V16QImode);
+  operands[9] = gen_reg_rtx (V16QImode);
+})
diff --git a/gcc/config/s390/vx-builtins.md b/gcc/config/s390/vx-builtins.md
index a7bb7ff..9b89b13 100644
--- a/gcc/config/s390/vx-builtins.md
+++ b/gcc/config/s390/vx-builtins.md
@@ -982,20 +982,18 @@
 
 ; vmhb, vmhh, vmhf, vmhg, vmhq
 (define_insn "vec_smulh<mode>"
-  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                       "=v")
-	(unspec:VIT_HW_VXE3_DT [(match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
-				(match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")]
-			       UNSPEC_VEC_SMULT_HI))]
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                              "=v")
+	(smul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")))]
   "TARGET_VX"
   "vmh<bhfgq>\t%v0,%v1,%v2"
   [(set_attr "op_type" "VRR")])
 
 ; vmlhb, vmlhh, vmlhf, vmlhg, vmlhq
 (define_insn "vec_umulh<mode>"
-  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                       "=v")
-	(unspec:VIT_HW_VXE3_DT [(match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
-				(match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")]
-			       UNSPEC_VEC_UMULT_HI))]
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                              "=v")
+	(umul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")))]
   "TARGET_VX"
   "vmlh<bhfgq>\t%v0,%v1,%v2"
   [(set_attr "op_type" "VRR")])
@@ -2136,23 +2134,22 @@
   "<vw>fche<sdx>bs\t%v2,%v0,%v1"
   [(set_attr "op_type" "VRR")])
 
-
 (define_insn "vfmin<mode>"
-  [(set (match_operand:VF_HW                0 "register_operand"  "=v")
-	(unspec:VF_HW [(match_operand:VF_HW 1 "register_operand"   "v")
-		       (match_operand:VF_HW 2 "register_operand"   "v")
-		       (match_operand:QI    3 "const_mask_operand" "C")]
-		      UNSPEC_VEC_VFMIN))]
+  [(set (match_operand:VFT_BFP                  0 "register_operand"  "=v")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand"   "v")
+		         (match_operand:VFT_BFP 2 "register_operand"   "v")
+		         (match_operand:QI      3 "const_mask_operand" "C")]
+		        UNSPEC_FMIN))]
   "TARGET_VXE"
   "<vw>fmin<sdx>b\t%v0,%v1,%v2,%b3"
   [(set_attr "op_type" "VRR")])
 
 (define_insn "vfmax<mode>"
-  [(set (match_operand:VF_HW                0 "register_operand"  "=v")
-	(unspec:VF_HW [(match_operand:VF_HW 1 "register_operand"   "v")
-		       (match_operand:VF_HW 2 "register_operand"   "v")
-		       (match_operand:QI    3 "const_mask_operand" "C")]
-		      UNSPEC_VEC_VFMAX))]
+  [(set (match_operand:VFT_BFP                  0 "register_operand"  "=v")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand"   "v")
+		         (match_operand:VFT_BFP 2 "register_operand"   "v")
+		         (match_operand:QI      3 "const_mask_operand" "C")]
+		        UNSPEC_FMAX))]
   "TARGET_VXE"
   "<vw>fmax<sdx>b\t%v0,%v1,%v2,%b3"
   [(set_attr "op_type" "VRR")])
diff --git a/gcc/config/sh/predicates.md b/gcc/config/sh/predicates.md
index 7349c97..e67ec8a 100644
--- a/gcc/config/sh/predicates.md
+++ b/gcc/config/sh/predicates.md
@@ -630,9 +630,7 @@
 ;; Same as treg_set_expr but disallow constants 0 and 1 which can be loaded
 ;; into the T bit.
 (define_predicate "treg_set_expr_not_const01"
-  (and (match_test "op != const0_rtx")
-       (match_test "op != const1_rtx")
-       (match_operand 0 "treg_set_expr")))
+  (match_test "sh_recog_treg_set_expr_not_01 (op, mode)"))
 
 ;; A predicate describing the T bit register in any form.
 (define_predicate "t_reg_operand"
diff --git a/gcc/config/sh/sh-protos.h b/gcc/config/sh/sh-protos.h
index c8cc19f..e78b669 100644
--- a/gcc/config/sh/sh-protos.h
+++ b/gcc/config/sh/sh-protos.h
@@ -261,6 +261,7 @@ extern rtx_insn* sh_peephole_emit_move_insn (rtx dst, rtx src);
 
 extern bool sh_in_recog_treg_set_expr (void);
 extern bool sh_recog_treg_set_expr (rtx op, machine_mode mode);
+extern bool sh_recog_treg_set_expr_not_01 (rtx op, machine_mode mode);
 
 /* Result value of sh_split_treg_set_expr.  Contains the first insn emitted
    and the optional trailing nott insn.  */
diff --git a/gcc/config/sh/sh.cc b/gcc/config/sh/sh.cc
index 1bc34e0..09e4ff7 100644
--- a/gcc/config/sh/sh.cc
+++ b/gcc/config/sh/sh.cc
@@ -12348,6 +12348,23 @@ sh_recog_treg_set_expr (rtx op, machine_mode mode)
   return result >= 0;
 }
 
+/* Return TRUE if OP is an expression for which there is a pattern to
+   set the T bit unless the expression is trivially loadable into
+   the T bit, FALSE otherwise.  */
+bool
+sh_recog_treg_set_expr_not_01 (rtx op, machine_mode mode)
+{
+  if (op == const0_rtx || op == const1_rtx)
+    return false;
+
+  /* A right shift of 31 will return 0 or 1.  */
+  if ((GET_CODE (op) == LSHIFTRT || GET_CODE (op) == ASHIFTRT)
+      && INTVAL (XEXP (op, 1)) == 31)
+    return false;
+
+  return sh_recog_treg_set_expr (op, mode);
+}
+
 /* Returns true when recog of a 'treg_set_expr' is currently in progress.
    This can be used as a condition for insn/split patterns to allow certain
    T bit setting patters only to be matched as sub expressions of other
diff --git a/gcc/config/vxworks-dummy.h b/gcc/config/vxworks-dummy.h
index 494799d..516728c 100644
--- a/gcc/config/vxworks-dummy.h
+++ b/gcc/config/vxworks-dummy.h
@@ -40,9 +40,21 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define TARGET_VXWORKS_RTP false
 #endif
 
+/* True if offsets between different segments may vary, so we must avoid
+   cross-segment GOT- and PC-relative address computations.  */
+#ifndef TARGET_VXWORKS_VAROFF
+#define TARGET_VXWORKS_VAROFF false
+#endif
+
 /* The symbol that points to an RTP's table of GOTs.  */
 #define VXWORKS_GOTT_BASE (gcc_unreachable (), "")
 
 /* The symbol that holds the index of the current module's GOT in
    VXWORKS_GOTT_BASE.  */
 #define VXWORKS_GOTT_INDEX (gcc_unreachable (), "")
+
+/* True if PIC relies on the GOTT_* symbols above.  As of VxWorks7, they are no
+   longer used.  */
+#ifndef TARGET_VXWORKS_GOTTPIC
+#define TARGET_VXWORKS_GOTTPIC false
+#endif
diff --git a/gcc/config/vxworks.h b/gcc/config/vxworks.h
index 204a8e0..d2b6025 100644
--- a/gcc/config/vxworks.h
+++ b/gcc/config/vxworks.h
@@ -159,6 +159,18 @@ extern void vxworks_driver_init (unsigned int *, struct cl_decoded_option **);
    Earlier versions did not, not even for RTPS.  */
 #define VXWORKS_HAVE_TLS TARGET_VXWORKS7
 
+/* RTP segments could be loaded with varying offsets, so cross-segment offsets
+   could not be assumed to be constant.  This rules out some PC- and
+   GOT-relative addressing.  */
+#undef TARGET_VXWORKS_VAROFF
+#define TARGET_VXWORKS_VAROFF (!TARGET_VXWORKS7 && TARGET_VXWORKS_RTP)
+
+/* GOTT_BASE and GOTT_INDEX symbols are only used by some ports up to VxWorks6.
+   This macro is only used by i386 so far.  Other ports seem to keep on using
+   GOTTPIC from VxWorks7 on, but they don't test this macro.  */
+#undef TARGET_VXWORKS_GOTTPIC
+#define TARGET_VXWORKS_GOTTPIC (!TARGET_VXWORKS7)
+
 /* On Vx6 and previous, the libraries to pick up depends on the architecture,
    so cannot be defined for all archs at once.  On Vx7, a VSB is always needed
    and its structure is fixed and does not depend on the arch.  We can thus
diff --git a/gcc/config/xtensa/xtensa.cc b/gcc/config/xtensa/xtensa.cc
index 92a2364..d75cba4 100644
--- a/gcc/config/xtensa/xtensa.cc
+++ b/gcc/config/xtensa/xtensa.cc
@@ -159,6 +159,10 @@ static void xtensa_asm_trampoline_template (FILE *);
 static void xtensa_trampoline_init (rtx, tree, rtx);
 static bool xtensa_output_addr_const_extra (FILE *, rtx);
 static bool xtensa_cannot_force_const_mem (machine_mode, rtx);
+static machine_mode xtensa_promote_function_mode (const_tree,
+						  machine_mode,
+						  int *, const_tree,
+						  int);
 
 static reg_class_t xtensa_preferred_reload_class (rtx, reg_class_t);
 static reg_class_t xtensa_preferred_output_reload_class (rtx, reg_class_t);
@@ -235,9 +239,7 @@ static HARD_REG_SET xtensa_zero_call_used_regs (HARD_REG_SET);
 #define TARGET_EXPAND_BUILTIN_VA_START xtensa_va_start
 
 #undef TARGET_PROMOTE_FUNCTION_MODE
-#define TARGET_PROMOTE_FUNCTION_MODE default_promote_function_mode_always_promote
-#undef TARGET_PROMOTE_PROTOTYPES
-#define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
+#define TARGET_PROMOTE_FUNCTION_MODE xtensa_promote_function_mode
 
 #undef TARGET_RETURN_IN_MEMORY
 #define TARGET_RETURN_IN_MEMORY xtensa_return_in_memory
@@ -421,12 +423,13 @@ xtensa_uimm8x4 (HOST_WIDE_INT v)
 }
 
 
-static bool
-xtensa_b4const (HOST_WIDE_INT v)
+bool
+xtensa_b4const_or_zero (HOST_WIDE_INT v)
 {
   switch (v)
     {
     case -1:
+    case 0:
     case 1:
     case 2:
     case 3:
@@ -449,15 +452,6 @@ xtensa_b4const (HOST_WIDE_INT v)
 
 
 bool
-xtensa_b4const_or_zero (HOST_WIDE_INT v)
-{
-  if (v == 0)
-    return true;
-  return xtensa_b4const (v);
-}
-
-
-bool
 xtensa_b4constu (HOST_WIDE_INT v)
 {
   switch (v)
@@ -607,8 +601,8 @@ constantpool_address_p (const_rtx addr)
 
       /* Make sure the address is word aligned.  */
       offset = XEXP (addr, 1);
-      if ((!CONST_INT_P (offset))
-	  || ((INTVAL (offset) & 3) != 0))
+      if (! CONST_INT_P (offset)
+	  || (INTVAL (offset) & 3) != 0)
 	return false;
 
       sym = XEXP (addr, 0);
@@ -617,6 +611,7 @@ constantpool_address_p (const_rtx addr)
   if (SYMBOL_REF_P (sym)
       && CONSTANT_POOL_ADDRESS_P (sym))
     return true;
+
   return false;
 }
 
@@ -4510,7 +4505,8 @@ xtensa_rtx_costs (rtx x, machine_mode mode, int outer_code,
 	    }
 	  break;
 	case COMPARE:
-	  if ((INTVAL (x) == 0) || xtensa_b4const (INTVAL (x)))
+	  if (xtensa_b4const_or_zero (INTVAL (x))
+	      || xtensa_b4constu (INTVAL (x)))
 	    {
 	      *total = 0;
 	      return true;
@@ -4699,29 +4695,56 @@ xtensa_rtx_costs (rtx x, machine_mode mode, int outer_code,
     }
 }
 
+/* Return TRUE if the specified insn corresponds to one or more L32R machine
+   instructions.  */
+
 static bool
 xtensa_is_insn_L32R_p (const rtx_insn *insn)
 {
-  rtx x = PATTERN (insn);
+  rtx pat, dest, src;
+  machine_mode mode;
 
-  if (GET_CODE (x) != SET)
+  /* RTX insns that are not "(set (reg) ...)" cannot become L32R instructions:
+     - it is permitted to apply PATTERN() to the insn without validation.
+       See insn_cost() in gcc/rtlanal.cc.
+     - it is used register_operand() instead of REG() to identify things that
+       don't look like REGs but will eventually become so as well.  */
+  if (GET_CODE (pat = PATTERN (insn)) != SET
+      || ! register_operand (dest = SET_DEST (pat), VOIDmode))
     return false;
 
-  x = XEXP (x, 1);
-  if (MEM_P (x))
-    {
-      x = XEXP (x, 0);
-      return (SYMBOL_REF_P (x) || CONST_INT_P (x))
-	     && CONSTANT_POOL_ADDRESS_P (x);
-    }
-
-  /* relaxed MOVI instructions, that will be converted to L32R by the
-     assembler.  */
-  if (CONST_INT_P (x)
-      && ! xtensa_simm12b (INTVAL (x)))
+  /* If the source is a reference to a literal pool entry, then the insn
+     obviously corresponds to an L32R instruction.  */
+  if (constantpool_mem_p (src = SET_SRC (pat)))
     return true;
 
-  return false;
+  /* Similarly, an insn whose source is not a constant obviously does not
+     correspond to L32R.  */
+  if (! CONSTANT_P (src))
+    return false;
+
+  /* If the source is a CONST_INT whose value fits into signed 12 bits, then
+     the insn corresponds to a MOVI instruction (rather than an L32R one),
+     regardless of the configuration of TARGET_CONST16 or
+     TARGET_AUTOLITPOOLS.  Note that the destination register can be non-
+     SImode.  */
+  if (((mode = GET_MODE (dest)) == SImode
+       || mode == HImode || mode == SFmode)
+      && CONST_INT_P (src) && xtensa_simm12b (INTVAL (src)))
+    return false;
+
+  /* If TARGET_CONST16 is configured, constants of the remaining forms
+     correspond to pairs of CONST16 instructions, not L32R.  */
+  if (TARGET_CONST16)
+    return false;
+
+  /* The last remaining form of constant is one of the following:
+     - CONST_INTs with large values
+     - floating-point constants
+     - symbolic constants
+     and is all handled by a relaxed MOVI instruction, which is later
+     converted to an L32R instruction by the assembler.  */
+  return true;
 }
 
 /* Compute a relative costs of RTL insns.  This is necessary in order to
@@ -4730,7 +4753,7 @@ xtensa_is_insn_L32R_p (const rtx_insn *insn)
 static int
 xtensa_insn_cost (rtx_insn *insn, bool speed)
 {
-  if (!(recog_memoized (insn) < 0))
+  if (! (recog_memoized (insn) < 0))
     {
       int len = get_attr_length (insn);
 
@@ -4743,7 +4766,7 @@ xtensa_insn_cost (rtx_insn *insn, bool speed)
 
 	  /* "L32R" may be particular slow (implementation-dependent).  */
 	  if (xtensa_is_insn_L32R_p (insn))
-	    return COSTS_N_INSNS (1 + xtensa_extra_l32r_costs);
+	    return COSTS_N_INSNS ((1 + xtensa_extra_l32r_costs) * n);
 
 	  /* Cost based on the pipeline model.  */
 	  switch (get_attr_type (insn))
@@ -4788,7 +4811,7 @@ xtensa_insn_cost (rtx_insn *insn, bool speed)
 	    {
 	      /* "L32R" itself plus constant in litpool.  */
 	      if (xtensa_is_insn_L32R_p (insn))
-		len = 3 + 4;
+		len += (len / 3) * 4;
 
 	      /* Consider fractional instruction length (for example, ".n"
 		 short instructions or "L32R" litpool constants.  */
@@ -4801,6 +4824,19 @@ xtensa_insn_cost (rtx_insn *insn, bool speed)
   return pattern_cost (PATTERN (insn), speed);
 }
 
+/* Worker function for TARGET_PROMOTE_FUNCTION_MODE.  */
+
+static machine_mode
+xtensa_promote_function_mode (const_tree type, machine_mode mode,
+			      int *punsignedp, const_tree, int)
+{
+  if (GET_MODE_CLASS (mode) == MODE_INT
+      && GET_MODE_SIZE (mode) < GET_MODE_SIZE (SImode))
+    return SImode;
+
+  return promote_mode (type, mode, punsignedp);
+}
+
 /* Worker function for TARGET_RETURN_IN_MEMORY.  */
 
 static bool
diff --git a/gcc/config/xtensa/xtensa.md b/gcc/config/xtensa/xtensa.md
index 029be99..629dfdd 100644
--- a/gcc/config/xtensa/xtensa.md
+++ b/gcc/config/xtensa/xtensa.md
@@ -1297,7 +1297,10 @@
       std::swap (operands[0], operands[1]);
       std::swap (operands[2], operands[3]);
     }
-})
+}
+  [(set_attr "type"	"move,move,load,load,store")
+   (set_attr "mode"	"DI")
+   (set_attr "length"	"6,12,6,6,6")])
 
 (define_split
   [(set (match_operand:DI 0 "register_operand")
@@ -1344,7 +1347,7 @@
    %v0s32i\t%1, %0
    rsr\t%0, ACCLO
    wsr\t%1, ACCLO"
-  [(set_attr "type"	"move,move,move,load,store,store,move,move,move,move,move,load,load,store,rsr,wsr")
+  [(set_attr "type"	"move,move,move,load,store,store,move,move,move,load,move,load,load,store,rsr,wsr")
    (set_attr "mode"	"SI")
    (set_attr "length"	"2,2,2,2,2,2,3,3,3,3,6,3,3,3,3,3")])
 
@@ -1410,7 +1413,7 @@
    %v0s16i\t%1, %0
    rsr\t%0, ACCLO
    wsr\t%1, ACCLO"
-  [(set_attr "type"	"move,move,move,move,move,load,load,store,rsr,wsr")
+  [(set_attr "type"	"move,move,move,move,load,load,load,store,rsr,wsr")
    (set_attr "mode"	"HI")
    (set_attr "length"	"2,2,3,3,3,3,3,3,3,3")])
 
@@ -1519,7 +1522,7 @@
    const16\t%0, %t1\;const16\t%0, %b1
    %v1l32i\t%0, %1
    %v0s32i\t%1, %0"
-  [(set_attr "type"	"farith,fload,fstore,move,load,load,store,move,farith,farith,move,move,load,store")
+  [(set_attr "type"	"farith,fload,fstore,move,load,load,store,move,farith,farith,load,move,load,store")
    (set_attr "mode"	"SF")
    (set_attr "length"	"3,3,3,2,3,2,2,3,3,3,3,6,3,3")])
 
@@ -1643,7 +1646,10 @@
       std::swap (operands[0], operands[1]);
       std::swap (operands[2], operands[3]);
     }
-})
+}
+  [(set_attr "type"	"move,load,move,load,load,store")
+   (set_attr "mode"	"DF")
+   (set_attr "length"	"6,6,12,6,6,6")])
 
 ;; Block moves
 
diff --git a/gcc/configure b/gcc/configure
index f056cfe..bacdd29 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -1064,6 +1064,7 @@ enable_versioned_jit
 enable_default_pie
 enable_cet
 enable_s390_excess_float_precision
+enable_x86_64_mfentry
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1842,6 +1843,7 @@ Optional Features:
   --enable-s390-excess-float-precision
                           on s390 targets, evaluate float with double
                           precision when in standards-conforming mode
+  --enable-x86-64-mfentry enable -mfentry by default on x86-64 targets
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -21520,7 +21522,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 21523 "configure"
+#line 21525 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -21626,7 +21628,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 21629 "configure"
+#line 21631 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -35022,6 +35024,46 @@ $as_echo "#define ENABLE_S390_EXCESS_FLOAT_PRECISION 1" >>confdefs.h
   ;;
 esac
 
+# On x86-64, when profiling is enabled with shrink wrapping, the mcount
+# call may not be placed at the function entry after
+#	pushq %rbp
+#	movq %rsp,%rbp
+# As the result, the profile data may be skewed which makes PGO less
+# effective:
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120881
+# Enable -mfentry by default on x86-64 to put the profiling counter call
+# before the prologue.
+# Check whether --enable-x86-64-mfentry was given.
+if test "${enable_x86_64_mfentry+set}" = set; then :
+  enableval=$enable_x86_64_mfentry; case "${enableval}" in
+   yes | no | auto)
+     enable_x86_64_mfentry=$enableval
+     ;;
+   *)
+     as_fn_error $? "'$enable_x86_64_mfentry' is an invalid value for --enable-x86-64-mfentry.  Valid choices are 'yes', 'no' and 'auto'." "$LINENO" 5
+     ;;
+   esac
+else
+  enable_x86_64_mfentry=auto
+fi
+
+
+if test x"$enable_x86_64_mfentry" = xauto; then
+  case "${target}" in
+  i?86-*-*gnu* | x86_64-*-*gnu*)
+    # Enable -mfentry by default with glibc on x86.
+    enable_x86_64_mfentry=yes
+    ;;
+  esac
+fi
+
+gif=`if test x$enable_x86_64_mfentry = xyes; then echo 1; else echo 0; fi`
+
+cat >>confdefs.h <<_ACEOF
+#define ENABLE_X86_64_MFENTRY $gif
+_ACEOF
+
+
 # Check if the linker supports '-z now'
 ld_now_support=no
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking linker -z now option" >&5
@@ -36456,7 +36498,7 @@ $as_echo "$as_me: executing $ac_file commands" >&6;}
     "depdir":C) $SHELL $ac_aux_dir/mkinstalldirs $DEPDIR ;;
     "gccdepdir":C)
   ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs build/$DEPDIR
-  for lang in $subdirs c-family common analyzer text-art rtl-ssa sym-exec
+  for lang in $subdirs c-family common analyzer diagnostics text-art rtl-ssa sym-exec
   do
       ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs $lang/$DEPDIR
   done ;;
diff --git a/gcc/configure.ac b/gcc/configure.ac
index 58bf63f..2c43b38 100644
--- a/gcc/configure.ac
+++ b/gcc/configure.ac
@@ -1397,7 +1397,7 @@ AC_CHECK_HEADERS(ext/hash_map)
 ZW_CREATE_DEPDIR
 AC_CONFIG_COMMANDS([gccdepdir],[
   ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs build/$DEPDIR
-  for lang in $subdirs c-family common analyzer text-art rtl-ssa sym-exec
+  for lang in $subdirs c-family common analyzer diagnostics text-art rtl-ssa sym-exec
   do
       ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs $lang/$DEPDIR
   done], [subdirs="$subdirs" ac_aux_dir=$ac_aux_dir DEPDIR=$DEPDIR])
@@ -7972,6 +7972,41 @@ standards-compatible mode on s390 targets.])
   ;;
 esac
 
+# On x86-64, when profiling is enabled with shrink wrapping, the mcount
+# call may not be placed at the function entry after
+#	pushq %rbp
+#	movq %rsp,%rbp
+# As the result, the profile data may be skewed which makes PGO less
+# effective:
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120881
+# Enable -mfentry by default on x86-64 to put the profiling counter call
+# before the prologue.
+AC_ARG_ENABLE(x86-64-mfentry,
+  [AS_HELP_STRING([--enable-x86-64-mfentry],
+		  [enable -mfentry by default on x86-64 targets])],
+  [case "${enableval}" in
+   yes | no | auto)
+     enable_x86_64_mfentry=$enableval
+     ;;
+   *)
+     AC_MSG_ERROR(['$enable_x86_64_mfentry' is an invalid value for --enable-x86-64-mfentry.  Valid choices are 'yes', 'no' and 'auto'.])
+     ;;
+   esac],
+  [enable_x86_64_mfentry=auto])
+
+if test x"$enable_x86_64_mfentry" = xauto; then
+  case "${target}" in
+  i?86-*-*gnu* | x86_64-*-*gnu*)
+    # Enable -mfentry by default with glibc on x86.
+    enable_x86_64_mfentry=yes
+    ;;
+  esac
+fi
+
+gif=`if test x$enable_x86_64_mfentry = xyes; then echo 1; else echo 0; fi`
+AC_DEFINE_UNQUOTED(ENABLE_X86_64_MFENTRY, $gif,
+[Define to enable -mfentry by default on x86-64.])
+
 # Check if the linker supports '-z now'
 ld_now_support=no
 AC_MSG_CHECKING(linker -z now option)
diff --git a/gcc/coretypes.h b/gcc/coretypes.h
index a11ebd1..b6a922a 100644
--- a/gcc/coretypes.h
+++ b/gcc/coretypes.h
@@ -164,8 +164,10 @@ struct cl_option;
 struct cl_decoded_option;
 struct cl_option_handlers;
 class rich_location;
-class diagnostic_context;
-class diagnostic_text_output_format;
+namespace diagnostics {
+  class context;
+  class text_sink;
+} // namespace diagnostics
 class pretty_printer;
 class diagnostic_event_id_t;
 typedef const char * (*diagnostic_input_charset_callback)(const char *);
diff --git a/gcc/coverage.cc b/gcc/coverage.cc
index dd3ed2e..75a24c6 100644
--- a/gcc/coverage.cc
+++ b/gcc/coverage.cc
@@ -238,6 +238,7 @@ read_counts_file (void)
 	  gcov_profile_info = profile_info = XCNEW (gcov_summary);
 	  profile_info->runs = gcov_read_unsigned ();
 	  profile_info->sum_max = gcov_read_unsigned ();
+	  profile_info->cutoff = 1;
 	}
       else if (GCOV_TAG_IS_COUNTER (tag) && fn_ident)
 	{
diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog
index ff7582d..3ab14f0 100644
--- a/gcc/cp/ChangeLog
+++ b/gcc/cp/ChangeLog
@@ -1,3 +1,691 @@
+2025-08-04  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121351
+	PR c++/119859
+	* class.cc (add_method): Substitute outer template arguments
+	into constraints before comparing them if the declarations are
+	from different classes.
+
+2025-08-04  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/120620
+	* constexpr.cc (cxx_dynamic_cast_fn_p): Return true only
+	for synthesized __dynamic_cast.
+
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/108080
+	* module.cc (trees_out::core_vals): Warn when streaming
+	target/optimize node; adjust comments.
+	(trees_in::core_vals): Don't stream a target/optimize node.
+
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121238
+	* module.cc (trees_in::fn_parms_fini): Merge properties for
+	definitions.
+
+2025-07-31  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120800
+	* constexpr.cc (cxx_eval_vec_init_1): Suppress access control.
+
+2025-07-31  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/120775
+	* constexpr.cc (cxx_eval_outermost_constant_expr): Use
+	extract_call_expr.
+	* cp-tree.h (CONSTEVAL_BLOCK_P, LAMBDA_EXPR_CONSTEVAL_BLOCK_P): Define.
+	(finish_static_assert): Adjust declaration.
+	(current_nonlambda_function): Likewise.
+	* lambda.cc (current_nonlambda_function): New parameter.  Only keep
+	iterating if the function represents a consteval block.
+	* parser.cc (cp_parser_lambda_expression): New parameter for
+	consteval blocks.  Use it.  Set LAMBDA_EXPR_CONSTEVAL_BLOCK_P.
+	(cp_parser_lambda_declarator_opt): Likewise.
+	(build_empty_string): New.
+	(cp_parser_next_tokens_are_consteval_block_p): New.
+	(cp_parser_consteval_block): New.
+	(cp_parser_block_declaration): Handle consteval blocks.
+	(cp_parser_static_assert): Use build_empty_string.
+	(cp_parser_member_declaration): Handle consteval blocks.
+	* pt.cc (tsubst_stmt): Adjust a call to finish_static_assert.
+	* semantics.cc (finish_fname): Warn for consteval blocks.
+	(finish_static_assert): New parameter for consteval blocks.  Set
+	CONSTEVAL_BLOCK_P.  Evaluate consteval blocks specially.
+
+2025-07-30  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121291
+	* constraint.cc (diagnose_trait_expr): Remove assumption about
+	failures returning error_mark_node.
+	* except.cc (explain_not_noexcept): Allow expr not being
+	noexcept.
+	* method.cc (build_invoke): Adjust comment.
+	(is_trivially_xible): Always note non-trivial components if expr
+	is not null or error_mark_node.
+	(is_nothrow_xible): Likewise for non-noexcept components.
+	(is_nothrow_convertible): Likewise.
+
+2025-07-30  Jason Merrill  <jason@redhat.com>
+
+	* pt.cc (convert_nontype_argument_function): Check
+	cxx_constant_value on failure.
+	(invalid_tparm_referent_p): Likewise.
+
+2025-07-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/121133
+	* parser.cc (cp_parser_unary_expression): Adjust
+	cp_parser_extension_opt caller and restore warn_long_long.
+	(cp_parser_declaration): Likewise.
+	(cp_parser_block_declaration): Likewise.
+	(cp_parser_member_declaration): Likewise.
+	(cp_parser_extension_opt): Add SAVED_LONG_LONG argument,
+	save previous warn_long_long state into it and clear it
+	for __extension__.
+
+2025-07-27  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	* cp-tree.h (struct lang_type): Add comment mentioning modules.
+	* module.cc (trees_out::lang_type_bools): Stream new flags, use
+	gcc_checking_assert.
+	(trees_in::lang_type_bools): Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* constexpr.cc: Update usage of "diagnostic_info" to explicitly
+	refer to "diagnostics::diagnostic_info".
+	* cp-tree.h: Likewise.
+	* error.cc: Likewise.
+	* module.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* call.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* constexpr.cc: Likewise.
+	* cp-tree.h: Likewise.
+	* decl.cc: Likewise.
+	* error.cc: Likewise.
+	* init.cc: Likewise.
+	* method.cc: Likewise.
+	* module.cc: Likewise.
+	* parser.cc: Likewise.
+	* pt.cc: Likewise.
+	* semantics.cc: Likewise.
+	* typeck.cc: Likewise.
+	* typeck2.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cp-tree.h: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+	* decl.cc: Likewise.
+	* error.cc: Likewise.
+	* name-lookup.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cp-tree.h: Update for diagnostic_context becoming
+	diagnostics::context.
+	* error.cc: Likewise.
+	* module.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* constexpr.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+	* error.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cp-tree.h: Update for move of diagnostics output formats into
+	namespace "diagnostics" as "sinks".
+	* error.cc: Likewise.
+
+2025-07-25  Patrick Palka  <ppalka@redhat.com>
+
+	* call.cc (build_new_op): If the selected candidate is
+	rewritten, communicate the LOOKUP_REWRITTEN/REVERSED flags to
+	the caller via the 'overload' out-parameter, and stop clearing
+	'*overload' in that case.
+	* tree.cc (build_min_non_dep_op_overload): Handle rebuilding all
+	C++20 rewritten comparison operator expressions.
+
+2025-07-25  Iain Sandoe  <iain@sandoe.co.uk>
+
+	PR c++/121219
+	* coroutines.cc
+	(cp_coroutine_transform::build_ramp_function): Reorder the return
+	expressions for the 'normal' and 'allocation failed' cases so that
+	NRV constraints are met.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/117294
+	PR c++/113854
+	* call.cc (implicit_conversion_error): Hide label when printing
+	a stub object.
+	(convert_like_internal): Likewise, and nest candidate
+	diagnostics.
+	* constexpr.cc (diagnose_failing_condition): Nest diagnostics,
+	attempt to provide more helpful diagnostics for traits.
+	* constraint.cc (satisfy_atom): Pass result before constant
+	evaluation to diagnose_atomic_constraint.
+	(diagnose_trait_expr): Adjust diagnostics for clarity and
+	detail.
+	(maybe_diagnose_standard_trait): New function.
+	(diagnose_atomic_constraint): Attempt to provide more helpful
+	diagnostics for more traits.
+	* cp-tree.h (explain_not_noexcept): Declare new function.
+	(is_trivially_xible): Add parameter.
+	(is_nothrow_xible): Likewise.
+	(is_xible): Likewise.
+	(is_convertible): Likewise.
+	(is_nothrow_convertible): Likewise.
+	(diagnose_trait_expr): Declare new function.
+	(maybe_diagnose_standard_trait): Declare new function.
+	* error.cc (dump_type) <case TREE_VEC>: Handle trait types.
+	* except.cc (explain_not_noexcept): New function.
+	* method.cc (build_trait_object): Add complain parameter.
+	(build_invoke): Propagate complain parameter.
+	(assignable_expr): Add explain parameter to show diagnostics.
+	(constructible_expr): Likewise.
+	(destructible_expr): Likewise.
+	(is_xible_helper): Replace trivial flag with explain flag,
+	add diagnostics.
+	(is_trivially_xible): New explain flag.
+	(is_nothrow_xible): Likewise.
+	(is_xible): Likewise.
+	(is_convertible_helper): Add complain flag.
+	(is_convertible): New explain flag.
+	(is_nothrow_convertible): Likewise.
+	* typeck.cc (cp_build_function_call_vec): Add handling for stub
+	objects.
+	(convert_arguments): Always return -1 on error.
+	* typeck2.cc (cxx_readonly_error): Add handling for stub
+	objects.
+
+2025-07-24  Jason Merrill  <jason@redhat.com>
+
+	* pt.cc (tsubst_lambda_expr): Revert r9-5971 change.
+
+2025-07-24  Jason Merrill  <jason@redhat.com>
+
+	PR c++/114632
+	PR c++/101233
+	* lambda.cc (maybe_add_lambda_conv_op): Not for xobj lambda.
+	* pt.cc (tsubst_function_decl): Add cp_evaluated.
+	(alias_ctad_tweaks): Revert PR101233 fix.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/120412
+	* module.cc (trees_out::core_vals): Write TU_LOCAL_ENTITY bits.
+	(trees_in::core_vals): Read it.
+	(trees_in::tree_node): Handle TU_LOCAL_ENTITY typedefs.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121179
+	* call.cc (build_new_op): Don't clear *overload for a simple
+	!= to == rewrite.
+	* tree.cc (build_min_non_dep_op_overload): Handle TRUTH_NOT_EXPR
+	appearing in a rewritten operator expression.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121055
+	* method.cc (build_invoke): Correct reference_wrapper handling.
+
+2025-07-22  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121068
+	* constexpr.cc (cxx_eval_store_expression): Allow ARRAY_REFs
+	when activating an array member of a union.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* semantics.cc (finish_asm_stmt): Pass null pointer to
+	parse_{input,output}_constraint().
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* pt.cc (tsubst_omp_clause_decl): Use OMP_ITERATOR_DECL_P.
+	* semantics.cc (handle_omp_array_sections): Likewise.
+	(finish_omp_clauses): Likewise.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* class.cc (add_method): Remove argument.
+	* cp-tree.h (maybe_version_functions): Ditto.
+	* decl.cc (decls_match): Ditto.
+	(maybe_version_functions): Ditto.
+
+2025-07-16  Jeremy Rifkin  <jeremy@rifkin.dev>
+
+	PR c/82134
+	* call.cc (build_call_a): Add suppress_warning
+	* cp-gimplify.cc (cp_gimplify_expr): Add suppress_warning
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/44677
+	* cp-gimplify.cc (cp_fold) [CLEANUP_POINT_EXPR]: Don't force rvalue.
+	[COMPOUND_EXPR]: Likewise.
+	* cvt.cc (convert_to_void): Call mark_exp_read later.
+	* expr.cc (mark_use): Turn off read_p for any void argument.
+	(mark_exp_read): Return early for void argument.
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120577
+	* constexpr.cc (cxx_eval_call_expression): Set
+	CONSTRUCTOR_NO_CLEARING on initial value for ctor.
+	(cxx_eval_component_reference): Make value-initialization
+	of an aggregate member explicit.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* cp-gimplify.cc (cp_fold): Clear DECL_READ_P on lhs of MODIFY_EXPR
+	after cp_fold_rvalue if it wasn't set before.
+	* decl.cc (poplevel): Use OPT_Wunused_but_set_variable_
+	instead of OPT_Wunused_but_set_variable.
+	(finish_function): Use OPT_Wunused_but_set_parameter_
+	instead of OPT_Wunused_but_set_parameter.
+	* expr.cc (mark_use): Clear read_p for {PRE,POST}{IN,DE}CREMENT_EXPR
+	cast to void on {VAR,PARM}_DECL for
+	-Wunused-but-set-{parameter,variable}={2,3}.
+	(mark_exp_read): Handle {PRE,POST}{IN,DE}CREMENT_EXPR and don't handle
+	it when cast to void.
+	* module.cc (trees_in::fn_parms_fini): Remove unused but set variable
+	ix.
+	* semantics.cc (finish_unary_op_expr): Return early for
+	PRE{IN,DE}CREMENT_EXPR.
+	* typeck.cc (cp_build_unary_op): Clear DECL_READ_P
+	after mark_lvalue_use for -Wunused-but-set-{parameter,variable}={2,3}
+	on PRE{IN,DE}CREMENT_EXPR argument.
+	(cp_build_modify_expr): Clear DECL_READ_P after cp_build_binary_op
+	for -Wunused-but-set-{parameter,variable}=3.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* cp-tree.h: Implement C++26 P2786R13 - Trivial Relocatability.
+	(struct lang_type): Add trivially_relocatable,
+	trivially_relocatable_computed, replaceable and replaceable_computed
+	bitfields.  Change width of dummy from 2 to 30.
+	(CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT,
+	CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED, CLASSTYPE_REPLACEABLE_BIT,
+	CLASSTYPE_REPLACEABLE_COMPUTED): Define.
+	(enum virt_specifier): Add VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE
+	and VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE enumerators.
+	(trivially_relocatable_type_p, replaceable_type_p): Declare.
+	* cp-trait.def (IS_NOTHROW_RELOCATABLE, IS_REPLACEABLE,
+	IS_TRIVIALLY_RELOCATABLE): New traits.
+	* parser.cc (cp_parser_class_property_specifier_seq_opt): Handle
+	trivially_relocatable_if_eligible,
+	__trivially_relocatable_if_eligible, replaceable_if_eligible and
+	__replaceable_if_eligible.
+	(cp_parser_class_head): Set CLASSTYPE_REPLACEABLE_BIT
+	and/or CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT if corresponding
+	conditional keywords were parsed and assert corresponding *_COMPUTED
+	macro is false.
+	* pt.cc (instantiate_class_template): Copy over also
+	CLASSTYPE_TRIVIALLY_RELOCATABLE_{BIT,COMPUTED} and
+	CLASSTYPE_REPLACEABLE_{BIT,COMPUTED} bits.
+	* semantics.cc (referenceable_type_p): Move definition earlier.
+	(trait_expr_value): Handle CPTK_IS_NOTHROW_RELOCATABLE,
+	CPTK_IS_REPLACEABLE and CPTK_IS_TRIVIALLY_RELOCATABLE.
+	(finish_trait_expr): Likewise.
+	* tree.cc (default_movable_type_p): New function.
+	(union_with_no_declared_special_member_fns): Likewise.
+	(trivially_relocatable_type_p): Likewise.
+	(replaceable_type_p): Likewise.
+	* constraint.cc (diagnose_trait_expr): Handle
+	CPTK_IS_NOTHROW_RELOCATABLE, CPTK_IS_REPLACEABLE and
+	CPTK_IS_TRIVIALLY_RELOCATABLE.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* cp-tree.h (struct lang_type): Add comment before key_method.
+	Remove lambda_expr.
+	(CLASSTYPE_KEY_METHOD): Give NULL_TREE if not TYPE_POLYMORPHIC_P.
+	(SET_CLASSTYPE_KEY_METHOD): Define.
+	(CLASSTYPE_LAMBDA_EXPR): Give NULL_TREE if TYPE_POLYMORPHIC_P.
+	Use key_method member instead of lambda_expr.
+	(SET_CLASSTYPE_LAMBDA_EXPR): Define.
+	* class.cc (determine_key_method): Use SET_CLASSTYPE_KEY_METHOD
+	macro.
+	* decl.cc (xref_tag): Use SET_CLASSTYPE_LAMBDA_EXPR macro.
+	* lambda.cc (begin_lambda_type): Likewise.
+	* module.cc (trees_in::read_class_def): Use SET_CLASSTYPE_LAMBDA_EXPR
+	and SET_CLASSTYPE_KEY_METHOD macros, assert lambda is NULL if
+	TYPE_POLYMORPHIC_P and otherwise assert key_method is NULL.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120628
+	* parser.cc (cp_parser_elaborated_type_specifier): Use
+	cp_parser_nth_token_starts_class_definition_p with extra argument 1
+	instead of cp_parser_next_token_starts_class_definition_p.
+	(cp_parser_class_property_specifier_seq_opt): For final conditional
+	keyword in C++98 check if the token after it isn't
+	cp_parser_nth_token_starts_class_definition_p nor CPP_NAME and in
+	that case break without consuming it nor warning.
+	(cp_parser_class_head): Use
+	cp_parser_nth_token_starts_class_definition_p with extra argument 1
+	instead of cp_parser_next_token_starts_class_definition_p.
+	(cp_parser_next_token_starts_class_definition_p): Renamed to ...
+	(cp_parser_nth_token_starts_class_definition_p): ... this.  Add N
+	argument.  Use cp_lexer_peek_nth_token instead of cp_lexer_peek_token.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120569
+	* parser.cc (cp_parser_class_property_specifier_seq_opt): New
+	function.
+	(cp_parser_class_head): Use it instead of
+	cp_parser_property_specifier_seq_opt.  Don't diagnose
+	VIRT_SPEC_OVERRIDE here.  Formatting fix.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117785
+	* constexpr.cc: Implement C++26 P3068R5 - constexpr exceptions.
+	(class constexpr_global_ctx): Add caught_exceptions and
+	uncaught_exceptions members.
+	(constexpr_global_ctx::constexpr_global_ctx): Initialize
+	uncaught_exceptions.
+	(returns, breaks, continues, switches): Move earlier.
+	(throws): New function.
+	(exception_what_str, diagnose_std_terminate,
+	diagnose_uncaught_exception): New functions.
+	(enum cxa_builtin): New type.
+	(cxx_cxa_builtin_fn_p, cxx_eval_cxa_builtin_fn): New functions.
+	(cxx_eval_builtin_function_call): Add jump_target argument.  Call
+	cxx_eval_cxa_builtin_fn for __builtin_eh_ptr_adjust_ref.  Adjust
+	cxx_eval_constant_expression calls, if it results in jmp_target,
+	set *jump_target to it and return.
+	(cxx_bind_parameters_in_call): Add jump_target argument.  Pass
+	it through to cxx_eval_constant_expression.  If it sets *jump_target,
+	break.
+	(fold_operand): Adjust cxx_eval_constant_expression caller.
+	(cxx_eval_assert): Likewise.  If it set jmp_target, return true.
+	(cxx_eval_internal_function): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression.  Return early if *jump_target
+	after recursing on args.
+	(cxx_eval_dynamic_cast_fn): Likewise.  Don't set reference_p for
+	C++26 with -fexceptions.
+	(cxx_eval_thunk_call): Add jump_target argument.  Pass it through
+	to cxx_eval_constant_expression.
+	(cxx_set_object_constness): Likewise.  Don't set TREE_READONLY if
+	throws (jump_target).
+	(cxx_eval_call_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_internal_function, cxx_eval_builtin_function_call,
+	cxx_eval_thunk_call, cxx_eval_dynamic_cast_fn and
+	cxx_set_object_constness.  Pass it through also
+	cxx_eval_constant_expression on arguments, cxx_bind_parameters_in_call
+	and cxx_fold_indirect_ref and for those cases return early
+	if *jump_target.  Call cxx_eval_cxa_builtin_fn for cxx_cxa_builtin_fn_p
+	functions.  For cxx_eval_constant_expression on body, pass address of
+	cleared jmp_target automatic variable, if it throws propagate
+	to *jump_target and make it non-cacheable.  For C++26 don't diagnose
+	calls to non-constexpr functions before cxx_bind_parameters_in_call
+	could report some argument throwing an exception.
+	(cxx_eval_unary_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression and return early
+	if *jump_target after the call.
+	(cxx_fold_pointer_plus_expression): Likewise.
+	(cxx_eval_binary_expression): Likewise and similarly for
+	cxx_fold_pointer_plus_expression call.
+	(cxx_eval_conditional_expression): Pass jump_target to
+	cxx_eval_constant_expression on first operand and return early
+	if *jump_target after the call.
+	(cxx_eval_vector_conditional_expression): Add jump_target argument.
+	Pass it through to cxx_eval_constant_expression for all 3 arguments
+	and return early if *jump_target after any of those calls.
+	(get_array_or_vector_nelts): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression.
+	(eval_and_check_array_index): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	each of them if *jump_target.
+	(cxx_eval_array_reference): Likewise.
+	(cxx_eval_component_reference): Likewise.
+	(cxx_eval_bit_field_ref): Likewise.
+	(cxx_eval_bit_cast): Likewise.  Assert CHECKING_P call doesn't
+	throw or return.
+	(cxx_eval_logical_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	each of them if *jump_target.
+	(cxx_eval_bare_aggregate): Likewise.
+	(cxx_eval_vec_init_1): Add jump_target argument.  Pass it through
+	to cxx_eval_bare_aggregate and recursive call.  Pass it through
+	to get_array_or_vector_nelts and cxx_eval_constant_expression
+	and return early after it if *jump_target.
+	(cxx_eval_vec_init): Add jump_target argument.  Pass it through
+	to cxx_eval_constant_expression and cxx_eval_vec_init_1.
+	(cxx_union_active_member): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression and return early after it
+	if *jump_target.
+	(cxx_fold_indirect_ref_1): Add jump_target argument.  Pass it
+	through to cxx_union_active_member and recursive calls.
+	(cxx_eval_indirect_ref): Add jump_target argument.  Pass it through
+	to cxx_fold_indirect_ref_1 calls and to recursive call, in which
+	case return early after it if *jump_target.
+	(cxx_fold_indirect_ref): Add jump_target argument.  Pass it through
+	to cxx_fold_indirect_ref and cxx_eval_constant_expression calls and
+	return early after those if *jump_target.
+	(cxx_eval_trinary_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	those if *jump_target.
+	(cxx_eval_store_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression and eval_and_check_array_index
+	calls and return early after those if *jump_target.
+	(cxx_eval_increment_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	those if *jump_target.
+	(label_matches): Handle VAR_DECL case.
+	(cxx_eval_statement_list): Remove local_target variable and
+	!jump_target handling.  Handle throws (jump_target) like returns or
+	breaks.
+	(cxx_eval_loop_expr): Remove local_target variable and !jump_target
+	handling.  Pass it through to cxx_eval_constant_expression.  Handle
+	throws (jump_target) like returns.
+	(cxx_eval_switch_expr): Pass jump_target through to
+	cxx_eval_constant_expression on cond, return early after it
+	if *jump_target.
+	(build_new_constexpr_heap_type): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls, return early after
+	those if *jump_target.
+	(merge_jump_target): New function.
+	(cxx_eval_constant_expression): Make jump_target argument no longer
+	defaulted, don't test jump_target for NULL.  Pass jump_target
+	through to recursive calls, cxx_eval_call_expression,
+	cxx_eval_store_expression, cxx_eval_indirect_ref,
+	cxx_eval_unary_expression, cxx_eval_binary_expression,
+	cxx_eval_logical_expression, cxx_eval_array_reference,
+	cxx_eval_component_reference, cxx_eval_bit_field_ref,
+	cxx_eval_vector_conditional_expression, cxx_eval_bare_aggregate,
+	cxx_eval_vec_init, cxx_eval_trinary_expression, cxx_fold_indirect_ref,
+	build_new_constexpr_heap_type, cxx_eval_increment_expression,
+	cxx_eval_bit_cast and return earlyu after some of those
+	if *jump_target as needed.
+	(cxx_eval_constant_expression) <case TARGET_EXPR>: For C++26 push
+	also CLEANUP_EH_ONLY cleanups, with NULL_TREE marker after them.
+	(cxx_eval_constant_expression) <case RETURN_EXPR>: Don't
+	override *jump_target if throws (jump_target).
+	(cxx_eval_constant_expression) <case TRY_CATCH_EXPR, case TRY_BLOCK,
+	case MUST_NOT_THROW_EXPR, case TRY_FINALLY_EXPR, case CLEANUP_STMT>:
+	Handle C++26 constant expressions.
+	(cxx_eval_constant_expression) <case CLEANUP_POINT_EXPR>: For C++26
+	with throws (jump_target) evaluate the CLEANUP_EH_ONLY cleanups as
+	well, and if not throws (jump_target) skip those.  Set *jump_target
+	if some of the cleanups threw.
+	(cxx_eval_constant_expression) <case THROW_EXPR>: Recurse on operand
+	for C++26.
+	(cxx_eval_outermost_constant_expr): Diagnose uncaught exceptions both
+	from main expression and cleanups, diagnose also
+	break/continue/returns from the main expression.  Handle
+	CLEANUP_EH_ONLY cleanup markers.  Don't diagnose mutable poison stuff
+	if non_constant_p.  Use different diagnostics for non-deleted heap
+	allocations if they were allocated by __cxa_allocate_exception.
+	(callee_might_throw): New function.
+	(struct check_for_return_continue_data): Add could_throw field.
+	(check_for_return_continue): Handle AGGR_INIT_EXPR and CALL_EXPR and
+	set d->could_throw if they could throw.
+	(potential_constant_expression_1): For CALL_EXPR allow
+	cxx_dynamic_cast_fn_p calls.  For C++26 set *jump_target to void_node
+	for calls that could throw.  For C++26 if call to non-constexpr call
+	is seen, try to evaluate arguments first and if they could throw,
+	don't diagnose call to non-constexpr function nor return false.
+	Adjust check_for_return_continue_data initializers and
+	set *jump_target to void_node if data.could_throw_p.  For C++26
+	recurse on THROW_EXPR argument.  Add comment explaining TRY_BLOCK
+	handling with C++26 exceptions.  Handle throws like returns in some
+	cases.
+	* cp-tree.h (MUST_NOT_THROW_NOEXCEPT_P, MUST_NOT_THROW_THROW_P,
+	MUST_NOT_THROW_CATCH_P, DECL_EXCEPTION_REFCOUNT): Define.
+	(DECL_LOCAL_DECL_P): Fix comment typo, VARIABLE_DECL -> VAR_DECL.
+	(enum cp_built_in_function): Add CP_BUILT_IN_EH_PTR_ADJUST_REF,
+	(handler_match_for_exception_type): Declare.
+	* call.cc (handler_match_for_exception_type): New function.
+	* except.cc (initialize_handler_parm): Set MUST_NOT_THROW_CATCH_P
+	on newly created MUST_NOT_THROW_EXPR.
+	(begin_eh_spec_block): Set MUST_NOT_THROW_NOEXCEPT_P.
+	(wrap_cleanups_r): Set MUST_NOT_THROW_THROW_P.
+	(build_throw): Add another TARGET_EXPR whose scope spans
+	until after the __cxa_throw call and copy pointer value from ptr
+	to it and use it in __cxa_throw argument.
+	* tree.cc (builtin_valid_in_constant_expr_p): Handle
+	CP_BUILT_IN_EH_PTR_ADJUST_REF.
+	* decl.cc (cxx_init_decl_processing): Initialize
+	__builtin_eh_ptr_adjust_ref FE builtin.
+	* pt.cc (tsubst_stmt) <case MUST_NOT_THROW_EXPR>: Copy the
+	MUST_NOT_THROW_NOEXCEPT_P, MUST_NOT_THROW_THROW_P and
+	MUST_NOT_THROW_CATCH_P flags.
+	* cp-gimplify.cc (cp_gimplify_expr) <case CALL_EXPR>: Error on
+	non-folded CP_BUILT_IN_EH_PTR_ADJUST_REF calls.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121012
+	PR c++/120917
+	* parser.cc (cp_parser_lambda_expression): Clear
+	parser->in_template_argument_list_p.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121008
+	PR c++/113563
+	* semantics.cc (finish_this_expr): Do check current_class_ref for
+	non-lambda.
+
+2025-07-09  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/119838
+	* parser.cc (cp_parser_nested_name_specifier_opt): New global_p
+	parameter.  Look for "template" when global_p is true.
+	(cp_parser_simple_type_specifier): Pass global_p to
+	cp_parser_nested_name_specifier_opt.
+
+2025-07-08  Marek Polacek  <polacek@redhat.com>
+	    Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c++/83469
+	PR c++/93809
+	* cp-tree.h (UNION_TYPE_P): Define.
+	(TYPENAME_IS_UNION_P): Define.
+	* decl.cc (struct typename_info): Add union_p field.
+	(struct typename_hasher::equal): Compare union_p field.
+	(build_typename_type): Use ti.union_p for union_type.  Set
+	TYPENAME_IS_UNION_P.
+	* error.cc (dump_type) <case TYPENAME_TYPE>: Handle
+	TYPENAME_IS_UNION_P.
+	* module.cc (trees_out::type_node): Likewise.
+	* parser.cc (cp_parser_check_class_key): Allow typename key for union
+	types and allow union keyword for typename types.
+	* pt.cc (tsubst) <case TYPENAME_TYPE>: Don't conflate unions with
+	class_type.  For TYPENAME_IS_CLASS_P, check NON_UNION_CLASS_TYPE_P
+	rather than CLASS_TYPE_P.  Add TYPENAME_IS_UNION_P handling.
+
+2025-07-08  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117784
+	* decl.cc: Implement part of C++26 P2686R4 - constexpr structured
+	bindings.
+	(cp_finish_decl): Pedwarn for C++23 and older on constinit on
+	structured bindings except for static/thread_local where it uses
+	earlier error.
+	(grokdeclarator): Pedwarn on constexpr structured bindings for
+	C++23 and older instead of emitting error always, don't clear
+	constexpr_p in that case.
+	* parser.cc (cp_parser_decomposition_declaration): Copy over
+	DECL_DECLARED_CONSTEXPR_P and DECL_DECLARED_CONSTINIT_P flags.
+
+2025-07-07  Alfie Richards  <alfie.richards@arm.com>
+
+	PR c++/119498
+	* decl.cc (duplicate_decls): Change logic to not always exclude FMV
+	annotated functions in cases of return type non-ambiguation.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* parser.cc (cp_parser_simple_type_specifier): Attach
+	auto in targ in parameter to -Wabbreviated-auto-in-template-arg.
+	(cp_parser_placeholder_type_specifier): Diagnose constrained auto in
+	template arg.
+
+2025-07-07  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/84009
+	* parser.cc (cp_parser_decomposition_declaration): Pedwarn
+	on thread_local, __thread or static in decl_specifiers for
+	for-range-declaration.
+	(cp_parser_init_declarator): Likewise, and also for extern
+	or register.
+
+2025-07-04  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120575
+	PR c++/116064
+	* parser.cc (cp_parser_abort_tentative_parse): Check seen_error
+	instead of errorcount.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120716
+	* lambda.cc (finish_lambda_function): Pass cur_stmt_list to
+	prune_lambda_captures.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120748
+	* lambda.cc (lambda_expr_this_capture): Don't return a FIELD_DECL.
+	* parser.cc (cp_parser_primary_expression): Ignore THIS_FORBIDDEN
+	if cp_unevaluated_operand.
+
+2025-07-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120940
+	* typeck.cc (cp_build_array_ref): Fix a pasto.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120684
+	PR c++/118856
+	* constexpr.cc (cxx_eval_constant_expression) [TARGET_EXPR]: Clear
+	the value first if is_complex.
+
 2025-07-01  Jakub Jelinek  <jakub@redhat.com>
 
 	PR c++/120471
diff --git a/gcc/cp/call.cc b/gcc/cp/call.cc
index 2c3ef3d..9283d97 100644
--- a/gcc/cp/call.cc
+++ b/gcc/cp/call.cc
@@ -220,7 +220,8 @@ static void add_candidates (tree, tree, const vec<tree, va_gc> *, tree, tree,
 			    bool, tree, tree, int, struct z_candidate **,
 			    tsubst_flags_t);
 static conversion *merge_conversion_sequences (conversion *, conversion *);
-static tree build_temp (tree, tree, int, diagnostic_t *, tsubst_flags_t);
+static tree build_temp (tree, tree, int, enum diagnostics::kind *,
+			tsubst_flags_t);
 static conversion *build_identity_conv (tree, tree);
 static inline bool conv_binds_to_array_of_unknown_bound (conversion *);
 static bool conv_is_prvalue (conversion *);
@@ -412,6 +413,7 @@ build_call_a (tree function, int n, tree *argarray)
 	      /* We're disconnecting the initializer from its target,
 		 don't create a temporary.  */
 	      arg = TARGET_EXPR_INITIAL (arg);
+	    suppress_warning (arg, OPT_Wunused_result);
 	    tree t = build0 (EMPTY_CLASS_EXPR, TREE_TYPE (arg));
 	    arg = build2 (COMPOUND_EXPR, TREE_TYPE (t), arg, t);
 	    CALL_EXPR_ARG (function, i) = arg;
@@ -1723,6 +1725,56 @@ involves_qualification_conversion_p (tree to, tree from)
   return false;
 }
 
+/* Return true if HANDLER is a match for exception object with EXCEPT_TYPE as
+   per [except.handle]/3.  */
+
+bool
+handler_match_for_exception_type (tree handler, tree except_type)
+{
+  tree handler_type = HANDLER_TYPE (handler);
+  if (handler_type == NULL_TREE)
+    return true; /* ... */
+  if (same_type_ignoring_top_level_qualifiers_p (handler_type, except_type))
+    return true;
+  if (CLASS_TYPE_P (except_type) && CLASS_TYPE_P (handler_type))
+    {
+      base_kind b_kind;
+      tree binfo = lookup_base (except_type, handler_type, ba_check, &b_kind,
+				tf_none);
+      if (binfo && binfo != error_mark_node)
+	return true;
+    }
+  if (TYPE_PTR_P (handler_type) || TYPE_PTRDATAMEM_P (handler_type))
+    {
+      if (TREE_CODE (except_type) == NULLPTR_TYPE)
+	return true;
+      if ((TYPE_PTR_P (handler_type) && TYPE_PTR_P (except_type))
+	  || (TYPE_PTRDATAMEM_P (handler_type)
+	      && TYPE_PTRDATAMEM_P (except_type)))
+	{
+	  conversion *conv
+	    = standard_conversion (handler_type, except_type, NULL_TREE,
+				   /*c_cast_p=*/false, 0, tf_none);
+	  if (conv && !conv->bad_p)
+	    {
+	      for (conversion *t = conv; t; t = next_conversion (t))
+		switch (t->kind)
+		  {
+		  case ck_ptr:
+		  case ck_fnptr:
+		  case ck_qual:
+		  case ck_identity:
+		    break;
+		  default:
+		    return false;
+		  }
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
 /* A reference of the indicated TYPE is being bound directly to the
    expression represented by the implicit conversion sequence CONV.
    Return a conversion sequence for this binding.  */
@@ -4876,6 +4928,11 @@ implicit_conversion_error (location_t loc, tree type, tree expr)
 	   && !CP_AGGREGATE_TYPE_P (type))
     error_at (loc, "designated initializers cannot be used with a "
 	      "non-aggregate type %qT", type);
+  else if (is_stub_object (expr))
+    /* The expression is generated by a trait check, we don't have
+       a useful location to highlight the label.  */
+    error_at (loc, "could not convert %qH to %qI",
+	      TREE_TYPE (expr), type);
   else
     {
       range_label_for_type_mismatch label (TREE_TYPE (expr), type);
@@ -6404,7 +6461,9 @@ build_conditional_expr (const op_location_t &loc,
 	       types when the enumeration is still being defined.  */;
 	  else if (complain & (cxx_dialect >= cxx26
 			       ? tf_warning_or_error : tf_warning))
-	    emit_diagnostic (cxx_dialect >= cxx26 ? DK_PEDWARN : DK_WARNING,
+	    emit_diagnostic ((cxx_dialect >= cxx26
+			      ? diagnostics::kind::pedwarn
+			      : diagnostics::kind::warning),
 			     loc, OPT_Wenum_compare, "enumerated mismatch "
 			     "in conditional expression: %qT vs %qT",
 			     arg2_type, arg3_type);
@@ -7435,7 +7494,16 @@ build_new_op (const op_location_t &loc, enum tree_code code, int flags,
       else if (TREE_CODE (cand->fn) == FUNCTION_DECL)
 	{
 	  if (overload)
-	    *overload = cand->fn;
+	    {
+	      if (cand->rewritten ())
+		/* build_min_non_dep_op_overload needs to know whether the
+		   candidate is rewritten/reversed.  */
+		*overload = build_tree_list (build_int_cst (integer_type_node,
+							    cand->flags),
+					     cand->fn);
+	      else
+		*overload = cand->fn;
+	    }
 
 	  if (resolve_args (arglist, complain) == NULL)
 	    result = error_mark_node;
@@ -7484,9 +7552,6 @@ build_new_op (const op_location_t &loc, enum tree_code code, int flags,
 	  /* If this was a C++20 rewritten comparison, adjust the result.  */
 	  if (cand->rewritten ())
 	    {
-	      /* FIXME build_min_non_dep_op_overload can't handle rewrites.  */
-	      if (overload)
-		*overload = NULL_TREE;
 	      switch (code)
 		{
 		case EQ_EXPR:
@@ -7585,8 +7650,9 @@ build_new_op (const op_location_t &loc, enum tree_code code, int flags,
 		      && (complain & tf_warning_or_error) == 0)
 		    result = error_mark_node;
 		  else if (cxx_dialect >= cxx26 || (complain & tf_warning))
-		    emit_diagnostic (cxx_dialect >= cxx26
-				     ? DK_PEDWARN : DK_WARNING,
+		    emit_diagnostic ((cxx_dialect >= cxx26
+				      ? diagnostics::kind::pedwarn
+				      : diagnostics::kind::warning),
 				     loc, OPT_Wenum_compare,
 				     "comparison between %q#T and %q#T",
 				     arg1_type, arg2_type);
@@ -8400,11 +8466,11 @@ complain_about_access (tree decl, tree diag_decl, tree diag_location,
 
 static tree
 build_temp (tree expr, tree type, int flags,
-	    diagnostic_t *diagnostic_kind, tsubst_flags_t complain)
+	    enum diagnostics::kind *diagnostic_kind, tsubst_flags_t complain)
 {
   int savew, savee;
 
-  *diagnostic_kind = DK_UNSPECIFIED;
+  *diagnostic_kind = diagnostics::kind::unspecified;
 
   /* If the source is a packed field, calling the copy constructor will require
      binding the field to the reference parameter to the copy constructor, and
@@ -8428,9 +8494,9 @@ build_temp (tree expr, tree type, int flags,
   expr = build_special_member_call (NULL_TREE, complete_ctor_identifier,
 				    &args, type, flags, complain);
   if (warningcount + werrorcount > savew)
-    *diagnostic_kind = DK_WARNING;
+    *diagnostic_kind = diagnostics::kind::warning;
   else if (errorcount > savee)
-    *diagnostic_kind = DK_ERROR;
+    *diagnostic_kind = diagnostics::kind::error;
   return expr;
 }
 
@@ -8642,9 +8708,10 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
 		       bool nested_p, tsubst_flags_t complain)
 {
   tree totype = convs->type;
-  diagnostic_t diag_kind;
+  enum diagnostics::kind diag_kind;
   int flags;
   location_t loc = cp_expr_loc_or_input_loc (expr);
+  const bool stub_object_p = is_stub_object (expr);
 
   if (convs->bad_p && !(complain & tf_error))
     return error_mark_node;
@@ -8721,7 +8788,10 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
 				      "from %qH to %qI", TREE_TYPE (expr),
 				      totype);
 	      if (complained)
-		print_z_candidate (loc, N_("candidate is:"), t->cand);
+		{
+		  auto_diagnostic_nesting_level sentinel;
+		  print_z_candidate (loc, N_("candidate is:"), t->cand);
+		}
 	      expr = convert_like (t, expr, fn, argnum,
 				   /*issue_conversion_warnings=*/false,
 				   /*c_cast_p=*/false, /*nested_p=*/true,
@@ -8746,7 +8816,14 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
 	  else if (t->kind == ck_identity)
 	    break;
 	}
-      if (!complained && expr != error_mark_node)
+      if (!complained && stub_object_p)
+	{
+	  /* An error diagnosed within a trait, don't give extra labels.  */
+	  error_at (loc, "invalid conversion from %qH to %qI",
+		    TREE_TYPE (expr), totype);
+	  complained = 1;
+	}
+      else if (!complained && expr != error_mark_node)
 	{
 	  range_label_for_type_mismatch label (TREE_TYPE (expr), totype);
 	  gcc_rich_location richloc (loc, &label, highlight_colors::percent_h);
@@ -9169,7 +9246,7 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
       if (convs->copy_init_p)
 	flags |= LOOKUP_ONLYCONVERTING;
       expr = build_temp (expr, totype, flags, &diag_kind, complain);
-      if (diag_kind && complain)
+      if (diag_kind != diagnostics::kind::unspecified && complain)
 	{
 	  auto_diagnostic_group d;
 	  maybe_print_user_conv_context (convs);
diff --git a/gcc/cp/class.cc b/gcc/cp/class.cc
index 9a41c00..14acb9c 100644
--- a/gcc/cp/class.cc
+++ b/gcc/cp/class.cc
@@ -1356,7 +1356,30 @@ add_method (tree type, tree method, bool via_using)
       if (!compparms (parms1, parms2))
 	continue;
 
-      if (!equivalently_constrained (fn, method))
+      tree fn_constraints = get_constraints (fn);
+      tree method_constraints = get_constraints (method);
+
+      if (fn_constraints && method_constraints
+	  && DECL_CONTEXT (fn) != type
+	  && !processing_template_decl)
+	{
+	  if (TREE_CODE (fn) == TEMPLATE_DECL)
+	    ++processing_template_decl;
+	  if (tree ti = CLASSTYPE_TEMPLATE_INFO (DECL_CONTEXT (fn)))
+	    fn_constraints = tsubst_constraint_info (fn_constraints,
+						     TI_ARGS (ti),
+						     tf_warning_or_error,
+						     fn);
+	  if (tree ti = CLASSTYPE_TEMPLATE_INFO (DECL_CONTEXT (method)))
+	    method_constraints = tsubst_constraint_info (method_constraints,
+							 TI_ARGS (ti),
+							 tf_warning_or_error,
+							 method);
+	  if (TREE_CODE (fn) == TEMPLATE_DECL)
+	    --processing_template_decl;
+	}
+
+      if (!equivalent_constraints (fn_constraints, method_constraints))
 	{
 	  if (processing_template_decl)
 	    /* We can't check satisfaction in dependent context, wait until
@@ -1407,7 +1430,7 @@ add_method (tree type, tree method, bool via_using)
       /* If these are versions of the same function, process and
 	 move on.  */
       if (TREE_CODE (fn) == FUNCTION_DECL
-	  && maybe_version_functions (method, fn, true))
+	  && maybe_version_functions (method, fn))
 	continue;
 
       if (DECL_INHERITED_CTOR (method))
@@ -7452,7 +7475,7 @@ determine_key_method (tree type)
 	&& ! DECL_DECLARED_INLINE_P (method)
 	&& ! DECL_PURE_VIRTUAL_P (method))
       {
-	CLASSTYPE_KEY_METHOD (type) = method;
+	SET_CLASSTYPE_KEY_METHOD (type, method);
 	break;
       }
 
diff --git a/gcc/cp/constexpr.cc b/gcc/cp/constexpr.cc
index f9066bc..142579a 100644
--- a/gcc/cp/constexpr.cc
+++ b/gcc/cp/constexpr.cc
@@ -153,7 +153,7 @@ static bool
 constexpr_error (location_t location, bool constexpr_fundef_p,
 		 const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
   va_start (ap, gmsgid);
@@ -161,14 +161,17 @@ constexpr_error (location_t location, bool constexpr_fundef_p,
   if (!constexpr_fundef_p)
     {
       /* Report an error that cannot be suppressed.  */
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_ERROR);
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::error);
       ret = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   else if (warn_invalid_constexpr)
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
-			   cxx_dialect < cxx23 ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Winvalid_constexpr;
+			   (cxx_dialect < cxx23
+			    ? diagnostics::kind::pedwarn
+			    : diagnostics::kind::warning));
+      diagnostic.m_option_id = OPT_Winvalid_constexpr;
       ret = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   else
@@ -1184,6 +1187,10 @@ public:
   /* Heap VAR_DECLs created during the evaluation of the outermost constant
      expression.  */
   auto_vec<tree, 16> heap_vars;
+  /* Vector of caught exceptions, including exceptions still not active at
+     the start of a handler (those are immediately followed up by HANDLER_TYPE
+     until __cxa_begin_catch finishes).  */
+  auto_vec<tree, 2> caught_exceptions;
   /* Cleanups that need to be evaluated at the end of CLEANUP_POINT_EXPR.  */
   vec<tree> *cleanups;
   /* If non-null, only allow modification of existing values of the variables
@@ -1191,10 +1198,13 @@ public:
   hash_set<tree> *modifiable;
   /* Number of heap VAR_DECL deallocations.  */
   unsigned heap_dealloc_count;
+  /* Number of uncaught exceptions.  */
+  unsigned uncaught_exceptions;
+
   /* Constructor.  */
   constexpr_global_ctx ()
     : constexpr_ops_count (0), cleanups (NULL), modifiable (nullptr),
-      heap_dealloc_count (0) {}
+      heap_dealloc_count (0), uncaught_exceptions (0) {}
 
   bool is_outside_lifetime (tree t)
   {
@@ -1308,6 +1318,48 @@ struct constexpr_ctx {
   mce_value manifestly_const_eval;
 };
 
+/* Predicates for the meaning of *jump_target.  */
+
+static bool
+returns (tree *jump_target)
+{
+  return *jump_target && TREE_CODE (*jump_target) == RETURN_EXPR;
+}
+
+static bool
+breaks (tree *jump_target)
+{
+  return (*jump_target
+	  && ((TREE_CODE (*jump_target) == LABEL_DECL
+	       && LABEL_DECL_BREAK (*jump_target))
+	      || TREE_CODE (*jump_target) == BREAK_STMT
+	      || TREE_CODE (*jump_target) == EXIT_EXPR));
+}
+
+static bool
+continues (tree *jump_target)
+{
+  return (*jump_target
+	  && ((TREE_CODE (*jump_target) == LABEL_DECL
+	       && LABEL_DECL_CONTINUE (*jump_target))
+	      || TREE_CODE (*jump_target) == CONTINUE_STMT));
+}
+
+static bool
+switches (tree *jump_target)
+{
+  return *jump_target && TREE_CODE (*jump_target) == INTEGER_CST;
+}
+
+static bool
+throws (tree *jump_target)
+{
+  /* void_node is for use in potential_constant_expression_1, otherwise
+     it should an artificial VAR_DECL created by constant evaluation
+     of __cxa_allocate_exception ().  */
+  return (*jump_target && (VAR_P (*jump_target) || *jump_target == void_node));
+}
+
 /* True if the constexpr relaxations afforded by P2280R4 for unknown
    references and objects are in effect.  */
 
@@ -1543,13 +1595,672 @@ enum value_cat {
 };
 
 static tree cxx_eval_constant_expression (const constexpr_ctx *, tree,
-					  value_cat, bool *, bool *, tree * = NULL);
+					  value_cat, bool *, bool *, tree *);
 static tree cxx_eval_bare_aggregate (const constexpr_ctx *, tree,
-				     value_cat, bool *, bool *);
+				     value_cat, bool *, bool *, tree *);
 static tree cxx_fold_indirect_ref (const constexpr_ctx *, location_t, tree, tree,
-				   bool * = NULL);
+				   bool *, tree *);
 static tree find_heap_var_refs (tree *, int *, void *);
 
+/* For exception object EXC if it has class type and usable what () method
+   which returns cv char * return the xmalloced string literal which it returns
+   if possible, otherwise return NULL.  */
+
+static char *
+exception_what_str (const constexpr_ctx *ctx, tree exc)
+{
+  tree type = strip_array_types (TREE_TYPE (exc));
+  if (!CLASS_TYPE_P (type))
+    return NULL;
+  tree std_exception = lookup_qualified_name (std_node, "exception",
+					      LOOK_want::NORMAL, false);
+  if (TREE_CODE (std_exception) != TYPE_DECL)
+    return NULL;
+  if (!CLASS_TYPE_P (TREE_TYPE (std_exception)))
+    return NULL;
+  base_kind b_kind;
+  tree binfo = lookup_base (type, TREE_TYPE (std_exception), ba_check, &b_kind,
+			    tf_none);
+  if (binfo == NULL_TREE || binfo == error_mark_node)
+    return NULL;
+  if (type != TREE_TYPE (exc))
+    exc = build4 (ARRAY_REF, type, exc, size_zero_node, NULL, NULL);
+  tree call
+    = finish_class_member_access_expr (exc, get_identifier ("what"), false,
+				       tf_none);
+  if (call == error_mark_node)
+    return NULL;
+  releasing_vec what_args;
+  call = finish_call_expr (call, &what_args, false, false, tf_none);
+  if (call == error_mark_node)
+    return NULL;
+  if (TREE_CODE (TREE_TYPE (call)) != POINTER_TYPE
+      || !INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (call)))
+      || !COMPLETE_TYPE_P (TREE_TYPE (TREE_TYPE (call)))
+      || !tree_int_cst_equal (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (call))),
+			      TYPE_SIZE_UNIT (char_type_node))
+      || TYPE_PRECISION (TREE_TYPE (TREE_TYPE (call))) != BITS_PER_UNIT)
+    return NULL;
+  if (!potential_constant_expression (call))
+    return NULL;
+  bool non_constant_p = false, overflow_p = false;
+  tree jmp_target = NULL;
+  tree ptr = cxx_eval_constant_expression (ctx, call, vc_prvalue,
+					   &non_constant_p, &overflow_p,
+					   &jmp_target);
+  if (throws (&jmp_target) || non_constant_p)
+    return NULL;
+  if (reduced_constant_expression_p (ptr))
+    if (const char *msg = c_getstr (ptr))
+      return xstrdup (msg);
+  auto_vec <char, 32> v;
+  for (unsigned i = 0; i < INT_MAX; ++i)
+    {
+      tree t = call;
+      if (i)
+	t = build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr), ptr, size_int (i));
+      t = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (t)), t);
+      non_constant_p = false;
+      overflow_p = false;
+      jmp_target = NULL;
+      tree t2 = cxx_eval_constant_expression (ctx, t, vc_prvalue,
+					      &non_constant_p, &overflow_p,
+					      &jmp_target);
+      if (throws (&jmp_target)
+	  || non_constant_p
+	  || !tree_fits_shwi_p (t2))
+	return NULL;
+      char c = tree_to_shwi (t2);
+      v.safe_push (c);
+      if (c == '\0')
+	break;
+    }
+  return xstrdup (v.address ());
+}
+
+/* Diagnose constant expression evaluation encountering call to
+   std::terminate due to exception EXC.  */
+
+static void
+diagnose_std_terminate (location_t loc, const constexpr_ctx *ctx, tree exc)
+{
+  tree type = strip_array_types (TREE_TYPE (exc));
+  if (char *str = exception_what_str (ctx, exc))
+    {
+      error_at (loc, "%qs called after throwing an exception of type %qT; "
+		     "%<what()%>: %qs", "std::terminate", type, str);
+      free (str);
+    }
+  else
+    {
+      if (type != TREE_TYPE (exc))
+	exc = build4 (ARRAY_REF, type, exc, size_zero_node, NULL, NULL);
+      bool non_constant_p = false, overflow_p = false;
+      tree jmp_target = NULL;
+      tree val = cxx_eval_constant_expression (ctx, exc, vc_prvalue,
+					       &non_constant_p, &overflow_p,
+					       &jmp_target);
+      gcc_assert (!throws (&jmp_target) && !non_constant_p);
+      if (reduced_constant_expression_p (val))
+	error_at (loc, "%qs called after throwing an exception %qE",
+		       "std::terminate", val);
+      else
+	error_at (loc, "%qs called after throwing an exception of type %qT",
+		       "std::terminate", type);
+    }
+}
+
+/* Diagnose constant expression evaluation encountering call to
+   uncaught exception EXC.  */
+
+static void
+diagnose_uncaught_exception (location_t loc, const constexpr_ctx *ctx, tree exc)
+{
+  tree type = strip_array_types (TREE_TYPE (exc));
+  if (char *str = exception_what_str (ctx, exc))
+    {
+      error_at (loc, "uncaught exception of type %qT; %<what()%>: %qs", type, str);
+      free (str);
+    }
+  else
+    {
+      if (type != TREE_TYPE (exc))
+	exc = build4 (ARRAY_REF, type, exc, size_zero_node, NULL, NULL);
+      bool non_constant_p = false, overflow_p = false;
+      tree jmp_target = NULL;
+      tree val = cxx_eval_constant_expression (ctx, exc, vc_prvalue,
+					       &non_constant_p, &overflow_p,
+					       &jmp_target);
+      gcc_assert (!throws (&jmp_target) && !non_constant_p);
+      if (reduced_constant_expression_p (val))
+	error_at (loc, "uncaught exception %qE", val);
+      else
+	error_at (loc, "uncaught exception of type %qT", type);
+    }
+}
+
+/* Kinds of __cxa_* functions (and a few other EH related ones) we handle as
+   magic constexpr functions for C++26.  */
+
+enum cxa_builtin {
+  CXA_NONE = 0,
+  CXA_ALLOCATE_EXCEPTION = 1,
+  CXA_FREE_EXCEPTION = 2,
+  CXA_THROW = 3,
+  CXA_BEGIN_CATCH = 4,
+  CXA_END_CATCH = 5,
+  CXA_RETHROW = 6,
+  CXA_GET_EXCEPTION_PTR = 7,
+  CXA_BAD_CAST = 8,
+  CXA_BAD_TYPEID = 9,
+  CXA_THROW_BAD_ARRAY_NEW_LENGTH = 10,
+  STD_UNCAUGHT_EXCEPTIONS = 11,
+  STD_CURRENT_EXCEPTION = 12,
+  STD_RETHROW_EXCEPTION = 13,
+  BUILTIN_EH_PTR_ADJUST_REF = 14
+};
+
+/* Return cxa_builtin if FNDECL is a __cxa_* function handled as
+   magic constexpr function for C++26.  Return CXA_NONE otherwise.  */
+
+static enum cxa_builtin
+cxx_cxa_builtin_fn_p (tree fndecl)
+{
+  if (cxx_dialect < cxx26)
+    return CXA_NONE;
+  if (DECL_LANGUAGE (fndecl) != lang_c)
+    {
+      if (!decl_in_std_namespace_p (fndecl))
+	return CXA_NONE;
+      if (id_equal (DECL_NAME (fndecl), "uncaught_exceptions"))
+	return STD_UNCAUGHT_EXCEPTIONS;
+      if (id_equal (DECL_NAME (fndecl), "current_exception"))
+	return STD_CURRENT_EXCEPTION;
+      if (id_equal (DECL_NAME (fndecl), "rethrow_exception"))
+	return STD_RETHROW_EXCEPTION;
+      return CXA_NONE;
+    }
+  if (!startswith (IDENTIFIER_POINTER (DECL_NAME (fndecl)), "__cxa_"))
+    return CXA_NONE;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_allocate_exception"))
+    return CXA_ALLOCATE_EXCEPTION;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_free_exception"))
+    return CXA_FREE_EXCEPTION;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_throw"))
+    return CXA_THROW;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_begin_catch"))
+    return CXA_BEGIN_CATCH;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_end_catch"))
+    return CXA_END_CATCH;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_rethrow"))
+    return CXA_RETHROW;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_get_exception_ptr"))
+    return CXA_GET_EXCEPTION_PTR;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_bad_cast"))
+    return CXA_BAD_CAST;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_bad_typeid"))
+    return CXA_BAD_TYPEID;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_throw_bad_array_new_length"))
+    return CXA_THROW_BAD_ARRAY_NEW_LENGTH;
+  return CXA_NONE;
+}
+
+/* Helper function for cxx_eval_cxa_builtin_fn.
+   Check if ARG is a valid first argument of __cxa_throw or
+   __cxa_free_exception or __builtin_eh_ptr_adjust_ref.  Return NULL_TREE if
+   not, otherwise return the artificial __cxa_allocate_exception allocated
+   VAR_DECL.  FREE_EXC is true for __cxa_free_exception, false otherwise.  */
+
+static tree
+cxa_check_throw_arg (tree arg, bool free_exc)
+{
+  STRIP_NOPS (arg);
+  if (TREE_CODE (arg) != ADDR_EXPR)
+    return NULL_TREE;
+  arg = TREE_OPERAND (arg, 0);
+  if (!VAR_P (arg)
+      || !DECL_ARTIFICIAL (arg)
+      || ((!free_exc || DECL_NAME (arg) != heap_uninit_identifier)
+	  && DECL_NAME (arg) != heap_identifier)
+      || !DECL_LANG_SPECIFIC (arg))
+    return NULL_TREE;
+  return arg;
+}
+
+/* Helper function for cxx_eval_cxa_builtin_fn.
+   "Allocate" on the constexpr heap an exception object of TYPE
+   with REFCOUNT.  */
+
+static tree
+cxa_allocate_exception (location_t loc, const constexpr_ctx *ctx, tree type,
+			tree refcount)
+{
+  tree var = build_decl (loc, VAR_DECL, heap_uninit_identifier, type);
+  DECL_ARTIFICIAL (var) = 1;
+  retrofit_lang_decl (var);
+  DECL_EXCEPTION_REFCOUNT (var) = refcount;
+  ctx->global->heap_vars.safe_push (var);
+  return var;
+}
+
+/* Evaluate various __cxa_* calls as magic constexpr builtins for
+   C++26 constexpr exception support (P3068R5).  */
+
+static tree
+cxx_eval_cxa_builtin_fn (const constexpr_ctx *ctx, tree call,
+			 enum cxa_builtin kind, tree fndecl,
+			 bool *non_constant_p, bool *overflow_p,
+			 tree *jump_target)
+{
+  int nargs = call_expr_nargs (call);
+  location_t loc = cp_expr_loc_or_input_loc (call);
+  tree args[4], arg;
+  if (nargs > 4)
+    {
+    invalid_nargs:
+      if (!ctx->quiet)
+	error_at (loc, "call to %qD function with incorrect"
+		  "number of arguments", fndecl);
+      *non_constant_p = true;
+      return call;
+    }
+  if ((kind == CXA_BEGIN_CATCH || kind == CXA_GET_EXCEPTION_PTR)
+      && nargs == 1
+      && (arg = CALL_EXPR_ARG (call, 0))
+      && TREE_CODE (arg) == CALL_EXPR
+      && call_expr_nargs (arg) == 1
+      && integer_zerop (CALL_EXPR_ARG (arg, 0)))
+    if (tree fun = get_function_named_in_call (arg))
+      if (fndecl_built_in_p (fun, BUILT_IN_EH_POINTER))
+	{
+	  if (ctx->global->caught_exceptions.length () < 2)
+	    {
+	    no_caught_exceptions:
+	      if (!ctx->quiet)
+		error_at (loc, "%qD called with no caught exceptions pending",
+			  fndecl);
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  /* Both __cxa_get_exception_ptr (__builtin_eh_pointer (0))
+	     and __cxa_begin_catch (__builtin_eh_pointer (0)) calls expect
+	     ctx->global->caught_exceptions vector to end with
+	     __cxa_allocate_exception created artificial VAR_DECL (the
+	     exception object) followed by handler type, pushed by TRY_BLOCK
+	     evaluation.  The only difference between the functions is that
+	     __cxa_begin_catch pops the handler type from the vector and keeps
+	     the VAR_DECL last and decreases uncaught_exceptions.  The
+	     VAR_DECL after __cxa_begin_catch serves as the current exception
+	     and is then popped in __cxa_end_catch evaluation.  */
+	  tree handler_type = ctx->global->caught_exceptions.last ();
+	  if (handler_type && VAR_P (handler_type))
+	    goto no_caught_exceptions;
+	  unsigned idx = ctx->global->caught_exceptions.length () - 2;
+	  arg = ctx->global->caught_exceptions[idx];
+	  gcc_assert (VAR_P (arg));
+	  if (kind == CXA_BEGIN_CATCH)
+	    {
+	      ctx->global->caught_exceptions.pop ();
+	      --ctx->global->uncaught_exceptions;
+	    }
+	  if (handler_type == NULL_TREE)
+	    /* Used for catch (...).  Just return void.  */
+	    return void_node;
+	  else if (POINTER_TYPE_P (handler_type))
+	    {
+	      /* Used for catch of a pointer.  */
+	      if (TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE)
+		arg = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (arg)), arg,
+			      size_zero_node, NULL_TREE, NULL_TREE);
+	      arg = cp_convert (handler_type, arg,
+				ctx->quiet ? tf_none : tf_warning_or_error);
+	      if (arg == error_mark_node)
+		{
+		  *non_constant_p = true;
+		  return call;
+		}
+	    }
+	  else
+	    {
+	      /* Used for catch of a non-pointer type.  */
+	      tree exc_type = strip_array_types (TREE_TYPE (arg));
+	      tree exc_ptr_type = build_pointer_type (exc_type);
+	      arg = build_fold_addr_expr_with_type (arg, exc_ptr_type);
+	      if (CLASS_TYPE_P (handler_type))
+		{
+		  tree ptr_type = build_pointer_type (handler_type);
+		  arg = cp_convert (ptr_type, arg,
+				    ctx->quiet ? tf_none
+				    : tf_warning_or_error);
+		  if (arg == error_mark_node)
+		    {
+		      *non_constant_p = true;
+		      return call;
+		    }
+		}
+	    }
+	  return cxx_eval_constant_expression (ctx, arg, vc_prvalue,
+					       non_constant_p, overflow_p,
+					       jump_target);
+	}
+  for (int i = 0; i < nargs; ++i)
+    {
+      args[i] = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (call, i),
+					      vc_prvalue, non_constant_p,
+					      overflow_p, jump_target);
+      if (*non_constant_p)
+	return call;
+      if (*jump_target)
+	return NULL_TREE;
+    }
+  switch (kind)
+    {
+    case CXA_ALLOCATE_EXCEPTION:
+      if (nargs != 1)
+	goto invalid_nargs;
+      if (!tree_fits_uhwi_p (args[0]))
+	{
+	  if (!ctx->quiet)
+	    error_at (loc, "cannot allocate exception: size not constant");
+	  *non_constant_p = true;
+	  return call;
+	}
+      else
+	{
+	  tree type = build_array_type_nelts (char_type_node,
+					      tree_to_uhwi (args[0]));
+	  tree var = cxa_allocate_exception (loc, ctx, type, size_zero_node);
+	  ctx->global->put_value (var, NULL_TREE);
+	  return fold_convert (ptr_type_node, build_address (var));
+	}
+    case CXA_FREE_EXCEPTION:
+      if (nargs != 1)
+	goto invalid_nargs;
+      arg = cxa_check_throw_arg (args[0], true);
+      if (arg == NULL_TREE)
+	{
+	invalid_ptr:
+	  if (!ctx->quiet)
+	    error_at (loc, "first argument to %qD function not result of "
+		      "%<__cxa_allocate_exception%>", fndecl);
+	  *non_constant_p = true;
+	  return call;
+	}
+      DECL_NAME (arg) = heap_deleted_identifier;
+      ctx->global->destroy_value (arg);
+      ctx->global->heap_dealloc_count++;
+      return void_node;
+    case CXA_THROW:
+      if (nargs != 3)
+	goto invalid_nargs;
+      arg = cxa_check_throw_arg (args[0], false);
+      if (arg == NULL_TREE)
+	goto invalid_ptr;
+      DECL_EXCEPTION_REFCOUNT (arg)
+	= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+		      size_one_node);
+      ++ctx->global->uncaught_exceptions;
+      *jump_target = arg;
+      return void_node;
+    case CXA_BEGIN_CATCH:
+    case CXA_GET_EXCEPTION_PTR:
+      goto invalid_nargs;
+    case CXA_END_CATCH:
+      if (nargs != 0)
+	goto invalid_nargs;
+      if (ctx->global->caught_exceptions.is_empty ())
+	{
+	no_active_exc:
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with no caught exceptions active",
+		      fndecl);
+	  *non_constant_p = true;
+	  return call;
+	}
+      else
+	{
+	  arg = ctx->global->caught_exceptions.pop ();
+	  if (arg == NULL_TREE || !VAR_P (arg))
+	    goto no_active_exc;
+	free_except:
+	  DECL_EXCEPTION_REFCOUNT (arg)
+	    = size_binop (MINUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+			  size_one_node);
+	  if (integer_zerop (DECL_EXCEPTION_REFCOUNT (arg)))
+	    {
+	      if (type_build_dtor_call (TREE_TYPE (arg)))
+		{
+		  tree cleanup
+		    = cxx_maybe_build_cleanup (arg, (ctx->quiet ? tf_none
+						     : tf_warning_or_error));
+		  if (cleanup == error_mark_node)
+		    *non_constant_p = true;
+		  tree jmp_target = NULL_TREE;
+		  cxx_eval_constant_expression (ctx, cleanup, vc_discard,
+						non_constant_p, overflow_p,
+						&jmp_target);
+		  if (throws (&jmp_target))
+		    *jump_target = jmp_target;
+		}
+	      DECL_NAME (arg) = heap_deleted_identifier;
+	      ctx->global->destroy_value (arg);
+	      ctx->global->heap_dealloc_count++;
+	    }
+	}
+      return void_node;
+    case CXA_RETHROW:
+      if (nargs != 0)
+	goto invalid_nargs;
+      unsigned idx;
+      FOR_EACH_VEC_ELT_REVERSE (ctx->global->caught_exceptions, idx, arg)
+	if (arg == NULL_TREE || !VAR_P (arg))
+	  --idx;
+	else
+	  break;
+      if (arg == NULL_TREE)
+	{
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with no caught exceptions active",
+		      fndecl);
+	  *non_constant_p = true;
+	  return call;
+	}
+      DECL_EXCEPTION_REFCOUNT (arg)
+	= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg), size_one_node);
+      ++ctx->global->uncaught_exceptions;
+      *jump_target = arg;
+      return void_node;
+    case CXA_BAD_CAST:
+    case CXA_BAD_TYPEID:
+    case CXA_THROW_BAD_ARRAY_NEW_LENGTH:
+      if (nargs != 0)
+	goto invalid_nargs;
+      else
+	{
+	  tree name;
+	  switch (kind)
+	    {
+	    case CXA_BAD_CAST:
+	      name = get_identifier ("bad_cast");
+	      break;
+	    case CXA_BAD_TYPEID:
+	      name = get_identifier ("bad_typeid");
+	      break;
+	    case CXA_THROW_BAD_ARRAY_NEW_LENGTH:
+	      name = get_identifier ("bad_array_new_length");
+	      break;
+	    default:
+	      gcc_unreachable ();
+	    }
+	  tree decl = lookup_qualified_name (std_node, name);
+	  if (TREE_CODE (decl) != TYPE_DECL
+	      || !CLASS_TYPE_P (TREE_TYPE (decl))
+	      || !type_build_ctor_call (TREE_TYPE (decl)))
+	    {
+	      if (!ctx->quiet)
+		error_at (loc, "%qD called without %<std::%D%> being defined",
+			  fndecl, name);
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  tree type = TREE_TYPE (decl);
+	  tree var = cxa_allocate_exception (loc, ctx, type, size_one_node);
+	  tree ctor
+	    = build_special_member_call (var, complete_ctor_identifier,
+					 NULL, type, LOOKUP_NORMAL,
+					 ctx->quiet ? tf_none
+					 : tf_warning_or_error);
+	  if (ctor == error_mark_node)
+	    {
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  if (TREE_CONSTANT (ctor))
+	    ctx->global->put_value (var, ctor);
+	  else
+	    {
+	      ctx->global->put_value (var, NULL_TREE);
+	      cxx_eval_constant_expression (ctx, ctor, vc_discard,
+					    non_constant_p, overflow_p,
+					    jump_target);
+	      if (*non_constant_p)
+		return call;
+	      if (throws (jump_target))
+		return NULL_TREE;
+	    }
+	  ++ctx->global->uncaught_exceptions;
+	  *jump_target = var;
+	}
+      return void_node;
+    case STD_UNCAUGHT_EXCEPTIONS:
+      if (nargs != 0)
+	goto invalid_nargs;
+      /* Similarly to __builtin_is_constant_evaluated (), we don't
+	 want to give a definite answer during mce_unknown evaluation,
+	 because that might prevent evaluation later on when some
+	 exceptions might be uncaught.  But unlike that, we don't
+	 want to constant fold it even during cp_fold, because at runtime
+	 std::uncaught_exceptions () might still be non-zero.  */
+      if (ctx->manifestly_const_eval != mce_true)
+	{
+	  *non_constant_p = true;
+	  return call;
+	}
+      return build_int_cst (integer_type_node,
+			    ctx->global->uncaught_exceptions);
+    case STD_CURRENT_EXCEPTION:
+      if (nargs != 0)
+	goto invalid_nargs;
+      else
+	{
+	  tree name = get_identifier ("exception_ptr");
+	  tree decl = lookup_qualified_name (std_node, name);
+	  tree fld;
+	  if (TREE_CODE (decl) != TYPE_DECL
+	      || !CLASS_TYPE_P (TREE_TYPE (decl))
+	      || !COMPLETE_TYPE_P (TREE_TYPE (decl))
+	      || !(fld = next_aggregate_field (TYPE_FIELDS (TREE_TYPE (decl))))
+	      || DECL_ARTIFICIAL (fld)
+	      || TREE_CODE (TREE_TYPE (fld)) != POINTER_TYPE
+	      || next_aggregate_field (DECL_CHAIN (fld))
+	      || !tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (decl)),
+				      TYPE_SIZE (TREE_TYPE (fld))))
+	    {
+	      if (!ctx->quiet)
+		error_at (loc, "%qD called without supportable %qs",
+			  fndecl, "std::exception_ptr");
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  FOR_EACH_VEC_ELT_REVERSE (ctx->global->caught_exceptions, idx, arg)
+	    if (arg == NULL_TREE || !VAR_P (arg))
+	      --idx;
+	    else
+	      break;
+	  /* Similarly to __builtin_is_constant_evaluated (), we don't
+	     want to give a definite answer during mce_unknown evaluation,
+	     because that might prevent evaluation later on when some
+	     exceptions might be current.  But unlike that, we don't
+	     want to constant fold it to null even during cp_fold, because
+	     at runtime std::current_exception () might still be non-null.  */
+	  if (ctx->manifestly_const_eval != mce_true && arg == NULL_TREE)
+	    {
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  if (arg == NULL_TREE)
+	    arg = build_zero_cst (TREE_TYPE (fld));
+	  else
+	    {
+	      DECL_EXCEPTION_REFCOUNT (arg)
+		= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+			      size_one_node);
+	      arg = fold_convert (ptr_type_node, build_address (arg));
+	    }
+	  return build_constructor_single (TREE_TYPE (decl), fld, arg);
+	}
+    case STD_RETHROW_EXCEPTION:
+      if (nargs != 1)
+	goto invalid_nargs;
+      if (TYPE_REF_P (TREE_TYPE (args[0])))
+	{
+	  arg = args[0];
+	  STRIP_NOPS (arg);
+	  if (TREE_CODE (arg) == ADDR_EXPR)
+	    {
+	      args[0]
+		= cxx_eval_constant_expression (ctx, TREE_OPERAND (arg, 0),
+						vc_prvalue, non_constant_p,
+						overflow_p, jump_target);
+	      if (*non_constant_p)
+		return call;
+	      if (*jump_target)
+		return NULL_TREE;
+	    }
+	}
+      if (TREE_CODE (args[0]) != CONSTRUCTOR
+	  || CONSTRUCTOR_NELTS (args[0]) != 1)
+	{
+	invalid_std_rethrow:
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with unexpected %qs argument",
+		      fndecl, "std::exception_ptr");
+	  *non_constant_p = true;
+	  return void_node;
+	}
+      arg = cxa_check_throw_arg (CONSTRUCTOR_ELT (args[0], 0)->value, false);
+      if (arg == NULL_TREE)
+	goto invalid_std_rethrow;
+      DECL_EXCEPTION_REFCOUNT (arg)
+	= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg), size_one_node);
+      ++ctx->global->uncaught_exceptions;
+      *jump_target = arg;
+      return void_node;
+    case BUILTIN_EH_PTR_ADJUST_REF:
+      if (nargs != 2)
+	goto invalid_nargs;
+      arg = cxa_check_throw_arg (args[0], false);
+      if (arg == NULL_TREE)
+	goto invalid_ptr;
+      if (integer_onep (args[1]))
+	DECL_EXCEPTION_REFCOUNT (arg)
+	  = size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+			size_one_node);
+      else if (integer_minus_onep (args[1]))
+	goto free_except;
+      else
+	{
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with second argument "
+		      "other than 1 or -1", fndecl);
+	  *non_constant_p = true;
+	}
+      return void_node;
+    default:
+      gcc_unreachable ();
+    }
+}
+
 /* Attempt to evaluate T which represents a call to a builtin function.
    We assume here that all builtin functions evaluate to scalar types
    represented by _CST nodes.  */
@@ -1557,7 +2268,8 @@ static tree find_heap_var_refs (tree *, int *, void *);
 static tree
 cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
 				value_cat lval,
-				bool *non_constant_p, bool *overflow_p)
+				bool *non_constant_p, bool *overflow_p,
+				tree *jump_target)
 {
   const int nargs = call_expr_nargs (t);
   tree *args = (tree *) alloca (nargs * sizeof (tree));
@@ -1603,6 +2315,12 @@ cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
       return fold_builtin_source_location (t);
     }
 
+  if (fndecl_built_in_p (fun, CP_BUILT_IN_EH_PTR_ADJUST_REF,
+			 BUILT_IN_FRONTEND))
+    return cxx_eval_cxa_builtin_fn (ctx, t, BUILTIN_EH_PTR_ADJUST_REF,
+				    fun, non_constant_p, overflow_p,
+				    jump_target);
+
   int strops = 0;
   int strret = 0;
   if (fndecl_built_in_p (fun, BUILT_IN_NORMAL))
@@ -1677,8 +2395,14 @@ cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
 	  || potential_constant_expression (arg))
 	{
 	  bool dummy1 = false, dummy2 = false;
+	  tree jmp_target = NULL_TREE;
 	  arg = cxx_eval_constant_expression (&new_ctx, arg, vc_prvalue,
-					      &dummy1, &dummy2);
+					      &dummy1, &dummy2, &jmp_target);
+	  if (jmp_target)
+	    {
+	      *jump_target = jmp_target;
+	      return NULL_TREE;
+	    }
 	}
 
       if (bi_const_p)
@@ -1767,7 +2491,8 @@ cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
     }
 
   return cxx_eval_constant_expression (&new_ctx, new_call, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 }
 
 /* TEMP is the constant value of a temporary object of type TYPE.  Adjust
@@ -1882,7 +2607,8 @@ addr_of_non_const_var (tree *tp, int *walk_subtrees, void *data)
 static tree
 cxx_bind_parameters_in_call (const constexpr_ctx *ctx, tree t, tree fun,
 			     tree orig_fun, bool *non_constant_p,
-			     bool *overflow_p, bool *non_constant_args)
+			     bool *overflow_p, bool *non_constant_args,
+			     tree *jump_target)
 {
   int nargs = call_expr_nargs (t);
   tree parms = DECL_ARGUMENTS (fun);
@@ -1958,14 +2684,16 @@ cxx_bind_parameters_in_call (const constexpr_ctx *ctx, tree t, tree fun,
 	  /* Undo convert_for_arg_passing work here.  */
 	  x = convert_from_reference (x);
 	  arg = cxx_eval_constant_expression (ctx, x, vc_glvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
 	}
       else
 	/* Normally we would strip a TARGET_EXPR in an initialization context
 	   such as this, but here we do the elision differently: we keep the
 	   TARGET_EXPR, and use its CONSTRUCTOR as the value of the parm.  */
 	arg = cxx_eval_constant_expression (ctx, x, vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
       /* Check we aren't dereferencing a null pointer when calling a non-static
 	 member function, which is undefined behaviour.  */
       if (i == 0 && DECL_OBJECT_MEMBER_FUNCTION_P (fun)
@@ -1983,6 +2711,8 @@ cxx_bind_parameters_in_call (const constexpr_ctx *ctx, tree t, tree fun,
       /* Don't VERIFY_CONSTANT here.  */
       if (*non_constant_p && ctx->quiet)
 	break;
+      if (*jump_target)
+	break;
       /* Just discard ellipsis args after checking their constantitude.  */
       if (!parms)
 	continue;
@@ -2094,9 +2824,10 @@ fold_operand (tree e, const constexpr_ctx *ctx)
   if (ctx)
     {
       bool new_non_constant_p = false, new_overflow_p = false;
+      tree jmp_target = NULL_TREE;
       e = cxx_eval_constant_expression (ctx, e, vc_prvalue,
 					&new_non_constant_p,
-					&new_overflow_p);
+					&new_overflow_p, &jmp_target);
     }
   else
     e = fold_non_dependent_expr (e, tf_none, /*manifestly_const_eval=*/true);
@@ -2155,10 +2886,15 @@ diagnose_failing_condition (tree bad, location_t cloc, bool show_expr_p,
   if (TREE_CODE (bad) == CLEANUP_POINT_EXPR)
     bad = TREE_OPERAND (bad, 0);
 
+  auto_diagnostic_nesting_level sentinel;
+
   /* Actually explain the failure if this is a concept check or a
      requires-expression.  */
   if (concept_check_p (bad) || TREE_CODE (bad) == REQUIRES_EXPR)
     diagnose_constraints (cloc, bad, NULL_TREE);
+  /* Similarly if this is a standard trait.  */
+  else if (maybe_diagnose_standard_trait (cloc, bad))
+    ;
   else if (COMPARISON_CLASS_P (bad)
 	   && ARITHMETIC_TYPE_P (TREE_TYPE (TREE_OPERAND (bad, 0))))
     {
@@ -2183,7 +2919,7 @@ cxx_eval_assert (const constexpr_ctx *ctx, tree arg, const char *msg,
   if (*non_constant_p)
     return true;
 
-  tree eval;
+  tree eval, jmp_target = NULL_TREE;
   if (!evaluated)
     {
       if (!potential_rvalue_constant_expression (arg))
@@ -2196,12 +2932,15 @@ cxx_eval_assert (const constexpr_ctx *ctx, tree arg, const char *msg,
       modifiable_tracker ms (new_ctx.global);
       eval = cxx_eval_constant_expression (&new_ctx, arg, vc_prvalue,
 					   &new_non_constant_p,
-					   &new_overflow_p);
+					   &new_overflow_p, &jmp_target);
     }
   else
     eval = cxx_eval_constant_expression (ctx, arg, vc_prvalue,
 					 non_constant_p,
-					 overflow_p);
+					 overflow_p, &jmp_target);
+  if (jmp_target)
+    return true;
+
   if (!*non_constant_p && integer_zerop (eval))
     {
       if (!ctx->quiet)
@@ -2233,7 +2972,8 @@ cxx_eval_assert (const constexpr_ctx *ctx, tree arg, const char *msg,
 static tree
 cxx_eval_internal_function (const constexpr_ctx *ctx, tree t,
 			    value_cat lval,
-			    bool *non_constant_p, bool *overflow_p)
+			    bool *non_constant_p, bool *overflow_p,
+			    tree *jump_target)
 {
   enum tree_code opcode = ERROR_MARK;
 
@@ -2266,13 +3006,15 @@ cxx_eval_internal_function (const constexpr_ctx *ctx, tree t,
     case IFN_LAUNDER:
       return cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 0),
 					   vc_prvalue, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
 
     case IFN_VEC_CONVERT:
       {
 	tree arg = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 0),
 						 vc_prvalue, non_constant_p,
-						 overflow_p);
+						 overflow_p, jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	if (TREE_CODE (arg) == VECTOR_CST)
 	  if (tree r = fold_const_call (CFN_VEC_CONVERT, TREE_TYPE (t), arg))
 	    return r;
@@ -2290,10 +3032,13 @@ cxx_eval_internal_function (const constexpr_ctx *ctx, tree t,
   /* Evaluate constant arguments using OPCODE and return a complex
      number containing the result and the overflow bit.  */
   tree arg0 = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 0), lval,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
   tree arg1 = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 1), lval,
-					    non_constant_p, overflow_p);
-
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (TREE_CODE (arg0) == INTEGER_CST && TREE_CODE (arg1) == INTEGER_CST)
     {
       location_t loc = cp_expr_loc_or_input_loc (t);
@@ -2495,7 +3240,11 @@ cxx_dynamic_cast_fn_p (tree fndecl)
 {
   return (cxx_dialect >= cxx20
 	  && id_equal (DECL_NAME (fndecl), "__dynamic_cast")
-	  && CP_DECL_CONTEXT (fndecl) == abi_node);
+	  && CP_DECL_CONTEXT (fndecl) == abi_node
+	  /* Only consider implementation-detail __dynamic_cast calls that
+	     correspond to a dynamic_cast, and ignore direct calls to
+	     abi::__dynamic_cast.  */
+	  && DECL_ARTIFICIAL (fndecl));
 }
 
 /* Often, we have an expression in the form of address + offset, e.g.
@@ -2566,7 +3315,8 @@ get_component_with_type (tree path, tree type, tree stop)
 
 static tree
 cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
-			  bool *non_constant_p, bool *overflow_p)
+			  bool *non_constant_p, bool *overflow_p,
+			  tree *jump_target)
 {
   /* T will be something like
       __dynamic_cast ((B*) b, &_ZTI1B, &_ZTI1D, 8)
@@ -2585,19 +3335,26 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
 
   /* TYPE can only be either T* or T&.  We can't know which of these it
      is by looking at TYPE, but OBJ will be "(T*) x" in the first case,
-     and something like "(T*)(T&)(T*) x" in the second case.  */
-  bool reference_p = false;
+     and something like "(T*)(T&)(T*) x" in the second case.
+     This is true for the reference cases in C++ < 26 or when exceptions
+     aren't enabled, in that case we should diagnose errors.  For C++26
+     with exceptions we should silently evaluate to null pointer and
+     let the callers call __cxa_bad_cast () later to throw an exception.  */
+  bool fail_for_non_constant_p = false;
   while (CONVERT_EXPR_P (obj) || TREE_CODE (obj) == SAVE_EXPR)
     {
-      reference_p |= TYPE_REF_P (TREE_TYPE (obj));
+      if (cxx_dialect < cxx26 || !flag_exceptions)
+	fail_for_non_constant_p |= TYPE_REF_P (TREE_TYPE (obj));
       obj = TREE_OPERAND (obj, 0);
     }
 
   /* Evaluate the object so that we know its dynamic type.  */
   obj = cxx_eval_constant_expression (ctx, obj, vc_prvalue, non_constant_p,
-				      overflow_p);
+				      overflow_p, jump_target);
   if (*non_constant_p)
     return call;
+  if (*jump_target)
+    return NULL_TREE;
 
   /* For dynamic_cast from classes with virtual bases we can get something
      like (virt_base *)(&d + 16) as OBJ.  Try to convert that into
@@ -2609,7 +3366,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
       if (TREE_CODE (objo) == POINTER_PLUS_EXPR)
 	{
 	  objo = cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (TREE_TYPE (obj)),
-					obj);
+					obj, NULL, jump_target);
 	  if (objo)
 	    obj = build_fold_addr_expr (objo);
 	}
@@ -2625,7 +3382,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
 		? TREE_OPERAND (obj, 1) : obj))
     if (TREE_CODE (t) != FIELD_DECL || !DECL_FIELD_IS_BASE (t))
       {
-	if (reference_p)
+	if (fail_for_non_constant_p)
 	  {
 	    if (!ctx->quiet)
 	      {
@@ -2647,9 +3404,12 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
      constructor or destructor's class.  */
   tree vtable = build_vfield_ref (obj, objtype);
   vtable = cxx_eval_constant_expression (ctx, vtable, vc_prvalue,
-					 non_constant_p, overflow_p);
+					 non_constant_p, overflow_p,
+					 jump_target);
   if (*non_constant_p)
     return call;
+  if (*jump_target)
+    return NULL_TREE;
   /* With -fsanitize=vptr, we initialize all vtable pointers to null,
      so it's possible that we got a null pointer now.  */
   if (integer_zerop (vtable))
@@ -2681,7 +3441,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
       /* If not accessible, give an error.  */
       if (t == error_mark_node)
 	{
-	  if (reference_p)
+	  if (fail_for_non_constant_p)
 	    {
 	      if (!ctx->quiet)
 		{
@@ -2714,7 +3474,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
   obj = get_component_with_type (obj, mdtype, NULL_TREE);
   if (obj == error_mark_node)
     {
-      if (reference_p)
+      if (fail_for_non_constant_p)
 	{
 	  if (!ctx->quiet)
 	    {
@@ -2736,7 +3496,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
   tree binfo = lookup_base (mdtype, type, ba_check, &b_kind, tf_none);
   if (!binfo || binfo == error_mark_node)
     {
-      if (reference_p)
+      if (fail_for_non_constant_p)
 	{
 	  if (!ctx->quiet)
 	    {
@@ -2832,7 +3592,7 @@ replace_decl (tree *tp, tree decl, tree replacement)
 static tree
 cxx_eval_thunk_call (const constexpr_ctx *ctx, tree t, tree thunk_fndecl,
 		     value_cat lval,
-		     bool *non_constant_p, bool *overflow_p)
+		     bool *non_constant_p, bool *overflow_p, tree *jump_target)
 {
   tree function = THUNK_TARGET (thunk_fndecl);
 
@@ -2875,7 +3635,8 @@ cxx_eval_thunk_call (const constexpr_ctx *ctx, tree t, tree thunk_fndecl,
 		       new_call, offset);
 
   return cxx_eval_constant_expression (ctx, new_call, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 }
 
 /* If OBJECT is of const class type, evaluate it to a CONSTRUCTOR and set
@@ -2885,7 +3646,7 @@ cxx_eval_thunk_call (const constexpr_ctx *ctx, tree t, tree thunk_fndecl,
 static void
 cxx_set_object_constness (const constexpr_ctx *ctx, tree object,
 			  bool readonly_p, bool *non_constant_p,
-			  bool *overflow_p)
+			  bool *overflow_p, tree *jump_target)
 {
   if (CLASS_TYPE_P (TREE_TYPE (object))
       && CP_TYPE_CONST_P (TREE_TYPE (object)))
@@ -2893,8 +3654,11 @@ cxx_set_object_constness (const constexpr_ctx *ctx, tree object,
       /* Subobjects might not be stored in ctx->global->values but we
 	 can get its CONSTRUCTOR by evaluating *this.  */
       tree e = cxx_eval_constant_expression (ctx, object, vc_prvalue,
-					     non_constant_p, overflow_p);
-      if (TREE_CODE (e) == CONSTRUCTOR && !*non_constant_p)
+					     non_constant_p, overflow_p,
+					     jump_target);
+      if (!*non_constant_p
+	  && !throws (jump_target)
+	  && TREE_CODE (e) == CONSTRUCTOR)
 	TREE_READONLY (e) = readonly_p;
     }
 }
@@ -2906,20 +3670,25 @@ cxx_set_object_constness (const constexpr_ctx *ctx, tree object,
 static tree
 cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 			  value_cat lval,
-			  bool *non_constant_p, bool *overflow_p)
+			  bool *non_constant_p, bool *overflow_p,
+			  tree *jump_target)
 {
   location_t loc = cp_expr_loc_or_input_loc (t);
   tree fun = get_function_named_in_call (t);
 
   if (fun == NULL_TREE)
     return cxx_eval_internal_function (ctx, t, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 
   if (TREE_CODE (fun) != FUNCTION_DECL)
     {
       /* Might be a constexpr function pointer.  */
       fun = cxx_eval_constant_expression (ctx, fun, vc_prvalue,
-					  non_constant_p, overflow_p);
+					  non_constant_p, overflow_p,
+					  jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       STRIP_NOPS (fun);
       if (TREE_CODE (fun) == ADDR_EXPR)
 	fun = TREE_OPERAND (fun, 0);
@@ -2971,9 +3740,12 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 
   if (fndecl_built_in_p (fun))
     return cxx_eval_builtin_function_call (ctx, t, fun,
-					   lval, non_constant_p, overflow_p);
+					   lval, non_constant_p, overflow_p,
+					   jump_target);
   if (DECL_THUNK_P (fun))
-    return cxx_eval_thunk_call (ctx, t, fun, lval, non_constant_p, overflow_p);
+    return cxx_eval_thunk_call (ctx, t, fun, lval, non_constant_p, overflow_p,
+				jump_target);
+  bool non_constexpr_call = false;
   if (!maybe_constexpr_fn (fun))
     {
       if (TREE_CODE (t) == CALL_EXPR
@@ -2988,7 +3760,10 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	    {
 	      tree arg = CALL_EXPR_ARG (t, i);
 	      arg = cxx_eval_constant_expression (ctx, arg, vc_prvalue,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
+	      if (*jump_target)
+		return NULL_TREE;
 	      /* Deleting a non-constant pointer has a better error message
 		 below.  */
 	      if (new_op_p || i != 0)
@@ -3103,7 +3878,10 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	    {
 	      tree arg = CALL_EXPR_ARG (t, i);
 	      arg = cxx_eval_constant_expression (ctx, arg, vc_prvalue,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
+	      if (*jump_target)
+		return NULL_TREE;
 	      if (i == 1)
 		arg1 = arg;
 	      else
@@ -3113,16 +3891,31 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	  return arg1;
 	}
       else if (cxx_dynamic_cast_fn_p (fun))
-	return cxx_eval_dynamic_cast_fn (ctx, t, non_constant_p, overflow_p);
+	return cxx_eval_dynamic_cast_fn (ctx, t, non_constant_p, overflow_p,
+					 jump_target);
+      else if (enum cxa_builtin kind = cxx_cxa_builtin_fn_p (fun))
+	return cxx_eval_cxa_builtin_fn (ctx, t, kind, fun,
+					non_constant_p, overflow_p,
+					jump_target);
 
-      if (!ctx->quiet)
+      /* Calls to non-constexpr functions can be diagnosed right away
+	 before C++26, though in C++26 evaluation of the arguments might
+	 throw and if caught it could be still constant expression.
+	 So for C++26 this is diagnosed only after
+	 cxx_bind_parameters_in_call.  */
+      if (cxx_dialect >= cxx26)
+	non_constexpr_call = true;
+      else
 	{
-	  if (!lambda_static_thunk_p (fun))
-	    error_at (loc, "call to non-%<constexpr%> function %qD", fun);
-	  explain_invalid_constexpr_fn (fun);
+	  if (!ctx->quiet)
+	    {
+	      if (!lambda_static_thunk_p (fun))
+		error_at (loc, "call to non-%<constexpr%> function %qD", fun);
+	      explain_invalid_constexpr_fn (fun);
+	    }
+	  *non_constant_p = true;
+	  return t;
 	}
-      *non_constant_p = true;
-      return t;
     }
 
   constexpr_ctx new_ctx = *ctx;
@@ -3158,7 +3951,8 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
   constexpr_call new_call;
   new_call.bindings
     = cxx_bind_parameters_in_call (ctx, t, fun, orig_fun, non_constant_p,
-				   overflow_p, &non_constant_args);
+				   overflow_p, &non_constant_args,
+				   jump_target);
 
   /* We build up the bindings list before we know whether we already have this
      call cached.  If we don't end up saving these bindings, ggc_free them when
@@ -3172,8 +3966,21 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
     void preserve () { bindings = NULL; }
   } fb (new_call.bindings);
 
+  if (*jump_target)
+    return NULL_TREE;
   if (*non_constant_p)
     return t;
+  if (non_constexpr_call)
+    {
+      if (!ctx->quiet)
+	{
+	  if (!lambda_static_thunk_p (fun))
+	    error_at (loc, "call to non-%<constexpr%> function %qD", fun);
+	  explain_invalid_constexpr_fn (fun);
+	}
+      *non_constant_p = true;
+      return t;
+    }
 
   /* We can't defer instantiating the function any longer.  */
   if (!DECL_INITIAL (fun)
@@ -3246,7 +4053,9 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
       new_obj = TREE_VEC_ELT (new_call.bindings, 0);
       bool empty_base = false;
       new_obj = cxx_fold_indirect_ref (ctx, loc, DECL_CONTEXT (fun), new_obj,
-				       &empty_base);
+				       &empty_base, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* If we're initializing an empty class, don't set constness, because
 	 cxx_fold_indirect_ref will return the wrong object to set constness
 	 of.  */
@@ -3395,7 +4204,7 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	     semantics are no longer in effect; see [class.dtor]p5.  */
 	  if (new_obj && DECL_DESTRUCTOR_P (fun))
 	    cxx_set_object_constness (ctx, new_obj, /*readonly_p=*/false,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p, jump_target);
 
 	  /* If this is a constructor, we are beginning the lifetime of the
 	     object we are initializing.  */
@@ -3404,21 +4213,30 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	      && TREE_CODE (new_obj) == COMPONENT_REF
 	      && TREE_CODE (TREE_TYPE (TREE_OPERAND (new_obj, 0))) == UNION_TYPE)
 	    {
+	      tree ctor = build_constructor (TREE_TYPE (new_obj), NULL);
+	      CONSTRUCTOR_NO_CLEARING (ctor) = true;
 	      tree activate = build2 (INIT_EXPR, TREE_TYPE (new_obj),
-				      new_obj,
-				      build_constructor (TREE_TYPE (new_obj),
-							 NULL));
+				      new_obj, ctor);
 	      cxx_eval_constant_expression (ctx, activate,
-					    lval, non_constant_p, overflow_p);
+					    lval, non_constant_p, overflow_p,
+					    jump_target);
 	      ggc_free (activate);
+	      if (*jump_target)
+		return NULL_TREE;
 	    }
 
-	  tree jump_target = NULL_TREE;
+	  tree jmp_target = NULL_TREE;
 	  cxx_eval_constant_expression (&call_ctx, body,
 					vc_discard, non_constant_p, overflow_p,
-					&jump_target);
+					&jmp_target);
 
-	  if (DECL_CONSTRUCTOR_P (fun))
+	  if (!*non_constant_p && throws (&jmp_target))
+	    {
+	      result = NULL_TREE;
+	      cacheable = false;
+	      *jump_target = jmp_target;
+	    }
+	  else if (DECL_CONSTRUCTOR_P (fun))
 	    /* This can be null for a subobject constructor call, in
 	       which case what we care about is the initialization
 	       side-effects rather than the value.  We could get at the
@@ -3446,7 +4264,7 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	     marking the CONSTRUCTOR TREE_READONLY.  */
 	  if (new_obj && DECL_CONSTRUCTOR_P (fun))
 	    cxx_set_object_constness (ctx, new_obj, /*readonly_p=*/true,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p, jump_target);
 
 	  /* Remove the parms/result from the values map.  */
 	  destroy_value_checked (ctx, res, non_constant_p);
@@ -3506,7 +4324,13 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 		    cacheable = false;
 		    result = cxx_eval_constant_expression (ctx, result, lval,
 							   non_constant_p,
-							   overflow_p);
+							   overflow_p,
+							   jump_target);
+		    if (*jump_target)
+		      {
+			cacheable = false;
+			result = NULL_TREE;
+		      }
 		  }
 	      }
 
@@ -3864,12 +4688,16 @@ cxx_eval_check_shift_p (location_t loc, const constexpr_ctx *ctx,
 static tree
 cxx_eval_unary_expression (const constexpr_ctx *ctx, tree t,
 			   bool /*lval*/,
-			   bool *non_constant_p, bool *overflow_p)
+			   bool *non_constant_p, bool *overflow_p,
+			   tree *jump_target)
 {
   tree r;
   tree orig_arg = TREE_OPERAND (t, 0);
   tree arg = cxx_eval_constant_expression (ctx, orig_arg, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg);
   location_t loc = EXPR_LOCATION (t);
   enum tree_code code = TREE_CODE (t);
@@ -3893,7 +4721,7 @@ cxx_eval_unary_expression (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_fold_pointer_plus_expression (const constexpr_ctx *ctx, tree t,
 				  tree lhs, tree rhs, bool *non_constant_p,
-				  bool *overflow_p)
+				  bool *overflow_p, tree *jump_target)
 {
   STRIP_NOPS (lhs);
   if (TREE_CODE (lhs) != ADDR_EXPR)
@@ -3915,9 +4743,12 @@ cxx_fold_pointer_plus_expression (const constexpr_ctx *ctx, tree t,
       t = fold_convert_loc (loc, ssizetype, TREE_OPERAND (lhs, 1));
       tree nelts = array_type_nelts_top (TREE_TYPE (TREE_OPERAND (lhs, 0)));
       nelts = cxx_eval_constant_expression (ctx, nelts, vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
       if (*non_constant_p)
 	return NULL_TREE;
+      if (*jump_target)
+	return NULL_TREE;
       /* Don't fold an out-of-bound access.  */
       if (!tree_int_cst_le (t, nelts))
 	return NULL_TREE;
@@ -3937,7 +4768,8 @@ cxx_fold_pointer_plus_expression (const constexpr_ctx *ctx, tree t,
       t = cp_build_addr_expr (t, tf_warning_or_error);
       t = cp_fold_convert (orig_type, t);
       return cxx_eval_constant_expression (ctx, t, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
     }
 
   return NULL_TREE;
@@ -3981,22 +4813,29 @@ cxx_maybe_fold_addr_pointer_plus (tree t)
 static tree
 cxx_eval_binary_expression (const constexpr_ctx *ctx, tree t,
 			    value_cat lval,
-			    bool *non_constant_p, bool *overflow_p)
+			    bool *non_constant_p, bool *overflow_p,
+			    tree *jump_target)
 {
   tree r = NULL_TREE;
   tree orig_lhs = TREE_OPERAND (t, 0);
   tree orig_rhs = TREE_OPERAND (t, 1);
   tree lhs, rhs;
   lhs = cxx_eval_constant_expression (ctx, orig_lhs, vc_prvalue,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
   /* Don't VERIFY_CONSTANT here, it's unnecessary and will break pointer
      subtraction.  */
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
   rhs = cxx_eval_constant_expression (ctx, orig_rhs, vc_prvalue,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
 
   location_t loc = EXPR_LOCATION (t);
   enum tree_code code = TREE_CODE (t);
@@ -4052,13 +4891,17 @@ cxx_eval_binary_expression (const constexpr_ctx *ctx, tree t,
       return t;
     }
   else if (code == POINTER_PLUS_EXPR)
-    r = cxx_fold_pointer_plus_expression (ctx, t, lhs, rhs, non_constant_p,
-					  overflow_p);
+    {
+      r = cxx_fold_pointer_plus_expression (ctx, t, lhs, rhs, non_constant_p,
+					    overflow_p, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
+    }
   else if (code == SPACESHIP_EXPR)
     {
       r = genericize_spaceship (loc, type, lhs, rhs);
       return cxx_eval_constant_expression (ctx, r, lval, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
     }
 
   if (r == NULL_TREE)
@@ -4117,7 +4960,10 @@ cxx_eval_conditional_expression (const constexpr_ctx *ctx, tree t,
 {
   tree val = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					   vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (val);
   if (TREE_CODE (t) == IF_STMT && IF_STMT_CONSTEVAL_P (t))
     {
@@ -4178,19 +5024,29 @@ cxx_eval_conditional_expression (const constexpr_ctx *ctx, tree t,
 
 static tree
 cxx_eval_vector_conditional_expression (const constexpr_ctx *ctx, tree t,
-					bool *non_constant_p, bool *overflow_p)
+					bool *non_constant_p, bool *overflow_p,
+					tree *jump_target)
 {
   tree arg1 = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					    vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg1);
   tree arg2 = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1),
 					    vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg2);
   tree arg3 = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 2),
 					    vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg3);
   location_t loc = EXPR_LOCATION (t);
   tree type = TREE_TYPE (t);
@@ -4578,7 +5434,8 @@ diag_array_subscript (location_t loc, const constexpr_ctx *ctx, tree array, tree
 
 static tree
 get_array_or_vector_nelts (const constexpr_ctx *ctx, tree type,
-			   bool *non_constant_p, bool *overflow_p)
+			   bool *non_constant_p, bool *overflow_p,
+			   tree *jump_target)
 {
   tree nelts;
   if (TREE_CODE (type) == ARRAY_TYPE)
@@ -4595,7 +5452,8 @@ get_array_or_vector_nelts (const constexpr_ctx *ctx, tree type,
 
   /* For VLAs, the number of elements won't be an integer constant.  */
   nelts = cxx_eval_constant_expression (ctx, nelts, vc_prvalue,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
   return nelts;
 }
 
@@ -4626,13 +5484,17 @@ extract_string_elt (tree string, unsigned chars_per_elt, unsigned index)
 static tree
 eval_and_check_array_index (const constexpr_ctx *ctx,
 			    tree t, bool allow_one_past,
-			    bool *non_constant_p, bool *overflow_p)
+			    bool *non_constant_p, bool *overflow_p,
+			    tree *jump_target)
 {
   location_t loc = cp_expr_loc_or_input_loc (t);
   tree ary = TREE_OPERAND (t, 0);
   t = TREE_OPERAND (t, 1);
   tree index = cxx_eval_constant_expression (ctx, t, vc_prvalue,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (index);
 
   if (!tree_fits_shwi_p (index)
@@ -4644,7 +5506,9 @@ eval_and_check_array_index (const constexpr_ctx *ctx,
     }
 
   tree nelts = get_array_or_vector_nelts (ctx, TREE_TYPE (ary), non_constant_p,
-					  overflow_p);
+					  overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (nelts);
   if (allow_one_past
       ? !tree_int_cst_le (index, nelts)
@@ -4664,14 +5528,18 @@ eval_and_check_array_index (const constexpr_ctx *ctx,
 static tree
 cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
 			  value_cat lval,
-			  bool *non_constant_p, bool *overflow_p)
+			  bool *non_constant_p, bool *overflow_p,
+			  tree *jump_target)
 {
   tree oldary = TREE_OPERAND (t, 0);
   tree ary = cxx_eval_constant_expression (ctx, oldary,
 					   lval,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
   if (!lval
       && TREE_CODE (ary) == VIEW_CONVERT_EXPR
       && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (ary, 0)))
@@ -4681,9 +5549,12 @@ cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
 
   tree oldidx = TREE_OPERAND (t, 1);
   tree index = eval_and_check_array_index (ctx, t, lval,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
 
   if (lval && ary == oldary && index == oldidx)
     return t;
@@ -4801,7 +5672,7 @@ cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
       ctx = &new_ctx;
     }
   t = cxx_eval_constant_expression (ctx, val, lval, non_constant_p,
-				    overflow_p);
+				    overflow_p, jump_target);
   if (new_ctor && t != ctx->ctor)
     free_constructor (ctx->ctor);
   return t;
@@ -4813,7 +5684,8 @@ cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
 			      value_cat lval,
-			      bool *non_constant_p, bool *overflow_p)
+			      bool *non_constant_p, bool *overflow_p,
+			      tree *jump_target)
 {
   unsigned HOST_WIDE_INT i;
   tree field;
@@ -4822,9 +5694,12 @@ cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
   tree orig_whole = TREE_OPERAND (t, 0);
   tree whole = cxx_eval_constant_expression (ctx, orig_whole,
 					     lval,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
   if (INDIRECT_REF_P (whole)
       && integer_zerop (TREE_OPERAND (whole, 0)))
     {
@@ -4930,10 +5805,23 @@ cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
     }
 
   /* If there's no explicit init for this field, it's value-initialized.  */
+
+  if (AGGREGATE_TYPE_P (TREE_TYPE (t)))
+    {
+      /* As in cxx_eval_store_expression, insert an empty CONSTRUCTOR
+	 and copy the flags.  */
+      constructor_elt *e = get_or_insert_ctor_field (whole, part);
+      e->value = value = build_constructor (TREE_TYPE (part), NULL);
+      CONSTRUCTOR_ZERO_PADDING_BITS (value)
+	= CONSTRUCTOR_ZERO_PADDING_BITS (whole);
+      return value;
+    }
+
   value = build_value_init (TREE_TYPE (t), tf_warning_or_error);
   return cxx_eval_constant_expression (ctx, value,
 				       lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 }
 
 /* Subroutine of cxx_eval_constant_expression.
@@ -4943,7 +5831,8 @@ cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_bit_field_ref (const constexpr_ctx *ctx, tree t,
 			value_cat lval,
-			bool *non_constant_p, bool *overflow_p)
+			bool *non_constant_p, bool *overflow_p,
+			tree *jump_target)
 {
   tree orig_whole = TREE_OPERAND (t, 0);
   tree retval, fldval, utype, mask;
@@ -4951,10 +5840,13 @@ cxx_eval_bit_field_ref (const constexpr_ctx *ctx, tree t,
   HOST_WIDE_INT istart, isize;
   tree whole = cxx_eval_constant_expression (ctx, orig_whole,
 					     lval,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
   tree start, field, value;
   unsigned HOST_WIDE_INT i;
 
+  if (*jump_target)
+    return NULL_TREE;
   if (whole == orig_whole)
     return t;
   /* Don't VERIFY_CONSTANT here; we only want to check that we got a
@@ -5235,7 +6127,7 @@ clear_uchar_or_std_byte_in_mask (location_t loc, tree t, unsigned char *mask)
 
 static tree
 cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
-		   bool *overflow_p)
+		   bool *overflow_p, tree *jump_target)
 {
   if (check_bit_cast_type (ctx, EXPR_LOCATION (t), TREE_TYPE (t),
 			   TREE_TYPE (t))
@@ -5249,9 +6141,12 @@ cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
     }
 
   tree op = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), vc_prvalue,
-					  non_constant_p, overflow_p);
+					  non_constant_p, overflow_p,
+					  jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
 
   location_t loc = EXPR_LOCATION (t);
   if (BITS_PER_UNIT != 8 || CHAR_BIT != 8)
@@ -5329,8 +6224,9 @@ cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
 	  if (CHECKING_P)
 	    {
 	      tree e = cxx_eval_bare_aggregate (ctx, r, vc_prvalue,
-						non_constant_p, overflow_p);
-	      gcc_checking_assert (e == r);
+						non_constant_p, overflow_p,
+						jump_target);
+	      gcc_checking_assert (e == r && !*jump_target);
 	      r = e;
 	    }
 	}
@@ -5371,19 +6267,24 @@ cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
 static tree
 cxx_eval_logical_expression (const constexpr_ctx *ctx, tree t,
                              tree bailout_value, tree continue_value,
-			     bool *non_constant_p, bool *overflow_p)
+			     bool *non_constant_p, bool *overflow_p,
+			     tree *jump_target)
 {
   tree r;
   tree lhs = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					   vc_prvalue, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (lhs);
   if (tree_int_cst_equal (lhs, bailout_value))
     return lhs;
   gcc_assert (tree_int_cst_equal (lhs, continue_value));
   r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1),
 				    vc_prvalue, non_constant_p,
-				    overflow_p);
+				    overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (r);
   return r;
 }
@@ -5540,7 +6441,8 @@ verify_ctor_sanity (const constexpr_ctx *ctx, tree type)
 static tree
 cxx_eval_bare_aggregate (const constexpr_ctx *ctx, tree t,
 			 value_cat lval,
-			 bool *non_constant_p, bool *overflow_p)
+			 bool *non_constant_p, bool *overflow_p,
+			 tree *jump_target)
 {
   vec<constructor_elt, va_gc> *v = CONSTRUCTOR_ELTS (t);
   bool changed = false;
@@ -5593,7 +6495,10 @@ cxx_eval_bare_aggregate (const constexpr_ctx *ctx, tree t,
 	get_or_insert_ctor_field (ctx->ctor, index);
       tree elt = cxx_eval_constant_expression (&new_ctx, value,
 					       lval,
-					       non_constant_p, overflow_p);
+					       non_constant_p, overflow_p,
+					       jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* Don't VERIFY_CONSTANT here.  */
       if (ctx->quiet && *non_constant_p)
 	break;
@@ -5683,7 +6588,8 @@ cxx_eval_bare_aggregate (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 		     bool value_init, value_cat lval,
-		     bool *non_constant_p, bool *overflow_p)
+		     bool *non_constant_p, bool *overflow_p,
+		     tree *jump_target)
 {
   tree elttype = TREE_TYPE (atype);
   verify_ctor_sanity (ctx, atype);
@@ -5694,7 +6600,10 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 
   if (init && TREE_CODE (init) == CONSTRUCTOR)
     return cxx_eval_bare_aggregate (ctx, init, lval,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p, jump_target);
+
+  /* We already checked access when building the VEC_INIT_EXPR.  */
+  deferring_access_check_sentinel acs (dk_deferred);
 
   /* For the default constructor, build up a call to the default
      constructor of the element type.  We only need to handle class types
@@ -5731,7 +6640,9 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
     }
 
   tree nelts = get_array_or_vector_nelts (ctx, atype, non_constant_p,
-					  overflow_p);
+					  overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   unsigned HOST_WIDE_INT max = tree_to_uhwi (nelts);
   for (i = 0; i < max; ++i)
     {
@@ -5757,9 +6668,9 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 	    }
 	  else
 	    eltinit = cp_build_array_ref (input_location, init, idx, complain);
-	  eltinit = cxx_eval_vec_init_1 (&new_ctx, elttype, eltinit, value_init,
-					 lval,
-					 non_constant_p, overflow_p);
+	  eltinit = cxx_eval_vec_init_1 (&new_ctx, elttype, eltinit,
+					 value_init, lval, non_constant_p,
+					 overflow_p, jump_target);
 	}
       else if (pre_init)
 	{
@@ -5773,7 +6684,8 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 	    /* Clarify what object is being initialized (118285).  */
 	    eltinit = build2 (INIT_EXPR, elttype, new_ctx.object, eltinit);
 	  eltinit = cxx_eval_constant_expression (&new_ctx, eltinit, lval,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
 	  reuse = i == 0;
 	}
       else
@@ -5789,8 +6701,11 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 	    /* Clarify what object is being initialized (118285).  */
 	    eltinit = build2 (INIT_EXPR, elttype, new_ctx.object, eltinit);
 	  eltinit = cxx_eval_constant_expression (&new_ctx, eltinit, lval,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
 	}
+      if (*jump_target)
+	return NULL_TREE;
       if (*non_constant_p)
 	break;
       if (no_slot)
@@ -5840,7 +6755,7 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 static tree
 cxx_eval_vec_init (const constexpr_ctx *ctx, tree t,
 		   value_cat lval,
-		   bool *non_constant_p, bool *overflow_p)
+		   bool *non_constant_p, bool *overflow_p, tree *jump_target)
 {
   tree atype = TREE_TYPE (t);
   tree init = VEC_INIT_EXPR_INIT (t);
@@ -5872,10 +6787,10 @@ cxx_eval_vec_init (const constexpr_ctx *ctx, tree t,
 	}
       init = expand_vec_init_expr (ctx->object, t, complain);
       return cxx_eval_constant_expression (ctx, init, lval, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
     }
   tree r = cxx_eval_vec_init_1 (ctx, atype, init, value_init,
-				lval, non_constant_p, overflow_p);
+				lval, non_constant_p, overflow_p, jump_target);
   if (*non_constant_p)
     return t;
   else
@@ -5904,14 +6819,16 @@ same_type_ignoring_tlq_and_bounds_p (tree type1, tree type2)
    otherwise return NULL_TREE.  */
 
 static tree
-cxx_union_active_member (const constexpr_ctx *ctx, tree t)
+cxx_union_active_member (const constexpr_ctx *ctx, tree t, tree *jump_target)
 {
   constexpr_ctx new_ctx = *ctx;
   new_ctx.quiet = true;
   bool non_constant_p = false, overflow_p = false;
   tree ctor = cxx_eval_constant_expression (&new_ctx, t, vc_prvalue,
 					    &non_constant_p,
-					    &overflow_p);
+					    &overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (TREE_CODE (ctor) == CONSTRUCTOR
       && CONSTRUCTOR_NELTS (ctor) == 1
       && CONSTRUCTOR_ELT (ctor, 0)->index
@@ -5924,7 +6841,8 @@ cxx_union_active_member (const constexpr_ctx *ctx, tree t)
 
 static tree
 cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
-			 tree op, unsigned HOST_WIDE_INT off, bool *empty_base)
+			 tree op, unsigned HOST_WIDE_INT off, bool *empty_base,
+			 tree *jump_target)
 {
   tree optype = TREE_TYPE (op);
   unsigned HOST_WIDE_INT const_nunits;
@@ -5941,7 +6859,8 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 	 than pointer type.  */
       if (tree ret = cxx_fold_indirect_ref_1 (ctx, loc,
 					      strip_array_types (optype),
-					      op, off, empty_base))
+					      op, off, empty_base,
+					      jump_target))
 	return fold_convert (type, ret);
     }
   else if (TREE_CODE (optype) == COMPLEX_TYPE
@@ -5987,7 +6906,7 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 	  op = build4_loc (loc, ARRAY_REF, TREE_TYPE (optype), op, index,
 			   NULL_TREE, NULL_TREE);
 	  return cxx_fold_indirect_ref_1 (ctx, loc, type, op, rem,
-					  empty_base);
+					  empty_base, jump_target);
 	}
     }
   /* ((foo *)&struct_with_foo_field)[x] => COMPONENT_REF */
@@ -5996,7 +6915,7 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
     {
       if (TREE_CODE (optype) == UNION_TYPE)
 	/* For unions prefer the currently active member.  */
-	if (tree field = cxx_union_active_member (ctx, op))
+	if (tree field = cxx_union_active_member (ctx, op, jump_target))
 	  {
 	    unsigned HOST_WIDE_INT el_sz
 	      = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (field)));
@@ -6005,7 +6924,8 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 		tree cop = build3 (COMPONENT_REF, TREE_TYPE (field),
 				   op, field, NULL_TREE);
 		if (tree ret = cxx_fold_indirect_ref_1 (ctx, loc, type, cop,
-							off, empty_base))
+							off, empty_base,
+							jump_target))
 		  return ret;
 	      }
 	  }
@@ -6050,7 +6970,8 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 				   op, field, NULL_TREE);
 		if (tree ret = cxx_fold_indirect_ref_1 (ctx, loc, type, cop,
 							off - upos,
-							empty_base))
+							empty_base,
+							jump_target))
 		  return ret;
 	      }
 	  }
@@ -6070,7 +6991,7 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 
 static tree
 cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
-		       tree op0, bool *empty_base /* = NULL*/)
+		       tree op0, bool *empty_base, tree *jump_target)
 {
   tree sub = op0;
   tree subtype;
@@ -6152,7 +7073,8 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
 	  tree off = integer_zero_node;
 	  canonicalize_obj_off (op, off);
 	  return cxx_fold_indirect_ref_1 (ctx, loc, type, op,
-					  tree_to_uhwi (off), empty_base);
+					  tree_to_uhwi (off), empty_base,
+					  jump_target);
 	}
     }
   else if (TREE_CODE (sub) == POINTER_PLUS_EXPR
@@ -6167,7 +7089,8 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
 	  tree obj = TREE_OPERAND (op00, 0);
 	  canonicalize_obj_off (obj, off);
 	  return cxx_fold_indirect_ref_1 (ctx, loc, type, obj,
-					  tree_to_uhwi (off), empty_base);
+					  tree_to_uhwi (off), empty_base,
+					  jump_target);
 	}
     }
   /* *(foo *)fooarrptr => (*fooarrptr)[0] */
@@ -6177,7 +7100,10 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
       tree type_domain;
       tree min_val = size_zero_node;
       tree newsub
-	= cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (subtype), sub, NULL);
+	= cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (subtype), sub, NULL,
+				 jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       if (newsub)
 	sub = newsub;
       else
@@ -6195,7 +7121,8 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
 static tree
 cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
 		       value_cat lval,
-		       bool *non_constant_p, bool *overflow_p)
+		       bool *non_constant_p, bool *overflow_p,
+		       tree *jump_target)
 {
   tree orig_op0 = TREE_OPERAND (t, 0);
   bool empty_base = false;
@@ -6213,13 +7140,17 @@ cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
 
   /* First try to simplify it directly.  */
   tree r = cxx_fold_indirect_ref (ctx, EXPR_LOCATION (t), TREE_TYPE (t),
-				  orig_op0, &empty_base);
+				  orig_op0, &empty_base, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (!r)
     {
       /* If that didn't work, evaluate the operand first.  */
       tree op0 = cxx_eval_constant_expression (ctx, orig_op0,
 					       vc_prvalue, non_constant_p,
-					       overflow_p);
+					       overflow_p, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* Don't VERIFY_CONSTANT here.  */
       if (*non_constant_p)
 	return t;
@@ -6233,7 +7164,9 @@ cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
 	}
 
       r = cxx_fold_indirect_ref (ctx, EXPR_LOCATION (t), TREE_TYPE (t), op0,
-				 &empty_base);
+				 &empty_base, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       if (r == NULL_TREE)
 	{
 	  /* We couldn't fold to a constant value.  Make sure it's not
@@ -6263,7 +7196,10 @@ cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
     }
 
   r = cxx_eval_constant_expression (ctx, r,
-				    lval, non_constant_p, overflow_p);
+				    lval, non_constant_p, overflow_p,
+				    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (*non_constant_p)
     return t;
 
@@ -6373,7 +7309,8 @@ non_const_var_error (location_t loc, tree r, bool fundef_p)
 static tree
 cxx_eval_trinary_expression (const constexpr_ctx *ctx, tree t,
 			     value_cat lval,
-			     bool *non_constant_p, bool *overflow_p)
+			     bool *non_constant_p, bool *overflow_p,
+			     tree *jump_target)
 {
   int i;
   tree args[3];
@@ -6383,7 +7320,10 @@ cxx_eval_trinary_expression (const constexpr_ctx *ctx, tree t,
     {
       args[i] = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, i),
 					      lval,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       VERIFY_CONSTANT (args[i]);
     }
 
@@ -6505,7 +7445,8 @@ modifying_const_object_p (tree_code code, tree obj, bool mutable_p)
 static tree
 cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 			   value_cat lval,
-			   bool *non_constant_p, bool *overflow_p)
+			   bool *non_constant_p, bool *overflow_p,
+			   tree *jump_target)
 {
   constexpr_ctx new_ctx = *ctx;
 
@@ -6531,7 +7472,10 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
       if (!SCALAR_TYPE_P (type))
 	new_ctx.ctor = new_ctx.object = NULL_TREE;
       init = cxx_eval_constant_expression (&new_ctx, init, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       if (*non_constant_p)
 	return t;
     }
@@ -6543,8 +7487,11 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	 as a whole; otherwise, only evaluate the innermost piece to avoid
 	 building up unnecessary *_REFs.  */
       target = cxx_eval_constant_expression (ctx, target, lval,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
       evaluated = true;
+      if (*jump_target)
+	return NULL_TREE;
       if (*non_constant_p)
 	return t;
     }
@@ -6570,7 +7517,10 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	    if (TREE_CODE (probe) == ARRAY_REF)
 	      {
 		elt = eval_and_check_array_index (ctx, probe, false,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
+		if (*jump_target)
+		  return NULL_TREE;
 		if (*non_constant_p)
 		  return t;
 	      }
@@ -6627,8 +7577,11 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 		  break;
 		}
 	      probe = cxx_eval_constant_expression (ctx, probe, vc_glvalue,
-						    non_constant_p, overflow_p);
+						    non_constant_p, overflow_p,
+						    jump_target);
 	      evaluated = true;
+	      if (*jump_target)
+		return NULL_TREE;
 	      if (*non_constant_p)
 		return t;
 	    }
@@ -6798,13 +7751,24 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	  CONSTRUCTOR_APPEND_ELT (CONSTRUCTOR_ELTS (*valp), first, NULL_TREE);
 
       /* Check for implicit change of active member for a union.  */
+
+      /* LWG3436, CWG2675, c++/121068: The array object model is confused.  For
+	 now allow initializing an array element to activate the array.  */
+      auto only_array_refs = [](const releasing_vec &refs)
+      {
+	for (unsigned i = 1; i < refs->length(); i += 3)
+	  if (TREE_CODE ((*refs)[i]) != INTEGER_CST)
+	    return false;
+	return true;
+      };
+
       if (code == UNION_TYPE
 	  && (CONSTRUCTOR_NELTS (*valp) == 0
 	      || CONSTRUCTOR_ELT (*valp, 0)->index != index)
 	  /* An INIT_EXPR of the last member in an access chain is always OK,
 	     but still check implicit change of members earlier on; see
 	     cpp2a/constexpr-union6.C.  */
-	  && !(TREE_CODE (t) == INIT_EXPR && refs->is_empty ()))
+	  && !(TREE_CODE (t) == INIT_EXPR && only_array_refs (refs)))
 	{
 	  bool has_active_member = CONSTRUCTOR_NELTS (*valp) != 0;
 	  tree inner = strip_array_types (reftype);
@@ -6972,7 +7936,10 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	if (tree tinit = TARGET_EXPR_INITIAL (init))
 	  init = tinit;
       init = cxx_eval_constant_expression (&new_ctx, init, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* The hash table might have moved since the get earlier, and the
 	 initializer might have mutated the underlying CONSTRUCTORs, so we must
 	 recompute VALP. */
@@ -7098,7 +8065,8 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
 			      value_cat lval,
-			      bool *non_constant_p, bool *overflow_p)
+			      bool *non_constant_p, bool *overflow_p,
+			      tree *jump_target)
 {
   enum tree_code code = TREE_CODE (t);
   tree type = TREE_TYPE (t);
@@ -7112,12 +8080,18 @@ cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
 
   /* The operand as an lvalue.  */
   op = cxx_eval_constant_expression (ctx, op, vc_glvalue,
-				     non_constant_p, overflow_p);
+				     non_constant_p, overflow_p,
+				     jump_target);
+  if (*jump_target)
+    return NULL_TREE;
 
   /* The operand as an rvalue.  */
   tree val
     = cxx_eval_constant_expression (ctx, op, vc_prvalue,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p,
+				    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   /* Don't VERIFY_CONSTANT if this might be dealing with a pointer to
      a local array in a constexpr function.  */
   bool ptr = INDIRECT_TYPE_P (TREE_TYPE (val));
@@ -7156,8 +8130,11 @@ cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
   tree store = build2_loc (cp_expr_loc_or_loc (t, input_location),
 			   MODIFY_EXPR, type, op, mod);
   mod = cxx_eval_constant_expression (ctx, store, lval,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
   ggc_free (store);
+  if (*jump_target)
+    return NULL_TREE;
   if (*non_constant_p)
     return t;
 
@@ -7171,42 +8148,6 @@ cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
     return val;
 }
 
-/* Predicates for the meaning of *jump_target.  */
-
-static bool
-returns (tree *jump_target)
-{
-  return *jump_target
-    && TREE_CODE (*jump_target) == RETURN_EXPR;
-}
-
-static bool
-breaks (tree *jump_target)
-{
-  return *jump_target
-    && ((TREE_CODE (*jump_target) == LABEL_DECL
-	 && LABEL_DECL_BREAK (*jump_target))
-	|| TREE_CODE (*jump_target) == BREAK_STMT
-	|| TREE_CODE (*jump_target) == EXIT_EXPR);
-}
-
-static bool
-continues (tree *jump_target)
-{
-  return *jump_target
-    && ((TREE_CODE (*jump_target) == LABEL_DECL
-	 && LABEL_DECL_CONTINUE (*jump_target))
-	|| TREE_CODE (*jump_target) == CONTINUE_STMT);
-
-}
-
-static bool
-switches (tree *jump_target)
-{
-  return *jump_target
-    && TREE_CODE (*jump_target) == INTEGER_CST;
-}
-
 /* Subroutine of cxx_eval_statement_list.  Determine whether the statement
    STMT matches *jump_target.  If we're looking for a case label and we see
    the default label, note it in ctx->css_state.  */
@@ -7254,6 +8195,11 @@ label_matches (const constexpr_ctx *ctx, tree *jump_target, tree stmt)
 	 breaks (jump_target) or continues (jump_target).  */
       break;
 
+    case VAR_DECL:
+      /* Uncaught exception.  This is handled by TRY_BLOCK evaluation
+	 and other places by testing throws (jump_target).  */
+      break;
+
     default:
       gcc_unreachable ();
     }
@@ -7268,15 +8214,9 @@ cxx_eval_statement_list (const constexpr_ctx *ctx, tree t,
 			 bool *non_constant_p, bool *overflow_p,
 			 tree *jump_target)
 {
-  tree local_target;
   /* In a statement-expression we want to return the last value.
      For empty statement expression return void_node.  */
   tree r = void_node;
-  if (!jump_target)
-    {
-      local_target = NULL_TREE;
-      jump_target = &local_target;
-    }
   for (tree_stmt_iterator i = tsi_start (t); !tsi_end_p (i); ++i)
     {
       tree stmt = *i;
@@ -7304,18 +8244,11 @@ cxx_eval_statement_list (const constexpr_ctx *ctx, tree t,
 					jump_target);
       if (*non_constant_p)
 	break;
-      if (returns (jump_target) || breaks (jump_target))
+      if (returns (jump_target)
+	  || breaks (jump_target)
+	  || throws (jump_target))
 	break;
     }
-  if (*jump_target && jump_target == &local_target)
-    {
-      /* We aren't communicating the jump to our caller, so give up.  We don't
-	 need to support evaluation of jumps out of statement-exprs.  */
-      if (!ctx->quiet)
-	error_at (cp_expr_loc_or_input_loc (r),
-		  "statement is not a constant expression");
-      *non_constant_p = true;
-    }
   return r;
 }
 
@@ -7327,13 +8260,6 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 		    bool *non_constant_p, bool *overflow_p,
 		    tree *jump_target)
 {
-  tree local_target;
-  if (!jump_target)
-    {
-      local_target = NULL_TREE;
-      jump_target = &local_target;
-    }
-
   tree body, cond = NULL_TREE, expr = NULL_TREE;
   tree cond_prep = NULL_TREE, cond_cleanup = NULL_TREE;
   unsigned cond_cleanup_depth = 0;
@@ -7389,7 +8315,7 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 	tree c;
 	FOR_EACH_VEC_ELT_REVERSE (cleanups, i, c)
 	  cxx_eval_constant_expression (ctx, c, vc_discard, non_constant_p,
-					overflow_p);
+					overflow_p, jump_target);
       }
     if (cond_prep)
       for (tree decl = BIND_EXPR_VARS (cond_prep);
@@ -7484,7 +8410,8 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 		      if (*non_constant_p
 			  || returns (jump_target)
 			  || breaks (jump_target)
-			  || continues (jump_target))
+			  || continues (jump_target)
+			  || throws (jump_target))
 			{
 			  depth = 1;
 			  break;
@@ -7531,6 +8458,7 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 	 && !breaks (jump_target)
 	 && !continues (jump_target)
 	 && (!switches (jump_target) || count == 0)
+	 && !throws (jump_target)
 	 && !*non_constant_p);
 
   cleanup_cond ();
@@ -7549,7 +8477,10 @@ cxx_eval_switch_expr (const constexpr_ctx *ctx, tree t,
   tree cond
     = TREE_CODE (t) == SWITCH_STMT ? SWITCH_STMT_COND (t) : SWITCH_COND (t);
   cond = cxx_eval_constant_expression (ctx, cond, vc_prvalue,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (cond);
   if (TREE_CODE (cond) != INTEGER_CST)
     {
@@ -7682,7 +8613,8 @@ maybe_warn_about_constant_value (location_t loc, tree decl)
 static tree
 build_new_constexpr_heap_type (const constexpr_ctx *ctx, tree elt_type,
 			       tree cookie_size, tree full_size, tree arg_size,
-			       bool *non_constant_p, bool *overflow_p)
+			       bool *non_constant_p, bool *overflow_p,
+			       tree *jump_target)
 {
   gcc_assert (cookie_size == NULL_TREE || tree_fits_uhwi_p (cookie_size));
   gcc_assert (tree_fits_uhwi_p (full_size));
@@ -7718,13 +8650,17 @@ build_new_constexpr_heap_type (const constexpr_ctx *ctx, tree elt_type,
 	  if (integer_zerop (op0))
 	    arg_size
 	      = cxx_eval_constant_expression (ctx, op1, vc_prvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
 	  else if (integer_zerop (op1))
 	    arg_size
 	      = cxx_eval_constant_expression (ctx, op0, vc_prvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
 	  else
 	    arg_size = NULL_TREE;
+	  if (*jump_target)
+	    return NULL_TREE;
 	}
       else
 	arg_size = NULL_TREE;
@@ -7745,6 +8681,38 @@ build_new_constexpr_heap_type (const constexpr_ctx *ctx, tree elt_type,
   return build_new_constexpr_heap_type (elt_type, cookie_size, itype2);
 }
 
+/* Handle the case when a cleanup of some expression throws.  JMP_TARGET
+   indicates whether the cleanup threw or not, *JUMP_TARGET indicates whether
+   the expression which needed the cleanup threw.  If both threw, diagnose
+   it and return NULL, otherwise return R.  If only the cleanup threw, set
+   *JUMP_TARGET to the exception object from the cleanup.  */
+
+static tree
+merge_jump_target (location_t loc, const constexpr_ctx *ctx, tree r,
+		   bool *non_constant_p, tree *jump_target, tree jmp_target)
+{
+  if (!throws (&jmp_target))
+    return r;
+  if (throws (jump_target))
+    {
+      /* [except.throw]/9 - If the exception handling mechanism
+	 handling an uncaught exception directly invokes a function
+	 that exits via an exception, the function std::terminate is
+	 invoked.  */
+      if (!ctx->quiet)
+	{
+	  auto_diagnostic_group d;
+	  diagnose_std_terminate (loc, ctx, *jump_target);
+	  inform (loc, "destructor exited with an exception");
+	}
+      *non_constant_p = true;
+      *jump_target = NULL_TREE;
+      return NULL_TREE;
+    }
+  *jump_target = jmp_target;
+  return r;
+}
+
 /* Attempt to reduce the expression T to a constant value.
    On failure, issue diagnostic and return error_mark_node.  */
 /* FIXME unify with c_fully_fold */
@@ -7754,9 +8722,9 @@ static tree
 cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 			      value_cat lval,
 			      bool *non_constant_p, bool *overflow_p,
-			      tree *jump_target /* = NULL */)
+			      tree *jump_target)
 {
-  if (jump_target && *jump_target)
+  if (*jump_target)
     {
       /* If we are jumping, ignore all statements/expressions except those
 	 that could have LABEL_EXPR or CASE_LABEL_EXPR in their bodies.  */
@@ -7880,7 +8848,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 		r = convert_from_reference (r);
 	    }
 	  return cxx_eval_constant_expression (ctx, r, lval, non_constant_p,
-					       overflow_p);
+					       overflow_p, jump_target);
 	}
       /* fall through */
     case CONST_DECL:
@@ -7958,7 +8926,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	  r = v;
 	  if (TREE_ADDRESSABLE (TREE_TYPE (t)))
 	    r = cxx_eval_constant_expression (ctx, r, vc_prvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
+	  if (*jump_target)
+	    return NULL_TREE;
 	}
       else if (lval)
 	/* Defer in case this is only used for its type.  */;
@@ -7991,7 +8962,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case CALL_EXPR:
     case AGGR_INIT_EXPR:
       r = cxx_eval_call_expression (ctx, t, lval,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p, jump_target);
       break;
 
     case DECL_EXPR:
@@ -8055,7 +9026,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	if (tree init = DECL_INITIAL (r))
 	  {
 	    init = cxx_eval_constant_expression (ctx, init, vc_prvalue,
-						 non_constant_p, overflow_p);
+						 non_constant_p, overflow_p,
+						 jump_target);
+	    if (*jump_target)
+	      return NULL_TREE;
 	    /* Don't share a CONSTRUCTOR that might be changed.  */
 	    init = unshare_constructor (init);
 	    /* Remember that a constant object's constructor has already
@@ -8125,9 +9099,12 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	/* Pass vc_prvalue because this indicates
 	   initialization of a temporary.  */
 	r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_prvalue,
-					  non_constant_p, overflow_p);
+					  non_constant_p, overflow_p,
+					  jump_target);
 	if (*non_constant_p)
 	  break;
+	if (*jump_target)
+	  return NULL_TREE;
 	if (!is_complex)
 	  {
 	    r = unshare_constructor (r);
@@ -8135,8 +9112,15 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    r = adjust_temp_type (type, r);
 	    ctx->global->put_value (slot, r);
 	  }
-	if (TARGET_EXPR_CLEANUP (t) && !CLEANUP_EH_ONLY (t))
-	  ctx->global->cleanups->safe_push (TARGET_EXPR_CLEANUP (t));
+	if (TARGET_EXPR_CLEANUP (t)
+	    && (!CLEANUP_EH_ONLY (t) || cxx_dialect >= cxx26))
+	  {
+	    ctx->global->cleanups->safe_push (TARGET_EXPR_CLEANUP (t));
+	    /* Mark CLEANUP_EH_ONLY cleanups by pushing NULL_TREE after
+	       them.  */
+	    if (CLEANUP_EH_ONLY (t))
+	      ctx->global->cleanups->safe_push (NULL_TREE);
+	  }
 	if (ctx->save_exprs)
 	  ctx->save_exprs->safe_push (slot);
 	if (lval)
@@ -8150,33 +9134,28 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case MODIFY_EXPR:
       gcc_assert (jump_target == NULL || *jump_target == NULL_TREE);
       r = cxx_eval_store_expression (ctx, t, lval,
-				     non_constant_p, overflow_p);
+				     non_constant_p, overflow_p, jump_target);
       break;
 
     case SCOPE_REF:
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1),
 					lval,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
       break;
 
     case RETURN_EXPR:
       if (TREE_OPERAND (t, 0) != NULL_TREE)
 	r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					  lval,
-					  non_constant_p, overflow_p);
-      /* FALLTHRU */
+					  non_constant_p, overflow_p,
+					  jump_target);
+      if (!throws (jump_target))
+	*jump_target = t;
+      break;
     case BREAK_STMT:
     case CONTINUE_STMT:
-      if (jump_target)
-	*jump_target = t;
-      else
-	{
-	  /* Can happen with ({ return true; }) && false; passed to
-	     maybe_constant_value.  There is nothing to jump over in this
-	     case, and the bug will be diagnosed later.  */
-	  gcc_assert (ctx->quiet);
-	  *non_constant_p = true;
-	}
+      *jump_target = t;
       break;
 
     case SAVE_EXPR:
@@ -8185,9 +9164,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	r = v;
       else
 	{
-	  r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), vc_prvalue,
-					    non_constant_p, overflow_p);
-	  if (*non_constant_p)
+	  r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
+					    vc_prvalue, non_constant_p,
+					    overflow_p, jump_target);
+	  if (*non_constant_p || *jump_target)
 	    break;
 	  ctx->global->put_value (t, r);
 	  if (ctx->save_exprs)
@@ -8195,16 +9175,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	}
       break;
 
-    case TRY_CATCH_EXPR:
-      if (TREE_OPERAND (t, 0) == NULL_TREE)
-	{
-	  r = void_node;
-	  break;
-	}
-      /* FALLTHRU */
     case NON_LVALUE_EXPR:
-    case TRY_BLOCK:
-    case MUST_NOT_THROW_EXPR:
     case EXPR_STMT:
     case EH_SPEC_BLOCK:
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
@@ -8213,6 +9184,42 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 					jump_target);
       break;
 
+    case TRY_BLOCK:
+      r = cxx_eval_constant_expression (ctx, TRY_STMTS (t), lval,
+					non_constant_p, overflow_p,
+					jump_target);
+      if (!*non_constant_p && throws (jump_target))
+	if (tree h = TRY_HANDLERS (t))
+	  {
+	    tree type = strip_array_types (TREE_TYPE (*jump_target));
+	    if (TREE_CODE (h) == STATEMENT_LIST)
+	      {
+		for (tree stmt : tsi_range (h))
+		  if (TREE_CODE (stmt) == HANDLER
+		      && handler_match_for_exception_type (stmt, type))
+		    {
+		      h = stmt;
+		      break;
+		    }
+		if (TREE_CODE (h) == STATEMENT_LIST)
+		  h = NULL_TREE;
+	      }
+	    else if (TREE_CODE (h) != HANDLER
+		     || !handler_match_for_exception_type (h, type))
+	      h = NULL_TREE;
+	    if (h)
+	      {
+		gcc_assert (VAR_P (*jump_target));
+		ctx->global->caught_exceptions.safe_push (*jump_target);
+		ctx->global->caught_exceptions.safe_push (HANDLER_TYPE (h));
+		*jump_target = NULL_TREE;
+		r = cxx_eval_constant_expression (ctx, HANDLER_BODY (h),
+						  vc_discard, non_constant_p,
+						  overflow_p, jump_target);
+	      }
+	  }
+      break;
+
     case CLEANUP_POINT_EXPR:
       {
 	auto_vec<tree, 2> cleanups;
@@ -8230,47 +9237,132 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 
 	ctx->global->cleanups = prev_cleanups;
 	unsigned int i;
-	tree cleanup;
+	tree cleanup, jmp_target = NULL_TREE;
+	bool eh = throws (jump_target);
 	/* Evaluate the cleanups.  */
 	FOR_EACH_VEC_ELT_REVERSE (cleanups, i, cleanup)
-	  cxx_eval_constant_expression (&new_ctx, cleanup, vc_discard,
-					non_constant_p, overflow_p);
+	  if (cleanup == NULL_TREE)
+	    {
+	      /* NULL_TREE cleanup is a marker that before it is
+		 CLEANUP_EH_ONLY cleanup.  Skip the cleanup before it
+		 if the body didn't throw.  */
+	      if (!eh)
+		--i;
+	    }
+	  else
+	    cxx_eval_constant_expression (&new_ctx, cleanup, vc_discard,
+					  non_constant_p, overflow_p,
+					  &jmp_target);
 
 	/* Forget SAVE_EXPRs and TARGET_EXPRs created by this
 	   full-expression.  */
 	for (tree save_expr : save_exprs)
 	  destroy_value_checked (ctx, save_expr, non_constant_p);
+	if (throws (&jmp_target))
+	  *jump_target = jmp_target;
       }
       break;
 
+    case MUST_NOT_THROW_EXPR:
+      r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
+					lval,
+					non_constant_p, overflow_p,
+					jump_target);
+      if (throws (jump_target))
+	{
+	  /* [except.handle]/7 - If the search for a handler exits the
+	     function body of a function with a non-throwing exception
+	     specification, the function std::terminate is invoked.  */
+	  if (!ctx->quiet)
+	    {
+	      auto_diagnostic_group d;
+	      diagnose_std_terminate (loc, ctx, *jump_target);
+	      if (MUST_NOT_THROW_NOEXCEPT_P (t)
+		  && ctx->call
+		  && ctx->call->fundef)
+		inform (loc, "uncaught exception exited from %<noexcept%> "
+			     "function %qD",
+			ctx->call->fundef->decl);
+	      else if (MUST_NOT_THROW_THROW_P (t))
+		inform (loc, "destructor exited with an exception after "
+			     "initializing the exception object");
+	      else if (MUST_NOT_THROW_CATCH_P (t))
+		inform (loc, "constructor exited with another exception while "
+			     "entering handler");
+	    }
+	  *non_constant_p = true;
+	  *jump_target = NULL_TREE;
+	  r = NULL_TREE;
+	}
+      break;
+
+    case TRY_CATCH_EXPR:
+      if (TREE_OPERAND (t, 0) == NULL_TREE)
+	{
+	  r = void_node;
+	  break;
+	}
+      r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
+					non_constant_p, overflow_p,
+					jump_target);
+      if (!*non_constant_p && throws (jump_target))
+	{
+	  tree jmp_target = NULL_TREE;
+	  cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_discard,
+					non_constant_p, overflow_p,
+					&jmp_target);
+	  r = merge_jump_target (loc, ctx, r, non_constant_p, jump_target,
+				 jmp_target);
+	}
+      break;
+
     case TRY_FINALLY_EXPR:
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
 					non_constant_p, overflow_p,
 					jump_target);
       if (!*non_constant_p)
-	/* Also evaluate the cleanup.  */
-	cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_discard,
-				      non_constant_p, overflow_p);
+	{
+	  tree jmp_target = NULL_TREE;
+	  /* Also evaluate the cleanup.  */
+	  if (TREE_CODE (TREE_OPERAND (t, 1)) == EH_ELSE_EXPR
+	      && throws (jump_target))
+	    cxx_eval_constant_expression (ctx,
+					  TREE_OPERAND (TREE_OPERAND (t, 1),
+							1), vc_discard,
+					  non_constant_p, overflow_p,
+					  &jmp_target);
+	  else
+	    cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_discard,
+					  non_constant_p, overflow_p,
+					  &jmp_target);
+	  r = merge_jump_target (loc, ctx, r, non_constant_p, jump_target,
+				 jmp_target);
+	}
       break;
 
     case EH_ELSE_EXPR:
       /* Evaluate any cleanup that applies to non-EH exits.  */
       cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), vc_discard,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p,
+				    jump_target);
 
-      /* We do not have constexpr exceptions yet, so skip the EH path.  */
+      /* The EH path is handled in TRY_FINALLY_EXPR handling above.  */
       break;
 
     case CLEANUP_STMT:
       r = cxx_eval_constant_expression (ctx, CLEANUP_BODY (t), lval,
 					non_constant_p, overflow_p,
 					jump_target);
-      if (!CLEANUP_EH_ONLY (t) && !*non_constant_p)
+      if ((!CLEANUP_EH_ONLY (t) || throws (jump_target)) && !*non_constant_p)
 	{
 	  iloc_sentinel ils (loc);
+	  tree jmp_target = NULL_TREE;
 	  /* Also evaluate the cleanup.  */
 	  cxx_eval_constant_expression (ctx, CLEANUP_EXPR (t), vc_discard,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					&jmp_target);
+	  r = merge_jump_target (loc, ctx, r, non_constant_p, jump_target,
+				 jmp_target);
 	}
       break;
 
@@ -8280,14 +9372,18 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case MEM_REF:
     case INDIRECT_REF:
       r = cxx_eval_indirect_ref (ctx, t, lval,
-				 non_constant_p, overflow_p);
+				 non_constant_p, overflow_p,
+				 jump_target);
       break;
 
     case ADDR_EXPR:
       {
 	tree oldop = TREE_OPERAND (t, 0);
 	tree op = cxx_eval_constant_expression (ctx, oldop, vc_glvalue,
-						non_constant_p, overflow_p);
+						non_constant_p, overflow_p,
+						jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	/* Don't VERIFY_CONSTANT here.  */
 	if (*non_constant_p)
 	  return t;
@@ -8307,7 +9403,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       if (lval)
 	{
 	  r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+	  if (*jump_target)
+	    return NULL_TREE;
 	  if (r == error_mark_node)
 	    ;
 	  else if (r == TREE_OPERAND (t, 0) || lval == vc_discard)
@@ -8328,7 +9427,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case FIXED_CONVERT_EXPR:
     case VEC_DUPLICATE_EXPR:
       r = cxx_eval_unary_expression (ctx, t, lval,
-				     non_constant_p, overflow_p);
+				     non_constant_p, overflow_p,
+				     jump_target);
       break;
 
     case SIZEOF_EXPR:
@@ -8366,6 +9466,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    cxx_eval_constant_expression (ctx, op0, vc_discard,
 					  non_constant_p, overflow_p,
 					  jump_target);
+	    if (*jump_target)
+	      return NULL_TREE;
 	    if (*non_constant_p)
 	      return t;
 	    op1 = TREE_OPERAND (t, 1);
@@ -8418,7 +9520,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case RANGE_EXPR:
     case COMPLEX_EXPR:
       r = cxx_eval_binary_expression (ctx, t, lval,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
       break;
 
       /* fold can introduce non-IF versions of these; still treat them as
@@ -8427,19 +9530,22 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case TRUTH_ANDIF_EXPR:
       r = cxx_eval_logical_expression (ctx, t, boolean_false_node,
 				       boolean_true_node,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
       break;
 
     case TRUTH_OR_EXPR:
     case TRUTH_ORIF_EXPR:
       r = cxx_eval_logical_expression (ctx, t, boolean_true_node,
 				       boolean_false_node,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
       break;
 
     case ARRAY_REF:
       r = cxx_eval_array_reference (ctx, t, lval,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p,
+				    jump_target);
       break;
 
     case COMPONENT_REF:
@@ -8454,17 +9560,19 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	  return t;
 	}
       r = cxx_eval_component_reference (ctx, t, lval,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
       break;
 
     case BIT_FIELD_REF:
       r = cxx_eval_bit_field_ref (ctx, t, lval,
-				  non_constant_p, overflow_p);
+				  non_constant_p, overflow_p,
+				  jump_target);
       break;
 
     case COND_EXPR:
     case IF_STMT:
-      if (jump_target && *jump_target)
+      if (*jump_target)
 	{
 	  tree orig_jump = *jump_target;
 	  tree arg = ((TREE_CODE (t) != IF_STMT || TREE_OPERAND (t, 1))
@@ -8502,7 +9610,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       break;
     case VEC_COND_EXPR:
       r = cxx_eval_vector_conditional_expression (ctx, t, non_constant_p,
-						  overflow_p);
+						  overflow_p, jump_target);
       break;
 
     case CONSTRUCTOR:
@@ -8514,7 +9622,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    return t;
 	}
       r = cxx_eval_bare_aggregate (ctx, t, lval,
-				   non_constant_p, overflow_p);
+				   non_constant_p, overflow_p, jump_target);
       break;
 
     case VEC_INIT_EXPR:
@@ -8524,12 +9632,13 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	 or xvalue of the same type, meaning direct-initialization from the
 	 corresponding member.  */
       r = cxx_eval_vec_init (ctx, t, lval,
-			     non_constant_p, overflow_p);
+			     non_constant_p, overflow_p, jump_target);
       break;
 
     case VEC_PERM_EXPR:
       r = cxx_eval_trinary_expression (ctx, t, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
       break;
 
     case PAREN_EXPR:
@@ -8537,7 +9646,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       /* A PAREN_EXPR resulting from __builtin_assoc_barrier has no effect in
          constant expressions since it's unaffected by -fassociative-math.  */
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
       break;
 
     case NOP_EXPR:
@@ -8561,7 +9671,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 						? vc_discard
 						: tcode == VIEW_CONVERT_EXPR
 						? lval : vc_prvalue,
-						non_constant_p, overflow_p);
+						non_constant_p, overflow_p,
+						jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	if (*non_constant_p)
 	  return t;
 	tree type = TREE_TYPE (t);
@@ -8618,7 +9731,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	      {
 		if (integer_zerop (sop))
 		  return build_int_cst (type, 0);
-		r = cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (type), sop);
+		r = cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (type), sop,
+					   NULL, jump_target);
+		if (*jump_target)
+		  return NULL_TREE;
 		if (r)
 		  {
 		    r = build1 (ADDR_EXPR, type, r);
@@ -8745,10 +9861,14 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 		if (cxx_replaceable_global_alloc_fn (fun)
 		    && IDENTIFIER_NEW_OP_P (DECL_NAME (fun)))
 		  arg_size = CALL_EXPR_ARG (oldop, 0);
-	    TREE_TYPE (var)
+	    tree new_type
 	      = build_new_constexpr_heap_type (ctx, elt_type, cookie_size,
 					       var_size, arg_size,
-					       non_constant_p, overflow_p);
+					       non_constant_p, overflow_p,
+					       jump_target);
+	    if (*jump_target)
+	      return NULL_TREE;
+	    TREE_TYPE (var) = new_type;
 	    TREE_TYPE (TREE_OPERAND (op, 0))
 	      = build_pointer_type (TREE_TYPE (var));
 	  }
@@ -8787,7 +9907,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 
 	tree op = cxx_eval_constant_expression (ctx, oldop,
 						lval,
-						non_constant_p, overflow_p);
+						non_constant_p, overflow_p,
+						jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	if (*non_constant_p)
 	  return t;
 	r = fold_convert (TREE_TYPE (t), op);
@@ -8824,14 +9947,20 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case PREDECREMENT_EXPR:
     case POSTDECREMENT_EXPR:
       return cxx_eval_increment_expression (ctx, t,
-					    lval, non_constant_p, overflow_p);
+					    lval, non_constant_p, overflow_p,
+					    jump_target);
 
+    case THROW_EXPR:
+      if (cxx_dialect >= cxx26)
+	return cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
+					     non_constant_p, overflow_p,
+					     jump_target);
+      /* FALLTHROUGH */
     case LAMBDA_EXPR:
     case NEW_EXPR:
     case VEC_NEW_EXPR:
     case DELETE_EXPR:
     case VEC_DELETE_EXPR:
-    case THROW_EXPR:
     case MODOP_EXPR:
       /* GCC internal stuff.  */
     case VA_ARG_EXPR:
@@ -8845,7 +9974,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case OBJ_TYPE_REF:
       /* Virtual function lookup.  We don't need to do anything fancy.  */
       return cxx_eval_constant_expression (ctx, OBJ_TYPE_REF_EXPR (t),
-					   lval, non_constant_p, overflow_p);
+					   lval, non_constant_p, overflow_p,
+					   jump_target);
 
     case PLACEHOLDER_EXPR:
       /* Use of the value or address of the current object.  */
@@ -8855,7 +9985,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    return ctor;
 	  else
 	    return cxx_eval_constant_expression (ctx, ctor, lval,
-						 non_constant_p, overflow_p);
+						 non_constant_p, overflow_p,
+						 jump_target);
 	}
       /* A placeholder without a referent.  We can get here when
 	 checking whether NSDMIs are noexcept, or in massage_init_elt;
@@ -8868,7 +9999,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       {
 	tree cond = TREE_OPERAND (t, 0);
 	cond = cxx_eval_constant_expression (ctx, cond, vc_prvalue,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	VERIFY_CONSTANT (cond);
 	if (integer_nonzerop (cond))
 	  *jump_target = t;
@@ -8980,7 +10114,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	  *non_constant_p = true;
 	  return t;
 	}
-      r = cxx_eval_bit_cast (ctx, t, non_constant_p, overflow_p);
+      r = cxx_eval_bit_cast (ctx, t, non_constant_p, overflow_p, jump_target);
       break;
 
     case OMP_PARALLEL:
@@ -9202,11 +10336,14 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 	{
 	  if (cxx_dialect < cxx20)
 	    return t;
-	  if (TREE_CODE (t) != CALL_EXPR && TREE_CODE (t) != AGGR_INIT_EXPR)
+	  /* We could have a COMPOUND_EXPR here coming from
+	     keep_unused_object_arg.  */
+	  tree x = extract_call_expr (t);
+	  if (x == NULL_TREE || x == error_mark_node)
 	    return t;
 	  /* Calls to immediate functions returning void need to be
 	     evaluated.  */
-	  tree fndecl = cp_get_callee_fndecl_nofold (t);
+	  tree fndecl = cp_get_callee_fndecl_nofold (x);
 	  if (fndecl == NULL_TREE || !DECL_IMMEDIATE_FUNCTION_P (fndecl))
 	    return t;
 	  else
@@ -9299,8 +10436,34 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 
   if (manifestly_const_eval == mce_true)
     instantiate_constexpr_fns (r);
+  tree jmp_target = NULL_TREE;
   r = cxx_eval_constant_expression (&ctx, r, vc_prvalue,
-				    &non_constant_p, &overflow_p);
+				    &non_constant_p, &overflow_p,
+				    &jmp_target);
+  if (throws (&jmp_target) && !non_constant_p)
+    {
+      if (!ctx.quiet)
+	diagnose_uncaught_exception (input_location, &ctx, jmp_target);
+      non_constant_p = true;
+      jmp_target = NULL_TREE;
+      r = t;
+    }
+  else if (!non_constant_p && jmp_target)
+    {
+      non_constant_p = true;
+      if (!ctx.quiet)
+	{
+	  if (breaks (&jmp_target))
+	    error ("%<break%> outside of a loop or %<switch%>");
+	  else if (continues (&jmp_target))
+	    error ("%<continue%> outside of a loop");
+	  else if (returns (&jmp_target))
+	    error ("%<return%> in a statement expression");
+	  else
+	    gcc_unreachable ();
+	}
+      r = t;
+    }
 
   /* If we got a non-simple TARGET_EXPR, the initializer was a sequence
      of statements, and the result ought to be stored in ctx.ctor.  */
@@ -9309,15 +10472,31 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 
   unsigned int i;
   tree cleanup;
+  jmp_target = NULL_TREE;
   /* Evaluate the cleanups.  */
   FOR_EACH_VEC_ELT_REVERSE (cleanups, i, cleanup)
-    cxx_eval_constant_expression (&ctx, cleanup, vc_discard,
-				  &non_constant_p, &overflow_p);
+    if (cleanup == NULL_TREE)
+      /* NULL_TREE cleanup is a marker that before it is
+	 CLEANUP_EH_ONLY cleanup.  Skip the cleanup before it.  */
+      --i;
+    else
+      cxx_eval_constant_expression (&ctx, cleanup, vc_discard,
+				    &non_constant_p, &overflow_p,
+				    &jmp_target);
+  if (throws (&jmp_target) && !non_constant_p)
+    {
+      if (!ctx.quiet)
+	diagnose_uncaught_exception (input_location, &ctx, jmp_target);
+      non_constant_p = true;
+      r = t;
+    }
 
   /* Mutable logic is a bit tricky: we want to allow initialization of
      constexpr variables with mutable members, but we can't copy those
      members to another constexpr variable.  */
-  if (TREE_CODE (r) == CONSTRUCTOR && CONSTRUCTOR_MUTABLE_POISON (r))
+  if (!non_constant_p
+      && TREE_CODE (r) == CONSTRUCTOR
+      && CONSTRUCTOR_MUTABLE_POISON (r))
     {
       if (!allow_non_constant)
 	error ("%qE is not a constant expression because it refers to "
@@ -9335,8 +10514,13 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 	{
 	  if (!allow_non_constant && !non_constant_p)
 	    {
-	      error ("%qE is not a constant expression because it refers to "
-		     "a result of %<operator new%>", t);
+	      if (DECL_LANG_SPECIFIC (heap_var))
+		error ("%qE is not a constant expression because it refers to "
+		       "exception object allocated with "
+		       "%<__cxa_allocate_exception%>", t);
+	      else
+		error ("%qE is not a constant expression because it refers to "
+		       "a result of %<operator new%>", t);
 	      inform (DECL_SOURCE_LOCATION (heap_var), "allocated here");
 	    }
 	  r = t;
@@ -9917,6 +11101,24 @@ cxx_constant_init (tree t, tree decl)
   return maybe_constant_init_1 (t, decl, false, mce_true);
 }
 
+/* Return true if CALL_EXPR T might throw during constant evaluation.  */
+
+static bool
+callee_might_throw (tree t)
+{
+  if (cxx_dialect < cxx26 || !flag_exceptions)
+    return false;
+  tree callee = cp_get_callee (t);
+  if (callee == NULL_TREE)
+    return false;
+  tree callee_fn = cp_get_fndecl_from_callee (callee, false);
+  return (!flag_enforce_eh_specs
+	  || type_dependent_expression_p (callee)
+	  || !POINTER_TYPE_P (TREE_TYPE (callee))
+	  || (!type_noexcept_p (TREE_TYPE (TREE_TYPE (callee)))
+	      && (callee_fn == NULL_TREE || !TREE_NOTHROW (callee_fn))));
+}
+
 #if 0
 /* FIXME see ADDR_EXPR section in potential_constant_expression_1.  */
 /* Return true if the object referred to by REF has automatic or thread
@@ -9949,11 +11151,13 @@ struct check_for_return_continue_data {
   hash_set<tree> *pset;
   tree continue_stmt;
   tree break_stmt;
+  bool could_throw;
 };
 
 /* Helper function for potential_constant_expression_1 SWITCH_STMT handling,
    called through cp_walk_tree.  Return the first RETURN_EXPR found, or note
-   the first CONTINUE_STMT and/or BREAK_STMT if RETURN_EXPR is not found.  */
+   the first CONTINUE_STMT and/or BREAK_STMT if RETURN_EXPR is not found.
+   For C++26 also note presence of possibly throwing calls.  */
 static tree
 check_for_return_continue (tree *tp, int *walk_subtrees, void *data)
 {
@@ -10038,6 +11242,13 @@ check_for_return_continue (tree *tp, int *walk_subtrees, void *data)
     case CONSTRUCTOR:
       break;
 
+    case AGGR_INIT_EXPR:
+    case CALL_EXPR:
+      /* In C++26 a function could throw.  */
+      if (callee_might_throw (t))
+	d->could_throw = true;
+      break;
+
     default:
       if (!EXPR_P (t))
 	*walk_subtrees = 0;
@@ -10243,8 +11454,27 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 			|| TREE_CODE (t) != CALL_EXPR
 			|| current_function_decl == NULL_TREE
 			|| !is_std_construct_at (current_function_decl))
-		    && !cxx_dynamic_cast_fn_p (fun))
+		    && !cxx_dynamic_cast_fn_p (fun)
+		    && !cxx_cxa_builtin_fn_p (fun))
 		  {
+		    /* In C++26 evaluation of the function arguments might
+		       throw and in that case it is irrelevant whether
+		       fun is constexpr or not.  */
+		    if (cxx_dialect >= cxx26)
+		      for (; i < nargs; ++i)
+			{
+			  tree x = get_nth_callarg (t, i);
+			  bool rv = processing_template_decl ? any : rval;
+			  bool sub_now = false;
+			  if (!potential_constant_expression_1 (x, rv, strict,
+								sub_now,
+								fundef_p,
+								flags,
+								jump_target))
+			    return false;
+			  if (throws (jump_target))
+			    return true;
+			}
 		    if ((flags & tf_error)
 			&& constexpr_error (loc, fundef_p,
 					    "call to non-%<constexpr%> "
@@ -10289,7 +11519,12 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 						  sub_now, fundef_p, flags,
 						  jump_target))
 	      return false;
+	    if (throws (jump_target))
+	      return true;
           }
+	/* In C++26 a function could throw.  */
+	if (*jump_target == NULL_TREE && callee_might_throw (t))
+	  *jump_target = void_node;
         return true;
       }
 
@@ -10512,11 +11747,13 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 		 a return.  */
 	      hash_set<tree> pset;
 	      check_for_return_continue_data data = { &pset, NULL_TREE,
-						      NULL_TREE };
+						      NULL_TREE, false };
 	      if (tree ret_expr
 		  = cp_walk_tree (&FOR_BODY (t), check_for_return_continue,
 				  &data, &pset))
 		*jump_target = ret_expr;
+	      if (data.could_throw)
+		*jump_target = void_node;
 	      return true;
 	    }
 	}
@@ -10556,11 +11793,13 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	     a return.  */
 	  hash_set<tree> pset;
 	  check_for_return_continue_data data = { &pset, NULL_TREE,
-						  NULL_TREE  };
+						  NULL_TREE, false };
 	  if (tree ret_expr
 	      = cp_walk_tree (&WHILE_BODY (t), check_for_return_continue,
 			      &data, &pset))
 	    *jump_target = ret_expr;
+	  if (data.could_throw)
+	    *jump_target = void_node;
 	  return true;
 	}
       if (!RECUR (WHILE_BODY (t), any))
@@ -10584,7 +11823,7 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	{
 	  hash_set<tree> pset;
 	  check_for_return_continue_data data = { &pset, NULL_TREE,
-						  NULL_TREE };
+						  NULL_TREE, false };
 	  if (tree ret_expr
 	      = cp_walk_tree (&SWITCH_STMT_BODY (t), check_for_return_continue,
 			      &data, &pset))
@@ -10593,6 +11832,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	  else if (data.continue_stmt)
 	    /* The switch can't return, but might continue.  */
 	    *jump_target = data.continue_stmt;
+	  if (data.could_throw)
+	    *jump_target = void_node;
 	}
       return true;
 
@@ -10622,7 +11863,6 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 
     case DYNAMIC_CAST_EXPR:
     case PSEUDO_DTOR_EXPR:
-    case THROW_EXPR:
     case OMP_PARALLEL:
     case OMP_TASK:
     case OMP_FOR:
@@ -10678,6 +11918,11 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	 constant.  */
       return true;
 
+    case THROW_EXPR:
+      if (cxx_dialect < cxx26)
+	goto fail;
+      return RECUR (TREE_OPERAND (t, 0), rval);
+
     case ASM_EXPR:
       if (flags & tf_error)
 	inline_asm_in_constexpr_error (loc, fundef_p);
@@ -10806,6 +12051,22 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
     case CLEANUP_POINT_EXPR:
     case MUST_NOT_THROW_EXPR:
     case TRY_CATCH_EXPR:
+      /* Even for C++26 handle TRY_BLOCK conservatively, if we detect the
+	 body could throw, even with catch (...) among handlers we'd need
+	 to analyze them in detail if they couldn't rethrow it.  More
+	 importantly though, throws (jump_target) is just conservative,
+	 and there could be e.g.
+	   try
+	     {
+	       possibly_throwing_fn (args);
+	       break;
+	     }
+	   catch (...)
+	     {
+	     }
+	 or continue or return instead of break.  So, clearing *jump_target
+	 because we see catch (...) handler might mean we missed break
+	 etc.  */
     case TRY_BLOCK:
     case EH_SPEC_BLOCK:
     case EXPR_STMT:
@@ -11047,9 +12308,9 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 					       want_rval, strict, now, fundef_p,
 					       tf_none, &this_jump_target))
 	    {
-	      if (returns (&this_jump_target))
+	      if (returns (&this_jump_target) || throws (&this_jump_target))
 		*jump_target = this_jump_target;
-	      else if (!returns (jump_target))
+	      else if (!returns (jump_target) && !throws (jump_target))
 		{
 		  if (breaks (&this_jump_target)
 		      || continues (&this_jump_target))
@@ -11061,7 +12322,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 			 couldn't return, break or continue.  */
 		      hash_set<tree> pset;
 		      check_for_return_continue_data data = { &pset, NULL_TREE,
-							      NULL_TREE };
+							      NULL_TREE,
+							      false };
 		      if (tree ret_expr
 			= cp_walk_tree (&TREE_OPERAND (t, 2),
 					check_for_return_continue, &data,
@@ -11074,6 +12336,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 			  else if (data.break_stmt)
 			    *jump_target = data.break_stmt;
 			}
+		      if (data.could_throw)
+			*jump_target = void_node;
 		    }
 		}
 	      return true;
diff --git a/gcc/cp/constraint.cc b/gcc/cp/constraint.cc
index c8eef24..cbdfafc 100644
--- a/gcc/cp/constraint.cc
+++ b/gcc/cp/constraint.cc
@@ -2490,10 +2490,11 @@ satisfy_atom (tree t, tree args, sat_info info)
   result = force_rvalue (result, info.complain);
   if (result == error_mark_node)
     return cache.save (inst_cache.save (error_mark_node));
+  tree substituted = result;
   if (!same_type_p (TREE_TYPE (result), boolean_type_node))
     {
       if (info.noisy ())
-	diagnose_atomic_constraint (t, args, result, info);
+	diagnose_atomic_constraint (t, args, substituted, info);
       return cache.save (inst_cache.save (error_mark_node));
     }
 
@@ -2511,7 +2512,7 @@ satisfy_atom (tree t, tree args, sat_info info)
     }
   result = satisfaction_value (result);
   if (result == boolean_false_node && info.diagnose_unsatisfaction_p ())
-    diagnose_atomic_constraint (t, args, result, info);
+    diagnose_atomic_constraint (t, args, substituted, info);
 
   return cache.save (inst_cache.save (result));
 }
@@ -3063,11 +3064,9 @@ get_constraint_error_location (tree t)
 
 /* Emit a diagnostic for a failed trait.  */
 
-static void
-diagnose_trait_expr (tree expr, tree args)
+void
+diagnose_trait_expr (location_t loc, tree expr, tree args)
 {
-  location_t loc = cp_expr_location (expr);
-
   /* Build a "fake" version of the instantiated trait, so we can
      get the instantiated types from result.  */
   ++processing_template_decl;
@@ -3076,209 +3075,245 @@ diagnose_trait_expr (tree expr, tree args)
 
   tree t1 = TRAIT_EXPR_TYPE1 (expr);
   tree t2 = TRAIT_EXPR_TYPE2 (expr);
-  if (t2 && TREE_CODE (t2) == TREE_VEC)
-    {
-      /* Convert the TREE_VEC of arguments into a TREE_LIST, since we can't
-	 directly print a TREE_VEC but we can a TREE_LIST via the E format
-	 specifier.  */
-      tree list = NULL_TREE;
-      for (tree t : tree_vec_range (t2))
-	list = tree_cons (NULL_TREE, t, list);
-      t2 = nreverse (list);
-    }
+
+  /* For traits intrinsically about the properties of user-defined types,
+     decl_loc will point to the declaration of that type.  */
+  location_t decl_loc = location_of (t1);
+  if (decl_loc == input_location)
+    decl_loc = loc;
+
   switch (TRAIT_EXPR_KIND (expr))
     {
     case CPTK_HAS_NOTHROW_ASSIGN:
-      inform (loc, "  %qT is not nothrow copy assignable", t1);
+      inform (decl_loc, "%qT is not nothrow copy assignable", t1);
       break;
     case CPTK_HAS_NOTHROW_CONSTRUCTOR:
-      inform (loc, "  %qT is not nothrow default constructible", t1);
+      inform (decl_loc, "%qT is not nothrow default constructible", t1);
       break;
     case CPTK_HAS_NOTHROW_COPY:
-      inform (loc, "  %qT is not nothrow copy constructible", t1);
+      inform (decl_loc, "%qT is not nothrow copy constructible", t1);
       break;
     case CPTK_HAS_TRIVIAL_ASSIGN:
-      inform (loc, "  %qT is not trivially copy assignable", t1);
+      inform (decl_loc, "%qT is not trivially copy assignable", t1);
       break;
     case CPTK_HAS_TRIVIAL_CONSTRUCTOR:
-      inform (loc, "  %qT is not trivially default constructible", t1);
+      inform (decl_loc, "%qT is not trivially default constructible", t1);
       break;
     case CPTK_HAS_TRIVIAL_COPY:
-      inform (loc, "  %qT is not trivially copy constructible", t1);
+      inform (decl_loc, "%qT is not trivially copy constructible", t1);
       break;
     case CPTK_HAS_TRIVIAL_DESTRUCTOR:
-      inform (loc, "  %qT is not trivially destructible", t1);
+      inform (decl_loc, "%qT is not trivially destructible", t1);
       break;
     case CPTK_HAS_UNIQUE_OBJ_REPRESENTATIONS:
-      inform (loc, "  %qT does not have unique object representations", t1);
+      inform (decl_loc, "%qT does not have unique object representations", t1);
       break;
     case CPTK_HAS_VIRTUAL_DESTRUCTOR:
-      inform (loc, "  %qT does not have a virtual destructor", t1);
+      {
+	location_t dtor_loc = decl_loc;
+	if (NON_UNION_CLASS_TYPE_P (t1))
+	  if (tree dtor = CLASSTYPE_DESTRUCTOR (t1))
+	    dtor_loc = DECL_SOURCE_LOCATION (dtor);
+	inform (dtor_loc, "%qT does not have a virtual destructor", t1);
+      }
       break;
     case CPTK_IS_ABSTRACT:
-      inform (loc, "  %qT is not an abstract class", t1);
+      inform (decl_loc, "%qT is not an abstract class", t1);
       break;
     case CPTK_IS_AGGREGATE:
-      inform (loc, "  %qT is not an aggregate", t1);
+      inform (decl_loc, "%qT is not an aggregate", t1);
       break;
     case CPTK_IS_ARRAY:
-      inform (loc, "  %qT is not an array", t1);
+      inform (loc, "%qT is not an array", t1);
       break;
     case CPTK_IS_ASSIGNABLE:
-      inform (loc, "  %qT is not assignable from %qT", t1, t2);
+      inform (loc, "%qT is not assignable from %qT, because", t1, t2);
+      is_xible (MODIFY_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_BASE_OF:
-      inform (loc, "  %qT is not a base of %qT", t1, t2);
+      inform (decl_loc, "%qT is not a base of %qT", t1, t2);
       break;
     case CPTK_IS_BOUNDED_ARRAY:
-      inform (loc, "  %qT is not a bounded array", t1);
+      inform (loc, "%qT is not a bounded array", t1);
       break;
     case CPTK_IS_CLASS:
-      inform (loc, "  %qT is not a class", t1);
+      inform (decl_loc, "%qT is not a class", t1);
       break;
     case CPTK_IS_CONST:
-      inform (loc, "  %qT is not a const type", t1);
+      inform (loc, "%qT is not a const type", t1);
       break;
     case CPTK_IS_CONSTRUCTIBLE:
-      if (!t2)
-	inform (loc, "  %qT is not default constructible", t1);
+      if (!TREE_VEC_LENGTH (t2))
+	inform (loc, "%qT is not default constructible, because", t1);
       else
-	inform (loc, "  %qT is not constructible from %qE", t1, t2);
+	inform (loc, "%qT is not constructible from %qT, because", t1, t2);
+      is_xible (INIT_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_CONVERTIBLE:
-      inform (loc, "  %qT is not convertible from %qE", t2, t1);
+      /* The errors produced here all seem to mention "convertible" in the
+	 diagnostic, so an extra inform here appears redundant.  */
+      is_convertible (t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_DESTRUCTIBLE:
-      inform (loc, "  %qT is not destructible", t1);
+      inform (loc, "%qT is not destructible, because", t1);
+      is_xible (BIT_NOT_EXPR, t1, NULL_TREE, /*explain=*/true);
       break;
     case CPTK_IS_EMPTY:
-      inform (loc, "  %qT is not an empty class", t1);
+      inform (decl_loc, "%qT is not an empty class", t1);
       break;
     case CPTK_IS_ENUM:
-      inform (loc, "  %qT is not an enum", t1);
+      inform (decl_loc, "%qT is not an enum", t1);
       break;
     case CPTK_IS_FINAL:
-      inform (loc, "  %qT is not a final class", t1);
+      inform (decl_loc, "%qT is not a final class", t1);
       break;
     case CPTK_IS_FUNCTION:
-      inform (loc, "  %qT is not a function", t1);
+      inform (loc, "%qT is not a function", t1);
       break;
     case CPTK_IS_INVOCABLE:
-      if (!t2)
-	inform (loc, "  %qT is not invocable", t1);
-      else
-	inform (loc, "  %qT is not invocable by %qE", t1, t2);
+      {
+	if (!TREE_VEC_LENGTH (t2))
+	  inform (loc, "%qT is not invocable, because", t1);
+	else
+	  inform (loc, "%qT is not invocable by %qT, because", t1, t2);
+	build_invoke (t1, t2, tf_error);
+      }
       break;
     case CPTK_IS_LAYOUT_COMPATIBLE:
-      inform (loc, "  %qT is not layout compatible with %qT", t1, t2);
+      inform (loc, "%qT is not layout compatible with %qT", t1, t2);
       break;
     case CPTK_IS_LITERAL_TYPE:
-      inform (loc, "  %qT is not a literal type", t1);
+      inform (decl_loc, "%qT is not a literal type", t1);
       break;
     case CPTK_IS_MEMBER_FUNCTION_POINTER:
-      inform (loc, "  %qT is not a member function pointer", t1);
+      inform (loc, "%qT is not a member function pointer", t1);
       break;
     case CPTK_IS_MEMBER_OBJECT_POINTER:
-      inform (loc, "  %qT is not a member object pointer", t1);
+      inform (loc, "%qT is not a member object pointer", t1);
       break;
     case CPTK_IS_MEMBER_POINTER:
-      inform (loc, "  %qT is not a member pointer", t1);
+      inform (loc, "%qT is not a member pointer", t1);
       break;
     case CPTK_IS_NOTHROW_ASSIGNABLE:
-      inform (loc, "  %qT is not nothrow assignable from %qT", t1, t2);
+      inform (loc, "%qT is not nothrow assignable from %qT, because", t1, t2);
+      is_nothrow_xible (MODIFY_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_CONSTRUCTIBLE:
-      if (!t2)
-	inform (loc, "  %qT is not nothrow default constructible", t1);
+      if (!TREE_VEC_LENGTH (t2))
+	inform (loc, "%qT is not nothrow default constructible, because", t1);
       else
-	inform (loc, "  %qT is not nothrow constructible from %qE", t1, t2);
+	inform (loc, "%qT is not nothrow constructible from %qT, because",
+		t1, t2);
+      is_nothrow_xible (INIT_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_CONVERTIBLE:
-      inform (loc, "  %qT is not nothrow convertible from %qE", t2, t1);
+      inform (loc, "%qT is not nothrow convertible from %qT, because", t1, t2);
+      is_nothrow_convertible (t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_DESTRUCTIBLE:
-      inform (loc, "  %qT is not nothrow destructible", t1);
+      inform (loc, "%qT is not nothrow destructible, because", t1);
+      is_nothrow_xible (BIT_NOT_EXPR, t1, NULL_TREE, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_INVOCABLE:
-      if (!t2)
-	inform (loc, "  %qT is not nothrow invocable", t1);
-      else
-	inform (loc, "  %qT is not nothrow invocable by %qE", t1, t2);
+      {
+	if (!TREE_VEC_LENGTH (t2))
+	  inform (loc, "%qT is not nothrow invocable, because", t1);
+	else
+	  inform (loc, "%qT is not nothrow invocable by %qT, because", t1, t2);
+	tree call = build_invoke (t1, t2, tf_error);
+	if (call != error_mark_node)
+	  explain_not_noexcept (call);
+      }
+      break;
+    case CPTK_IS_NOTHROW_RELOCATABLE:
+      inform (loc, "%qT is not nothrow relocatable", t1);
       break;
     case CPTK_IS_OBJECT:
-      inform (loc, "  %qT is not an object type", t1);
+      inform (loc, "%qT is not an object type", t1);
       break;
     case CPTK_IS_POINTER_INTERCONVERTIBLE_BASE_OF:
-      inform (loc, "  %qT is not pointer-interconvertible base of %qT",
+      inform (decl_loc, "%qT is not a pointer-interconvertible base of %qT",
 	      t1, t2);
       break;
     case CPTK_IS_POD:
-      inform (loc, "  %qT is not a POD type", t1);
+      inform (loc, "%qT is not a POD type", t1);
       break;
     case CPTK_IS_POINTER:
-      inform (loc, "  %qT is not a pointer", t1);
+      inform (loc, "%qT is not a pointer", t1);
       break;
     case CPTK_IS_POLYMORPHIC:
-      inform (loc, "  %qT is not a polymorphic type", t1);
+      inform (decl_loc, "%qT is not a polymorphic type", t1);
       break;
     case CPTK_IS_REFERENCE:
-      inform (loc, "  %qT is not a reference", t1);
+      inform (loc, "%qT is not a reference", t1);
+      break;
+    case CPTK_IS_REPLACEABLE:
+      inform (loc, "%qT is not replaceable", t1);
       break;
     case CPTK_IS_SAME:
-      inform (loc, "  %qT is not the same as %qT", t1, t2);
+      inform (loc, "%q#T is not the same as %q#T", t1, t2);
       break;
     case CPTK_IS_SCOPED_ENUM:
-      inform (loc, "  %qT is not a scoped enum", t1);
+      inform (decl_loc, "%qT is not a scoped enum", t1);
       break;
     case CPTK_IS_STD_LAYOUT:
-      inform (loc, "  %qT is not an standard layout type", t1);
+      inform (decl_loc, "%qT is not a standard layout type", t1);
       break;
     case CPTK_IS_TRIVIAL:
-      inform (loc, "  %qT is not a trivial type", t1);
+      inform (decl_loc, "%qT is not a trivial type", t1);
       break;
     case CPTK_IS_TRIVIALLY_ASSIGNABLE:
-      inform (loc, "  %qT is not trivially assignable from %qT", t1, t2);
+      inform (loc, "%qT is not trivially assignable from %qT, because", t1, t2);
+      is_trivially_xible (MODIFY_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_TRIVIALLY_CONSTRUCTIBLE:
-      if (!t2)
-	inform (loc, "  %qT is not trivially default constructible", t1);
+      if (!TREE_VEC_LENGTH (t2))
+	inform (loc, "%qT is not trivially default constructible, because", t1);
       else
-	inform (loc, "  %qT is not trivially constructible from %qE", t1, t2);
+	inform (loc, "%qT is not trivially constructible from %qT, because",
+		t1, t2);
+      is_trivially_xible (INIT_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_TRIVIALLY_COPYABLE:
-      inform (loc, "  %qT is not trivially copyable", t1);
+      inform (decl_loc, "%qT is not trivially copyable", t1);
       break;
     case CPTK_IS_TRIVIALLY_DESTRUCTIBLE:
-      inform (loc, "  %qT is not trivially destructible", t1);
+      inform (loc, "%qT is not trivially destructible, because", t1);
+      is_trivially_xible (BIT_NOT_EXPR, t1, NULL_TREE, /*explain=*/true);
+      break;
+    case CPTK_IS_TRIVIALLY_RELOCATABLE:
+      inform (loc, "%qT is not trivially relocatable", t1);
       break;
     case CPTK_IS_UNBOUNDED_ARRAY:
-      inform (loc, "  %qT is not an unbounded array", t1);
+      inform (loc, "%qT is not an unbounded array", t1);
       break;
     case CPTK_IS_UNION:
-      inform (loc, "  %qT is not a union", t1);
+      inform (decl_loc, "%qT is not a union", t1);
       break;
     case CPTK_IS_VIRTUAL_BASE_OF:
-      inform (loc, "  %qT is not a virtual base of %qT", t1, t2);
+      inform (decl_loc, "%qT is not a virtual base of %qT", t1, t2);
+      if (CLASS_TYPE_P (t2))
+	inform (location_of (t2), "%qT declared here", t2);
       break;
     case CPTK_IS_VOLATILE:
-      inform (loc, "  %qT is not a volatile type", t1);
+      inform (loc, "%qT is not a volatile type", t1);
       break;
     case CPTK_RANK:
-      inform (loc, "  %qT cannot yield a rank", t1);
+      inform (loc, "%qT cannot yield a rank", t1);
       break;
     case CPTK_TYPE_ORDER:
-      inform (loc, "  %qT and %qT cannot be ordered", t1, t2);
+      inform (loc, "%qT and %qT cannot be ordered", t1, t2);
       break;
     case CPTK_REF_CONSTRUCTS_FROM_TEMPORARY:
-      inform (loc, "  %qT is not a reference that binds to a temporary "
+      inform (loc, "%qT is not a reference that binds to a temporary "
 	      "object of type %qT (direct-initialization)", t1, t2);
       break;
     case CPTK_REF_CONVERTS_FROM_TEMPORARY:
-      inform (loc, "  %qT is not a reference that binds to a temporary "
+      inform (loc, "%qT is not a reference that binds to a temporary "
 	      "object of type %qT (copy-initialization)", t1, t2);
       break;
     case CPTK_IS_DEDUCIBLE:
-      inform (loc, "  %qD is not deducible from %qT", t1, t2);
+      inform (loc, "%qD is not deducible from %qT", t1, t2);
       break;
 #define DEFTRAIT_TYPE(CODE, NAME, ARITY) \
     case CPTK_##CODE:
@@ -3291,10 +3326,50 @@ diagnose_trait_expr (tree expr, tree args)
     }
 }
 
+/* Attempt to detect if this is a standard type trait, defined in terms
+   of a compiler builtin (above).  If so, this will allow us to provide
+   more helpful diagnostics.   */
+
+bool
+maybe_diagnose_standard_trait (location_t loc, tree expr)
+{
+  gcc_assert (TREE_CODE (expr) != TRAIT_EXPR);
+  expr = tree_strip_nop_conversions (expr);
+
+  /* TODO: in some cases it would be possible to provide more helpful
+     diagnostics for negations of traits, e.g. '!is_same_v<T1, T2>'.  */
+
+  tree args = NULL_TREE;
+  if (VAR_P (expr) && DECL_LANG_SPECIFIC (expr) && DECL_USE_TEMPLATE (expr))
+    {
+      tree tinfo = DECL_TEMPLATE_INFO (expr);
+      if (PRIMARY_TEMPLATE_P (TI_TEMPLATE (tinfo)) && TI_PARTIAL_INFO (tinfo))
+	tinfo = TI_PARTIAL_INFO (tinfo);
+      else if (DECL_TEMPLATE_SPECIALIZATION (expr))
+	/* In an explicit specialisation we no longer know what the original
+	   initializer looked like.  */
+	tinfo = NULL_TREE;
+
+      if (tinfo)
+	{
+	  expr = DECL_INITIAL (DECL_TEMPLATE_RESULT (TI_TEMPLATE (tinfo)));
+	  args = TI_ARGS (tinfo);
+	}
+    }
+
+  if (TREE_CODE (expr) == TRAIT_EXPR)
+    {
+      diagnose_trait_expr (loc, expr, args);
+      return true;
+    }
+
+  return false;
+}
+
 /* Diagnose a substitution failure in the atomic constraint T using ARGS.  */
 
 static void
-diagnose_atomic_constraint (tree t, tree args, tree result, sat_info info)
+diagnose_atomic_constraint (tree t, tree args, tree substituted, sat_info info)
 {
   /* If the constraint is already ill-formed, we've previously diagnosed
      the reason.  We should still say why the constraints aren't satisfied.  */
@@ -3315,25 +3390,26 @@ diagnose_atomic_constraint (tree t, tree args, tree result, sat_info info)
   /* Generate better diagnostics for certain kinds of expressions.  */
   tree expr = ATOMIC_CONSTR_EXPR (t);
   STRIP_ANY_LOCATION_WRAPPER (expr);
-  switch (TREE_CODE (expr))
+
+  if (TREE_CODE (expr) == REQUIRES_EXPR)
     {
-    case TRAIT_EXPR:
-      diagnose_trait_expr (expr, args);
-      break;
-    case REQUIRES_EXPR:
       gcc_checking_assert (info.diagnose_unsatisfaction_p ());
       /* Clear in_decl before replaying the substitution to avoid emitting
 	 seemingly unhelpful "in declaration ..." notes that follow some
 	 substitution failure error messages.  */
       info.in_decl = NULL_TREE;
       tsubst_requires_expr (expr, args, info);
-      break;
-    default:
-      if (!same_type_p (TREE_TYPE (result), boolean_type_node))
-	error_at (loc, "constraint %qE has type %qT, not %<bool%>",
-		  t, TREE_TYPE (result));
+    }
+  else if (!same_type_p (TREE_TYPE (substituted), boolean_type_node))
+    error_at (loc, "constraint %qE has type %qT, not %<bool%>",
+	      t, TREE_TYPE (substituted));
+  else
+    {
+      inform (loc, "the expression %qE evaluated to %<false%>", t);
+      if (TREE_CODE (expr) == TRAIT_EXPR)
+	diagnose_trait_expr (loc, expr, args);
       else
-	inform (loc, "the expression %qE evaluated to %<false%>", t);
+	maybe_diagnose_standard_trait (loc, substituted);
     }
 }
 
diff --git a/gcc/cp/coroutines.cc b/gcc/cp/coroutines.cc
index 52cc186..690e510 100644
--- a/gcc/cp/coroutines.cc
+++ b/gcc/cp/coroutines.cc
@@ -5050,6 +5050,8 @@ cp_coroutine_transform::build_ramp_function ()
      check the returned pointer and call the func if it's null.
      Otherwise, no check, and we fail for noexcept/fno-exceptions cases.  */
 
+  tree grooaf_if_stmt = NULL_TREE;
+  tree alloc_ok_scope = NULL_TREE;
   if (grooaf)
     {
       /* [dcl.fct.def.coroutine] / 10 (part 3)
@@ -5057,20 +5059,11 @@ cp_coroutine_transform::build_ramp_function ()
 	 control to the caller of the coroutine and the return value is
 	 obtained by a call to T::get_return_object_on_allocation_failure(),
 	 where T is the promise type.  */
-      tree if_stmt = begin_if_stmt ();
       tree cond = build1 (CONVERT_EXPR, frame_ptr_type, nullptr_node);
-      cond = build2 (EQ_EXPR, boolean_type_node, coro_fp, cond);
-      finish_if_stmt_cond (cond, if_stmt);
-      r = NULL_TREE;
-      if (void_ramp_p)
-	/* Execute the get-return-object-on-alloc-fail call...  */
-	finish_expr_stmt (grooaf);
-      else
-	/* Get the fallback return object.  */
-	r = grooaf;
-      finish_return_stmt (r);
-      finish_then_clause (if_stmt);
-      finish_if_stmt (if_stmt);
+      cond = build2 (NE_EXPR, boolean_type_node, coro_fp, cond);
+      grooaf_if_stmt = begin_if_stmt ();
+      finish_if_stmt_cond (cond, grooaf_if_stmt);
+      alloc_ok_scope = begin_compound_stmt (BCS_NORMAL);
     }
 
   /* Dereference the frame pointer, to use in member access code.  */
@@ -5295,7 +5288,6 @@ cp_coroutine_transform::build_ramp_function ()
 	 a temp which is then used to intialize the return object, including
 	 NVRO.  */
 
-      /* Temporary var to hold the g_r_o across the function body.  */
       coro_gro
 	= coro_build_and_push_artificial_var (loc, "_Coro_gro", gro_type,
 					      orig_fn_decl, NULL_TREE);
@@ -5328,9 +5320,28 @@ cp_coroutine_transform::build_ramp_function ()
   /* The ramp is done, we just need the return statement, which we build from
      the return object we constructed before we called the actor.  */
 
+  /* This is our 'normal' exit.  */
   r = void_ramp_p ? NULL_TREE : convert_from_reference (coro_gro);
   finish_return_stmt (r);
 
+  if (grooaf)
+    {
+      finish_compound_stmt (alloc_ok_scope);
+      finish_then_clause (grooaf_if_stmt);
+
+      begin_else_clause (grooaf_if_stmt);
+      /* We come here if the frame allocation failed.  */
+      r = NULL_TREE;
+      if (void_ramp_p)
+	/* Execute the get-return-object-on-alloc-fail call...  */
+	finish_expr_stmt (grooaf);
+      else
+	/* Get the fallback return object.  */
+	r = grooaf;
+      finish_return_stmt (r);
+      finish_if_stmt (grooaf_if_stmt);
+    }
+
   finish_compound_stmt (ramp_fnbody);
   return true;
 }
diff --git a/gcc/cp/cp-gimplify.cc b/gcc/cp/cp-gimplify.cc
index ce69bd6..4ff8f36a 100644
--- a/gcc/cp/cp-gimplify.cc
+++ b/gcc/cp/cp-gimplify.cc
@@ -690,6 +690,7 @@ cp_gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 		    && (REFERENCE_CLASS_P (op1) || DECL_P (op1)))
 		  op1 = build_fold_addr_expr (op1);
 
+		suppress_warning (op1, OPT_Wunused_result);
 		gimplify_and_add (op1, pre_p);
 	      }
 	    gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
@@ -889,6 +890,12 @@ cp_gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 			(EXPR_LOCATION (*expr_p), call_expr_nargs (*expr_p),
 			 &CALL_EXPR_ARG (*expr_p, 0));
 		break;
+	      case CP_BUILT_IN_EH_PTR_ADJUST_REF:
+		error_at (EXPR_LOCATION (*expr_p),
+			  "%qs used outside of constant expressions",
+			  "__builtin_eh_ptr_adjust_ref");
+		*expr_p = void_node;
+		break;
 	      default:
 		break;
 	      }
@@ -3022,7 +3029,7 @@ cp_fold (tree x, fold_flags_t flags)
     case CLEANUP_POINT_EXPR:
       /* Strip CLEANUP_POINT_EXPR if the expression doesn't have side
 	 effects.  */
-      r = cp_fold_rvalue (TREE_OPERAND (x, 0), flags);
+      r = cp_fold (TREE_OPERAND (x, 0), flags);
       if (!TREE_SIDE_EFFECTS (r))
 	x = r;
       break;
@@ -3211,7 +3218,16 @@ cp_fold (tree x, fold_flags_t flags)
 
       loc = EXPR_LOCATION (x);
       op0 = cp_fold_maybe_rvalue (TREE_OPERAND (x, 0), rval_ops, flags);
-      op1 = cp_fold_rvalue (TREE_OPERAND (x, 1), flags);
+      bool clear_decl_read;
+      clear_decl_read = false;
+      if (code == MODIFY_EXPR
+	  && (VAR_P (op0) || TREE_CODE (op0) == PARM_DECL)
+	  && !DECL_READ_P (op0))
+	clear_decl_read = true;
+      op1 = cp_fold_maybe_rvalue (TREE_OPERAND (x, 1),
+				  code != COMPOUND_EXPR, flags);
+      if (clear_decl_read)
+	DECL_READ_P (op0) = 0;
 
       /* decltype(nullptr) has only one value, so optimize away all comparisons
 	 with that type right away, keeping them in the IL causes troubles for
diff --git a/gcc/cp/cp-trait.def b/gcc/cp/cp-trait.def
index e71b28c..9fedfd7 100644
--- a/gcc/cp/cp-trait.def
+++ b/gcc/cp/cp-trait.def
@@ -87,12 +87,14 @@ DEFTRAIT_EXPR (IS_NOTHROW_CONSTRUCTIBLE, "__is_nothrow_constructible", -1)
 DEFTRAIT_EXPR (IS_NOTHROW_CONVERTIBLE, "__is_nothrow_convertible", 2)
 DEFTRAIT_EXPR (IS_NOTHROW_DESTRUCTIBLE, "__is_nothrow_destructible", 1)
 DEFTRAIT_EXPR (IS_NOTHROW_INVOCABLE, "__is_nothrow_invocable", -1)
+DEFTRAIT_EXPR (IS_NOTHROW_RELOCATABLE, "__builtin_is_nothrow_relocatable", 1)
 DEFTRAIT_EXPR (IS_OBJECT, "__is_object", 1)
 DEFTRAIT_EXPR (IS_POINTER_INTERCONVERTIBLE_BASE_OF, "__is_pointer_interconvertible_base_of", 2)
 DEFTRAIT_EXPR (IS_POD, "__is_pod", 1)
 DEFTRAIT_EXPR (IS_POINTER, "__is_pointer", 1)
 DEFTRAIT_EXPR (IS_POLYMORPHIC, "__is_polymorphic", 1)
 DEFTRAIT_EXPR (IS_REFERENCE, "__is_reference", 1)
+DEFTRAIT_EXPR (IS_REPLACEABLE, "__builtin_is_replaceable", 1)
 DEFTRAIT_EXPR (IS_SAME, "__is_same", 2)
 DEFTRAIT_EXPR (IS_SCOPED_ENUM, "__is_scoped_enum", 1)
 DEFTRAIT_EXPR (IS_STD_LAYOUT, "__is_standard_layout", 1)
@@ -101,6 +103,7 @@ DEFTRAIT_EXPR (IS_TRIVIALLY_ASSIGNABLE, "__is_trivially_assignable", 2)
 DEFTRAIT_EXPR (IS_TRIVIALLY_CONSTRUCTIBLE, "__is_trivially_constructible", -1)
 DEFTRAIT_EXPR (IS_TRIVIALLY_COPYABLE, "__is_trivially_copyable", 1)
 DEFTRAIT_EXPR (IS_TRIVIALLY_DESTRUCTIBLE, "__is_trivially_destructible", 1)
+DEFTRAIT_EXPR (IS_TRIVIALLY_RELOCATABLE, "__builtin_is_trivially_relocatable", 1)
 DEFTRAIT_EXPR (IS_UNBOUNDED_ARRAY, "__is_unbounded_array", 1)
 DEFTRAIT_EXPR (IS_UNION, "__is_union", 1)
 DEFTRAIT_EXPR (IS_VIRTUAL_BASE_OF, "__builtin_is_virtual_base_of", 2)
diff --git a/gcc/cp/cp-tree.h b/gcc/cp/cp-tree.h
index 1b893e2..fb8e0d8 100644
--- a/gcc/cp/cp-tree.h
+++ b/gcc/cp/cp-tree.h
@@ -452,6 +452,9 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       contract_semantic (in ASSERTION_, PRECONDITION_, POSTCONDITION_STMT)
       RETURN_EXPR_LOCAL_ADDR_P (in RETURN_EXPR)
       PACK_INDEX_PARENTHESIZED_P (in PACK_INDEX_*)
+      MUST_NOT_THROW_NOEXCEPT_P (in MUST_NOT_THROW_EXPR)
+      CONSTEVAL_BLOCK_P (in STATIC_ASSERT)
+      LAMBDA_EXPR_CONSTEVAL_BLOCK_P (in LAMBDA_EXPR)
    1: IDENTIFIER_KIND_BIT_1 (in IDENTIFIER_NODE)
       TI_PENDING_TEMPLATE_FLAG.
       TEMPLATE_PARMS_FOR_INLINE.
@@ -472,6 +475,7 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       BIND_EXPR_VEC_DTOR (in BIND_EXPR)
       ATOMIC_CONSTR_EXPR_FROM_CONCEPT_P (in ATOMIC_CONSTR)
       STATIC_INIT_DECOMP_BASE_P (in the TREE_LIST for {static,tls}_aggregates)
+      MUST_NOT_THROW_THROW_P (in MUST_NOT_THROW_EXPR)
    2: IDENTIFIER_KIND_BIT_2 (in IDENTIFIER_NODE)
       ICS_THIS_FLAG (in _CONV)
       DECL_INITIALIZED_BY_CONSTANT_EXPRESSION_P (in VAR_DECL)
@@ -493,6 +497,7 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       contract_semantic (in ASSERTION_, PRECONDITION_, POSTCONDITION_STMT)
       STATIC_INIT_DECOMP_NONBASE_P (in the TREE_LIST
 				    for {static,tls}_aggregates)
+      MUST_NOT_THROW_CATCH_P (in MUST_NOT_THROW_EXPR)
    3: IMPLICIT_RVALUE_P (in NON_LVALUE_EXPR or STATIC_CAST_EXPR)
       ICS_BAD_FLAG (in _CONV)
       FN_TRY_BLOCK_P (in TRY_BLOCK)
@@ -506,6 +511,7 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       LAMBDA_EXPR_STATIC_P (in LAMBDA_EXPR)
       TARGET_EXPR_ELIDING_P (in TARGET_EXPR)
       contract_semantic (in ASSERTION_, PRECONDITION_, POSTCONDITION_STMT)
+      TYPENAME_IS_UNION_P (in TYPENAME_TYPE)
    4: IDENTIFIER_MARKED (IDENTIFIER_NODEs)
       TREE_HAS_CONSTRUCTOR (in INDIRECT_REF, SAVE_EXPR, CONSTRUCTOR,
 	  CALL_EXPR, or FIELD_DECL).
@@ -1429,6 +1435,10 @@ struct GTY (()) tree_deferred_noexcept {
 #define STATIC_ASSERT_SOURCE_LOCATION(NODE) \
   (((struct tree_static_assert *)STATIC_ASSERT_CHECK (NODE))->location)
 
+/* True if this static assert represents a C++26 consteval block.  */
+#define CONSTEVAL_BLOCK_P(NODE) \
+  TREE_LANG_FLAG_0 (STATIC_ASSERT_CHECK (NODE))
+
 struct GTY (()) tree_static_assert {
   struct tree_base base;
   tree condition;
@@ -1543,6 +1553,10 @@ enum cp_lambda_default_capture_mode_type {
 #define LAMBDA_EXPR_THIS_CAPTURE(NODE) \
   (((struct tree_lambda_expr *)LAMBDA_EXPR_CHECK (NODE))->this_capture)
 
+/* True iff this lambda was created for a consteval block.  */
+#define LAMBDA_EXPR_CONSTEVAL_BLOCK_P(NODE) \
+  TREE_LANG_FLAG_0 (LAMBDA_EXPR_CHECK (NODE))
+
 /* True iff uses of a const variable capture were optimized away.  */
 #define LAMBDA_EXPR_CAPTURE_OPTIMIZED(NODE) \
   TREE_LANG_FLAG_2 (LAMBDA_EXPR_CHECK (NODE))
@@ -2354,6 +2368,10 @@ enum languages { lang_c, lang_cplusplus };
 #define NON_UNION_CLASS_TYPE_P(T) \
   (TREE_CODE (T) == RECORD_TYPE && TYPE_LANG_FLAG_5 (T))
 
+/* Nonzero if T is a class type and is a union.  */
+#define UNION_TYPE_P(T) \
+  (TREE_CODE (T) == UNION_TYPE && TYPE_LANG_FLAG_5 (T))
+
 /* Keep these checks in ascending code order.  */
 #define RECORD_OR_UNION_CODE_P(T)	\
   ((T) == RECORD_TYPE || (T) == UNION_TYPE)
@@ -2492,15 +2510,22 @@ struct GTY(()) lang_type {
   bool erroneous : 1;
   bool non_pod_aggregate : 1;
   bool non_aggregate_pod : 1;
+  bool trivially_relocatable : 1;
+  bool trivially_relocatable_computed : 1;
+
+  bool replaceable : 1;
+  bool replaceable_computed : 1;
 
   /* When adding a flag here, consider whether or not it ought to
      apply to a template instance if it applies to the template.  If
-     so, make sure to copy it in instantiate_class_template!  */
+     so, make sure to copy it in instantiate_class_template!
+
+     Also make sure new flags here are streamed in module.cc.  */
 
   /* There are some bits left to fill out a 32-bit word.  Keep track
      of this by updating the size of this bitfield whenever you add or
      remove a flag.  */
-  unsigned dummy : 2;
+  unsigned dummy : 30;
 
   tree primary_base;
   vec<tree_pair_s, va_gc> *vcall_indices;
@@ -2511,11 +2536,11 @@ struct GTY(()) lang_type {
   vec<tree, va_gc> *pure_virtuals;
   tree friend_classes;
   vec<tree, va_gc> * GTY((reorder ("resort_type_member_vec"))) members;
+  /* CLASSTYPE_KEY_METHOD for TYPE_POLYMORPHIC_P types, CLASSTYPE_LAMBDA_EXPR
+     otherwise.  */
   tree key_method;
   tree decl_list;
   tree befriending_classes;
-  /* FIXME reuse another field?  */
-  tree lambda_expr;
   union maybe_objc_info {
     /* If not c_dialect_objc, this part is not even allocated.  */
     char GTY((tag ("0"))) non_objc;
@@ -2638,7 +2663,13 @@ struct GTY(()) lang_type {
 /* The member function with which the vtable will be emitted:
    the first noninline non-pure-virtual member function.  NULL_TREE
    if there is no key function or if this is a class template */
-#define CLASSTYPE_KEY_METHOD(NODE) (LANG_TYPE_CLASS_CHECK (NODE)->key_method)
+#define CLASSTYPE_KEY_METHOD(NODE) \
+  (TYPE_POLYMORPHIC_P (NODE)					\
+   ? LANG_TYPE_CLASS_CHECK (NODE)->key_method			\
+   : NULL_TREE)
+#define SET_CLASSTYPE_KEY_METHOD(NODE, VALUE) \
+  (gcc_checking_assert (TYPE_POLYMORPHIC_P (NODE)),		\
+   LANG_TYPE_CLASS_CHECK (NODE)->key_method = (VALUE))
 
 /* Vector of members.  During definition, it is unordered and only
    member functions are present.  After completion it is sorted and
@@ -2770,7 +2801,12 @@ struct GTY(()) lang_type {
 
 /* The associated LAMBDA_EXPR that made this class.  */
 #define CLASSTYPE_LAMBDA_EXPR(NODE) \
-  (LANG_TYPE_CLASS_CHECK (NODE)->lambda_expr)
+  (TYPE_POLYMORPHIC_P (NODE)				\
+   ? NULL_TREE						\
+   : LANG_TYPE_CLASS_CHECK (NODE)->key_method)
+#define SET_CLASSTYPE_LAMBDA_EXPR(NODE, VALUE) \
+  (gcc_checking_assert (!TYPE_POLYMORPHIC_P (NODE)),	\
+   LANG_TYPE_CLASS_CHECK (NODE)->key_method = (VALUE))
 /* The extra mangling scope for this closure type.  */
 #define LAMBDA_TYPE_EXTRA_SCOPE(NODE) \
   (LAMBDA_EXPR_EXTRA_SCOPE (CLASSTYPE_LAMBDA_EXPR (NODE)))
@@ -2836,6 +2872,29 @@ struct GTY(()) lang_type {
    above (c++/120012).  This could also be a hash_set.  */
 #define CLASSTYPE_NON_AGGREGATE_POD(NODE) \
   (LANG_TYPE_CLASS_CHECK (NODE)->non_aggregate_pod)
+
+/* If CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED, true if this class is
+   trivially relocatable.
+   If !CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED, true if this class
+   is marked with trivially_relocatable_if_eligible conditional keyword.  */
+#define CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->trivially_relocatable)
+
+/* True if whether this class is trivially relocatable or not
+   has been computed already.  */
+#define CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->trivially_relocatable_computed)
+
+/* If CLASSTYPE_REPLACEABLE_COMPUTED, true if this class is replaceable.
+   If !CLASSTYPE_REPLACEABLE_COMPUTED, true if this class is marked with
+   replaceable_if_eligible conditional keyword.  */
+#define CLASSTYPE_REPLACEABLE_BIT(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->replaceable)
+
+/* True if whether this class is replaceable or not has been computed
+   already.  */
+#define CLASSTYPE_REPLACEABLE_COMPUTED(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->replaceable_computed)
 
 /* Additional macros for inheritance information.  */
 
@@ -3011,6 +3070,8 @@ struct GTY(()) lang_decl_min {
      In a lambda-capture proxy VAR_DECL, this is DECL_CAPTURED_VARIABLE.
      In a function-scope TREE_STATIC VAR_DECL or IMPLICIT_TYPEDEF_P TYPE_DECL,
      this is DECL_DISCRIMINATOR.
+     In constexpr exception artificial VAR_DECL, this is
+     DECL_EXCEPTION_REFCOUNT.
      In a DECL_LOCAL_DECL_P decl, this is the namespace decl it aliases.
      Otherwise, in a class-scope DECL, this is DECL_ACCESS.   */
   tree access;
@@ -4465,6 +4526,23 @@ get_vec_init_expr (tree t)
 #define MUST_NOT_THROW_COND(NODE) \
   TREE_OPERAND (MUST_NOT_THROW_EXPR_CHECK (NODE), 1)
 
+/* Reasons why MUST_NOT_THROW_EXPR has been created.  */
+
+/* Indicates MUST_NOT_THROW_EXPR has been created to wrap body of
+   a noexcept function.  */
+#define MUST_NOT_THROW_NOEXCEPT_P(NODE) \
+  TREE_LANG_FLAG_0 (MUST_NOT_THROW_EXPR_CHECK (NODE))
+
+/* Indicates MUST_NOT_THROW_EXPR has been created to wrap construction of
+   exception object during throw.  */
+#define MUST_NOT_THROW_THROW_P(NODE) \
+  TREE_LANG_FLAG_1 (MUST_NOT_THROW_EXPR_CHECK (NODE))
+
+/* Indicates MUST_NOT_THROW_EXPR has been created to wrap construction of
+   handler parameter during catch.  */
+#define MUST_NOT_THROW_CATCH_P(NODE) \
+  TREE_LANG_FLAG_2 (MUST_NOT_THROW_EXPR_CHECK (NODE))
+
 /* The TYPE_MAIN_DECL for a class template type is a TYPE_DECL, not a
    TEMPLATE_DECL.  This macro determines whether or not a given class
    type is really a template type, as opposed to an instantiation or
@@ -4485,11 +4563,14 @@ get_vec_init_expr (tree t)
 #define TYPENAME_IS_ENUM_P(NODE) \
   (TREE_LANG_FLAG_0 (TYPENAME_TYPE_CHECK (NODE)))
 
-/* True if a TYPENAME_TYPE was declared as a "class", "struct", or
-   "union".  */
+/* True if a TYPENAME_TYPE was declared as a "class" or "struct".  */
 #define TYPENAME_IS_CLASS_P(NODE) \
   (TREE_LANG_FLAG_1 (TYPENAME_TYPE_CHECK (NODE)))
 
+/* True if a TYPENAME_TYPE was declared as a "union".  */
+#define TYPENAME_IS_UNION_P(NODE) \
+  (TREE_LANG_FLAG_3 (TYPENAME_TYPE_CHECK (NODE)))
+
 /* True if a TYPENAME_TYPE is in the process of being resolved.  */
 #define TYPENAME_IS_RESOLVING_P(NODE) \
   (TREE_LANG_FLAG_2 (TYPENAME_TYPE_CHECK (NODE)))
@@ -4504,7 +4585,7 @@ get_vec_init_expr (tree t)
 #define TYPE_CONTAINS_VPTR_P(NODE)		\
   (TYPE_POLYMORPHIC_P (NODE) || CLASSTYPE_VBASECLASSES (NODE))
 
-/* Nonzero if NODE is a FUNCTION_DECL or VARIABLE_DECL (for a decl
+/* Nonzero if NODE is a FUNCTION_DECL or VAR_DECL (for a decl
    with namespace scope) declared in a local scope.  */
 #define DECL_LOCAL_DECL_P(NODE) \
   DECL_LANG_FLAG_0 (VAR_OR_FUNCTION_DECL_CHECK (NODE))
@@ -5145,6 +5226,10 @@ get_vec_init_expr (tree t)
    protected_access_node will appear in the DECL_ACCESS for the node.  */
 #define DECL_ACCESS(NODE) (LANG_DECL_MIN_CHECK (NODE)->access)
 
+/* In artificial VAR_DECL created by cxa_allocate_exception
+   this is reference count.  */
+#define DECL_EXCEPTION_REFCOUNT(NODE) (LANG_DECL_MIN_CHECK (NODE)->access)
+
 /* Nonzero if the FUNCTION_DECL is a global constructor.  */
 #define DECL_GLOBAL_CTOR_P(NODE) \
   (LANG_DECL_FN_CHECK (NODE)->global_ctor_p)
@@ -6463,7 +6548,9 @@ enum virt_specifier
   {
     VIRT_SPEC_UNSPECIFIED = 0x0,
     VIRT_SPEC_FINAL       = 0x1,
-    VIRT_SPEC_OVERRIDE    = 0x2
+    VIRT_SPEC_OVERRIDE    = 0x2,
+    VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE = 0x4,
+    VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE = 0x8
   };
 
 /* A type-qualifier, or bitmask therefore, using the VIRT_SPEC
@@ -6805,6 +6892,7 @@ enum cp_built_in_function {
   CP_BUILT_IN_IS_CORRESPONDING_MEMBER,
   CP_BUILT_IN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS,
   CP_BUILT_IN_SOURCE_LOCATION,
+  CP_BUILT_IN_EH_PTR_ADJUST_REF,
   CP_BUILT_IN_LAST
 };
 
@@ -6985,6 +7073,7 @@ extern bool type_has_extended_temps		(tree);
 extern tree strip_top_quals			(tree);
 extern bool reference_related_p			(tree, tree);
 extern bool reference_compatible_p		(tree, tree);
+extern bool handler_match_for_exception_type	(tree, tree);
 extern int remaining_arguments			(tree);
 extern tree build_implicit_conv_flags		(tree, tree, int);
 extern tree perform_implicit_conversion		(tree, tree, tsubst_flags_t);
@@ -7155,7 +7244,7 @@ extern void determine_local_discriminator	(tree, tree = NULL_TREE);
 extern bool member_like_constrained_friend_p	(tree);
 extern bool fns_correspond			(tree, tree);
 extern int decls_match				(tree, tree, bool = true);
-extern bool maybe_version_functions		(tree, tree, bool);
+extern bool maybe_version_functions		(tree, tree);
 extern bool validate_constexpr_redeclaration	(tree, tree);
 extern bool merge_default_template_args		(tree, tree, bool);
 extern tree duplicate_decls			(tree, tree,
@@ -7378,7 +7467,7 @@ extern void maybe_warn_variadic_templates       (void);
 extern void maybe_warn_cpp0x			(cpp0x_warn_str str,
 						 location_t = input_location);
 extern bool pedwarn_cxx98                       (location_t,
-						 diagnostic_option_id option_id,
+						 diagnostics::option_id option_id,
 						 const char *, ...) ATTRIBUTE_GCC_DIAG(3,4);
 extern location_t location_of                   (tree);
 extern void qualified_name_lookup_error		(tree, tree, tree,
@@ -7404,6 +7493,7 @@ extern int nothrow_libfn_p			(const_tree);
 extern void check_handlers			(tree);
 extern tree finish_noexcept_expr		(tree, tsubst_flags_t);
 extern bool expr_noexcept_p			(tree, tsubst_flags_t);
+extern void explain_not_noexcept		(tree);
 extern void perform_deferred_noexcept_checks	(void);
 extern bool nothrow_spec_p			(const_tree);
 extern bool type_noexcept_p			(const_tree);
@@ -7525,11 +7615,14 @@ extern void finish_thunk			(tree);
 extern void use_thunk				(tree, bool);
 extern bool trivial_fn_p			(tree);
 extern tree forward_parm			(tree);
-extern bool is_trivially_xible			(enum tree_code, tree, tree);
-extern bool is_nothrow_xible			(enum tree_code, tree, tree);
-extern bool is_xible				(enum tree_code, tree, tree);
-extern bool is_convertible			(tree, tree);
-extern bool is_nothrow_convertible		(tree, tree);
+extern bool is_trivially_xible			(enum tree_code, tree, tree,
+						 bool = false);
+extern bool is_nothrow_xible			(enum tree_code, tree, tree,
+						 bool = false);
+extern bool is_xible				(enum tree_code, tree, tree,
+						 bool = false);
+extern bool is_convertible			(tree, tree, bool = false);
+extern bool is_nothrow_convertible		(tree, tree, bool = false);
 extern bool ref_xes_from_temporary		(tree, tree, bool);
 extern tree get_defaulted_eh_spec		(tree, tsubst_flags_t = tf_warning_or_error);
 extern bool maybe_explain_implicit_delete	(tree);
@@ -8160,7 +8253,7 @@ extern bool cxx_omp_create_clause_info		(tree, tree, bool, bool,
 						 bool, bool);
 extern tree baselink_for_fns                    (tree);
 extern void finish_static_assert                (tree, tree, location_t,
-						 bool, bool);
+						 bool, bool, bool = false);
 extern tree finish_decltype_type                (tree, bool, tsubst_flags_t);
 extern tree fold_builtin_is_corresponding_member (location_t, int, tree *);
 extern tree fold_builtin_is_pointer_inverconvertible_with_class (location_t, int, tree *);
@@ -8185,7 +8278,7 @@ extern void register_capture_members		(tree);
 extern tree lambda_expr_this_capture            (tree, int);
 extern void maybe_generic_this_capture		(tree, tree);
 extern tree maybe_resolve_dummy			(tree, bool);
-extern tree current_nonlambda_function		(void);
+extern tree current_nonlambda_function		(bool = false);
 extern tree nonlambda_method_basetype		(void);
 extern tree current_nonlambda_scope		(bool = false);
 extern tree current_lambda_expr			(void);
@@ -8235,6 +8328,8 @@ extern bool pod_type_p				(const_tree);
 extern bool layout_pod_type_p			(const_tree);
 extern bool std_layout_type_p			(const_tree);
 extern bool trivial_type_p			(const_tree);
+extern bool trivially_relocatable_type_p	(tree);
+extern bool replaceable_type_p			(tree);
 extern bool trivially_copyable_p		(const_tree);
 extern bool type_has_unique_obj_representations (const_tree);
 extern bool scalarish_type_p			(const_tree);
@@ -8380,9 +8475,9 @@ extern void cxx_print_xnode			(FILE *, tree, int);
 extern void cxx_print_decl			(FILE *, tree, int);
 extern void cxx_print_type			(FILE *, tree, int);
 extern void cxx_print_identifier		(FILE *, tree, int);
-extern void cxx_print_error_function		(diagnostic_text_output_format &,
+extern void cxx_print_error_function		(diagnostics::text_sink &,
 						 const char *,
-						 const diagnostic_info *);
+						 const diagnostics::diagnostic_info *);
 
 /* in typeck.cc */
 /* Says how we should behave when comparing two arrays one of which
@@ -8569,7 +8664,7 @@ extern void maybe_warn_pessimizing_move	     (tree, tree, bool);
 /* in typeck2.cc */
 extern void require_complete_eh_spec_types	(tree, tree);
 extern bool cxx_incomplete_type_diagnostic	(location_t, const_tree,
-						 const_tree, diagnostic_t);
+						 const_tree, enum diagnostics::kind);
 inline location_t
 loc_or_input_loc (location_t loc)
 {
@@ -8617,7 +8712,7 @@ cp_expr_loc_or_input_loc (const_tree t)
 
 inline bool
 cxx_incomplete_type_diagnostic (const_tree value, const_tree type,
-				diagnostic_t diag_kind)
+				enum diagnostics::kind diag_kind)
 {
   return cxx_incomplete_type_diagnostic (cp_expr_loc_or_input_loc (value),
 					 value, type, diag_kind);
@@ -8628,7 +8723,7 @@ extern void cxx_incomplete_type_error		(location_t, const_tree,
 inline void
 cxx_incomplete_type_error (const_tree value, const_tree type)
 {
-  cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+  cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
 }
 
 extern void cxx_incomplete_type_inform 	        (const_tree);
@@ -8699,7 +8794,7 @@ extern alias_set_type cxx_get_alias_set		(tree);
 extern bool cxx_warn_unused_global_decl		(const_tree);
 extern size_t cp_tree_size			(enum tree_code);
 extern bool cp_var_mod_type_p			(tree, tree);
-extern void cxx_initialize_diagnostics		(diagnostic_context *);
+extern void cxx_initialize_diagnostics		(diagnostics::context *);
 extern int cxx_types_compatible_p		(tree, tree);
 extern bool cxx_block_may_fallthru		(const_tree);
 
@@ -8844,6 +8939,8 @@ extern bool constraints_equivalent_p            (tree, tree);
 extern bool atomic_constraints_identical_p	(tree, tree);
 extern hashval_t iterative_hash_constraint      (tree, hashval_t);
 extern hashval_t hash_atomic_constraint         (tree);
+extern void diagnose_trait_expr			(location_t, tree, tree);
+extern bool maybe_diagnose_standard_trait	(location_t, tree);
 extern void diagnose_constraints                (location_t, tree, tree);
 
 extern void note_failed_type_completion		(tree, tsubst_flags_t);
diff --git a/gcc/cp/cvt.cc b/gcc/cp/cvt.cc
index f663a6d..55be12d 100644
--- a/gcc/cp/cvt.cc
+++ b/gcc/cp/cvt.cc
@@ -1186,13 +1186,6 @@ convert_to_void (tree expr, impl_conv_void implicit, tsubst_flags_t complain)
 
   expr = maybe_undo_parenthesized_ref (expr);
 
-  expr = mark_discarded_use (expr);
-  if (implicit == ICV_CAST)
-    /* An explicit cast to void avoids all -Wunused-but-set* warnings.  */
-    mark_exp_read (expr);
-
-  if (!TREE_TYPE (expr))
-    return expr;
   if (invalid_nonstatic_memfn_p (loc, expr, complain))
     return error_mark_node;
   if (TREE_CODE (expr) == PSEUDO_DTOR_EXPR)
@@ -1209,6 +1202,12 @@ convert_to_void (tree expr, impl_conv_void implicit, tsubst_flags_t complain)
 
   if (VOID_TYPE_P (TREE_TYPE (expr)))
     return expr;
+
+  expr = mark_discarded_use (expr);
+  if (implicit == ICV_CAST)
+    /* An explicit cast to void avoids all -Wunused-but-set* warnings.  */
+    mark_exp_read (expr);
+
   switch (TREE_CODE (expr))
     {
     case COND_EXPR:
diff --git a/gcc/cp/decl.cc b/gcc/cp/decl.cc
index 83c8e28..cb3ebff 100644
--- a/gcc/cp/decl.cc
+++ b/gcc/cp/decl.cc
@@ -110,7 +110,7 @@ static void initialize_local_var (tree, tree, bool);
 static void expand_static_init (tree, tree);
 static location_t smallest_type_location (const cp_decl_specifier_seq*);
 static bool identify_goto (tree, location_t, const location_t *,
-			   diagnostic_t, bool);
+			   enum diagnostics::kind, bool);
 
 /* The following symbols are subsumed in the cp_global_trees array, and
    listed here individually for documentation purposes.
@@ -747,11 +747,11 @@ poplevel (int keep, int reverse, int functionbody)
 	      {
 		if (!DECL_NAME (decl) && DECL_DECOMPOSITION_P (decl))
 		  warning_at (DECL_SOURCE_LOCATION (decl),
-			      OPT_Wunused_but_set_variable, "structured "
+			      OPT_Wunused_but_set_variable_, "structured "
 			      "binding declaration set but not used");
 		else
 		  warning_at (DECL_SOURCE_LOCATION (decl),
-			      OPT_Wunused_but_set_variable,
+			      OPT_Wunused_but_set_variable_,
 			      "variable %qD set but not used", decl);
 		unused_but_set_errorcount = errorcount;
 	      }
@@ -1214,9 +1214,7 @@ decls_match (tree newdecl, tree olddecl, bool record_versions /* = true */)
 	  && targetm.target_option.function_versions (newdecl, olddecl))
 	{
 	  if (record_versions)
-	    maybe_version_functions (newdecl, olddecl,
-				     (!DECL_FUNCTION_VERSIONED (newdecl)
-				      || !DECL_FUNCTION_VERSIONED (olddecl)));
+	    maybe_version_functions (newdecl, olddecl);
 	  return 0;
 	}
     }
@@ -1283,11 +1281,11 @@ maybe_mark_function_versioned (tree decl)
 }
 
 /* NEWDECL and OLDDECL have identical signatures.  If they are
-   different versions adjust them and return true.
-   If RECORD is set to true, record function versions.  */
+   different versions adjust them, record function versions, and return
+   true.  */
 
 bool
-maybe_version_functions (tree newdecl, tree olddecl, bool record)
+maybe_version_functions (tree newdecl, tree olddecl)
 {
   if (!targetm.target_option.function_versions (newdecl, olddecl))
     return false;
@@ -1310,16 +1308,13 @@ maybe_version_functions (tree newdecl, tree olddecl, bool record)
       maybe_mark_function_versioned (newdecl);
     }
 
-  if (record)
-    {
-      /* Add the new version to the function version structure.  */
-      cgraph_node *fn_node = cgraph_node::get_create (olddecl);
-      cgraph_function_version_info *fn_v = fn_node->function_version ();
-      if (!fn_v)
-	fn_v = fn_node->insert_new_function_version ();
+  /* Add the new version to the function version structure.  */
+  cgraph_node *fn_node = cgraph_node::get_create (olddecl);
+  cgraph_function_version_info *fn_v = fn_node->function_version ();
+  if (!fn_v)
+    fn_v = fn_node->insert_new_function_version ();
 
-      cgraph_node::add_function_version (fn_v, newdecl);
-    }
+  cgraph_node::add_function_version (fn_v, newdecl);
 
   return true;
 }
@@ -2014,8 +2009,10 @@ duplicate_decls (tree newdecl, tree olddecl, bool hiding, bool was_hidden)
 	    }
 	  /* For function versions, params and types match, but they
 	     are not ambiguous.  */
-	  else if ((!DECL_FUNCTION_VERSIONED (newdecl)
-		    && !DECL_FUNCTION_VERSIONED (olddecl))
+	  else if (((!DECL_FUNCTION_VERSIONED (newdecl)
+		     && !DECL_FUNCTION_VERSIONED (olddecl))
+		    || !same_type_p (fndecl_declared_return_type (newdecl),
+				     fndecl_declared_return_type (olddecl)))
 		   /* Let constrained hidden friends coexist for now, we'll
 		      check satisfaction later.  */
 		   && !member_like_constrained_friend_p (newdecl)
@@ -3740,10 +3737,10 @@ decl_jump_unsafe (tree decl)
 
 static bool
 identify_goto (tree decl, location_t loc, const location_t *locus,
-	       diagnostic_t diag_kind, bool computed)
+	       enum diagnostics::kind diag_kind, bool computed)
 {
   if (computed)
-    diag_kind = DK_WARNING;
+    diag_kind = diagnostics::kind::warning;
   bool complained
     = emit_diagnostic (diag_kind, loc, 0,
 		       decl ? G_("jump to label %qD")
@@ -3779,7 +3776,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
 
   if (exited_omp)
     {
-      complained = identify_goto (decl, input_location, locus, DK_ERROR,
+      complained = identify_goto (decl, input_location, locus,
+				  diagnostics::kind::error,
 				  computed);
       if (complained)
 	inform (input_location, "  exits OpenMP structured block");
@@ -3801,7 +3799,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
 
 	  if (!identified)
 	    {
-	      complained = identify_goto (decl, input_location, locus, DK_ERROR,
+	      complained = identify_goto (decl, input_location, locus,
+					  diagnostics::kind::error,
 					  computed);
 	      identified = 2;
 	    }
@@ -3869,7 +3868,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
       if (inf)
 	{
 	  if (identified < 2)
-	    complained = identify_goto (decl, input_location, locus, DK_ERROR,
+	    complained = identify_goto (decl, input_location, locus,
+					diagnostics::kind::error,
 					computed);
 	  identified = 2;
 	  if (complained)
@@ -3880,7 +3880,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
   if (!vec_safe_is_empty (computed))
     {
       if (!identified)
-	complained = identify_goto (decl, input_location, locus, DK_ERROR,
+	complained = identify_goto (decl, input_location, locus,
+				    diagnostics::kind::error,
 				    computed);
       identified = 2;
       if (complained)
@@ -3952,14 +3953,14 @@ check_goto_1 (named_label_entry *ent, bool computed)
       || ent->in_omp_scope || ent->in_stmt_expr
       || !vec_safe_is_empty (ent->bad_decls))
     {
-      diagnostic_t diag_kind = DK_PERMERROR;
+      enum diagnostics::kind diag_kind = diagnostics::kind::permerror;
       if (ent->in_try_scope || ent->in_catch_scope || ent->in_constexpr_if
 	  || ent->in_consteval_if || ent->in_transaction_scope
 	  || ent->in_omp_scope || ent->in_stmt_expr)
-	diag_kind = DK_ERROR;
+	diag_kind = diagnostics::kind::error;
       complained = identify_goto (decl, DECL_SOURCE_LOCATION (decl),
 				  &input_location, diag_kind, computed);
-      identified = 1 + (diag_kind == DK_ERROR);
+      identified = 1 + (diag_kind == diagnostics::kind::error);
     }
 
   FOR_EACH_VEC_SAFE_ELT (ent->bad_decls, ix, bad)
@@ -3972,7 +3973,9 @@ check_goto_1 (named_label_entry *ent, bool computed)
 	  if (identified == 1)
 	    {
 	      complained = identify_goto (decl, DECL_SOURCE_LOCATION (decl),
-					  &input_location, DK_ERROR, computed);
+					  &input_location,
+					  diagnostics::kind::error,
+					  computed);
 	      identified = 2;
 	    }
 	  if (complained)
@@ -4016,7 +4019,8 @@ check_goto_1 (named_label_entry *ent, bool computed)
 	      {
 		complained = identify_goto (decl,
 					    DECL_SOURCE_LOCATION (decl),
-					    &input_location, DK_ERROR,
+					    &input_location,
+					    diagnostics::kind::error,
 					    computed);
 		identified = 2;
 	      }
@@ -4040,7 +4044,8 @@ check_goto_1 (named_label_entry *ent, bool computed)
 		{
 		  complained
 		    = identify_goto (decl, DECL_SOURCE_LOCATION (decl),
-				     &input_location, DK_ERROR, computed);
+				     &input_location, diagnostics::kind::error,
+				     computed);
 		  identified = 2;
 		}
 	      if (complained)
@@ -4056,7 +4061,8 @@ check_goto_1 (named_label_entry *ent, bool computed)
 		    {
 		      complained
 			= identify_goto (decl, DECL_SOURCE_LOCATION (decl),
-					 &input_location, DK_ERROR, computed);
+					 &input_location, diagnostics::kind::error,
+					 computed);
 		      identified = 2;
 		    }
 		  if (complained)
@@ -4368,6 +4374,7 @@ struct typename_info {
   tree template_id;
   bool enum_p;
   bool class_p;
+  bool union_p;
 };
 
 struct typename_hasher : ggc_ptr_hash<tree_node>
@@ -4406,7 +4413,8 @@ struct typename_hasher : ggc_ptr_hash<tree_node>
 	    && TYPE_CONTEXT (t1) == t2->scope
 	    && TYPENAME_TYPE_FULLNAME (t1) == t2->template_id
 	    && TYPENAME_IS_ENUM_P (t1) == t2->enum_p
-	    && TYPENAME_IS_CLASS_P (t1) == t2->class_p);
+	    && TYPENAME_IS_CLASS_P (t1) == t2->class_p
+	    && TYPENAME_IS_UNION_P (t1) == t2->union_p);
   }
 };
 
@@ -4430,9 +4438,8 @@ build_typename_type (tree context, tree name, tree fullname,
   ti.name = name;
   ti.template_id = fullname;
   ti.enum_p = tag_type == enum_type;
-  ti.class_p = (tag_type == class_type
-		|| tag_type == record_type
-		|| tag_type == union_type);
+  ti.class_p = (tag_type == class_type || tag_type == record_type);
+  ti.union_p = tag_type == union_type;
   hashval_t hash = typename_hasher::hash (&ti);
 
   /* See if we already have this type.  */
@@ -4448,6 +4455,7 @@ build_typename_type (tree context, tree name, tree fullname,
       TYPENAME_TYPE_FULLNAME (t) = ti.template_id;
       TYPENAME_IS_ENUM_P (t) = ti.enum_p;
       TYPENAME_IS_CLASS_P (t) = ti.class_p;
+      TYPENAME_IS_UNION_P (t) = ti.union_p;
 
       /* Build the corresponding TYPE_DECL.  */
       tree d = build_decl (input_location, TYPE_DECL, name, t);
@@ -5078,6 +5086,18 @@ cxx_init_decl_processing (void)
 			    BUILT_IN_FRONTEND, NULL, NULL_TREE);
   set_call_expr_flags (decl, ECF_CONST | ECF_NOTHROW | ECF_LEAF);
 
+  if (cxx_dialect >= cxx26)
+    {
+      tree void_ptrintftype
+	= build_function_type_list (void_type_node, ptr_type_node,
+				    integer_type_node, NULL_TREE);
+      decl = add_builtin_function ("__builtin_eh_ptr_adjust_ref",
+				   void_ptrintftype,
+				   CP_BUILT_IN_EH_PTR_ADJUST_REF,
+				   BUILT_IN_FRONTEND, NULL, NULL_TREE);
+      set_call_expr_flags (decl, ECF_NOTHROW | ECF_LEAF);
+    }
+
   integer_two_node = build_int_cst (NULL_TREE, 2);
 
   /* Guess at the initial static decls size.  */
@@ -9172,6 +9192,10 @@ cp_finish_decl (tree decl, tree init, bool init_const_expr_p,
 
       if (decomp)
 	{
+	  if (DECL_DECLARED_CONSTINIT_P (decl) && cxx_dialect < cxx26)
+	    pedwarn (DECL_SOURCE_LOCATION (decl), OPT_Wc__26_extensions,
+		     "%<constinit%> can be applied to structured binding "
+		     "only with %<-std=c++2c%> or %<-std=gnu++2c%>");
 	  cp_maybe_mangle_decomp (decl, decomp);
 	  if (TREE_STATIC (decl) && !DECL_FUNCTION_SCOPE_P (decl))
 	    {
@@ -11260,12 +11284,12 @@ grokfndecl (tree ctype,
 	   t && t != void_list_node; t = TREE_CHAIN (t))
 	if (TREE_PURPOSE (t))
 	  {
-	    diagnostic_t diag_kind = DK_PERMERROR;
+	    enum diagnostics::kind diag_kind = diagnostics::kind::permerror;
 	    /* For templates, mark the default argument as erroneous and give a
 	       hard error.  */
 	    if (processing_template_decl)
 	      {
-		diag_kind = DK_ERROR;
+		diag_kind = diagnostics::kind::error;
 		TREE_PURPOSE (t) = error_mark_node;
 	      }
 	    if (!has_errored)
@@ -11273,7 +11297,7 @@ grokfndecl (tree ctype,
 		has_errored = true;
 		emit_diagnostic (diag_kind,
 				 DECL_SOURCE_LOCATION (decl),
-				 /*diagnostic_option_id=*/0,
+				 /*diagnostics::option_id=*/0,
 				 "friend declaration of %qD specifies default "
 				 "arguments and isn%'t a definition", decl);
 	      }
@@ -13619,9 +13643,10 @@ grokdeclarator (const cp_declarator *declarator,
       if (typedef_p)
 	error_at (declspecs->locations[ds_typedef],
 		  "structured binding declaration cannot be %qs", "typedef");
-      if (constexpr_p && !concept_p)
-	error_at (declspecs->locations[ds_constexpr], "structured "
-		  "binding declaration cannot be %qs", "constexpr");
+      if (constexpr_p && !concept_p && cxx_dialect < cxx26)
+	pedwarn (declspecs->locations[ds_constexpr], OPT_Wc__26_extensions,
+		 "structured binding declaration can be %qs only with "
+		 "%<-std=c++2c%> or %<-std=gnu++2c%>", "constexpr");
       if (consteval_p)
 	error_at (declspecs->locations[ds_consteval], "structured "
 		  "binding declaration cannot be %qs", "consteval");
@@ -13632,8 +13657,11 @@ grokdeclarator (const cp_declarator *declarator,
 		 declspecs->gnu_thread_keyword_p
 		 ? "__thread" : "thread_local");
       if (concept_p)
-	error_at (declspecs->locations[ds_concept],
-		  "structured binding declaration cannot be %qs", "concept");
+	{
+	  error_at (declspecs->locations[ds_concept],
+		    "structured binding declaration cannot be %qs", "concept");
+	  constexpr_p = 0;
+	}
       /* [dcl.struct.bind] "A cv that includes volatile is deprecated."  */
       if (type_quals & TYPE_QUAL_VOLATILE)
 	warning_at (declspecs->locations[ds_volatile], OPT_Wvolatile,
@@ -13688,7 +13716,6 @@ grokdeclarator (const cp_declarator *declarator,
 		 "%<auto%> type %qT", type);
       inlinep = 0;
       typedef_p = 0;
-      constexpr_p = 0;
       consteval_p = 0;
       concept_p = 0;
       if (storage_class != sc_static)
@@ -17266,7 +17293,7 @@ xref_tag (enum tag_types tag_code, tree name,
       if (IDENTIFIER_LAMBDA_P (name))
 	/* Mark it as a lambda type right now.  Our caller will
 	   correct the value.  */
-	CLASSTYPE_LAMBDA_EXPR (t) = error_mark_node;
+	SET_CLASSTYPE_LAMBDA_EXPR (t, error_mark_node);
       t = pushtag (name, t, how);
     }
   else
@@ -17379,7 +17406,8 @@ xref_basetypes (tree ref, tree base_list)
 	 compatibility.  */
       if (processing_template_decl
 	  && CLASS_TYPE_P (basetype) && TYPE_BEING_DEFINED (basetype))
-	cxx_incomplete_type_diagnostic (NULL_TREE, basetype, DK_PEDWARN);
+	cxx_incomplete_type_diagnostic (NULL_TREE, basetype,
+					diagnostics::kind::pedwarn);
       if (!dependent_type_p (basetype)
 	  && !complete_type_or_else (basetype, NULL))
 	/* An incomplete type.  Remove it from the list.  */
@@ -19473,14 +19501,14 @@ finish_function (bool inline_p)
 	    && !DECL_READ_P (decl)
 	    && DECL_NAME (decl)
 	    && !DECL_ARTIFICIAL (decl)
-	    && !warning_suppressed_p (decl,OPT_Wunused_but_set_parameter)
+	    && !warning_suppressed_p (decl, OPT_Wunused_but_set_parameter_)
 	    && !DECL_IN_SYSTEM_HEADER (decl)
 	    && TREE_TYPE (decl) != error_mark_node
 	    && !TYPE_REF_P (TREE_TYPE (decl))
 	    && (!CLASS_TYPE_P (TREE_TYPE (decl))
 	        || !TYPE_HAS_NONTRIVIAL_DESTRUCTOR (TREE_TYPE (decl))))
 	  warning_at (DECL_SOURCE_LOCATION (decl),
-		      OPT_Wunused_but_set_parameter,
+		      OPT_Wunused_but_set_parameter_,
 		      "parameter %qD set but not used", decl);
       unused_but_set_errorcount = errorcount;
     }
diff --git a/gcc/cp/error.cc b/gcc/cp/error.cc
index abeb028..c427163 100644
--- a/gcc/cp/error.cc
+++ b/gcc/cp/error.cc
@@ -24,7 +24,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cp-tree.h"
 #include "stringpool.h"
 #include "tree-diagnostic.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 #include "langhooks-def.h"
 #include "intl.h"
 #include "cxx-pretty-print.h"
@@ -38,7 +38,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cp-name-hint.h"
 #include "attribs.h"
 #include "pretty-print-format-impl.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 #define pp_separate_with_comma(PP) pp_cxx_separate_with (PP, ',')
 #define pp_separate_with_semicolon(PP) pp_cxx_separate_with (PP, ';')
@@ -96,17 +96,17 @@ static void dump_scope (cxx_pretty_printer *, tree, int);
 static void dump_template_parms (cxx_pretty_printer *, tree, int, int);
 static int get_non_default_template_args_count (tree, int);
 static const char *function_category (tree);
-static void maybe_print_constexpr_context (diagnostic_text_output_format &);
-static void maybe_print_instantiation_context (diagnostic_text_output_format &);
-static void print_instantiation_full_context (diagnostic_text_output_format &);
-static void print_instantiation_partial_context (diagnostic_text_output_format &,
+static void maybe_print_constexpr_context (diagnostics::text_sink &);
+static void maybe_print_instantiation_context (diagnostics::text_sink &);
+static void print_instantiation_full_context (diagnostics::text_sink &);
+static void print_instantiation_partial_context (diagnostics::text_sink &,
 						 struct tinst_level *,
 						 location_t);
-static void maybe_print_constraint_context (diagnostic_text_output_format &);
-static void cp_diagnostic_text_starter (diagnostic_text_output_format &,
-					const diagnostic_info *);
-static void cp_print_error_function (diagnostic_text_output_format &,
-				     const diagnostic_info *);
+static void maybe_print_constraint_context (diagnostics::text_sink &);
+static void cp_diagnostic_text_starter (diagnostics::text_sink &,
+					const diagnostics::diagnostic_info *);
+static void cp_print_error_function (diagnostics::text_sink &,
+				     const diagnostics::diagnostic_info *);
 
 static bool cp_printer (pretty_printer *, text_info *, const char *,
 			int, bool, bool, bool, bool *, pp_token_list &);
@@ -242,7 +242,7 @@ get_current_template ()
 
 erroneous_templates_t *erroneous_templates;
 
-/* Callback function diagnostic_context::m_adjust_diagnostic_info.
+/* Callback function diagnostics::context::m_adjust_diagnostic_info.
 
    Errors issued when parsing a template are automatically treated like
    permerrors associated with the -Wtemplate-body flag and can be
@@ -250,16 +250,16 @@ erroneous_templates_t *erroneous_templates;
    issue an error if we later need to instantiate the template.  */
 
 static void
-cp_adjust_diagnostic_info (diagnostic_context *context,
-			   diagnostic_info *diagnostic)
+cp_adjust_diagnostic_info (diagnostics::context *context,
+			   diagnostics::diagnostic_info *diagnostic)
 {
-  if (diagnostic->kind == DK_ERROR)
+  if (diagnostic->m_kind == diagnostics::kind::error)
     if (tree tmpl = get_current_template ())
       {
-	diagnostic->option_id = OPT_Wtemplate_body;
+	diagnostic->m_option_id = OPT_Wtemplate_body;
 
 	if (context->m_permissive)
-	  diagnostic->kind = DK_WARNING;
+	  diagnostic->m_kind = diagnostics::kind::warning;
 
 	bool existed;
 	location_t &error_loc
@@ -269,7 +269,7 @@ cp_adjust_diagnostic_info (diagnostic_context *context,
 	  /* Remember that this template had a parse-time error so
 	     that we'll ensure a hard error has been issued upon
 	     its instantiation.  */
-	  error_loc = diagnostic->richloc->get_loc ();
+	  error_loc = diagnostic->m_richloc->get_loc ();
       }
 }
 
@@ -298,14 +298,14 @@ cp_seen_error ()
    capacities.  */
 
 void
-cxx_initialize_diagnostics (diagnostic_context *context)
+cxx_initialize_diagnostics (diagnostics::context *context)
 {
   cxx_pretty_printer *pp = new cxx_pretty_printer ();
   pp->set_format_postprocessor (std::make_unique<cxx_format_postprocessor> ());
   context->set_pretty_printer (std::unique_ptr<pretty_printer> (pp));
 
   c_common_diagnostics_set_defaults (context);
-  diagnostic_text_starter (context) = cp_diagnostic_text_starter;
+  diagnostics::text_starter (context) = cp_diagnostic_text_starter;
   /* diagnostic_finalizer is already c_diagnostic_text_finalizer.  */
   context->set_format_decoder (cp_printer);
   context->set_adjust_diagnostic_info_callback (cp_adjust_diagnostic_info);
@@ -704,6 +704,20 @@ dump_type (cxx_pretty_printer *pp, tree t, int flags)
 	}
       break;
 
+    case TREE_VEC:
+      {
+	/* A list of types used for a trait.  */
+	bool need_comma = false;
+	for (tree arg : tree_vec_range (t))
+	  {
+	    if (need_comma)
+	      pp_separate_with_comma (pp);
+	    dump_type (pp, arg, flags);
+	    need_comma = true;
+	  }
+      }
+      break;
+
     case TREE_LIST:
       /* A list of function parms.  */
       dump_parameters (pp, t, flags);
@@ -810,6 +824,7 @@ dump_type (cxx_pretty_printer *pp, tree t, int flags)
       pp_cxx_ws_string (pp,
 			 TYPENAME_IS_ENUM_P (t) ? "enum"
 			 : TYPENAME_IS_CLASS_P (t) ? "class"
+			 : TYPENAME_IS_UNION_P (t) ? "union"
 			 : "typename");
       dump_typename (pp, t, flags);
       break;
@@ -3746,9 +3761,9 @@ eh_spec_to_string (tree p, int /*v*/)
 
 /* Langhook for print_error_function.  */
 void
-cxx_print_error_function (diagnostic_text_output_format &text_output,
+cxx_print_error_function (diagnostics::text_sink &text_output,
 			  const char *file,
-			  const diagnostic_info *diagnostic)
+			  const diagnostics::diagnostic_info *diagnostic)
 {
   char *prefix;
   if (file)
@@ -3762,8 +3777,8 @@ cxx_print_error_function (diagnostic_text_output_format &text_output,
 }
 
 static void
-cp_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-			    const diagnostic_info *diagnostic)
+cp_diagnostic_text_starter (diagnostics::text_sink &text_output,
+			    const diagnostics::diagnostic_info *diagnostic)
 {
   pp_set_prefix (text_output.get_printer (),
 		 text_output.build_indent_prefix (true));
@@ -3779,8 +3794,8 @@ cp_diagnostic_text_starter (diagnostic_text_output_format &text_output,
 /* Print current function onto BUFFER, in the process of reporting
    a diagnostic message.  Called from cp_diagnostic_starter.  */
 static void
-cp_print_error_function (diagnostic_text_output_format &text_output,
-			 const diagnostic_info *diagnostic)
+cp_print_error_function (diagnostics::text_sink &text_output,
+			 const diagnostics::diagnostic_info *diagnostic)
 {
   /* If we are in an instantiation context, current_function_decl is likely
      to be wrong, so just rely on print_instantiation_full_context.  */
@@ -3789,7 +3804,7 @@ cp_print_error_function (diagnostic_text_output_format &text_output,
   /* The above is true for constraint satisfaction also.  */
   if (current_failed_constraint)
     return;
-  diagnostic_context *const context = &text_output.get_context ();
+  diagnostics::context *const context = &text_output.get_context ();
   if (diagnostic_last_function_changed (context, diagnostic))
     {
       pretty_printer *const pp = text_output.get_printer ();
@@ -3927,7 +3942,7 @@ function_category (tree fn)
 /* Report the full context of a current template instantiation,
    onto BUFFER.  */
 static void
-print_instantiation_full_context (diagnostic_text_output_format &text_output)
+print_instantiation_full_context (diagnostics::text_sink &text_output)
 {
   struct tinst_level *p = current_instantiation ();
   location_t location = input_location;
@@ -3955,7 +3970,7 @@ print_instantiation_full_context (diagnostic_text_output_format &text_output)
 }
 
 static void
-print_location (diagnostic_text_output_format &text_output,
+print_location (diagnostics::text_sink &text_output,
 		location_t loc)
 {
   expanded_location xloc = expand_location (loc);
@@ -3969,7 +3984,7 @@ print_location (diagnostic_text_output_format &text_output,
 }
 
 /* A RAII class for use when emitting a line of contextual information
-   via pp_verbatim to a diagnostic_text_output_format to add before/after
+   via pp_verbatim to a diagnostics::text_sink to add before/after
    behaviors to the pp_verbatim calls.
 
    If the text output has show_nesting_p (), then the ctor prints
@@ -3984,7 +3999,7 @@ print_location (diagnostic_text_output_format &text_output,
 class auto_context_line
 {
 public:
-  auto_context_line (diagnostic_text_output_format &text_output,
+  auto_context_line (diagnostics::text_sink &text_output,
 		     location_t loc,
 		     bool show_locus = false)
   : m_text_output (text_output),
@@ -4015,7 +4030,7 @@ public:
 	    diagnostic_show_locus (&m_text_output.get_context (),
 				   m_text_output.get_source_printing_options (),
 				   &rich_loc,
-				   DK_NOTE, pp);
+				   diagnostics::kind::note, pp);
 	    pp_set_prefix (pp, saved_prefix);
 	  }
       }
@@ -4027,12 +4042,12 @@ public:
 	diagnostic_show_locus (&m_text_output.get_context (),
 			       m_text_output.get_source_printing_options (),
 			       &rich_loc,
-			       DK_NOTE, pp);
+			       diagnostics::kind::note, pp);
 	pp_set_prefix (pp, saved_prefix);
       }
   }
 private:
-  diagnostic_text_output_format &m_text_output;
+  diagnostics::text_sink &m_text_output;
   location_t m_loc;
   bool m_show_locus;
 };
@@ -4041,7 +4056,7 @@ private:
    prints a single line of instantiation context.  */
 
 static void
-print_instantiation_partial_context_line (diagnostic_text_output_format &text_output,
+print_instantiation_partial_context_line (diagnostics::text_sink &text_output,
 					  struct tinst_level *t,
 					  location_t loc, bool recursive_p)
 {
@@ -4079,7 +4094,7 @@ print_instantiation_partial_context_line (diagnostic_text_output_format &text_ou
 /* Same as print_instantiation_full_context but less verbose.  */
 
 static void
-print_instantiation_partial_context (diagnostic_text_output_format &text_output,
+print_instantiation_partial_context (diagnostics::text_sink &text_output,
 				     struct tinst_level *t0, location_t loc)
 {
   struct tinst_level *t;
@@ -4150,7 +4165,7 @@ print_instantiation_partial_context (diagnostic_text_output_format &text_output,
 
 /* Called from cp_thing to print the template context for an error.  */
 static void
-maybe_print_instantiation_context (diagnostic_text_output_format &text_output)
+maybe_print_instantiation_context (diagnostics::text_sink &text_output)
 {
   if (!problematic_instantiation_changed () || current_instantiation () == 0)
     return;
@@ -4162,7 +4177,7 @@ maybe_print_instantiation_context (diagnostic_text_output_format &text_output)
 /* Report what constexpr call(s) we're trying to expand, if any.  */
 
 void
-maybe_print_constexpr_context (diagnostic_text_output_format &text_output)
+maybe_print_constexpr_context (diagnostics::text_sink &text_output)
 {
   vec<tree> call_stack = cx_error_context ();
   unsigned ix;
@@ -4182,7 +4197,7 @@ maybe_print_constexpr_context (diagnostic_text_output_format &text_output)
 
 
 static void
-print_constrained_decl_info (diagnostic_text_output_format &text_output,
+print_constrained_decl_info (diagnostics::text_sink &text_output,
 			     tree decl)
 {
   auto_context_line sentinel (text_output, DECL_SOURCE_LOCATION (decl));
@@ -4191,7 +4206,7 @@ print_constrained_decl_info (diagnostic_text_output_format &text_output,
 }
 
 static void
-print_concept_check_info (diagnostic_text_output_format &text_output,
+print_concept_check_info (diagnostics::text_sink &text_output,
 			  tree expr, tree map, tree args)
 {
   gcc_assert (concept_check_p (expr));
@@ -4216,7 +4231,7 @@ print_concept_check_info (diagnostic_text_output_format &text_output,
    context, if any.  */
 
 static tree
-print_constraint_context_head (diagnostic_text_output_format &text_output,
+print_constraint_context_head (diagnostics::text_sink &text_output,
 			       tree cxt, tree args)
 {
   tree src = TREE_VALUE (cxt);
@@ -4240,7 +4255,7 @@ print_constraint_context_head (diagnostic_text_output_format &text_output,
 }
 
 static void
-print_requires_expression_info (diagnostic_text_output_format &text_output,
+print_requires_expression_info (diagnostics::text_sink &text_output,
 				tree constr, tree args)
 {
 
@@ -4270,7 +4285,7 @@ print_requires_expression_info (diagnostic_text_output_format &text_output,
 }
 
 void
-maybe_print_single_constraint_context (diagnostic_text_output_format &text_output,
+maybe_print_single_constraint_context (diagnostics::text_sink &text_output,
 				       tree failed)
 {
   if (!failed)
@@ -4301,7 +4316,7 @@ maybe_print_single_constraint_context (diagnostic_text_output_format &text_outpu
 }
 
 void
-maybe_print_constraint_context (diagnostic_text_output_format &text_output)
+maybe_print_constraint_context (diagnostics::text_sink &text_output)
 {
   if (!current_failed_constraint)
     return;
@@ -4914,18 +4929,20 @@ maybe_warn_variadic_templates (void)
    C++0x.  */
 bool
 pedwarn_cxx98 (location_t location,
-	       diagnostic_option_id option_id,
+	       diagnostics::option_id option_id,
 	       const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool ret;
   rich_location richloc (line_table, location);
 
   va_start (ap, gmsgid);
   diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
-		       (cxx_dialect == cxx98) ? DK_PEDWARN : DK_WARNING);
-  diagnostic.option_id = option_id;
+		       (cxx_dialect == cxx98
+			? diagnostics::kind::pedwarn
+			: diagnostics::kind::warning));
+  diagnostic.m_option_id = option_id;
   ret = diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
   return ret;
diff --git a/gcc/cp/except.cc b/gcc/cp/except.cc
index a9d8e2f..204769f 100644
--- a/gcc/cp/except.cc
+++ b/gcc/cp/except.cc
@@ -367,6 +367,8 @@ initialize_handler_parm (tree decl, tree exp)
 	 MUST_NOT_THROW_EXPR.  */
       init = fold_build_cleanup_point_expr (TREE_TYPE (init), init);
       init = build_must_not_throw_expr (init, NULL_TREE);
+      if (init && TREE_CODE (init) == MUST_NOT_THROW_EXPR)
+	MUST_NOT_THROW_CATCH_P (init) = 1;
     }
 
   decl = pushdecl (decl);
@@ -523,6 +525,7 @@ begin_eh_spec_block (void)
       r = build_stmt (spec_location, MUST_NOT_THROW_EXPR,
 		      NULL_TREE, NULL_TREE);
       TREE_SIDE_EFFECTS (r) = 1;
+      MUST_NOT_THROW_NOEXCEPT_P (r) = 1;
     }
   else
     r = build_stmt (spec_location, EH_SPEC_BLOCK, NULL_TREE, NULL_TREE);
@@ -614,6 +617,7 @@ wrap_cleanups_r (tree *tp, int *walk_subtrees, void * /*data*/)
     {
       cleanup = build2 (MUST_NOT_THROW_EXPR, void_type_node, cleanup,
 			NULL_TREE);
+      MUST_NOT_THROW_THROW_P (cleanup) = 1;
       TARGET_EXPR_CLEANUP (exp) = cleanup;
     }
 
@@ -712,6 +716,11 @@ build_throw (location_t loc, tree exp, tsubst_flags_t complain)
       allocate_expr = do_allocate_exception (temp_type);
       if (allocate_expr == error_mark_node)
 	return error_mark_node;
+      /* Copy ptr inside of the CLEANUP_POINT_EXPR
+	 added below to a TARGET_EXPR slot added outside of it,
+	 otherwise during constant evaluation of throw expression
+	 we'd diagnose accessing ptr outside of its lifetime.  */
+      tree ptr_copy = get_internal_target_expr (null_pointer_node);
       allocate_expr = get_internal_target_expr (allocate_expr);
       ptr = TARGET_EXPR_SLOT (allocate_expr);
       TARGET_EXPR_CLEANUP (allocate_expr) = do_free_exception (ptr);
@@ -763,10 +772,17 @@ build_throw (location_t loc, tree exp, tsubst_flags_t complain)
       /* Prepend the allocation.  */
       exp = build2 (COMPOUND_EXPR, TREE_TYPE (exp), allocate_expr, exp);
 
+      exp = build2 (COMPOUND_EXPR, void_type_node, exp,
+		    build2 (MODIFY_EXPR, void_type_node,
+			    TARGET_EXPR_SLOT (ptr_copy), ptr));
+      ptr = TARGET_EXPR_SLOT (ptr_copy);
+
       /* Force all the cleanups to be evaluated here so that we don't have
 	 to do them during unwinding.  */
       exp = build1 (CLEANUP_POINT_EXPR, void_type_node, exp);
 
+      exp = build2 (COMPOUND_EXPR, TREE_TYPE (exp), ptr_copy, exp);
+
       throw_type = build_eh_type_type (prepare_eh_type (TREE_TYPE (object)));
 
       cleanup = NULL_TREE;
@@ -1202,6 +1218,20 @@ expr_noexcept_p (tree expr, tsubst_flags_t complain)
     return true;
 }
 
+/* If EXPR is not noexcept, explain why.  */
+
+void
+explain_not_noexcept (tree expr)
+{
+  tree fn = cp_walk_tree_without_duplicates (&expr, check_noexcept_r, 0);
+  if (!fn)
+    /* The call was noexcept, nothing to do.  */;
+  else if (DECL_P (fn))
+    inform (DECL_SOURCE_LOCATION (fn), "%qD is not %<noexcept%>", fn);
+  else
+    inform (location_of (fn), "%qT is not %<noexcept%>", TREE_TYPE (fn));
+}
+
 /* Return true iff SPEC is throw() or noexcept(true).  */
 
 bool
diff --git a/gcc/cp/expr.cc b/gcc/cp/expr.cc
index 2157cfb..32dc3ee 100644
--- a/gcc/cp/expr.cc
+++ b/gcc/cp/expr.cc
@@ -102,6 +102,9 @@ mark_use (tree expr, bool rvalue_p, bool read_p,
   if (reject_builtin && reject_gcc_builtin (expr, loc))
     return error_mark_node;
 
+  if (TREE_TYPE (expr) && VOID_TYPE_P (TREE_TYPE (expr)))
+    read_p = false;
+
   if (read_p)
     mark_exp_read (expr);
 
@@ -211,7 +214,7 @@ mark_use (tree expr, bool rvalue_p, bool read_p,
 	    }
 	  return expr;
 	}
-      gcc_fallthrough();
+      gcc_fallthrough ();
     CASE_CONVERT:
       recurse_op[0] = true;
       break;
@@ -352,6 +355,9 @@ mark_exp_read (tree exp)
   if (exp == NULL)
     return;
 
+  if (TREE_TYPE (exp) && VOID_TYPE_P (TREE_TYPE (exp)))
+    return;
+
   switch (TREE_CODE (exp))
     {
     case VAR_DECL:
@@ -361,16 +367,20 @@ mark_exp_read (tree exp)
     case PARM_DECL:
       DECL_READ_P (exp) = 1;
       break;
+    CASE_CONVERT:
     case ARRAY_REF:
     case COMPONENT_REF:
     case MODIFY_EXPR:
     case REALPART_EXPR:
     case IMAGPART_EXPR:
-    CASE_CONVERT:
     case ADDR_EXPR:
     case INDIRECT_REF:
     case FLOAT_EXPR:
     case VIEW_CONVERT_EXPR:
+    case PREINCREMENT_EXPR:
+    case PREDECREMENT_EXPR:
+    case POSTINCREMENT_EXPR:
+    case POSTDECREMENT_EXPR:
       mark_exp_read (TREE_OPERAND (exp, 0));
       break;
     case COMPOUND_EXPR:
diff --git a/gcc/cp/init.cc b/gcc/cp/init.cc
index 0a389fb..09fb4f3 100644
--- a/gcc/cp/init.cc
+++ b/gcc/cp/init.cc
@@ -4181,7 +4181,8 @@ build_vec_delete_1 (location_t loc, tree base, tree maxindex, tree type,
 			  "possible problem detected in invocation of "
 			  "operator %<delete []%>"))
 	    {
-	      cxx_incomplete_type_diagnostic (base, type, DK_WARNING);
+	      cxx_incomplete_type_diagnostic (base, type,
+					      diagnostics::kind::warning);
 	      inform (loc, "neither the destructor nor the "
 		      "class-specific operator %<delete []%> will be called, "
 		      "even if they are declared when the class is defined");
@@ -5286,7 +5287,8 @@ build_delete (location_t loc, tree otype, tree addr,
 				  "possible problem detected in invocation of "
 				  "%<operator delete%>"))
 		    {
-		      cxx_incomplete_type_diagnostic (addr, type, DK_WARNING);
+		      cxx_incomplete_type_diagnostic (addr, type,
+						      diagnostics::kind::warning);
 		      inform (loc,
 			      "neither the destructor nor the class-specific "
 			      "%<operator delete%> will be called, even if "
diff --git a/gcc/cp/lambda.cc b/gcc/cp/lambda.cc
index 2a9061a..c798967 100644
--- a/gcc/cp/lambda.cc
+++ b/gcc/cp/lambda.cc
@@ -150,7 +150,7 @@ begin_lambda_type (tree lambda)
 
   /* Cross-reference the expression and the type.  */
   LAMBDA_EXPR_CLOSURE (lambda) = type;
-  CLASSTYPE_LAMBDA_EXPR (type) = lambda;
+  SET_CLASSTYPE_LAMBDA_EXPR (type, lambda);
 
   /* In C++17, assume the closure is literal; we'll clear the flag later if
      necessary.  */
@@ -823,6 +823,14 @@ lambda_expr_this_capture (tree lambda, int add_capture_p)
   if (cp_unevaluated_operand)
     add_capture_p = false;
 
+  /* If we captured 'this' but don't have a capture proxy yet, look up the
+     captured 'this' again.  */
+  if (this_capture && TREE_CODE (this_capture) == FIELD_DECL)
+    {
+      gcc_assert (!add_capture_p);
+      this_capture = NULL_TREE;
+    }
+
   /* Try to default capture 'this' if we can.  */
   if (!this_capture)
     {
@@ -940,6 +948,9 @@ lambda_expr_this_capture (tree lambda, int add_capture_p)
       result = rvalue (result);
     }
 
+  gcc_checking_assert (!result || result == error_mark_node
+		       || TYPE_PTR_P (TREE_TYPE (result)));
+
   return result;
 }
 
@@ -1027,13 +1038,19 @@ maybe_generic_this_capture (tree object, tree fns)
       }
 }
 
-/* Returns the innermost non-lambda function.  */
+/* Returns the innermost non-lambda function.  If ONLY_SKIP_CONSTEVAL_BLOCK_P,
+   we only skip lambda functions that represent consteval blocks.  */
 
 tree
-current_nonlambda_function (void)
+current_nonlambda_function (bool only_skip_consteval_block_p/*=false*/)
 {
   tree fn = current_function_decl;
-  while (fn && LAMBDA_FUNCTION_P (fn))
+  tree lam;
+  while (fn && LAMBDA_FUNCTION_P (fn)
+	 && (!only_skip_consteval_block_p
+	     /* Only keep going if FN represents a consteval block.  */
+	     || ((lam = CLASSTYPE_LAMBDA_EXPR (CP_DECL_CONTEXT (fn)))
+		 && LAMBDA_EXPR_CONSTEVAL_BLOCK_P (lam))))
     fn = decl_function_context (fn);
   return fn;
 }
@@ -1132,7 +1149,9 @@ maybe_add_lambda_conv_op (tree type)
   tree lam = CLASSTYPE_LAMBDA_EXPR (type);
 
   if (LAMBDA_EXPR_CAPTURE_LIST (lam) != NULL_TREE
-      || LAMBDA_EXPR_DEFAULT_CAPTURE_MODE (lam) != CPLD_NONE)
+      || LAMBDA_EXPR_DEFAULT_CAPTURE_MODE (lam) != CPLD_NONE
+      /* CWG2561 ...and no explicit object parameter.  */
+      || DECL_XOBJ_MEMBER_FUNCTION_P (callop))
     return;
 
   if (processing_template_decl)
@@ -1939,7 +1958,7 @@ finish_lambda_function (tree body)
 {
   finish_function_body (body);
 
-  prune_lambda_captures (body);
+  prune_lambda_captures (cur_stmt_list);
 
   /* Finish the function and generate code for it if necessary.  */
   tree fn = finish_function (/*inline_p=*/true);
diff --git a/gcc/cp/method.cc b/gcc/cp/method.cc
index a4089c5..397e496 100644
--- a/gcc/cp/method.cc
+++ b/gcc/cp/method.cc
@@ -1187,15 +1187,15 @@ early_check_defaulted_comparison (tree fn)
   if (!DECL_OVERLOADED_OPERATOR_IS (fn, SPACESHIP_EXPR)
       && !same_type_p (TREE_TYPE (TREE_TYPE (fn)), boolean_type_node))
     {
-      diagnostic_t kind = DK_UNSPECIFIED;
+      enum diagnostics::kind kind = diagnostics::kind::unspecified;
       int opt = 0;
       if (is_auto (TREE_TYPE (fn)))
-	kind = DK_PEDWARN;
+	kind = diagnostics::kind::pedwarn;
       else
-	kind = DK_ERROR;
+	kind = diagnostics::kind::error;
       emit_diagnostic (kind, loc, opt,
 		       "defaulted %qD must return %<bool%>", fn);
-      if (kind == DK_ERROR)
+      if (kind == diagnostics::kind::error)
 	ok = false;
     }
 
@@ -1928,8 +1928,8 @@ is_stub_object (tree expr)
 
 /* Build a std::declval<TYPE>() expression and return it.  */
 
-tree
-build_trait_object (tree type)
+static tree
+build_trait_object (tree type, tsubst_flags_t complain)
 {
   /* TYPE can't be a function with cv-/ref-qualifiers: std::declval is
      defined as
@@ -1942,13 +1942,18 @@ build_trait_object (tree type)
   if (FUNC_OR_METHOD_TYPE_P (type)
       && (type_memfn_quals (type) != TYPE_UNQUALIFIED
 	  || type_memfn_rqual (type) != REF_QUAL_NONE))
-    return error_mark_node;
+    {
+      if (complain & tf_error)
+	error ("object cannot have qualified function type %qT", type);
+      return error_mark_node;
+    }
 
   return build_stub_object (type);
 }
 
 /* [func.require] Build an expression of INVOKE(FN_TYPE, ARG_TYPES...).  If the
-   given is not invocable, returns error_mark_node.  */
+   given is not invocable, returns error_mark_node, unless COMPLAIN includes
+   tf_error.  */
 
 tree
 build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
@@ -2036,22 +2041,26 @@ build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
 	      const_tree name = DECL_NAME (datum_decl);
 	      if (name && (id_equal (name, "reference_wrapper")))
 		{
-		  /* 1.2 & 1.5: Retrieve T from std::reference_wrapper<T>,
+		  /* 1.2 & 1.5: Retrieve T& from std::reference_wrapper<T>,
 		     i.e., decltype(datum.get()).  */
 		  datum_type =
 		    TREE_VEC_ELT (TYPE_TI_ARGS (non_ref_datum_type), 0);
+		  datum_type = cp_build_reference_type (datum_type, false);
 		  datum_is_refwrap = true;
 		}
 	    }
 	}
 
-      tree datum_expr = build_trait_object (datum_type);
+      tree datum_expr = build_trait_object (datum_type, complain);
       if (!ptrmem_is_same_or_base_of_datum && !datum_is_refwrap)
 	/* 1.3 & 1.6: Try to dereference datum_expr.  */
 	datum_expr = build_x_indirect_ref (UNKNOWN_LOCATION, datum_expr,
 					   RO_UNARY_STAR, NULL_TREE, complain);
 
-      tree fn_expr = build_trait_object (fn_type);
+      if (error_operand_p (datum_expr))
+	return error_mark_node;
+
+      tree fn_expr = build_trait_object (fn_type, complain);
       ptrmem_expr = build_m_component_ref (datum_expr, fn_expr, complain);
 
       if (error_operand_p (ptrmem_expr))
@@ -2068,7 +2077,9 @@ build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
   for (int i = is_ptrmemfunc ? 1 : 0; i < TREE_VEC_LENGTH (arg_types); ++i)
     {
       tree arg_type = TREE_VEC_ELT (arg_types, i);
-      tree arg = build_trait_object (arg_type);
+      tree arg = build_trait_object (arg_type, complain);
+      if (error_operand_p (arg))
+	return error_mark_node;
       vec_safe_push (args, arg);
     }
 
@@ -2077,8 +2088,8 @@ build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
     invoke_expr = build_offset_ref_call_from_tree (ptrmem_expr, &args,
 						   complain);
   else  /* 1.7.  */
-    invoke_expr = finish_call_expr (build_trait_object (fn_type), &args, false,
-				    false, complain);
+    invoke_expr = finish_call_expr (build_trait_object (fn_type, complain),
+				    &args, false, false, complain);
   return invoke_expr;
 }
 
@@ -2227,12 +2238,20 @@ check_nontriv (tree *tp, int *, void *)
 /* Return declval<T>() = declval<U>() treated as an unevaluated operand.  */
 
 static tree
-assignable_expr (tree to, tree from)
+assignable_expr (tree to, tree from, bool explain)
 {
   cp_unevaluated cp_uneval_guard;
-  to = build_trait_object (to);
-  from = build_trait_object (from);
-  tree r = cp_build_modify_expr (input_location, to, NOP_EXPR, from, tf_none);
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
+
+  to = build_trait_object (to, complain);
+  if (to == error_mark_node)
+    return error_mark_node;
+
+  from = build_trait_object (from, complain);
+  if (from == error_mark_node)
+    return error_mark_node;
+
+  tree r = cp_build_modify_expr (input_location, to, NOP_EXPR, from, complain);
   return r;
 }
 
@@ -2244,10 +2263,11 @@ assignable_expr (tree to, tree from)
    Return something equivalent in well-formedness and triviality.  */
 
 static tree
-constructible_expr (tree to, tree from)
+constructible_expr (tree to, tree from, bool explain)
 {
   tree expr;
   cp_unevaluated cp_uneval_guard;
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
   const int len = TREE_VEC_LENGTH (from);
   if (CLASS_TYPE_P (to))
     {
@@ -2259,14 +2279,14 @@ constructible_expr (tree to, tree from)
 	to = cp_build_reference_type (to, /*rval*/false);
       tree ob = build_stub_object (to);
       if (len == 0)
-	expr = build_value_init (ctype, tf_none);
+	expr = build_value_init (ctype, complain);
       else
 	{
 	  vec_alloc (args, len);
 	  for (tree arg : tree_vec_range (from))
 	    args->quick_push (build_stub_object (arg));
 	  expr = build_special_member_call (ob, complete_ctor_identifier, &args,
-					    ctype, LOOKUP_NORMAL, tf_none);
+					    ctype, LOOKUP_NORMAL, complain);
 	}
       if (expr == error_mark_node)
 	return error_mark_node;
@@ -2276,7 +2296,7 @@ constructible_expr (tree to, tree from)
 	{
 	  tree dtor = build_special_member_call (ob, complete_dtor_identifier,
 						 NULL, ctype, LOOKUP_NORMAL,
-						 tf_none);
+						 complain);
 	  if (dtor == error_mark_node)
 	    return error_mark_node;
 	  if (!TYPE_HAS_TRIVIAL_DESTRUCTOR (ctype))
@@ -2286,12 +2306,15 @@ constructible_expr (tree to, tree from)
   else
     {
       if (len == 0)
-	return build_value_init (strip_array_types (to), tf_none);
+	return build_value_init (strip_array_types (to), complain);
       if (len > 1)
 	{
 	  if (cxx_dialect < cxx20)
-	    /* Too many initializers.  */
-	    return error_mark_node;
+	    {
+	      if (explain)
+		error ("too many initializers for non-class type %qT", to);
+	      return error_mark_node;
+	    }
 
 	  /* In C++20 this is well-formed:
 	       using T = int[2];
@@ -2312,9 +2335,11 @@ constructible_expr (tree to, tree from)
 	}
       else
 	from = build_stub_object (TREE_VEC_ELT (from, 0));
+
+      tree orig_from = from;
       expr = perform_direct_initialization_if_possible (to, from,
 							/*cast*/false,
-							tf_none);
+							complain);
       /* If t(e) didn't work, maybe t{e} will.  */
       if (expr == NULL_TREE
 	  && len == 1
@@ -2326,7 +2351,24 @@ constructible_expr (tree to, tree from)
 	  CONSTRUCTOR_IS_PAREN_INIT (from) = true;
 	  expr = perform_direct_initialization_if_possible (to, from,
 							    /*cast*/false,
-							    tf_none);
+							    complain);
+	}
+
+      if (expr == NULL_TREE && explain)
+	{
+	  if (len > 1)
+	    error ("too many initializers for non-class type %qT", to);
+	  else
+	    {
+	      /* Redo the implicit conversion for diagnostics.  */
+	      int count = errorcount + warningcount;
+	      perform_implicit_conversion_flags (to, orig_from, complain,
+						 LOOKUP_NORMAL);
+	      if (count == errorcount + warningcount)
+		/* The message may have been suppressed due to -w + -fpermissive,
+		   emit a generic response instead.  */
+		error ("the conversion is invalid");
+	    }
 	}
     }
   return expr;
@@ -2340,20 +2382,25 @@ constructible_expr (tree to, tree from)
    valid or error_mark_node if not.  */
 
 static tree
-destructible_expr (tree to)
+destructible_expr (tree to, bool explain)
 {
   cp_unevaluated cp_uneval_guard;
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
   int flags = LOOKUP_NORMAL|LOOKUP_DESTRUCTOR;
   if (TYPE_REF_P (to))
     return void_node;
   if (!COMPLETE_TYPE_P (complete_type (to)))
-    return error_mark_node;
+    {
+      if (explain)
+	error_at (location_of (to), "%qT is incomplete", to);
+      return error_mark_node;
+    }
   to = strip_array_types (to);
   if (CLASS_TYPE_P (to))
     {
-      to = build_trait_object (to);
+      to = build_trait_object (to, complain);
       return build_delete (input_location, TREE_TYPE (to), to,
-			     sfk_complete_destructor, flags, 0, tf_none);
+			   sfk_complete_destructor, flags, 0, complain);
     }
   /* [expr.prim.id.dtor] If the id-expression names a pseudo-destructor, T
      shall be a scalar type.... */
@@ -2365,70 +2412,95 @@ destructible_expr (tree to)
 
 /* Returns a tree iff TO is assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN is
+   set, emit a diagnostic explaining why the operation failed.  */
 
 static tree
-is_xible_helper (enum tree_code code, tree to, tree from, bool trivial)
+is_xible_helper (enum tree_code code, tree to, tree from, bool explain)
 {
   to = complete_type (to);
   deferring_access_check_sentinel acs (dk_no_deferred);
-  if (VOID_TYPE_P (to)
-      || (from && FUNC_OR_METHOD_TYPE_P (from)
-	  && (TYPE_READONLY (from) || FUNCTION_REF_QUALIFIED (from))))
-    return error_mark_node;
+
+  if (VOID_TYPE_P (to))
+    {
+      if (explain)
+	error_at (location_of (to), "%qT is incomplete", to);
+      return error_mark_node;
+    }
+  if (from
+      && FUNC_OR_METHOD_TYPE_P (from)
+      && (TYPE_READONLY (from) || FUNCTION_REF_QUALIFIED (from)))
+    {
+      if (explain)
+	error ("%qT is a qualified function type", from);
+      return error_mark_node;
+    }
+
   tree expr;
   if (code == MODIFY_EXPR)
-    expr = assignable_expr (to, from);
+    expr = assignable_expr (to, from, explain);
   else if (code == BIT_NOT_EXPR)
-    expr = destructible_expr (to);
-  else if (trivial && TREE_VEC_LENGTH (from) > 1
-	   && cxx_dialect < cxx20)
-    return error_mark_node; // only 0- and 1-argument ctors can be trivial
-			    // before C++20 aggregate paren init
+    expr = destructible_expr (to, explain);
   else if (TREE_CODE (to) == ARRAY_TYPE && !TYPE_DOMAIN (to))
-    return error_mark_node; // can't construct an array of unknown bound
+    {
+      if (explain)
+	error ("cannot construct an array of unknown bound");
+      return error_mark_node;
+    }
   else
-    expr = constructible_expr (to, from);
+    expr = constructible_expr (to, from, explain);
   return expr;
 }
 
 /* Returns true iff TO is trivially assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN, diagnose
+   why we returned false.  */
 
 bool
-is_trivially_xible (enum tree_code code, tree to, tree from)
+is_trivially_xible (enum tree_code code, tree to, tree from,
+		    bool explain/*=false*/)
 {
-  tree expr = is_xible_helper (code, to, from, /*trivial*/true);
+  tree expr = is_xible_helper (code, to, from, explain);
   if (expr == NULL_TREE || expr == error_mark_node)
     return false;
+
   tree nt = cp_walk_tree_without_duplicates (&expr, check_nontriv, NULL);
+  if (explain && nt)
+    inform (location_of (nt), "%qE is non-trivial", nt);
   return !nt;
 }
 
 /* Returns true iff TO is nothrow assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN, diagnose
+   why we returned false.  */
 
 bool
-is_nothrow_xible (enum tree_code code, tree to, tree from)
+is_nothrow_xible (enum tree_code code, tree to, tree from,
+		  bool explain/*=false*/)
 {
   ++cp_noexcept_operand;
-  tree expr = is_xible_helper (code, to, from, /*trivial*/false);
+  tree expr = is_xible_helper (code, to, from, explain);
   --cp_noexcept_operand;
   if (expr == NULL_TREE || expr == error_mark_node)
     return false;
-  return expr_noexcept_p (expr, tf_none);
+
+  bool is_noexcept = expr_noexcept_p (expr, tf_none);
+  if (explain && !is_noexcept)
+    explain_not_noexcept (expr);
+  return is_noexcept;
 }
 
 /* Returns true iff TO is assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN, diagnose
+   why we returned false.  */
 
 bool
-is_xible (enum tree_code code, tree to, tree from)
+is_xible (enum tree_code code, tree to, tree from, bool explain/*=false*/)
 {
-  tree expr = is_xible_helper (code, to, from, /*trivial*/false);
+  tree expr = is_xible_helper (code, to, from, explain);
   if (expr == error_mark_node)
     return false;
   return !!expr;
@@ -2453,7 +2525,7 @@ ref_xes_from_temporary (tree to, tree from, bool direct_init_p)
     return false;
   /* We don't check is_constructible<T, U>: if T isn't constructible
      from U, we won't be able to create a conversion.  */
-  tree val = build_trait_object (from);
+  tree val = build_trait_object (from, tf_none);
   if (val == error_mark_node)
     return false;
   if (!TYPE_REF_P (from) && TREE_CODE (from) != FUNCTION_TYPE)
@@ -2462,25 +2534,36 @@ ref_xes_from_temporary (tree to, tree from, bool direct_init_p)
 }
 
 /* Worker for is_{,nothrow_}convertible.  Attempt to perform an implicit
-   conversion from FROM to TO and return the result.  */
+   conversion from FROM to TO and return the result.  If EXPLAIN, emit a
+   diagnostic about why the conversion failed.  */
 
 static tree
-is_convertible_helper (tree from, tree to)
+is_convertible_helper (tree from, tree to, bool explain)
 {
   if (VOID_TYPE_P (from) && VOID_TYPE_P (to))
     return integer_one_node;
   cp_unevaluated u;
-  tree expr = build_trait_object (from);
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
+
   /* std::is_{,nothrow_}convertible test whether the imaginary function
      definition
 
        To test() { return std::declval<From>(); }
 
      is well-formed.  A function can't return a function.  */
-  if (FUNC_OR_METHOD_TYPE_P (to) || expr == error_mark_node)
+  if (FUNC_OR_METHOD_TYPE_P (to))
+    {
+      if (explain)
+	error ("%qT is a function type", to);
+      return error_mark_node;
+    }
+
+  tree expr = build_trait_object (from, complain);
+  if (expr == error_mark_node)
     return error_mark_node;
+
   deferring_access_check_sentinel acs (dk_no_deferred);
-  return perform_implicit_conversion (to, expr, tf_none);
+  return perform_implicit_conversion (to, expr, complain);
 }
 
 /* Return true if FROM can be converted to TO using implicit conversions,
@@ -2489,9 +2572,9 @@ is_convertible_helper (tree from, tree to)
    to either type" restriction.  */
 
 bool
-is_convertible (tree from, tree to)
+is_convertible (tree from, tree to, bool explain/*=false*/)
 {
-  tree expr = is_convertible_helper (from, to);
+  tree expr = is_convertible_helper (from, to, explain);
   if (expr == error_mark_node)
     return false;
   return !!expr;
@@ -2500,12 +2583,16 @@ is_convertible (tree from, tree to)
 /* Like is_convertible, but the conversion is also noexcept.  */
 
 bool
-is_nothrow_convertible (tree from, tree to)
+is_nothrow_convertible (tree from, tree to, bool explain/*=false*/)
 {
-  tree expr = is_convertible_helper (from, to);
+  tree expr = is_convertible_helper (from, to, explain);
   if (expr == NULL_TREE || expr == error_mark_node)
     return false;
-  return expr_noexcept_p (expr, tf_none);
+
+  bool is_noexcept = expr_noexcept_p (expr, tf_none);
+  if (explain && !is_noexcept)
+    explain_not_noexcept (expr);
+  return is_noexcept;
 }
 
 /* Categorize various special_function_kinds.  */
@@ -3586,21 +3673,24 @@ maybe_delete_defaulted_fn (tree fn, tree implicit_fn)
 	      the program is ill-formed"  */
 	   || !TYPE_REF_P (parmtype)));
   /* Decide if we want to emit a pedwarn, error, or a warning.  */
-  diagnostic_t diag_kind;
+  enum diagnostics::kind diag_kind;
   int opt;
   if (illformed_p)
     {
-      diag_kind = DK_ERROR;
+      diag_kind = diagnostics::kind::error;
       opt = 0;
     }
   else
     {
-      diag_kind = cxx_dialect >= cxx20 ? DK_WARNING : DK_PEDWARN;
+      diag_kind = (cxx_dialect >= cxx20
+		   ? diagnostics::kind::warning
+		   : diagnostics::kind::pedwarn);
       opt = OPT_Wdefaulted_function_deleted;
     }
 
   /* Don't warn for template instantiations.  */
-  if (DECL_TEMPLATE_INSTANTIATION (fn) && diag_kind == DK_WARNING)
+  if (DECL_TEMPLATE_INSTANTIATION (fn)
+      && diag_kind == diagnostics::kind::warning)
     return;
 
   const char *wmsg;
diff --git a/gcc/cp/module.cc b/gcc/cp/module.cc
index c8e79f3..9412f78 100644
--- a/gcc/cp/module.cc
+++ b/gcc/cp/module.cc
@@ -3883,9 +3883,9 @@ class GTY((chain_next ("%h.parent"), for_user)) module_state {
   void write_macro_maps (elf_out *to, range_t &, unsigned *crc_ptr);
   bool read_macro_maps (line_map_uint_t);
 
-  void write_diagnostic_classification (elf_out *, diagnostic_context *,
+  void write_diagnostic_classification (elf_out *, diagnostics::context *,
 					unsigned *);
-  bool read_diagnostic_classification (diagnostic_context *);
+  bool read_diagnostic_classification (diagnostics::context *);
 
  private:
   void write_define (bytes_out &, const cpp_macro *);
@@ -4822,7 +4822,8 @@ noisy_p ()
     return false;
 
   pp_needs_newline (global_dc->get_reference_printer ()) = true;
-  diagnostic_set_last_function (global_dc, (diagnostic_info *) NULL);
+  diagnostic_set_last_function (global_dc,
+				(diagnostics::diagnostic_info *) nullptr);
 
   return true;
 }
@@ -6157,7 +6158,7 @@ trees_out::lang_type_bools (tree t, bits_out& bits)
   WB (lang->declared_class);
   WB (lang->diamond_shaped);
   WB (lang->repeated_base);
-  gcc_assert (!lang->being_defined);
+  gcc_checking_assert (!lang->being_defined);
   // lang->debug_requested
   WB (lang->fields_readonly);
   WB (lang->ptrmemfunc_flag);
@@ -6183,6 +6184,14 @@ trees_out::lang_type_bools (tree t, bits_out& bits)
   WB (lang->has_constexpr_ctor);
   WB (lang->unique_obj_representations);
   WB (lang->unique_obj_representations_set);
+  gcc_checking_assert (!lang->erroneous);
+  WB (lang->non_pod_aggregate);
+  WB (lang->non_aggregate_pod);
+  WB (lang->trivially_relocatable);
+  WB (lang->trivially_relocatable_computed);
+
+  WB (lang->replaceable);
+  WB (lang->replaceable_computed);
 #undef WB
 }
 
@@ -6227,8 +6236,8 @@ trees_in::lang_type_bools (tree t, bits_in& bits)
   RB (lang->declared_class);
   RB (lang->diamond_shaped);
   RB (lang->repeated_base);
-  gcc_assert (!lang->being_defined);
-  gcc_assert (!lang->debug_requested);
+  gcc_checking_assert (!lang->being_defined);
+  gcc_checking_assert (!lang->debug_requested);
   RB (lang->fields_readonly);
   RB (lang->ptrmemfunc_flag);
 
@@ -6253,6 +6262,14 @@ trees_in::lang_type_bools (tree t, bits_in& bits)
   RB (lang->has_constexpr_ctor);
   RB (lang->unique_obj_representations);
   RB (lang->unique_obj_representations_set);
+  gcc_checking_assert (!lang->erroneous);
+  RB (lang->non_pod_aggregate);
+  RB (lang->non_aggregate_pod);
+  RB (lang->trivially_relocatable);
+  RB (lang->trivially_relocatable_computed);
+
+  RB (lang->replaceable);
+  RB (lang->replaceable_computed);
 #undef RB
   return !get_overrun ();
 }
@@ -6543,8 +6560,14 @@ trees_out::core_vals (tree t)
 	}
 
       WT (t->function_decl.personality);
-      WT (t->function_decl.function_specific_target);
-      WT (t->function_decl.function_specific_optimization);
+      /* Rather than streaming target/optimize nodes, we should reconstruct
+	 them on stream-in from any attributes applied to the function.  */
+      if (streaming_p () && t->function_decl.function_specific_target)
+	warning_at (DECL_SOURCE_LOCATION (t), 0,
+		    "%<target%> attribute currently unsupported in modules");
+      if (streaming_p () && t->function_decl.function_specific_optimization)
+	warning_at (DECL_SOURCE_LOCATION (t), 0,
+		    "%<optimize%> attribute currently unsupported in modules");
       WT (t->function_decl.vindex);
 
       if (DECL_HAS_DEPENDENT_EXPLICIT_SPEC_P (t))
@@ -6634,11 +6657,12 @@ trees_out::core_vals (tree t)
     case TARGET_OPTION_NODE:
       // FIXME: Our representation for these two nodes is a cache of
       // the resulting set of options.  Not a record of the options
-      // that got changed by a particular attribute or pragma.  Should
-      // we record that, or should we record the diff from the command
-      // line options?  The latter seems the right behaviour, but is
-      // (a) harder, and I guess could introduce strangeness if the
-      // importer has set some incompatible set of optimization flags?
+      // that got changed by a particular attribute or pragma.  Instead
+      // of recording that, we probably should just rebuild the options
+      // on stream-in from the function attributes.  This could introduce
+      // strangeness if the importer has some incompatible set of flags
+      // but we currently assume users "know what they're doing" in such
+      // a case anyway.
       gcc_unreachable ();
       break;
 
@@ -6785,6 +6809,13 @@ trees_out::core_vals (tree t)
       if (streaming_p ())
 	WU (((lang_tree_node *)t)->trait_expression.kind);
       break;
+
+    case TU_LOCAL_ENTITY:
+      WT (((lang_tree_node *)t)->tu_local_entity.name);
+      if (state)
+	state->write_location
+	  (*this, ((lang_tree_node *)t)->tu_local_entity.loc);
+      break;
     }
 
   if (CODE_CONTAINS_STRUCT (code, TS_TYPED))
@@ -7090,8 +7121,10 @@ trees_in::core_vals (tree t)
 	  }
 
 	RT (t->function_decl.personality);
-	RT (t->function_decl.function_specific_target);
-	RT (t->function_decl.function_specific_optimization);
+	/* These properties are not streamed, and should be reconstructed
+	   from any function attributes.  */
+	// t->function_decl.function_specific_target);
+	// t->function_decl.function_specific_optimization);
 	RT (t->function_decl.vindex);
 
 	if (DECL_HAS_DEPENDENT_EXPLICIT_SPEC_P (t))
@@ -7197,7 +7230,7 @@ trees_in::core_vals (tree t)
 
     case OPTIMIZATION_NODE:
     case TARGET_OPTION_NODE:
-      /* Not yet implemented, see trees_out::core_vals.  */
+      /* Not implemented, see trees_out::core_vals.  */
       gcc_unreachable ();
       break;
 
@@ -7328,6 +7361,11 @@ trees_in::core_vals (tree t)
       RT (((lang_tree_node *)t)->trait_expression.type2);
       RUC (cp_trait_kind, ((lang_tree_node *)t)->trait_expression.kind);
       break;
+
+    case TU_LOCAL_ENTITY:
+      RT (((lang_tree_node *)t)->tu_local_entity.name);
+      ((lang_tree_node *)t)->tu_local_entity.loc
+	= state->read_location (*this);
     }
 
   if (CODE_CONTAINS_STRUCT (code, TS_TYPED))
@@ -9619,6 +9657,8 @@ trees_out::type_node (tree type)
 	      tag_type = enum_type;
 	    else if (TYPENAME_IS_CLASS_P (type))
 	      tag_type = class_type;
+	    else if (TYPENAME_IS_UNION_P (type))
+	      tag_type = union_type;
 	    u (int (tag_type));
 	  }
 	}
@@ -10266,7 +10306,8 @@ trees_in::tree_node (bool is_use)
 	    && dump ("Read %stypedef %C:%N",
 		     DECL_IMPLICIT_TYPEDEF_P (res) ? "implicit " : "",
 		     TREE_CODE (res), res);
-	  res = TREE_TYPE (res);
+	  if (TREE_CODE (res) != TU_LOCAL_ENTITY)
+	    res = TREE_TYPE (res);
 	}
       break;
 
@@ -11122,8 +11163,7 @@ trees_in::fn_parms_fini (int tag, tree fn, tree existing, bool is_defn)
 {
   tree existing_parm = existing ? DECL_ARGUMENTS (existing) : NULL_TREE;
   tree parms = DECL_ARGUMENTS (fn);
-  unsigned ix = 0;
-  for (tree parm = parms; parm; parm = DECL_CHAIN (parm), ix++)
+  for (tree parm = parms; parm; parm = DECL_CHAIN (parm))
     {
       if (existing_parm)
 	{
@@ -11133,6 +11173,20 @@ trees_in::fn_parms_fini (int tag, tree fn, tree existing, bool is_defn)
 		 names of the parms from us.  */
 	      DECL_NAME (existing_parm) = DECL_NAME (parm);
 	      DECL_SOURCE_LOCATION (existing_parm) = DECL_SOURCE_LOCATION (parm);
+
+	      /* And some other flags important for codegen are only set
+		 by the definition.  */
+	      TREE_ADDRESSABLE (existing_parm) = TREE_ADDRESSABLE (parm);
+	      DECL_BY_REFERENCE (existing_parm) = DECL_BY_REFERENCE (parm);
+	      DECL_NONLOCAL (existing_parm) = DECL_NONLOCAL (parm);
+	      DECL_ARG_TYPE (existing_parm) = DECL_ARG_TYPE (parm);
+
+	      /* Invisiref parms had their types adjusted by cp_genericize. */
+	      if (DECL_BY_REFERENCE (parm))
+		{
+		  TREE_TYPE (existing_parm) = TREE_TYPE (parm);
+		  relayout_decl (existing_parm);
+		}
 	    }
 
 	  back_refs[~tag] = existing_parm;
@@ -13391,13 +13445,19 @@ trees_in::read_class_def (tree defn, tree maybe_template)
 
       if (TYPE_LANG_SPECIFIC (type))
 	{
-	  CLASSTYPE_LAMBDA_EXPR (type) = lambda;
+	  if (!TYPE_POLYMORPHIC_P (type))
+	    SET_CLASSTYPE_LAMBDA_EXPR (type, lambda);
+	  else
+	    gcc_checking_assert (lambda == NULL_TREE);
 
 	  CLASSTYPE_MEMBER_VEC (type) = member_vec;
 	  CLASSTYPE_PURE_VIRTUALS (type) = pure_virts;
 	  CLASSTYPE_VCALL_INDICES (type) = vcall_indices;
 
-	  CLASSTYPE_KEY_METHOD (type) = key_method;
+	  if (TYPE_POLYMORPHIC_P (type))
+	    SET_CLASSTYPE_KEY_METHOD (type, key_method);
+	  else
+	    gcc_checking_assert (key_method == NULL_TREE);
 
 	  CLASSTYPE_VBASECLASSES (type) = vbase_vec;
 
@@ -18318,22 +18378,23 @@ module_state::write_ordinary_maps (elf_out *to, range_t &info,
 /* Return the prefix to use for dumping a #pragma diagnostic change to DK.  */
 
 static const char *
-dk_string (diagnostic_t dk)
+dk_string (enum diagnostics::kind dk)
 {
-  gcc_assert (dk > DK_UNSPECIFIED && dk < DK_LAST_DIAGNOSTIC_KIND);
-  if (dk == DK_IGNORED)
-    /* diagnostic.def has an empty string for ignored.  */
+  gcc_assert (dk > diagnostics::kind::unspecified
+	      && dk < diagnostics::kind::last_diagnostic_kind);
+  if (dk == diagnostics::kind::ignored)
+    /* diagnostics/kinds.def has an empty string for ignored.  */
     return "ignored: ";
   else
-    return get_diagnostic_kind_text (dk);
+    return diagnostics::get_text_for_kind (dk);
 }
 
 /* Dump one #pragma GCC diagnostic entry.  */
 
 static bool
-dump_dc_change (unsigned index, unsigned opt, diagnostic_t dk)
+dump_dc_change (unsigned index, unsigned opt, enum diagnostics::kind dk)
 {
-  if (dk == DK_POP)
+  if (dk == diagnostics::kind::pop)
     return dump (" Index %u: pop from %d", index, opt);
   else
     return dump (" Index %u: %s%s", index, dk_string (dk),
@@ -18344,7 +18405,7 @@ dump_dc_change (unsigned index, unsigned opt, diagnostic_t dk)
 
 void
 module_state::write_diagnostic_classification (elf_out *to,
-					       diagnostic_context *dc,
+					       diagnostics::context *dc,
 					       unsigned *crc_p)
 {
   auto &changes = dc->get_classification_history ();
@@ -18360,9 +18421,10 @@ module_state::write_diagnostic_classification (elf_out *to,
   unsigned len = changes.length ();
 
   /* We don't want to write out any entries that came from one of our imports.
-     But then we need to adjust the total, and change DK_POP targets to match
-     the index in our actual output.  So remember how many lines we had skipped
-     at each step, where -1 means this line itself is skipped.  */
+     But then we need to adjust the total, and change diagnostics::kind::pop
+     targets to match the index in our actual output.  So remember how many
+     lines we had skipped at each step, where -1 means this line itself
+     is skipped.  */
   int skips = 0;
   auto_vec<int> skips_at (len);
   skips_at.safe_grow (len);
@@ -18395,10 +18457,10 @@ module_state::write_diagnostic_classification (elf_out *to,
       if (sec.streaming_p ())
 	{
 	  unsigned opt = c.option;
-	  if (c.kind == DK_POP)
+	  if (c.kind == diagnostics::kind::pop)
 	    opt -= skips_at[opt];
 	  sec.u (opt);
-	  sec.u (c.kind);
+	  sec.u (static_cast<unsigned> (c.kind));
 	  dump () && dump_dc_change (i - skips_at[i], opt, c.kind);
 	}
     }
@@ -18413,7 +18475,7 @@ module_state::write_diagnostic_classification (elf_out *to,
 /* Read any #pragma GCC diagnostic info from the .dgc section.  */
 
 bool
-module_state::read_diagnostic_classification (diagnostic_context *dc)
+module_state::read_diagnostic_classification (diagnostics::context *dc)
 {
   bytes_in sec;
 
@@ -18433,8 +18495,8 @@ module_state::read_diagnostic_classification (diagnostic_context *dc)
     {
       location_t loc = read_location (sec);
       int opt = sec.u ();
-      diagnostic_t kind = (diagnostic_t) sec.u ();
-      if (kind == DK_POP)
+      enum diagnostics::kind kind = (enum diagnostics::kind) sec.u ();
+      if (kind == diagnostics::kind::pop)
 	/* For a pop, opt is the 'changes' index to return to.  */
 	opt += offset;
       changes.quick_push ({ loc, opt, kind });
@@ -18449,7 +18511,7 @@ module_state::read_diagnostic_classification (diagnostic_context *dc)
 	gcc_checking_assert (i >= offset);
 
 	const auto &c = changes[i];
-	if (c.kind != DK_POP)
+	if (c.kind != diagnostics::kind::pop)
 	  break;
 	else if (c.option == offset)
 	  {
@@ -18466,7 +18528,7 @@ module_state::read_diagnostic_classification (diagnostic_context *dc)
       /* It didn't, so add a pop at its last location to avoid affecting later
 	 imports.  */
       location_t last_loc = ordinary_locs.first + ordinary_locs.second - 1;
-      changes.quick_push ({ last_loc, offset, DK_POP });
+      changes.quick_push ({ last_loc, offset, diagnostics::kind::pop });
       dump () && dump (" Adding final pop from index %d", offset);
     }
 
diff --git a/gcc/cp/name-lookup.cc b/gcc/cp/name-lookup.cc
index 9aa7c16..f5b36c9 100644
--- a/gcc/cp/name-lookup.cc
+++ b/gcc/cp/name-lookup.cc
@@ -7195,7 +7195,7 @@ suggest_alternatives_for_1 (location_t location, tree name,
   /* Look for exact matches for builtin defines that would have been
      defined if the user had passed a command-line option (e.g. -fopenmp
      for "_OPENMP").  */
-  diagnostic_option_id option_id
+  diagnostics::option_id option_id
     = get_option_for_builtin_define (IDENTIFIER_POINTER (name));
   if (option_id.m_idx > 0)
     return name_hint
diff --git a/gcc/cp/parser.cc b/gcc/cp/parser.cc
index b1626ac..860f3f0 100644
--- a/gcc/cp/parser.cc
+++ b/gcc/cp/parser.cc
@@ -2519,7 +2519,7 @@ static cp_expr cp_parser_id_expression
 static cp_expr cp_parser_unqualified_id
   (cp_parser *, bool, bool, bool, bool);
 static tree cp_parser_nested_name_specifier_opt
-  (cp_parser *, bool, bool, bool, bool, bool = false);
+  (cp_parser *, bool, bool, bool, bool, bool = false, bool = false);
 static tree cp_parser_nested_name_specifier
   (cp_parser *, bool, bool, bool, bool);
 static tree cp_parser_qualifying_entity
@@ -2576,11 +2576,11 @@ static cp_expr cp_parser_constant_expression
 static cp_expr cp_parser_builtin_offsetof
   (cp_parser *);
 static cp_expr cp_parser_lambda_expression
-  (cp_parser *);
+  (cp_parser *, bool = false);
 static void cp_parser_lambda_introducer
   (cp_parser *, tree);
 static bool cp_parser_lambda_declarator_opt
-  (cp_parser *, tree);
+  (cp_parser *, tree, bool = false);
 static void cp_parser_lambda_body
   (cp_parser *, tree);
 
@@ -2921,7 +2921,7 @@ static size_t cp_parser_skip_std_attribute_spec_seq
 static size_t cp_parser_skip_attributes_opt
   (cp_parser *, size_t);
 static bool cp_parser_extension_opt
-  (cp_parser *, int *);
+  (cp_parser *, int *, int *);
 static void cp_parser_label_declaration
   (cp_parser *);
 
@@ -3091,8 +3091,8 @@ static cp_token *cp_parser_require_keyword
   (cp_parser *, enum rid, required_token);
 static bool cp_parser_token_starts_function_definition_p
   (cp_token *);
-static bool cp_parser_next_token_starts_class_definition_p
-  (cp_parser *);
+static bool cp_parser_nth_token_starts_class_definition_p
+  (cp_parser *, size_t);
 static bool cp_parser_next_token_ends_template_argument_p
   (cp_parser *);
 static bool cp_parser_nth_token_starts_template_argument_list_p
@@ -5266,7 +5266,7 @@ cp_parser_userdef_numeric_literal (cp_parser *parser)
 	      && (id_equal (suffix_id, "i")
 		  || id_equal (suffix_id, "if")
 		  || id_equal (suffix_id, "il")));
-  diagnostic_t kind = DK_ERROR;
+  enum diagnostics::kind kind = diagnostics::kind::error;
   int opt = 0;
 
   if (i14 && ext)
@@ -5276,7 +5276,7 @@ cp_parser_userdef_numeric_literal (cp_parser *parser)
       if (cxlit == error_mark_node)
 	{
 	  /* No <complex>, so pedwarn and use GNU semantics.  */
-	  kind = DK_PEDWARN;
+	  kind = diagnostics::kind::pedwarn;
 	  opt = OPT_Wpedantic;
 	}
     }
@@ -5303,7 +5303,7 @@ cp_parser_userdef_numeric_literal (cp_parser *parser)
 	      "to enable more built-in suffixes");
   }
 
-  if (kind == DK_ERROR)
+  if (kind == diagnostics::kind::error)
     value = error_mark_node;
   else
     {
@@ -6307,7 +6307,10 @@ cp_parser_primary_expression (cp_parser *parser,
 	  /* Recognize the `this' keyword.  */
 	case RID_THIS:
 	  cp_lexer_consume_token (parser->lexer);
-	  if (parser->local_variables_forbidden_p & THIS_FORBIDDEN)
+	  if ((parser->local_variables_forbidden_p & THIS_FORBIDDEN)
+	      /* It's OK to refer to 'this' in an unevaluated operand in a
+		 lambda default argument (lambda-targ16.C).  */
+	      && !cp_unevaluated_operand)
 	    {
 	      error_at (token->location,
 			"%<this%> may not be used in this context");
@@ -6644,7 +6647,8 @@ cp_parser_primary_expression (cp_parser *parser,
    member template.  */
 
 static void
-missing_template_diag (location_t loc, diagnostic_t diag_kind = DK_WARNING)
+missing_template_diag (location_t loc,
+		       enum diagnostics::kind diag_kind = diagnostics::kind::warning)
 {
   if (warning_suppressed_at (loc, OPT_Wmissing_template_keyword))
     return;
@@ -7239,18 +7243,22 @@ check_template_keyword_in_nested_name_spec (tree name)
      nested-name-specifier template [opt] simple-template-id ::
 
    PARSER->SCOPE should be set appropriately before this function is
-   called.  TYPENAME_KEYWORD_P is TRUE if the `typename' keyword is in
-   effect.  TYPE_P is TRUE if we non-type bindings should be ignored
-   in name lookups.
+   called.  TYPENAME_KEYWORD_P is true if the `typename' keyword is in
+   effect.  TYPE_P is true if we non-type bindings should be ignored
+   in name lookups.  TEMPLATE_KEYWORD_P is true if the `template' keyword
+   was seen.  GLOBAL_P is true if `::' has already been parsed.
+   TODO: This function doesn't handle the C++14 change to make `::'
+   a nested-name-specifier by itself.  If it did, GLOBAL_P could probably
+   go.
 
    Sets PARSER->SCOPE to the class (TYPE) or namespace
    (NAMESPACE_DECL) specified by the nested-name-specifier, or leaves
    it unchanged if there is no nested-name-specifier.  Returns the new
    scope iff there is a nested-name-specifier, or NULL_TREE otherwise.
 
-   If CHECK_DEPENDENCY_P is FALSE, names are looked up in dependent scopes.
+   If CHECK_DEPENDENCY_P is false, names are looked up in dependent scopes.
 
-   If IS_DECLARATION is TRUE, the nested-name-specifier is known to be
+   If IS_DECLARATION is true, the nested-name-specifier is known to be
    part of a declaration and/or decl-specifier.  */
 
 static tree
@@ -7259,7 +7267,8 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
 				     bool check_dependency_p,
 				     bool type_p,
 				     bool is_declaration,
-				     bool template_keyword_p /* = false */)
+				     bool template_keyword_p /* = false */,
+				     bool global_p /* = false */)
 {
   bool success = false;
   cp_token_position start = 0;
@@ -7307,8 +7316,9 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
 
       /* Spot cases that cannot be the beginning of a
 	 nested-name-specifier.  On the second and subsequent times
-	 through the loop, we look for the `template' keyword.  */
-      if (success && token->keyword == RID_TEMPLATE)
+	 (or the first, if '::' has already been parsed) through the
+	 loop, we look for the `template' keyword.  */
+      if ((success || global_p) && token->keyword == RID_TEMPLATE)
 	;
       /* A template-id can start a nested-name-specifier.  */
       else if (token->type == CPP_TEMPLATE_ID)
@@ -7356,8 +7366,11 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
       cp_parser_parse_tentatively (parser);
 
       /* Look for the optional `template' keyword, if this isn't the
-	 first time through the loop.  */
-      if (success)
+	 first time through the loop, or if we've already parsed '::';
+	 this is then the
+	   nested-name-specifier template [opt] simple-template-id ::
+	 production.  */
+      if (success || global_p)
 	{
 	  template_keyword_p = cp_parser_optional_template_keyword (parser);
 	  /* DR1710: "In a qualified-id used as the name in
@@ -8842,8 +8855,10 @@ cp_parser_dot_deref_incomplete (tree *scope, cp_expr *postfix_expression,
 {
   /* In a template, be permissive by treating an object expression
      of incomplete type as dependent (after a pedwarn).  */
-  diagnostic_t kind = (processing_template_decl
-		       && MAYBE_CLASS_TYPE_P (*scope) ? DK_PEDWARN : DK_ERROR);
+  enum diagnostics::kind kind = ((processing_template_decl
+				  && MAYBE_CLASS_TYPE_P (*scope))
+				 ? diagnostics::kind::pedwarn
+				 : diagnostics::kind::error);
 
   switch (TREE_CODE (*postfix_expression))
     {
@@ -8855,27 +8870,27 @@ cp_parser_dot_deref_incomplete (tree *scope, cp_expr *postfix_expression,
     case IMPLICIT_CONV_EXPR:
     case VIEW_CONVERT_EXPR:
     case NON_LVALUE_EXPR:
-      kind = DK_ERROR;
+      kind = diagnostics::kind::error;
       break;
     case OVERLOAD:
       /* Don't emit any diagnostic for OVERLOADs.  */
-      kind = DK_IGNORED;
+      kind = diagnostics::kind::ignored;
       break;
     default:
       /* Avoid clobbering e.g. DECLs.  */
       if (!EXPR_P (*postfix_expression))
-	kind = DK_ERROR;
+	kind = diagnostics::kind::error;
       break;
     }
 
-  if (kind == DK_IGNORED)
+  if (kind == diagnostics::kind::ignored)
     return false;
 
   location_t exploc = location_of (*postfix_expression);
   cxx_incomplete_type_diagnostic (exploc, *postfix_expression, *scope, kind);
   if (!MAYBE_CLASS_TYPE_P (*scope))
     return true;
-  if (kind == DK_ERROR)
+  if (kind == diagnostics::kind::error)
     *scope = *postfix_expression = error_mark_node;
   else if (processing_template_decl)
     {
@@ -9489,21 +9504,23 @@ cp_parser_unary_expression (cp_parser *parser, cp_id_kind * pidk,
 	case RID_EXTENSION:
 	  {
 	    /* The saved value of the PEDANTIC flag.  */
-	    int saved_pedantic;
+	    int saved_pedantic, saved_long_long;
 	    tree expr;
 
 	    /* Save away the PEDANTIC flag.  */
-	    cp_parser_extension_opt (parser, &saved_pedantic);
+	    cp_parser_extension_opt (parser, &saved_pedantic,
+				     &saved_long_long);
 	    /* Also suppress -Wconditionally-supported.  */
 	    diagnostic_push_diagnostics (global_dc, input_location);
 	    diagnostic_classify_diagnostic
 	      (global_dc, OPT_Wconditionally_supported,
-	       DK_IGNORED, input_location);
+	       diagnostics::kind::ignored, input_location);
 	    /* Parse the cast-expression.  */
 	    expr = cp_parser_simple_cast_expression (parser);
 	    /* Restore the PEDANTIC flag.  */
 	    diagnostic_pop_diagnostics (global_dc, input_location);
 	    pedantic = saved_pedantic;
+	    warn_long_long = saved_long_long;
 
 	    return expr;
 	  }
@@ -11727,10 +11744,14 @@ cp_parser_trait (cp_parser* parser, const cp_trait* trait)
      lambda-introducer < template-parameter-list > requires-clause [opt]
        lambda-declarator [opt] compound-statement
 
+   If CONSTEVAL_BLOCK_P is true, we are parsing a consteval block, which
+   is syntactic sugar for a consteval lambda.
+
    Returns a representation of the expression.  */
 
 static cp_expr
-cp_parser_lambda_expression (cp_parser* parser)
+cp_parser_lambda_expression (cp_parser* parser,
+			     bool consteval_block_p/*=false*/)
 {
   tree lambda_expr = build_lambda_expr ();
   tree type;
@@ -11739,6 +11760,7 @@ cp_parser_lambda_expression (cp_parser* parser)
   cp_token_position start = 0;
 
   LAMBDA_EXPR_LOCATION (lambda_expr) = token->location;
+  LAMBDA_EXPR_CONSTEVAL_BLOCK_P (lambda_expr) = consteval_block_p;
 
   if (cxx_dialect >= cxx20)
     {
@@ -11782,9 +11804,12 @@ cp_parser_lambda_expression (cp_parser* parser)
      it now.  */
   push_deferring_access_checks (dk_no_deferred);
 
-  cp_parser_lambda_introducer (parser, lambda_expr);
-  if (cp_parser_error_occurred (parser))
-    return error_mark_node;
+  if (!consteval_block_p)
+    {
+      cp_parser_lambda_introducer (parser, lambda_expr);
+      if (cp_parser_error_occurred (parser))
+	return error_mark_node;
+    }
 
   {
     /* OK, this is a bit tricksy.  cp_parser_requires_expression sets
@@ -11826,6 +11851,7 @@ cp_parser_lambda_expression (cp_parser* parser)
     bool auto_is_implicit_function_template_parm_p
         = parser->auto_is_implicit_function_template_parm_p;
     bool saved_omp_array_section_p = parser->omp_array_section_p;
+    bool saved_in_targ = parser->in_template_argument_list_p;
 
     parser->num_template_parameter_lists = 0;
     parser->in_statement = 0;
@@ -11835,6 +11861,7 @@ cp_parser_lambda_expression (cp_parser* parser)
     parser->implicit_template_scope = 0;
     parser->auto_is_implicit_function_template_parm_p = false;
     parser->omp_array_section_p = false;
+    parser->in_template_argument_list_p = false;
 
     /* Inside the lambda, outside unevaluated context do not apply.  */
     cp_evaluated ev;
@@ -11854,7 +11881,8 @@ cp_parser_lambda_expression (cp_parser* parser)
     if (cp_parser_start_tentative_firewall (parser))
       start = token;
 
-    ok &= cp_parser_lambda_declarator_opt (parser, lambda_expr);
+    ok &= cp_parser_lambda_declarator_opt (parser, lambda_expr,
+					   consteval_block_p);
 
     if (ok && cp_parser_error_occurred (parser))
       ok = false;
@@ -11889,6 +11917,7 @@ cp_parser_lambda_expression (cp_parser* parser)
     parser->auto_is_implicit_function_template_parm_p
 	= auto_is_implicit_function_template_parm_p;
     parser->omp_array_section_p = saved_omp_array_section_p;
+    parser->in_template_argument_list_p = saved_in_targ;
   }
 
   /* This lambda shouldn't have any proxies left at this point.  */
@@ -12236,10 +12265,13 @@ cp_parser_lambda_introducer (cp_parser* parser, tree lambda_expr)
      decl-specifier-seq [opt] noexcept-specifier [opt]
        attribute-specifier-seq [opt] trailing-return-type [opt]
 
-   LAMBDA_EXPR is the current representation of the lambda expression.  */
+   LAMBDA_EXPR is the current representation of the lambda expression.
+   If CONSTEVAL_BLOCK_P is true, we are parsing a consteval block, which
+   is syntactic sugar for a consteval lambda.  */
 
 static bool
-cp_parser_lambda_declarator_opt (cp_parser* parser, tree lambda_expr)
+cp_parser_lambda_declarator_opt (cp_parser* parser, tree lambda_expr,
+				 bool consteval_block_p/*=false*/)
 {
   /* 5.1.1.4 of the standard says:
        If a lambda-expression does not include a lambda-declarator, it is as if
@@ -12342,6 +12374,18 @@ cp_parser_lambda_declarator_opt (cp_parser* parser, tree lambda_expr)
 				  CP_PARSER_FLAGS_ONLY_MUTABLE_OR_CONSTEXPR,
 				  &lambda_specs, &declares_class_or_enum);
 
+  /* [dcl.pre] For a consteval-block-declaration D, the expression E
+     corresponding to D is:
+       [] -> void static consteval compound-statement ()
+     Make it so.  */
+  if (consteval_block_p)
+    {
+      return_type = void_type_node;
+      lambda_specs.storage_class = sc_static;
+      set_and_check_decl_spec_loc (&lambda_specs, ds_consteval,
+				   cp_lexer_peek_token (parser->lexer));
+    }
+
   if (omitted_parms_loc && lambda_specs.any_specifiers_p)
     {
       pedwarn (omitted_parms_loc, OPT_Wc__23_extensions,
@@ -16029,15 +16073,16 @@ cp_parser_declaration_seq_opt (cp_parser* parser)
 static void
 cp_parser_declaration (cp_parser* parser, tree prefix_attrs)
 {
-  int saved_pedantic;
+  int saved_pedantic, saved_long_long;
 
   /* Check for the `__extension__' keyword.  */
-  if (cp_parser_extension_opt (parser, &saved_pedantic))
+  if (cp_parser_extension_opt (parser, &saved_pedantic, &saved_long_long))
     {
       /* Parse the qualified declaration.  */
       cp_parser_declaration (parser, prefix_attrs);
       /* Restore the PEDANTIC flag.  */
       pedantic = saved_pedantic;
+      warn_long_long = saved_long_long;
 
       return;
     }
@@ -16279,6 +16324,56 @@ cp_parser_toplevel_declaration (cp_parser* parser)
     cp_parser_declaration (parser, NULL_TREE);
 }
 
+/* Build an empty string for static_assert.  */
+
+static tree
+build_empty_string ()
+{
+  tree message = build_string (1, "");
+  TREE_TYPE (message) = char_array_type_node;
+  fix_string_type (message);
+  return message;
+}
+
+/* Return true iff the next tokens start a C++26 consteval block.  */
+
+static bool
+cp_parser_next_tokens_are_consteval_block_p (cp_parser *parser)
+{
+  return (cxx_dialect >= cxx26
+	  && cp_lexer_next_token_is_keyword (parser->lexer, RID_CONSTEVAL)
+	  && cp_lexer_nth_token_is (parser->lexer, 2, CPP_OPEN_BRACE));
+}
+
+/* Parse a consteval-block-declaration.
+
+   consteval-block-declaration:
+     consteval compound-statement
+
+   If MEMBER_P, this consteval block is a member declaration.  */
+
+static void
+cp_parser_consteval_block (cp_parser *parser, bool member_p)
+{
+  const location_t loc = cp_lexer_peek_token (parser->lexer)->location;
+  /* Consume the 'consteval'.  */
+  cp_lexer_consume_token (parser->lexer);
+
+  /* We know the next token is '{'.  Let cp_parser_lambda_body handle it.  */
+  cp_expr lam = cp_parser_lambda_expression (parser,
+					     /*consteval_block_p=*/true);
+  if (!cp_parser_error_occurred (parser))
+    {
+      releasing_vec args;
+      tree call = finish_call_expr (lam, &args,
+				    /*disallow_virtual=*/false,
+				    /*koenig_p=*/false,
+				    tf_warning_or_error);
+      finish_static_assert (call, build_empty_string (), loc, member_p,
+			    /*show_expr_p=*/false, /*consteval_block_p=*/true);
+    }
+}
+
 /* Parse a block-declaration.
 
    block-declaration:
@@ -16286,18 +16381,18 @@ cp_parser_toplevel_declaration (cp_parser* parser)
      asm-definition
      namespace-alias-definition
      using-declaration
+     using-enum-declaration
      using-directive
+     static_assert-declaration
+     consteval-block-declaration
+     alias-declaration
+     opaque-enum-declaration
 
    GNU Extension:
 
    block-declaration:
      __extension__ block-declaration
 
-   C++0x Extension:
-
-   block-declaration:
-     static_assert-declaration
-
    If STATEMENT_P is TRUE, then this block-declaration is occurring as
    part of a declaration-statement.  */
 
@@ -16305,15 +16400,16 @@ static void
 cp_parser_block_declaration (cp_parser *parser,
 			     bool      statement_p)
 {
-  int saved_pedantic;
+  int saved_pedantic, saved_long_long;
 
   /* Check for the `__extension__' keyword.  */
-  if (cp_parser_extension_opt (parser, &saved_pedantic))
+  if (cp_parser_extension_opt (parser, &saved_pedantic, &saved_long_long))
     {
       /* Parse the qualified declaration.  */
       cp_parser_block_declaration (parser, statement_p);
       /* Restore the PEDANTIC flag.  */
       pedantic = saved_pedantic;
+      warn_long_long = saved_long_long;
 
       return;
     }
@@ -16374,6 +16470,8 @@ cp_parser_block_declaration (cp_parser *parser,
   /* If the next token is `static_assert' we have a static assertion.  */
   else if (token1->keyword == RID_STATIC_ASSERT)
     cp_parser_static_assert (parser, /*member_p=*/false);
+  else if (cp_parser_next_tokens_are_consteval_block_p (parser))
+    cp_parser_consteval_block (parser, /*member_p=*/false);
   else
     {
       size_t attr_idx = cp_parser_skip_std_attribute_spec_seq (parser, 1);
@@ -16867,7 +16965,11 @@ cp_parser_decomposition_declaration (cp_parser *parser,
 	  decl = error_mark_node;
 	}
       else
-	prev = decl2;
+	{
+	  prev = decl2;
+	  DECL_DECLARED_CONSTEXPR_P (decl2) = DECL_DECLARED_CONSTEXPR_P (decl);
+	  DECL_DECLARED_CONSTINIT_P (decl2) = DECL_DECLARED_CONSTINIT_P (decl);
+	}
       if (elt_pushed_scope)
 	pop_scope (elt_pushed_scope);
     }
@@ -16916,6 +17018,15 @@ cp_parser_decomposition_declaration (cp_parser *parser,
       /* Ensure DECL_VALUE_EXPR is created for all the decls but
 	 the underlying DECL.  */
       cp_finish_decomp (decl, &decomp);
+      if (decl_spec_seq_has_spec_p (decl_specifiers, ds_thread))
+	pedwarn (decl_specifiers->locations[ds_thread],
+		 0, "for-range-declaration cannot be %qs",
+		 decl_specifiers->gnu_thread_keyword_p
+		 ? "__thread" : "thread_local");
+      else if (decl_specifiers->storage_class == sc_static)
+	pedwarn (decl_specifiers->locations[ds_storage_class],
+		 0, "for-range-declaration cannot be %qs",
+		 "static");
     }
 
   if (pushed_scope)
@@ -17664,9 +17775,7 @@ cp_parser_static_assert (cp_parser *parser, bool member_p)
 		 "only available with %<-std=c++17%> or %<-std=gnu++17%>");
       /* Eat the ')'  */
       cp_lexer_consume_token (parser->lexer);
-      message = build_string (1, "");
-      TREE_TYPE (message) = char_array_type_node;
-      fix_string_type (message);
+      message = build_empty_string ();
     }
   else
     {
@@ -21009,9 +21118,6 @@ cp_parser_simple_type_specifier (cp_parser* parser,
 			 "only available with "
 			 "%<-std=c++14%> or %<-std=gnu++14%>");
 	    }
-	  else if (parser->in_template_argument_list_p)
-	    error_at (token->location,
-		     "use of %<auto%> in template argument");
 	  else if (!flag_concepts)
 	    pedwarn (token->location, OPT_Wc__20_extensions,
 		     "use of %<auto%> in parameter declaration "
@@ -21021,6 +21127,11 @@ cp_parser_simple_type_specifier (cp_parser* parser,
 		     "use of %<auto%> in parameter declaration "
 		     "only available with "
 		     "%<-std=c++14%> or %<-std=gnu++14%>");
+
+	  if (parser->in_template_argument_list_p)
+	    permerror_opt (token->location,
+			   OPT_Wabbreviated_auto_in_template_arg,
+			   "use of %<auto%> in template argument");
 	}
       else
 	type = make_auto ();
@@ -21149,7 +21260,9 @@ cp_parser_simple_type_specifier (cp_parser* parser,
 						/*typename_keyword_p=*/false,
 						/*check_dependency_p=*/true,
 						/*type_p=*/false,
-						/*is_declaration=*/false)
+						/*is_declaration=*/false,
+						/*template_keyword_p=*/false,
+						global_p)
 	   != NULL_TREE);
       /* If we have seen a nested-name-specifier, and the next token
 	 is `template', then we are using the template-id production.  */
@@ -21467,6 +21580,10 @@ cp_parser_placeholder_type_specifier (cp_parser *parser, location_t loc,
 	  error_at (loc, "cannot declare a parameter with %<decltype(auto)%>");
 	  return error_mark_node;
 	}
+      if (parser->in_template_argument_list_p)
+	permerror_opt (placeholder->location,
+		       OPT_Wabbreviated_auto_in_template_arg,
+		       "use of %<auto%> in template argument");
       tree parm = build_constrained_parameter (con, proto, args);
       return synthesize_implicit_template_parm (parser, parm);
     }
@@ -21995,7 +22112,7 @@ cp_parser_elaborated_type_specifier (cp_parser* parser,
 
 	  bool template_p =
 	    (template_parm_lists_apply
-	     && (cp_parser_next_token_starts_class_definition_p (parser)
+	     && (cp_parser_nth_token_starts_class_definition_p (parser, 1)
 		 || cp_lexer_next_token_is (parser->lexer, CPP_SEMICOLON)));
 	  /* An unqualified name was used to reference this type, so
 	     there were no qualifying templates.  */
@@ -24159,7 +24276,26 @@ cp_parser_init_declarator (cp_parser* parser,
 	  && token->type != CPP_SEMICOLON)
 	{
 	  if (maybe_range_for_decl && *maybe_range_for_decl != error_mark_node)
-	    range_for_decl_p = true;
+	    {
+	      range_for_decl_p = true;
+	      if (decl_spec_seq_has_spec_p (decl_specifiers, ds_thread))
+		pedwarn (decl_specifiers->locations[ds_thread],
+			 0, "for-range-declaration cannot be %qs",
+			 decl_specifiers->gnu_thread_keyword_p
+			 ? "__thread" : "thread_local");
+	      else if (decl_specifiers->storage_class == sc_static)
+		pedwarn (decl_specifiers->locations[ds_storage_class],
+			 0, "for-range-declaration cannot be %qs",
+			 "static");
+	      else if (decl_specifiers->storage_class == sc_extern)
+		pedwarn (decl_specifiers->locations[ds_storage_class],
+			 0, "for-range-declaration cannot be %qs",
+			 "extern");
+	      else if (decl_specifiers->storage_class == sc_register)
+		pedwarn (decl_specifiers->locations[ds_storage_class],
+			 0, "for-range-declaration cannot be %qs",
+			 "register");
+	    }
 	  else
 	    {
 	      if (!maybe_range_for_decl)
@@ -28013,6 +28149,98 @@ cp_parser_class_specifier (cp_parser* parser)
   return type;
 }
 
+/* Parse an (optional) class-property-specifier-seq.
+
+   class-property-specifier-seq:
+     class-property-specifier class-property-specifier-seq [opt]
+
+   class-property-specifier:
+     final
+     trivially_relocatable_if_eligible (C++26)
+     replaceable_if_eligible (C++26)
+
+   Returns a bitmask representing the class-property-specifiers.  */
+
+static cp_virt_specifiers
+cp_parser_class_property_specifier_seq_opt (cp_parser *parser)
+{
+  cp_virt_specifiers virt_specifiers = VIRT_SPEC_UNSPECIFIED;
+
+  while (true)
+    {
+      cp_token *token;
+      cp_virt_specifiers virt_specifier;
+
+      /* Peek at the next token.  */
+      token = cp_lexer_peek_token (parser->lexer);
+      /* See if it's a class-property-specifier.  */
+      if (token->type != CPP_NAME)
+	break;
+      if (id_equal (token->u.value, "final"))
+	{
+	  /* For C++98, quietly ignore final in e.g.
+	     struct S final = 24;  */
+	  if (cxx_dialect == cxx98
+	      && virt_specifiers == VIRT_SPEC_UNSPECIFIED
+	      && !cp_parser_nth_token_starts_class_definition_p (parser, 2)
+	      && !cp_lexer_nth_token_is (parser->lexer, 2, CPP_NAME))
+	    break;
+	  maybe_warn_cpp0x (CPP0X_OVERRIDE_CONTROLS);
+	  virt_specifier = VIRT_SPEC_FINAL;
+	}
+      else if (id_equal (token->u.value, "__final"))
+	virt_specifier = VIRT_SPEC_FINAL;
+      else if (id_equal (token->u.value, "trivially_relocatable_if_eligible"))
+	{
+	  if (cxx_dialect < cxx26)
+	    {
+	      /* Warn about the C++26 conditional keyword (but don't parse
+		 it).  */
+	      warning_at (token->location, OPT_Wc__26_compat,
+			  "identifier %qE is a conditional keyword in C++26",
+			  token->u.value);
+	      break;
+	    }
+	  virt_specifier = VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE;
+	}
+      else if (id_equal (token->u.value,
+			 "__trivially_relocatable_if_eligible"))
+	virt_specifier = VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE;
+      else if (id_equal (token->u.value, "replaceable_if_eligible"))
+	{
+	  if (cxx_dialect < cxx26)
+	    {
+	      /* Warn about the C++26 conditional keyword (but don't parse
+		 it).  */
+	      warning_at (token->location, OPT_Wc__26_compat,
+			  "identifier %qE is a conditional keyword in C++26",
+			  token->u.value);
+	      break;
+	    }
+	  virt_specifier = VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE;
+	}
+      else if (id_equal (token->u.value,
+			 "__replaceable_if_eligible"))
+	virt_specifier = VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE;
+      else
+	break;
+
+      if (virt_specifiers & virt_specifier)
+	{
+	  gcc_rich_location richloc (token->location);
+	  richloc.add_fixit_remove ();
+	  error_at (&richloc, "duplicate %qD specifier", token->u.value);
+	  cp_lexer_purge_token (parser->lexer);
+	}
+      else
+	{
+	  cp_lexer_consume_token (parser->lexer);
+	  virt_specifiers |= virt_specifier;
+	}
+    }
+  return virt_specifiers;
+}
+
 /* Parse a class-head.
 
    class-head:
@@ -28203,18 +28431,16 @@ cp_parser_class_head (cp_parser* parser,
   pop_deferring_access_checks ();
 
   if (id)
-    {
-      cp_parser_check_for_invalid_template_id (parser, id,
-					       class_key,
-                                               type_start_token->location);
-    }
-  virt_specifiers = cp_parser_virt_specifier_seq_opt (parser);
+    cp_parser_check_for_invalid_template_id (parser, id,
+					     class_key,
+					     type_start_token->location);
+  virt_specifiers = cp_parser_class_property_specifier_seq_opt (parser);
 
   /* If it's not a `:' or a `{' then we can't really be looking at a
      class-head, since a class-head only appears as part of a
      class-specifier.  We have to detect this situation before calling
      xref_tag, since that has irreversible side-effects.  */
-  if (!cp_parser_next_token_starts_class_definition_p (parser))
+  if (!cp_parser_nth_token_starts_class_definition_p (parser, 1))
     {
       cp_parser_error (parser, "expected %<{%> or %<:%>");
       type = error_mark_node;
@@ -28224,13 +28450,6 @@ cp_parser_class_head (cp_parser* parser,
   /* At this point, we're going ahead with the class-specifier, even
      if some other problem occurs.  */
   cp_parser_commit_to_tentative_parse (parser);
-  if (virt_specifiers & VIRT_SPEC_OVERRIDE)
-    {
-      cp_parser_error (parser,
-                       "cannot specify %<override%> for a class");
-      type = error_mark_node;
-      goto out;
-    }
   /* Issue the error about the overly-qualified name now.  */
   if (qualified_p)
     {
@@ -28558,6 +28777,16 @@ cp_parser_class_head (cp_parser* parser,
     DECL_SOURCE_LOCATION (TYPE_NAME (type)) = type_start_token->location;
   if (type && (virt_specifiers & VIRT_SPEC_FINAL))
     CLASSTYPE_FINAL (type) = 1;
+  if (type && (virt_specifiers & VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE))
+    {
+      gcc_assert (!CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (type));
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (type) = 1;
+    }
+  if (type && (virt_specifiers & VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE))
+    {
+      gcc_assert (!CLASSTYPE_REPLACEABLE_COMPUTED (type));
+      CLASSTYPE_REPLACEABLE_BIT (type) = 1;
+    }
  out:
   parser->colon_corrects_to_scope_p = saved_colon_corrects_to_scope_p;
   return type;
@@ -28676,12 +28905,20 @@ cp_parser_member_specification_opt (cp_parser* parser)
 /* Parse a member-declaration.
 
    member-declaration:
-     decl-specifier-seq [opt] member-declarator-list [opt] ;
-     function-definition ; [opt]
-     :: [opt] nested-name-specifier template [opt] unqualified-id ;
+     attribute-specifier-seq [opt] decl-specifier-seq [opt]
+       member-declarator-list [opt] ;
+     function-definition
+     friend-type-declaration
      using-declaration
+     using-enum-declaration
+     static_assert-declaration
+     consteval-block-declaration
      template-declaration
+     explicit-specialization
+     deduction-guide
      alias-declaration
+     opaque-enum-declaration
+     empty-declaration
 
    member-declarator-list:
      member-declarator
@@ -28700,12 +28937,7 @@ cp_parser_member_specification_opt (cp_parser* parser)
    member-declarator:
      declarator attributes [opt] pure-specifier [opt]
      declarator attributes [opt] constant-initializer [opt]
-     identifier [opt] attributes [opt] : constant-expression
-
-   C++0x Extensions:
-
-   member-declaration:
-     static_assert-declaration  */
+     identifier [opt] attributes [opt] : constant-expression  */
 
 static void
 cp_parser_member_declaration (cp_parser* parser)
@@ -28718,16 +28950,17 @@ cp_parser_member_declaration (cp_parser* parser)
   cp_token *token = NULL;
   cp_token *decl_spec_token_start = NULL;
   cp_token *initializer_token_start = NULL;
-  int saved_pedantic;
+  int saved_pedantic, saved_long_long;
   bool saved_colon_corrects_to_scope_p = parser->colon_corrects_to_scope_p;
 
   /* Check for the `__extension__' keyword.  */
-  if (cp_parser_extension_opt (parser, &saved_pedantic))
+  if (cp_parser_extension_opt (parser, &saved_pedantic, &saved_long_long))
     {
       /* Recurse.  */
       cp_parser_member_declaration (parser);
       /* Restore the old value of the PEDANTIC flag.  */
       pedantic = saved_pedantic;
+      warn_long_long = saved_long_long;
 
       return;
     }
@@ -28804,6 +29037,12 @@ cp_parser_member_declaration (cp_parser* parser)
       return;
     }
 
+  if (cp_parser_next_tokens_are_consteval_block_p (parser))
+    {
+      cp_parser_consteval_block (parser, /*member_p=*/true);
+      return;
+    }
+
   parser->colon_corrects_to_scope_p = false;
 
   cp_omp_declare_simd_data odsd;
@@ -31869,13 +32108,16 @@ cp_parser_skip_attributes_opt (cp_parser *parser, size_t n)
    present, and FALSE otherwise.  *SAVED_PEDANTIC is set to the
    current value of the PEDANTIC flag, regardless of whether or not
    the `__extension__' keyword is present.  The caller is responsible
-   for restoring the value of the PEDANTIC flag.  */
+   for restoring the value of the PEDANTIC flag.  Similarly *SAVED_LONG_LONG
+   for warn_long_long flag.  */
 
 static bool
-cp_parser_extension_opt (cp_parser* parser, int* saved_pedantic)
+cp_parser_extension_opt (cp_parser *parser, int *saved_pedantic,
+			 int *saved_long_long)
 {
   /* Save the old value of the PEDANTIC flag.  */
   *saved_pedantic = pedantic;
+  *saved_long_long = warn_long_long;
 
   if (cp_lexer_next_token_is_keyword (parser->lexer, RID_EXTENSION))
     {
@@ -31884,6 +32126,8 @@ cp_parser_extension_opt (cp_parser* parser, int* saved_pedantic)
       /* We're not being pedantic while the `__extension__' keyword is
 	 in effect.  */
       pedantic = 0;
+      /* And we don't want -Wlong-long warning.  */
+      warn_long_long = 0;
 
       return true;
     }
@@ -33558,7 +33802,9 @@ cp_parser_constructor_declarator_p (cp_parser *parser, cp_parser_flags flags,
       && cp_lexer_peek_token (parser->lexer)->type == CPP_TEMPLATE_ID)
     {
       auto_diagnostic_group d;
-      if (emit_diagnostic (cxx_dialect >= cxx20 ? DK_PEDWARN : DK_WARNING,
+      if (emit_diagnostic ((cxx_dialect >= cxx20
+			    ? diagnostics::kind::pedwarn
+			    : diagnostics::kind::warning),
 			   input_location, OPT_Wtemplate_id_cdtor,
 			   "template-id not allowed for constructor in C++20"))
 	inform (input_location, "remove the %qs", "< >");
@@ -35599,15 +35845,15 @@ cp_parser_token_starts_function_definition_p (cp_token* token)
 	  || token->keyword == RID_RETURN);
 }
 
-/* Returns TRUE iff the next token is the ":" or "{" beginning a class
+/* Returns TRUE iff the Nth token is the ":" or "{" beginning a class
    definition.  */
 
 static bool
-cp_parser_next_token_starts_class_definition_p (cp_parser *parser)
+cp_parser_nth_token_starts_class_definition_p (cp_parser *parser, size_t n)
 {
   cp_token *token;
 
-  token = cp_lexer_peek_token (parser->lexer);
+  token = cp_lexer_peek_nth_token (parser->lexer, n);
   return (token->type == CPP_OPEN_BRACE
 	  || (token->type == CPP_COLON
 	      && !parser->colon_doesnt_start_class_def_p));
@@ -35887,7 +36133,9 @@ cp_parser_check_class_key (cp_parser *parser, location_t key_loc,
     return;
 
   bool seen_as_union = TREE_CODE (type) == UNION_TYPE;
-  if (seen_as_union != (class_key == union_type))
+  if (class_key != typename_type
+      && TREE_CODE (type) != TYPENAME_TYPE
+      && seen_as_union != (class_key == union_type))
     {
       auto_diagnostic_group d;
       if (permerror (input_location, "%qs tag used in naming %q#T",
@@ -36661,7 +36909,7 @@ static void
 cp_parser_abort_tentative_parse (cp_parser* parser)
 {
   gcc_assert (parser->context->status != CP_PARSER_STATUS_KIND_COMMITTED
-	      || errorcount > 0);
+	      || seen_error ());
   cp_parser_simulate_error (parser);
   /* Now, pretend that we want to see if the construct was
      successfully parsed.  */
@@ -52898,11 +53146,14 @@ cp_parser_omp_error (cp_parser *parser, cp_token *pragma_tok,
       if (msg == NULL)
 	msg = _("<message unknown at compile time>");
     }
+  const enum diagnostics::kind diag_kind = (severity_fatal
+					    ? diagnostics::kind::error
+					    : diagnostics::kind::warning);
   if (msg)
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered: %s", msg);
   else
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered");
   return false;
 }
diff --git a/gcc/cp/pt.cc b/gcc/cp/pt.cc
index 3362a6f..acfeb81 100644
--- a/gcc/cp/pt.cc
+++ b/gcc/cp/pt.cc
@@ -6952,14 +6952,22 @@ convert_nontype_argument_function (tree type, tree expr,
 	{
 	  auto_diagnostic_group d;
 	  location_t loc = cp_expr_loc_or_input_loc (expr);
-	  error_at (loc, "%qE is not a valid template argument for type %qT",
-		    expr, type);
-	  if (TYPE_PTR_P (type))
-	    inform (loc, "it must be the address of a function "
-		    "with external linkage");
+	  tree c;
+	  if (cxx_dialect >= cxx17
+	      && (c = cxx_constant_value (fn),
+		  c == error_mark_node))
+	    ;
 	  else
-	    inform (loc, "it must be the name of a function with "
-		    "external linkage");
+	    {
+	      error_at (loc, "%qE is not a valid template argument for "
+			"type %qT", expr, type);
+	      if (TYPE_PTR_P (type))
+		inform (loc, "it must be the address of a function "
+			"with external linkage");
+	      else
+		inform (loc, "it must be the name of a function with "
+			"external linkage");
+	    }
 	}
       return NULL_TREE;
     }
@@ -7402,22 +7410,22 @@ invalid_tparm_referent_p (tree type, tree expr, tsubst_flags_t complain)
 	/* Null pointer values are OK in C++11.  */;
       else
 	{
-	  if (VAR_P (expr))
-	    {
-	      if (complain & tf_error)
-		error ("%qD is not a valid template argument "
-		       "because %qD is a variable, not the address of "
-		       "a variable", expr, expr);
-	      return true;
-	    }
+	  tree c;
+	  if (!(complain & tf_error))
+	    ;
+	  else if (cxx_dialect >= cxx17
+		   && (c = cxx_constant_value (expr),
+		       c == error_mark_node))
+	    ;
+	  else if (VAR_P (expr))
+	    error ("%qD is not a valid template argument "
+		   "because %qD is a variable, not the address of "
+		   "a variable", expr, expr);
 	  else
-	    {
-	      if (complain & tf_error)
-		error ("%qE is not a valid template argument for %qT "
-		       "because it is not the address of a variable",
-		       expr, type);
-	      return true;
-	    }
+	    error ("%qE is not a valid template argument for %qT "
+		   "because it is not the address of a variable",
+		   expr, type);
+	  return true;
 	}
     }
   return false;
@@ -12699,7 +12707,17 @@ instantiate_class_template (tree type)
       determine_visibility (TYPE_MAIN_DECL (type));
     }
   if (CLASS_TYPE_P (type))
-    CLASSTYPE_FINAL (type) = CLASSTYPE_FINAL (pattern);
+    {
+      CLASSTYPE_FINAL (type) = CLASSTYPE_FINAL (pattern);
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (type)
+	= CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (pattern);
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (type)
+	= CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (pattern);
+      CLASSTYPE_REPLACEABLE_BIT (type)
+	= CLASSTYPE_REPLACEABLE_BIT (pattern);
+      CLASSTYPE_REPLACEABLE_COMPUTED (type)
+	= CLASSTYPE_REPLACEABLE_COMPUTED (pattern);
+    }
 
   pbinfo = TYPE_BINFO (pattern);
 
@@ -14906,6 +14924,9 @@ tsubst_function_decl (tree t, tree args, tsubst_flags_t complain,
       argvec = NULL_TREE;
     }
 
+  /* Make sure tsubst_decl substitutes all the parameters.  */
+  cp_evaluated ev;
+
   tree closure = (lambda_fntype ? TYPE_METHOD_BASETYPE (lambda_fntype)
 		  : NULL_TREE);
   tree ctx = closure ? closure : DECL_CONTEXT (t);
@@ -17250,13 +17271,14 @@ tsubst (tree t, tree args, tsubst_flags_t complain, tree in_decl)
 	      return error_mark_node;
 	  }
 
-	/* FIXME: TYPENAME_IS_CLASS_P conflates 'class' vs 'struct' vs 'union'
-	   tags.  TYPENAME_TYPE should probably remember the exact tag that
-	   was written.  */
+	/* FIXME: TYPENAME_IS_CLASS_P conflates 'class' vs 'struct' tags.
+	   TYPENAME_TYPE should probably remember the exact tag that
+	   was written for -Wmismatched-tags.  */
 	enum tag_types tag_type
-	  = TYPENAME_IS_CLASS_P (t) ? class_type
-	  : TYPENAME_IS_ENUM_P (t) ? enum_type
-	  : typename_type;
+	  = (TYPENAME_IS_CLASS_P (t) ? class_type
+	     : TYPENAME_IS_UNION_P (t) ? union_type
+	     : TYPENAME_IS_ENUM_P (t) ? enum_type
+	     : typename_type);
 	tsubst_flags_t tcomplain = complain | tf_keep_type_decl;
 	tcomplain |= tst_ok_flag | qualifying_scope_flag;
 	f = make_typename_type (ctx, f, tag_type, tcomplain);
@@ -17278,10 +17300,18 @@ tsubst (tree t, tree args, tsubst_flags_t complain, tree in_decl)
 		else
 		  return error_mark_node;
 	      }
-	    else if (TYPENAME_IS_CLASS_P (t) && !CLASS_TYPE_P (f))
+	    else if (TYPENAME_IS_CLASS_P (t) && !NON_UNION_CLASS_TYPE_P (f))
 	      {
 		if (complain & tf_error)
-		  error ("%qT resolves to %qT, which is not a class type",
+		  error ("%qT resolves to %qT, which is not a non-union "
+			 "class type", t, f);
+		else
+		  return error_mark_node;
+	      }
+	    else if (TYPENAME_IS_UNION_P (t) && !UNION_TYPE_P (f))
+	      {
+		if (complain & tf_error)
+		  error ("%qT resolves to %qT, which is not a union type",
 			 t, f);
 		else
 		  return error_mark_node;
@@ -17967,9 +17997,7 @@ tsubst_omp_clause_decl (tree decl, tree args, tsubst_flags_t complain,
     return decl;
 
   /* Handle OpenMP iterators.  */
-  if (TREE_CODE (decl) == TREE_LIST
-      && TREE_PURPOSE (decl)
-      && TREE_CODE (TREE_PURPOSE (decl)) == TREE_VEC)
+  if (OMP_ITERATOR_DECL_P (decl))
     {
       tree ret;
       if (iterator_cache[0] == TREE_PURPOSE (decl))
@@ -19573,7 +19601,8 @@ tsubst_stmt (tree t, tree args, tsubst_flags_t complain, tree in_decl)
 
 	finish_static_assert (condition, message,
                               STATIC_ASSERT_SOURCE_LOCATION (t),
-			      /*member_p=*/false, /*show_expr_p=*/true);
+			      /*member_p=*/false, /*show_expr_p=*/true,
+			      CONSTEVAL_BLOCK_P (t));
       }
       break;
 
@@ -20138,7 +20167,14 @@ tsubst_stmt (tree t, tree args, tsubst_flags_t complain, tree in_decl)
       {
 	tree op0 = RECUR (TREE_OPERAND (t, 0));
 	tree cond = RECUR (MUST_NOT_THROW_COND (t));
-	RETURN (build_must_not_throw_expr (op0, cond));
+	stmt = build_must_not_throw_expr (op0, cond);
+	if (stmt && TREE_CODE (stmt) == MUST_NOT_THROW_EXPR)
+	  {
+	    MUST_NOT_THROW_NOEXCEPT_P (stmt) = MUST_NOT_THROW_NOEXCEPT_P (t);
+	    MUST_NOT_THROW_THROW_P (stmt) = MUST_NOT_THROW_THROW_P (t);
+	    MUST_NOT_THROW_CATCH_P (stmt) = MUST_NOT_THROW_CATCH_P (t);
+	  }
+	RETURN (stmt);
       }
 
     case EXPR_PACK_EXPANSION:
@@ -20470,11 +20506,6 @@ tsubst_lambda_expr (tree t, tree args, tsubst_flags_t complain, tree in_decl)
     r = error_mark_node;
   else
     {
-      /* The body of a lambda-expression is not a subexpression of the
-	 enclosing expression.  Parms are to have DECL_CHAIN tsubsted,
-	 which would be skipped if cp_unevaluated_operand.  */
-      cp_evaluated ev;
-
       /* Fix the type of 'this'.
 	 For static and xobj member functions we use this to transport the
 	 lambda's closure type.  It appears that in the regular case the
@@ -20500,6 +20531,10 @@ tsubst_lambda_expr (tree t, tree args, tsubst_flags_t complain, tree in_decl)
       /* Let finish_function set this.  */
       DECL_DECLARED_CONSTEXPR_P (fn) = false;
 
+      /* The body of a lambda-expression is not a subexpression of the
+	 enclosing expression.  */
+      cp_evaluated ev;
+
       bool nested = cfun;
       if (nested)
 	push_function_context ();
@@ -31190,14 +31225,8 @@ alias_ctad_tweaks (tree tmpl, tree uguides)
 	  /* Substitute the deduced arguments plus the rewritten template
 	     parameters into f to get g.  This covers the type, copyness,
 	     guideness, and explicit-specifier.  */
-	  tree g;
-	    {
-	      /* Parms are to have DECL_CHAIN tsubsted, which would be skipped
-		 if cp_unevaluated_operand.  */
-	      cp_evaluated ev;
-	      g = tsubst_decl (DECL_TEMPLATE_RESULT (f), targs, complain,
+	  tree g = tsubst_decl (DECL_TEMPLATE_RESULT (f), targs, complain,
 			       /*use_spec_table=*/false);
-	    }
 	  if (g == error_mark_node)
 	    continue;
 	  DECL_NAME (g) = name;
@@ -31627,7 +31656,9 @@ do_class_deduction (tree ptype, tree tmpl, tree init, tree outer_targs,
 	  /* Be permissive with equivalent alias templates.  */
 	  tree u = get_underlying_template (tmpl);
 	  auto_diagnostic_group d;
-	  diagnostic_t dk = (u == tmpl) ? DK_ERROR : DK_PEDWARN;
+	  const enum diagnostics::kind dk = ((u == tmpl)
+					     ? diagnostics::kind::error
+					     : diagnostics::kind::pedwarn);
 	  bool complained
 	    = emit_diagnostic (dk, input_location, 0,
 			       "alias template deduction only available "
diff --git a/gcc/cp/semantics.cc b/gcc/cp/semantics.cc
index 28baf7b..be79b50 100644
--- a/gcc/cp/semantics.cc
+++ b/gcc/cp/semantics.cc
@@ -2338,7 +2338,8 @@ finish_asm_stmt (location_t loc, int volatile_p, tree string,
 	  oconstraints[i] = constraint;
 
 	  if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-				       &allows_mem, &allows_reg, &is_inout))
+				       &allows_mem, &allows_reg, &is_inout,
+				       nullptr))
 	    {
 	      /* If the operand is going to end up in memory,
 		 mark it addressable.  */
@@ -2397,7 +2398,8 @@ finish_asm_stmt (location_t loc, int volatile_p, tree string,
 	  constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
 	  bool constraint_parsed
 	    = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				      oconstraints, &allows_mem, &allows_reg);
+				      oconstraints, &allows_mem, &allows_reg,
+				      nullptr);
 	  /* If the operand is going to end up in memory, don't call
 	     decay_conversion.  */
 	  if (constraint_parsed && !allows_reg && allows_mem)
@@ -3605,11 +3607,13 @@ finish_this_expr (void)
 {
   tree result = NULL_TREE;
 
-  if (current_class_type && LAMBDA_TYPE_P (current_class_type))
+  if (current_class_ref && !LAMBDA_TYPE_P (TREE_TYPE (current_class_ref)))
+    result = current_class_ptr;
+  else if (current_class_type && LAMBDA_TYPE_P (current_class_type))
     result = (lambda_expr_this_capture
 	      (CLASSTYPE_LAMBDA_EXPR (current_class_type), /*add*/true));
-  else if (current_class_ptr)
-    result = current_class_ptr;
+  else
+    gcc_checking_assert (!current_class_ptr);
 
   if (result)
     /* The keyword 'this' is a prvalue expression.  */
@@ -3739,6 +3743,11 @@ finish_unary_op_expr (location_t op_loc, enum tree_code code, cp_expr expr,
   if (!(complain & tf_warning))
     return result;
 
+  /* These will never fold into a constant, so no need to check for
+     overflow for them.  */
+  if (code == PREINCREMENT_EXPR || code == PREDECREMENT_EXPR)
+    return result;
+
   tree result_ovl = result;
   tree expr_ovl = expr;
 
@@ -3983,9 +3992,15 @@ finish_compound_literal (tree type, tree compound_literal,
 tree
 finish_fname (tree id)
 {
-  tree decl;
-
-  decl = fname_decl (input_location, C_RID_CODE (id), id);
+  tree decl = fname_decl (input_location, C_RID_CODE (id), id);
+  /* [expr.prim.lambda.closure]/16 "Unless the compound-statement is that
+     of a consteval-block-declaration, a variable __func__ is implicitly
+     defined...".  We could be in a consteval block in a function, though,
+     and then we shouldn't warn.  */
+  if (current_function_decl
+      && !current_nonlambda_function (/*only_skip_consteval_block_p=*/true))
+    pedwarn (input_location, 0, "%qD is not defined outside of function scope",
+	     decl);
   if (processing_template_decl && current_function_decl
       && decl != error_mark_node)
     decl = DECL_NAME (decl);
@@ -6295,9 +6310,7 @@ handle_omp_array_sections (tree &c, enum c_omp_region_type ort)
   tree *tp = &OMP_CLAUSE_DECL (c);
   if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
        || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_AFFINITY)
-      && TREE_CODE (*tp) == TREE_LIST
-      && TREE_PURPOSE (*tp)
-      && TREE_CODE (TREE_PURPOSE (*tp)) == TREE_VEC)
+      && OMP_ITERATOR_DECL_P (*tp))
     tp = &TREE_VALUE (*tp);
   tree first = handle_omp_array_sections_1 (c, *tp, types,
 					    maybe_zero_len, first_non_one,
@@ -8817,9 +8830,7 @@ finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	  /* FALLTHRU */
 	case OMP_CLAUSE_AFFINITY:
 	  t = OMP_CLAUSE_DECL (c);
-	  if (TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (OMP_ITERATOR_DECL_P (t))
 	    {
 	      if (TREE_PURPOSE (t) != last_iterators)
 		last_iterators_remove
@@ -12593,11 +12604,14 @@ cexpr_str::extract (location_t location, const char * & msg, int &len)
    CONDITION and the message text MESSAGE.  LOCATION is the location
    of the static assertion in the source code.  When MEMBER_P, this
    static assertion is a member of a class.  If SHOW_EXPR_P is true,
-   print the condition (because it was instantiation-dependent).  */
+   print the condition (because it was instantiation-dependent).
+   If CONSTEVAL_BLOCK_P is true, this static assertion represents
+   a consteval block.  */
 
 void
 finish_static_assert (tree condition, tree message, location_t location,
-		      bool member_p, bool show_expr_p)
+		      bool member_p, bool show_expr_p,
+		      bool consteval_block_p/*=false*/)
 {
   tsubst_flags_t complain = tf_warning_or_error;
 
@@ -12625,6 +12639,7 @@ finish_static_assert (tree condition, tree message, location_t location,
       STATIC_ASSERT_CONDITION (assertion) = orig_condition;
       STATIC_ASSERT_MESSAGE (assertion) = cstr.message;
       STATIC_ASSERT_SOURCE_LOCATION (assertion) = location;
+      CONSTEVAL_BLOCK_P (assertion) = consteval_block_p;
 
       if (member_p)
         maybe_add_class_template_decl_list (current_class_type,
@@ -12636,6 +12651,13 @@ finish_static_assert (tree condition, tree message, location_t location,
       return;
     }
 
+  /* Evaluate the consteval { }.  This must be done only once.  */
+  if (consteval_block_p)
+    {
+      cxx_constant_value (condition);
+      return;
+    }
+
   /* Fold the expression and convert it to a boolean value. */
   condition = contextual_conv_bool (condition, complain);
   condition = fold_non_dependent_expr (condition, complain,
@@ -13359,6 +13381,18 @@ object_type_p (const_tree type)
           && !VOID_TYPE_P (type));
 }
 
+/* [defns.referenceable] True iff TYPE is a referenceable type.  */
+
+static bool
+referenceable_type_p (const_tree type)
+{
+  return (TYPE_REF_P (type)
+	  || object_type_p (type)
+	  || (FUNC_OR_METHOD_TYPE_P (type)
+	      && type_memfn_quals (type) == TYPE_UNQUALIFIED
+	      && type_memfn_rqual (type) == REF_QUAL_NONE));
+}
+
 /* Actually evaluates the trait.  */
 
 static bool
@@ -13528,6 +13562,21 @@ trait_expr_value (cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_NOTHROW_INVOCABLE:
       return expr_noexcept_p (build_invoke (type1, type2, tf_none), tf_none);
 
+    case CPTK_IS_NOTHROW_RELOCATABLE:
+      if (trivially_relocatable_type_p (type1))
+	return true;
+      else
+	{
+	  type1 = strip_array_types (type1);
+	  if (!referenceable_type_p (type1))
+	    return false;
+	  tree arg = make_tree_vec (1);
+	  TREE_VEC_ELT (arg, 0)
+	    = cp_build_reference_type (type1, /*rval=*/true);
+	  return (is_nothrow_xible (INIT_EXPR, type1, arg)
+		  && is_nothrow_xible (BIT_NOT_EXPR, type1, NULL_TREE));
+	}
+
     case CPTK_IS_OBJECT:
       return object_type_p (type1);
 
@@ -13546,6 +13595,9 @@ trait_expr_value (cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_REFERENCE:
       return type_code1 == REFERENCE_TYPE;
 
+    case CPTK_IS_REPLACEABLE:
+      return replaceable_type_p (type1);
+
     case CPTK_IS_SAME:
       return same_type_p (type1, type2);
 
@@ -13570,6 +13622,9 @@ trait_expr_value (cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_TRIVIALLY_DESTRUCTIBLE:
       return is_trivially_xible (BIT_NOT_EXPR, type1, NULL_TREE);
 
+    case CPTK_IS_TRIVIALLY_RELOCATABLE:
+      return trivially_relocatable_type_p (type1);
+
     case CPTK_IS_UNBOUNDED_ARRAY:
       return array_of_unknown_bound_p (type1);
 
@@ -13653,7 +13708,8 @@ check_trait_type (tree type, int kind = 1)
 
   type = complete_type (strip_array_types (type));
   if (!COMPLETE_TYPE_P (type)
-      && cxx_incomplete_type_diagnostic (NULL_TREE, type, DK_PERMERROR)
+      && cxx_incomplete_type_diagnostic (NULL_TREE, type,
+					 diagnostics::kind::permerror)
       && !flag_permissive)
     return false;
   return true;
@@ -13700,18 +13756,6 @@ same_type_ref_bind_p (cp_trait_kind kind, tree type1, tree type2)
 	      (non_reference (to), non_reference (from))));
 }
 
-/* [defns.referenceable] True iff TYPE is a referenceable type.  */
-
-static bool
-referenceable_type_p (const_tree type)
-{
-  return (TYPE_REF_P (type)
-	  || object_type_p (type)
-	  || (FUNC_OR_METHOD_TYPE_P (type)
-	      && (type_memfn_quals (type) == TYPE_UNQUALIFIED
-		  && type_memfn_rqual (type) == REF_QUAL_NONE)));
-}
-
 /* Process a trait expression.  */
 
 tree
@@ -13760,8 +13804,11 @@ finish_trait_expr (location_t loc, cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_LITERAL_TYPE:
     case CPTK_IS_POD:
     case CPTK_IS_STD_LAYOUT:
+    case CPTK_IS_REPLACEABLE:
+    case CPTK_IS_NOTHROW_RELOCATABLE:
     case CPTK_IS_TRIVIAL:
     case CPTK_IS_TRIVIALLY_COPYABLE:
+    case CPTK_IS_TRIVIALLY_RELOCATABLE:
     case CPTK_HAS_UNIQUE_OBJ_REPRESENTATIONS:
       if (!check_trait_type (type1, /* kind = */ 2))
 	return error_mark_node;
diff --git a/gcc/cp/tree.cc b/gcc/cp/tree.cc
index 5863b68..d56d73f 100644
--- a/gcc/cp/tree.cc
+++ b/gcc/cp/tree.cc
@@ -488,6 +488,7 @@ builtin_valid_in_constant_expr_p (const_tree decl)
 	  case CP_BUILT_IN_SOURCE_LOCATION:
 	  case CP_BUILT_IN_IS_CORRESPONDING_MEMBER:
 	  case CP_BUILT_IN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS:
+	  case CP_BUILT_IN_EH_PTR_ADJUST_REF:
 	    return true;
 	  default:
 	    break;
@@ -3695,6 +3696,58 @@ build_min_non_dep_op_overload (enum tree_code op,
   int nargs, expected_nargs;
   tree fn, call, obj = NULL_TREE;
 
+  releasing_vec args;
+  va_start (p, overload);
+
+  bool negated = false, rewritten = false, reversed = false;
+  if (cxx_dialect >= cxx20 && TREE_CODE (overload) == TREE_LIST)
+    {
+      /* Handle rebuilding a C++20 rewritten comparison operator expression,
+	 e.g. !(x == y), y <=> x, (x <=> y) @ 0 etc, that resolved to a call
+	 to a user-defined operator<=>/==.  */
+      gcc_checking_assert (TREE_CODE_CLASS (op) == tcc_comparison
+			   || op == SPACESHIP_EXPR);
+      int flags = TREE_INT_CST_LOW (TREE_PURPOSE (overload));
+      if (TREE_CODE (non_dep) == TRUTH_NOT_EXPR)
+	{
+	  negated = true;
+	  non_dep = TREE_OPERAND (non_dep, 0);
+	}
+      if (flags & LOOKUP_REWRITTEN)
+	rewritten = true;
+      if (flags & LOOKUP_REVERSED)
+	reversed = true;
+      if (rewritten
+	  && DECL_OVERLOADED_OPERATOR_IS (TREE_VALUE (overload),
+					  SPACESHIP_EXPR))
+	{
+	  /* Handle (x <=> y) @ 0 and 0 @ (y <=> x) by recursing to first
+	     rebuild the <=>.  Note that both OVERLOAD and the provided arguments
+	     in this case already correspond to the selected operator<=>.  */
+
+	  tree spaceship_non_dep = CALL_EXPR_ARG (non_dep, reversed ? 1 : 0);
+	  gcc_checking_assert (TREE_CODE (spaceship_non_dep) == CALL_EXPR);
+	  tree spaceship_op0 = va_arg (p, tree);
+	  tree spaceship_op1 = va_arg (p, tree);
+	  if (reversed)
+	    std::swap (spaceship_op0, spaceship_op1);
+
+	  /* Push the correct arguments for the operator OP expression, and
+	     set OVERLOAD appropriately.  */
+	  tree op0 = build_min_non_dep_op_overload (SPACESHIP_EXPR,
+						    spaceship_non_dep,
+						    TREE_VALUE (overload),
+						    spaceship_op0,
+						    spaceship_op1);
+	  tree op1 = CALL_EXPR_ARG (non_dep, reversed ? 0 : 1);
+	  gcc_checking_assert (integer_zerop (op1));
+	  vec_safe_push (args, op0);
+	  vec_safe_push (args, op1);
+	  overload = CALL_EXPR_FN (non_dep);
+	}
+      else
+	overload = TREE_VALUE (overload);
+    }
   non_dep = extract_call_expr (non_dep);
 
   nargs = call_expr_nargs (non_dep);
@@ -3715,32 +3768,40 @@ build_min_non_dep_op_overload (enum tree_code op,
     expected_nargs += 1;
   gcc_assert (nargs == expected_nargs);
 
-  releasing_vec args;
-  va_start (p, overload);
-
   if (!DECL_OBJECT_MEMBER_FUNCTION_P (overload))
     {
       fn = overload;
-      if (op == ARRAY_REF)
-	obj = va_arg (p, tree);
-      for (int i = 0; i < nargs; i++)
+      if (vec_safe_length (args) != 0)
+	/* The correct arguments were already pushed above.  */
+	gcc_checking_assert (rewritten);
+      else
 	{
-	  tree arg = va_arg (p, tree);
-	  vec_safe_push (args, arg);
+	  if (op == ARRAY_REF)
+	    obj = va_arg (p, tree);
+	  for (int i = 0; i < nargs; i++)
+	    {
+	      tree arg = va_arg (p, tree);
+	      vec_safe_push (args, arg);
+	    }
 	}
+      if (reversed)
+	std::swap ((*args)[0], (*args)[1]);
     }
   else
     {
+      gcc_checking_assert (vec_safe_length (args) == 0);
       tree object = va_arg (p, tree);
-      tree binfo = TYPE_BINFO (TREE_TYPE (object));
-      tree method = build_baselink (binfo, binfo, overload, NULL_TREE);
-      fn = build_min (COMPONENT_REF, TREE_TYPE (overload),
-		      object, method, NULL_TREE);
       for (int i = 0; i < nargs; i++)
 	{
 	  tree arg = va_arg (p, tree);
 	  vec_safe_push (args, arg);
 	}
+      if (reversed)
+	std::swap (object, (*args)[0]);
+      tree binfo = TYPE_BINFO (TREE_TYPE (object));
+      tree method = build_baselink (binfo, binfo, overload, NULL_TREE);
+      fn = build_min (COMPONENT_REF, TREE_TYPE (overload),
+		      object, method, NULL_TREE);
     }
 
   va_end (p);
@@ -3752,6 +3813,8 @@ build_min_non_dep_op_overload (enum tree_code op,
   CALL_EXPR_ORDERED_ARGS (call_expr) = CALL_EXPR_ORDERED_ARGS (non_dep);
   CALL_EXPR_REVERSE_ARGS (call_expr) = CALL_EXPR_REVERSE_ARGS (non_dep);
 
+  if (negated)
+    call = build_min (TRUTH_NOT_EXPR, boolean_type_node, call);
   if (obj)
     return keep_unused_object_arg (call, obj, overload);
   return call;
@@ -4715,6 +4778,293 @@ trivial_type_p (const_tree t)
     return scalarish_type_p (t);
 }
 
+/* Returns 1 iff type T is a default-movable type, as defined in
+   [class.prop].  */
+
+static bool
+default_movable_type_p (tree t)
+{
+  if (!CLASS_TYPE_P (t) || !COMPLETE_TYPE_P (t))
+    return false;
+  if (CLASSTYPE_LAZY_DESTRUCTOR (t))
+    lazily_declare_fn (sfk_destructor, t);
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (user_provided_p (dtor) || DECL_DELETED_FN (dtor))
+      return false;
+
+  tree copy_ctor = NULL_TREE, move_ctor = NULL_TREE;
+  tree copy_assign = NULL_TREE, move_assign = NULL_TREE;
+  if (CLASSTYPE_LAZY_MOVE_CTOR (t))
+    move_ctor = lazily_declare_fn (sfk_move_constructor, t);
+  if (CLASSTYPE_LAZY_MOVE_ASSIGN (t))
+    move_assign = lazily_declare_fn (sfk_move_assignment, t);
+  if (!move_ctor)
+    for (ovl_iterator iter (CLASSTYPE_CONSTRUCTORS (t)); iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_ctor = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_ctor = *iter;
+	      break;
+	    }
+	}
+  if (!move_assign)
+    for (ovl_iterator iter (get_class_binding_direct (t,
+						      assign_op_identifier));
+	 iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_assign = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_assign = *iter;
+	      break;
+	    }
+	}
+  if (!move_ctor)
+    {
+      if (CLASSTYPE_LAZY_COPY_CTOR (t))
+	copy_ctor = lazily_declare_fn (sfk_copy_constructor, t);
+      if (!copy_ctor)
+	return false;
+      if (user_provided_p (copy_ctor)
+	  || DECL_DELETED_FN (copy_ctor)
+	  || DECL_CONTEXT (copy_ctor) != t
+	  || DECL_INHERITED_CTOR (copy_ctor))
+	return false;
+    }
+  else if (user_provided_p (move_ctor)
+	   || DECL_DELETED_FN (move_ctor)
+	   || DECL_CONTEXT (move_ctor) != t
+	   || DECL_INHERITED_CTOR (move_ctor))
+    return false;
+  if (!move_assign)
+    {
+      if (CLASSTYPE_LAZY_COPY_ASSIGN (t))
+	copy_assign = lazily_declare_fn (sfk_copy_assignment, t);
+      if (!copy_assign)
+	return false;
+      if (user_provided_p (copy_assign)
+	  || DECL_DELETED_FN (copy_assign)
+	  || DECL_CONTEXT (copy_assign) != t)
+	return false;
+    }
+  else if (user_provided_p (move_assign)
+	   || DECL_DELETED_FN (move_assign)
+	   || DECL_CONTEXT (move_assign) != t)
+    return false;
+  return true;
+}
+
+/* Returns 1 iff type T is a union with no user declared special member
+   functions.  */
+
+static bool
+union_with_no_declared_special_member_fns (tree t)
+{
+  if (TREE_CODE (t) != UNION_TYPE)
+    return false;
+
+  for (ovl_iterator iter (CLASSTYPE_CONSTRUCTORS (t)); iter; ++iter)
+    if (TREE_CODE (*iter) == FUNCTION_DECL
+	&& !DECL_ARTIFICIAL (*iter)
+	&& (default_ctor_p (*iter) || copy_fn_p (*iter) || move_fn_p (*iter)))
+      return false;
+
+  for (ovl_iterator iter (get_class_binding_direct (t, assign_op_identifier));
+       iter; ++iter)
+    if (TREE_CODE (*iter) == FUNCTION_DECL
+	&& !DECL_ARTIFICIAL (*iter)
+	&& (copy_fn_p (*iter) || move_fn_p (*iter)))
+      return false;
+
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (!DECL_ARTIFICIAL (dtor))
+      return false;
+
+  return true;
+}
+
+/* Returns 1 iff type T is a trivially relocatable type, as defined in
+   [basic.types.general] and [class.prop].  */
+
+bool
+trivially_relocatable_type_p (tree t)
+{
+  t = strip_array_types (t);
+
+  if (!CLASS_TYPE_P (t))
+    return scalarish_type_p (t);
+
+  t = TYPE_MAIN_VARIANT (t);
+  if (CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (t))
+    return CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t);
+  if (!COMPLETE_TYPE_P (t))
+    return false;
+
+  if (!CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t)
+      && !union_with_no_declared_special_member_fns (t)
+      && !default_movable_type_p (t))
+    {
+    nontriv:
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t) = 0;
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (t) = 1;
+      return false;
+    }
+
+  if (CLASSTYPE_VBASECLASSES (t))
+    goto nontriv;
+
+  if (CLASSTYPE_LAZY_DESTRUCTOR (t))
+    lazily_declare_fn (sfk_destructor, t);
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (DECL_DELETED_FN (dtor))
+      goto nontriv;
+
+  tree binfo, base_binfo;
+  unsigned int i;
+  for (binfo = TYPE_BINFO (t), i = 0;
+       BINFO_BASE_ITERATE (binfo, i, base_binfo); i++)
+    {
+      tree basetype = TREE_TYPE (base_binfo);
+      if (!trivially_relocatable_type_p (basetype))
+	goto nontriv;
+    }
+
+  for (tree field = TYPE_FIELDS (t); field; field = DECL_CHAIN (field))
+    if (TREE_CODE (field) == FIELD_DECL
+	&& !DECL_ARTIFICIAL (field)
+	&& !DECL_UNNAMED_BIT_FIELD (field))
+      {
+	tree type = TREE_TYPE (field);
+	if (type == error_mark_node)
+	  goto nontriv;
+	if (!TYPE_REF_P (type) && !trivially_relocatable_type_p (type))
+	  goto nontriv;
+      }
+
+  CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t) = 1;
+  CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (t) = 1;
+  return true;
+}
+
+/* Returns 1 iff type T is a replaceable type, as defined in [basic.types]
+   and [class].  */
+
+bool
+replaceable_type_p (tree t)
+{
+  t = strip_array_types (t);
+
+  if (cv_qualified_p (t))
+    return false;
+
+  if (!CLASS_TYPE_P (t))
+    return scalarish_type_p (t);
+
+  t = TYPE_MAIN_VARIANT (t);
+  if (CLASSTYPE_REPLACEABLE_COMPUTED (t))
+    return CLASSTYPE_REPLACEABLE_BIT (t);
+  if (!COMPLETE_TYPE_P (t))
+    return false;
+
+  if (!CLASSTYPE_REPLACEABLE_BIT (t)
+      && !union_with_no_declared_special_member_fns (t)
+      && !default_movable_type_p (t))
+    {
+    nonrepl:
+      CLASSTYPE_REPLACEABLE_BIT (t) = 0;
+      CLASSTYPE_REPLACEABLE_COMPUTED (t) = 1;
+      return false;
+    }
+
+  if (CLASSTYPE_LAZY_DESTRUCTOR (t))
+    lazily_declare_fn (sfk_destructor, t);
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (DECL_DELETED_FN (dtor))
+      goto nonrepl;
+
+  tree copy_ctor = NULL_TREE, move_ctor = NULL_TREE;
+  tree copy_assign = NULL_TREE, move_assign = NULL_TREE;
+  if (CLASSTYPE_LAZY_MOVE_CTOR (t))
+    move_ctor = lazily_declare_fn (sfk_move_constructor, t);
+  if (CLASSTYPE_LAZY_MOVE_ASSIGN (t))
+    move_assign = lazily_declare_fn (sfk_move_assignment, t);
+  if (!move_ctor)
+    for (ovl_iterator iter (CLASSTYPE_CONSTRUCTORS (t)); iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_ctor = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_ctor = *iter;
+	      break;
+	    }
+	}
+  if (!move_assign)
+    for (ovl_iterator iter (get_class_binding_direct (t,
+						      assign_op_identifier));
+	 iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_assign = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_assign = *iter;
+	      break;
+	    }
+	}
+  if (!move_ctor)
+    {
+      if (CLASSTYPE_LAZY_COPY_CTOR (t))
+	copy_ctor = lazily_declare_fn (sfk_copy_constructor, t);
+      if (!copy_ctor || DECL_DELETED_FN (copy_ctor))
+	goto nonrepl;
+    }
+  else if (DECL_DELETED_FN (move_ctor))
+    goto nonrepl;
+  if (!move_assign)
+    {
+      if (CLASSTYPE_LAZY_COPY_ASSIGN (t))
+	copy_assign = lazily_declare_fn (sfk_copy_assignment, t);
+      if (!copy_assign || DECL_DELETED_FN (copy_assign))
+	goto nonrepl;
+    }
+  else if (DECL_DELETED_FN (move_assign))
+    goto nonrepl;
+
+  tree binfo, base_binfo;
+  unsigned int i;
+  for (binfo = TYPE_BINFO (t), i = 0;
+       BINFO_BASE_ITERATE (binfo, i, base_binfo); i++)
+    {
+      tree basetype = TREE_TYPE (base_binfo);
+      if (!replaceable_type_p (basetype))
+	goto nonrepl;
+    }
+
+  for (tree field = TYPE_FIELDS (t); field; field = DECL_CHAIN (field))
+    if (TREE_CODE (field) == FIELD_DECL
+	&& !DECL_ARTIFICIAL (field)
+	&& !DECL_UNNAMED_BIT_FIELD (field))
+      {
+	tree type = TREE_TYPE (field);
+	if (type == error_mark_node)
+	  goto nonrepl;
+	if (!replaceable_type_p (type))
+	  goto nonrepl;
+      }
+
+  CLASSTYPE_REPLACEABLE_BIT (t) = 1;
+  CLASSTYPE_REPLACEABLE_COMPUTED (t) = 1;
+  return true;
+}
+
 /* Returns 1 iff type T is a POD type, as defined in [basic.types].  */
 
 bool
diff --git a/gcc/cp/typeck.cc b/gcc/cp/typeck.cc
index 447fe81..f592894 100644
--- a/gcc/cp/typeck.cc
+++ b/gcc/cp/typeck.cc
@@ -155,7 +155,7 @@ complete_type_or_maybe_complain (tree type, tree value, tsubst_flags_t complain)
   else if (!COMPLETE_TYPE_P (type))
     {
       if (complain & tf_error)
-	cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+	cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
       note_failed_type_completion (type, complain);
       return NULL_TREE;
     }
@@ -618,7 +618,7 @@ type_after_usual_arithmetic_conversions (tree t1, tree t2)
 
 static void
 composite_pointer_error (const op_location_t &location,
-			 diagnostic_t kind, tree t1, tree t2,
+			 enum diagnostics::kind kind, tree t1, tree t2,
 			 composite_pointer_operation operation)
 {
   switch (operation)
@@ -690,7 +690,7 @@ composite_pointer_type_r (const op_location_t &location,
   else
     {
       if (complain & tf_error)
-	composite_pointer_error (location, DK_PERMERROR,
+	composite_pointer_error (location, diagnostics::kind::permerror,
 				 t1, t2, operation);
       else
 	return error_mark_node;
@@ -716,7 +716,7 @@ composite_pointer_type_r (const op_location_t &location,
 			TYPE_PTRMEM_CLASS_TYPE (t2)))
 	{
 	  if (complain & tf_error)
-	    composite_pointer_error (location, DK_PERMERROR,
+	    composite_pointer_error (location, diagnostics::kind::permerror,
 				     t1, t2, operation);
 	  else
 	    return error_mark_node;
@@ -851,7 +851,8 @@ composite_pointer_type (const op_location_t &location,
       else
         {
           if (complain & tf_error)
-	    composite_pointer_error (location, DK_ERROR, t1, t2, operation);
+	    composite_pointer_error (location, diagnostics::kind::error,
+				     t1, t2, operation);
           return error_mark_node;
         }
     }
@@ -4004,7 +4005,7 @@ cp_build_array_ref (location_t loc, tree array, tree idx,
       tree op0, op1, op2;
       op0 = TREE_OPERAND (array, 0);
       op1 = TREE_OPERAND (array, 1);
-      op2 = TREE_OPERAND (array, 1);
+      op2 = TREE_OPERAND (array, 2);
       if (TREE_SIDE_EFFECTS (idx) || !tree_invariant_p (idx))
 	{
 	  /* If idx could possibly have some SAVE_EXPRs, turning
@@ -4530,7 +4531,11 @@ cp_build_function_call_vec (tree function, vec<tree, va_gc> **params,
     {
       if (complain & tf_error)
 	{
-	  if (!flag_diagnostics_show_caret)
+	  if (is_stub_object (original))
+	    error_at (input_location,
+		      "%qT cannot be used as a function",
+		      TREE_TYPE (original));
+	  else if (!flag_diagnostics_show_caret)
 	    error_at (input_location,
 		      "%qE cannot be used as a function", original);
 	  else if (DECL_P (original))
@@ -4672,12 +4677,8 @@ convert_arguments (tree typelist, vec<tree, va_gc> **values, tree fndecl,
       if (type == void_type_node)
 	{
           if (complain & tf_error)
-            {
-	      error_args_num (input_location, fndecl, /*too_many_p=*/true);
-              return i;
-            }
-          else
-            return -1;
+	    error_args_num (input_location, fndecl, /*too_many_p=*/true);
+	  return -1;
 	}
 
       /* build_c_cast puts on a NOP_EXPR to make the result not an lvalue.
@@ -7680,7 +7681,18 @@ cp_build_unary_op (enum tree_code code, tree xarg, bool noconvert,
       if (val != 0)
 	goto return_build_unary_op;
 
-      arg = mark_lvalue_use (arg);
+      tree stripped_arg;
+      stripped_arg = tree_strip_any_location_wrapper (arg);
+      if ((VAR_P (stripped_arg) || TREE_CODE (stripped_arg) == PARM_DECL)
+	  && !DECL_READ_P (stripped_arg)
+	  && (VAR_P (stripped_arg) ? warn_unused_but_set_variable
+				   : warn_unused_but_set_parameter) > 1)
+	{
+	  arg = mark_lvalue_use (arg);
+	  DECL_READ_P (stripped_arg) = 0;
+	}
+      else
+	arg = mark_lvalue_use (arg);
 
       /* Increment or decrement the real part of the value,
 	 and don't change the imaginary part.  */
@@ -9796,7 +9808,22 @@ cp_build_modify_expr (location_t loc, tree lhs, enum tree_code modifycode,
 	  {
 	    auto_diagnostic_group d;
 	    rhs = stabilize_expr (rhs, &init);
+	    bool clear_decl_read = false;
+	    tree stripped_lhs = tree_strip_any_location_wrapper (lhs);
+	    if ((VAR_P (stripped_lhs) || TREE_CODE (stripped_lhs) == PARM_DECL)
+		&& !DECL_READ_P (stripped_lhs)
+		&& (VAR_P (stripped_lhs) ? warn_unused_but_set_variable
+					 : warn_unused_but_set_parameter) > 2
+		&& !CLASS_TYPE_P (TREE_TYPE (lhs))
+		&& !CLASS_TYPE_P (TREE_TYPE (rhs)))
+	      {
+		mark_exp_read (rhs);
+		if (!DECL_READ_P (stripped_lhs))
+		  clear_decl_read = true;
+	      }
 	    newrhs = cp_build_binary_op (loc, modifycode, lhs, rhs, complain);
+	    if (clear_decl_read)
+	      DECL_READ_P (stripped_lhs) = 0;
 	    if (newrhs == error_mark_node)
 	      {
 		if (complain & tf_error)
@@ -10779,7 +10806,9 @@ maybe_warn_about_returning_address_of_local (tree retval, location_t loc)
     {
       if (TYPE_REF_P (valtype))
 	/* P2748 made this an error in C++26.  */
-	emit_diagnostic (cxx_dialect >= cxx26 ? DK_PERMERROR : DK_WARNING,
+	emit_diagnostic ((cxx_dialect >= cxx26
+			  ? diagnostics::kind::permerror
+			  : diagnostics::kind::warning),
 			 loc, OPT_Wreturn_local_addr,
 			 "returning reference to temporary");
       else if (TYPE_PTR_P (valtype))
diff --git a/gcc/cp/typeck2.cc b/gcc/cp/typeck2.cc
index 45edd18..faaf1df 100644
--- a/gcc/cp/typeck2.cc
+++ b/gcc/cp/typeck2.cc
@@ -119,6 +119,11 @@ cxx_readonly_error (location_t loc, tree arg, enum lvalue_use errstring)
 			  G_("increment of read-only reference %qD"),
 			  G_("decrement of read-only reference %qD"),
 			  TREE_OPERAND (arg, 0));
+  else if (is_stub_object (arg))
+    {
+      gcc_assert (errstring == lv_assign);
+      error_at (loc, "assignment to read-only type %qT", TREE_TYPE (arg));
+    }
   else
     readonly_error (loc, arg, errstring);
 }
@@ -290,11 +295,12 @@ cxx_incomplete_type_inform (const_tree type)
 /* Print an error message for invalid use of an incomplete type.
    VALUE is the expression that was used (or 0 if that isn't known)
    and TYPE is the type that was invalid.  DIAG_KIND indicates the
-   type of diagnostic (see diagnostic.def).  */
+   type of diagnostic (see diagnostics/kinds.def).  */
 
 bool
 cxx_incomplete_type_diagnostic (location_t loc, const_tree value,
-				const_tree type, diagnostic_t diag_kind)
+				const_tree type,
+				enum diagnostics::kind diag_kind)
 {
   bool is_decl = false, complained = false;
 
@@ -440,7 +446,7 @@ cxx_incomplete_type_diagnostic (location_t loc, const_tree value,
 void
 cxx_incomplete_type_error (location_t loc, const_tree value, const_tree type)
 {
-  cxx_incomplete_type_diagnostic (loc, value, type, DK_ERROR);
+  cxx_incomplete_type_diagnostic (loc, value, type, diagnostics::kind::error);
 }
 
 
@@ -2627,7 +2633,7 @@ add_exception_specifier (tree list, tree spec, tsubst_flags_t complain)
   bool ok;
   tree core = spec;
   bool is_ptr;
-  diagnostic_t diag_type = DK_UNSPECIFIED; /* none */
+  enum diagnostics::kind diag_type = diagnostics::kind::unspecified; /* none */
 
   if (spec == error_mark_node)
     return list;
@@ -2659,7 +2665,7 @@ add_exception_specifier (tree list, tree spec, tsubst_flags_t complain)
 	 and calls.  So just give a pedwarn at this point; we will give an
 	 error later if we hit one of those two cases.  */
       if (!COMPLETE_TYPE_P (complete_type (core)))
-	diag_type = DK_PEDWARN; /* pedwarn */
+	diag_type = diagnostics::kind::pedwarn; /* pedwarn */
     }
 
   if (ok)
@@ -2673,9 +2679,9 @@ add_exception_specifier (tree list, tree spec, tsubst_flags_t complain)
 	list = tree_cons (NULL_TREE, spec, list);
     }
   else
-    diag_type = DK_ERROR; /* error */
+    diag_type = diagnostics::kind::error; /* error */
 
-  if (diag_type != DK_UNSPECIFIED
+  if (diag_type != diagnostics::kind::unspecified
       && (complain & tf_warning_or_error))
     cxx_incomplete_type_diagnostic (NULL_TREE, core, diag_type);
 
diff --git a/gcc/cprop.cc b/gcc/cprop.cc
index bc72e64..dfe3462 100644
--- a/gcc/cprop.cc
+++ b/gcc/cprop.cc
@@ -1525,6 +1525,7 @@ static bool
 bypass_block (basic_block bb, rtx_insn *setcc, rtx_insn *jump)
 {
   rtx_insn *insn;
+  rtx setcc_src, setcc_dest;
   rtx note;
   edge e, edest;
   bool change;
@@ -1533,7 +1534,19 @@ bypass_block (basic_block bb, rtx_insn *setcc, rtx_insn *jump)
   unsigned i;
   edge_iterator ei;
 
-  insn = (setcc != NULL) ? setcc : jump;
+  if (setcc != NULL)
+    {
+      rtx set = single_set (setcc);
+      setcc_dest = SET_DEST (set);
+      setcc_src = SET_SRC (set);
+      insn = setcc;
+    }
+  else
+    {
+      setcc_dest = NULL;
+      setcc_src = NULL;
+      insn = jump;
+    }
 
   /* Determine set of register uses in INSN.  */
   reg_use_count = 0;
@@ -1608,9 +1621,7 @@ bypass_block (basic_block bb, rtx_insn *setcc, rtx_insn *jump)
 	  src = SET_SRC (pc_set (jump));
 
 	  if (setcc != NULL)
-	    src = simplify_replace_rtx (src,
-					SET_DEST (PATTERN (setcc)),
-					SET_SRC (PATTERN (setcc)));
+	    src = simplify_replace_rtx (src, setcc_dest, setcc_src);
 
 	  new_rtx = simplify_replace_rtx (src, reg_used, set->src);
 
@@ -1716,10 +1727,11 @@ bypass_conditional_jumps (void)
 	      {
 		if (setcc)
 		  break;
-		if (GET_CODE (PATTERN (insn)) != SET)
+		rtx singleset = single_set (insn);
+		if (singleset == NULL_RTX)
 		  break;
 
-		dest = SET_DEST (PATTERN (insn));
+		dest = SET_DEST (singleset);
 		if (REG_P (dest))
 		  setcc = insn;
 		else
diff --git a/gcc/d/ChangeLog b/gcc/d/ChangeLog
index b6815d3..4fe6418 100644
--- a/gcc/d/ChangeLog
+++ b/gcc/d/ChangeLog
@@ -1,3 +1,22 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* d-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* d-diagnostic.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* d-diagnostic.cc (d_diagnostic_report_diagnostic): Update to add
+	"m_" prefix to fields of diagnostic_info throughout.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* toir.cc: Pass null pointer to
+	parse_{input,output}_constraint().
+
 2025-04-29  Iain Buclaw  <ibuclaw@gdcproject.org>
 
 	PR d/103044
diff --git a/gcc/d/d-diagnostic.cc b/gcc/d/d-diagnostic.cc
index 55cb42e..e666ddf 100644
--- a/gcc/d/d-diagnostic.cc
+++ b/gcc/d/d-diagnostic.cc
@@ -185,7 +185,7 @@ escape_d_format (const char *format)
 static void ATTRIBUTE_GCC_DIAG(3,0)
 d_diagnostic_report_diagnostic (const SourceLoc &loc, int opt,
 				const char *format, va_list ap,
-				diagnostic_t kind, bool verbatim)
+				enum diagnostics::kind kind, bool verbatim)
 {
   va_list argp;
   va_copy (argp, ap);
@@ -193,13 +193,13 @@ d_diagnostic_report_diagnostic (const SourceLoc &loc, int opt,
   if (loc.filename.length != 0 || !verbatim)
     {
       rich_location rich_loc (line_table, make_location_t (loc));
-      diagnostic_info diagnostic;
+      diagnostics::diagnostic_info diagnostic;
       char *xformat = expand_d_format (format);
 
       diagnostic_set_info_translated (&diagnostic, xformat, &argp,
 				      &rich_loc, kind);
       if (opt != 0)
-	diagnostic.option_id = opt;
+	diagnostic.m_option_id = opt;
 
       diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
@@ -224,7 +224,7 @@ void D_ATTRIBUTE_FORMAT(2,0) ATTRIBUTE_GCC_DIAG(2,0)
 verrorReport (const SourceLoc loc, const char *format, va_list ap,
 	      ErrorKind kind, const char *prefix1, const char *prefix2)
 {
-  diagnostic_t diag_kind = DK_UNSPECIFIED;
+  enum diagnostics::kind diag_kind = diagnostics::kind::unspecified;
   int opt = 0;
   bool verbatim = false;
   char *xformat;
@@ -238,7 +238,9 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
       if (global.gag && !global.params.v.showGaggedErrors)
 	return;
 
-      diag_kind = global.gag ? DK_ANACHRONISM : DK_ERROR;
+      diag_kind = (global.gag
+		   ? diagnostics::kind::anachronism
+		   : diagnostics::kind::error);
     }
   else if (kind == ErrorKind::warning)
     {
@@ -254,7 +256,7 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
       if (global.params.useWarnings == DIAGNOSTICerror)
 	global.warnings++;
 
-      diag_kind = DK_WARNING;
+      diag_kind = diagnostics::kind::warning;
     }
   else if (kind == ErrorKind::deprecation)
     {
@@ -270,11 +272,11 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
 	}
 
       opt = OPT_Wdeprecated;
-      diag_kind = DK_WARNING;
+      diag_kind = diagnostics::kind::warning;
     }
   else if (kind == ErrorKind::message)
     {
-      diag_kind = DK_NOTE;
+      diag_kind = diagnostics::kind::note;
       verbatim = true;
     }
   else if (kind == ErrorKind::tip)
@@ -282,7 +284,7 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
       if (global.gag)
 	return;
 
-      diag_kind = DK_DEBUG;
+      diag_kind = diagnostics::kind::debug;
       verbatim = true;
     }
   else
@@ -328,7 +330,8 @@ verrorReportSupplemental (const SourceLoc loc, const char* format, va_list ap,
   else
     gcc_unreachable ();
 
-  d_diagnostic_report_diagnostic (loc, 0, format, ap, DK_NOTE, false);
+  d_diagnostic_report_diagnostic (loc, 0, format, ap, diagnostics::kind::note,
+				  false);
 }
 
 /* Call this after printing out fatal error messages to clean up and
diff --git a/gcc/d/toir.cc b/gcc/d/toir.cc
index b70db7a..554399b 100644
--- a/gcc/d/toir.cc
+++ b/gcc/d/toir.cc
@@ -1450,7 +1450,8 @@ public:
 	    oconstraints[i] = constraint;
 
 	    if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-					 &allows_mem, &allows_reg, &is_inout))
+					 &allows_mem, &allows_reg, &is_inout,
+					 nullptr))
 	      {
 		/* If the output argument is going to end up in memory.  */
 		if (!allows_reg)
@@ -1469,7 +1470,8 @@ public:
 	      = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
 
 	    if (parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-					oconstraints, &allows_mem, &allows_reg))
+					oconstraints, &allows_mem, &allows_reg,
+					nullptr))
 	      {
 		/* If the input argument is going to end up in memory.  */
 		if (!allows_reg && allows_mem)
diff --git a/gcc/defaults.h b/gcc/defaults.h
index 16f6dc2..f807ef6 100644
--- a/gcc/defaults.h
+++ b/gcc/defaults.h
@@ -1158,7 +1158,7 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #endif
 
 #ifndef MAX_FIXED_MODE_SIZE
-#define MAX_FIXED_MODE_SIZE GET_MODE_BITSIZE (DImode)
+#define MAX_FIXED_MODE_SIZE MAX (BITS_PER_WORD * 2, 64)
 #endif
 
 /* Nonzero if structures and unions should be returned in memory.
diff --git a/gcc/diagnostic-core.h b/gcc/diagnostic-core.h
index 1e89c9f..fc32f48 100644
--- a/gcc/diagnostic-core.h
+++ b/gcc/diagnostic-core.h
@@ -23,21 +23,8 @@ along with GCC; see the file COPYING3.  If not see
 #define GCC_DIAGNOSTIC_CORE_H
 
 #include "bversion.h"
-
-/* Constants used to discriminate diagnostics.  */
-typedef enum
-{
-#define DEFINE_DIAGNOSTIC_KIND(K, msgid, C) K,
-#include "diagnostic.def"
-#undef DEFINE_DIAGNOSTIC_KIND
-  DK_LAST_DIAGNOSTIC_KIND,
-  /* This is used for tagging pragma pops in the diagnostic
-     classification history chain.  */
-  DK_POP,
-  /* This is used internally to note that a diagnostic is enabled
-     without mandating any specific type.  */
-  DK_ANY,
-} diagnostic_t;
+#include "diagnostics/kinds.h"
+#include "diagnostics/option-id.h"
 
 /* RAII-style class for grouping related diagnostics within global_dc.  */
 
@@ -61,28 +48,9 @@ class auto_diagnostic_nesting_level
 };
 
 /* Forward decl.  */
-class diagnostic_metadata; /* See diagnostic-metadata.h.  */
-
-/* A class to use for the ID of an option that controls
-   a particular diagnostic.
-   This is just a wrapper around "int", but better documents
-   the intent of the code.  */
-
-struct diagnostic_option_id
-{
-  diagnostic_option_id () : m_idx (0) {}
-
-  diagnostic_option_id (int idx) : m_idx (idx) {}
-  /* Ideally we'd take an enum opt_code here, but we don't
-     want to depend on its decl.  */
-
-  bool operator== (diagnostic_option_id other) const
-  {
-    return m_idx == other.m_idx;
-  }
-
-  int m_idx;
-};
+namespace diagnostics {
+   class metadata; /* See diagnostics/metadata.h.  */
+} // namespace diagnostics
 
 extern const char *progname;
 
@@ -109,29 +77,29 @@ extern void internal_error (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2)
 extern void internal_error_no_backtrace (const char *, ...)
      ATTRIBUTE_GCC_DIAG(1,2) ATTRIBUTE_NORETURN;
 /* Pass one of the OPT_W* from options.h as the first parameter.  */
-extern bool warning (diagnostic_option_id,
+extern bool warning (diagnostics::option_id,
 		     const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern bool warning_n (location_t,
-		       diagnostic_option_id,
+		       diagnostics::option_id,
 		       unsigned HOST_WIDE_INT,
 		       const char *, const char *, ...)
     ATTRIBUTE_GCC_DIAG(4,6) ATTRIBUTE_GCC_DIAG(5,6);
 extern bool warning_n (rich_location *,
-		       diagnostic_option_id,
+		       diagnostics::option_id,
 		       unsigned HOST_WIDE_INT,
 		       const char *, const char *, ...)
     ATTRIBUTE_GCC_DIAG(4, 6) ATTRIBUTE_GCC_DIAG(5, 6);
 extern bool warning_at (location_t,
-			diagnostic_option_id,
+			diagnostics::option_id,
 			const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
 extern bool warning_at (rich_location *,
-			diagnostic_option_id,
+			diagnostics::option_id,
 			const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
 extern bool warning_meta (rich_location *,
-			  const diagnostic_metadata &,
-			  diagnostic_option_id,
+			  const diagnostics::metadata &,
+			  diagnostics::option_id,
 			  const char *, ...)
     ATTRIBUTE_GCC_DIAG(4,5);
 extern void error (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
@@ -141,29 +109,29 @@ extern void error_n (location_t, unsigned HOST_WIDE_INT, const char *,
 extern void error_at (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern void error_at (rich_location *, const char *, ...)
   ATTRIBUTE_GCC_DIAG(2,3);
-extern void error_meta (rich_location *, const diagnostic_metadata &,
+extern void error_meta (rich_location *, const diagnostics::metadata &,
 			const char *, ...)
   ATTRIBUTE_GCC_DIAG(3,4);
 extern void fatal_error (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3)
      ATTRIBUTE_NORETURN;
 /* Pass one of the OPT_W* from options.h as the second parameter.  */
 extern bool pedwarn (location_t,
-		     diagnostic_option_id,
+		     diagnostics::option_id,
 		     const char *, ...)
      ATTRIBUTE_GCC_DIAG(3,4);
 extern bool pedwarn (rich_location *,
-		     diagnostic_option_id,
+		     diagnostics::option_id,
 		     const char *, ...)
      ATTRIBUTE_GCC_DIAG(3,4);
 extern bool permerror (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern bool permerror (rich_location *, const char *,
 				   ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern bool permerror_opt (location_t,
-			   diagnostic_option_id,
+			   diagnostics::option_id,
 			   const char *, ...)
   ATTRIBUTE_GCC_DIAG(3,4);
 extern bool permerror_opt (rich_location *,
-			   diagnostic_option_id,
+			   diagnostics::option_id,
 			   const char *, ...)
   ATTRIBUTE_GCC_DIAG(3,4);
 extern void sorry (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
@@ -174,23 +142,23 @@ extern void inform_n (location_t, unsigned HOST_WIDE_INT, const char *,
 		      const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,5) ATTRIBUTE_GCC_DIAG(4,5);
 extern void verbatim (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
-extern bool emit_diagnostic (diagnostic_t,
+extern bool emit_diagnostic (enum diagnostics::kind,
 			     location_t,
-			     diagnostic_option_id,
+			     diagnostics::option_id,
 			     const char *, ...) ATTRIBUTE_GCC_DIAG(4,5);
-extern bool emit_diagnostic (diagnostic_t,
+extern bool emit_diagnostic (enum diagnostics::kind,
 			     rich_location *,
-			     diagnostic_option_id,
+			     diagnostics::option_id,
 			     const char *, ...) ATTRIBUTE_GCC_DIAG(4,5);
-extern bool emit_diagnostic_valist (diagnostic_t,
+extern bool emit_diagnostic_valist (enum diagnostics::kind,
 				    location_t,
-				    diagnostic_option_id,
+				    diagnostics::option_id,
 				    const char *, va_list *)
   ATTRIBUTE_GCC_DIAG (4,0);
-extern bool emit_diagnostic_valist_meta (diagnostic_t,
+extern bool emit_diagnostic_valist_meta (enum diagnostics::kind,
 					 rich_location *,
-					 const diagnostic_metadata *,
-					 diagnostic_option_id,
+					 const diagnostics::metadata *,
+					 diagnostics::option_id,
 					 const char *,
 					 va_list *) ATTRIBUTE_GCC_DIAG (5,0);
 extern bool seen_error (void);
diff --git a/gcc/diagnostic-global-context.cc b/gcc/diagnostic-global-context.cc
index 1165915..500f19c 100644
--- a/gcc/diagnostic-global-context.cc
+++ b/gcc/diagnostic-global-context.cc
@@ -27,11 +27,11 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 
-/* A diagnostic_context surrogate for stderr.  */
-static diagnostic_context global_diagnostic_context;
-diagnostic_context *global_dc = &global_diagnostic_context;
+/* A diagnostics::context surrogate for stderr.  */
+static diagnostics::context global_diagnostic_context;
+diagnostics::context *global_dc = &global_diagnostic_context;
 
 /* Standard error reporting routines in increasing order of severity.  */
 
@@ -49,13 +49,13 @@ verbatim (const char *gmsgid, ...)
   va_end (ap);
 }
 
-/* Wrapper around diagnostic_context::diagnostic_impl
+/* Wrapper around diagnostics::context::diagnostic_impl
    implying global_dc and taking a variable argument list.  */
 
 bool
-emit_diagnostic (diagnostic_t kind,
+emit_diagnostic (enum diagnostics::kind kind,
 		 location_t location,
-		 diagnostic_option_id option_id,
+		 diagnostics::option_id option_id,
 		 const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -71,9 +71,9 @@ emit_diagnostic (diagnostic_t kind,
 /* As above, but for rich_location *.  */
 
 bool
-emit_diagnostic (diagnostic_t kind,
+emit_diagnostic (enum diagnostics::kind kind,
 		 rich_location *richloc,
-		 diagnostic_option_id option_id,
+		 diagnostics::option_id option_id,
 		 const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -88,9 +88,9 @@ emit_diagnostic (diagnostic_t kind,
 /* As above, but taking a variable argument list.  */
 
 bool
-emit_diagnostic_valist (diagnostic_t kind,
+emit_diagnostic_valist (enum diagnostics::kind kind,
 			location_t location,
-			diagnostic_option_id option_id,
+			diagnostics::option_id option_id,
 			const char *gmsgid, va_list *ap)
 {
   rich_location richloc (line_table, location);
@@ -101,10 +101,10 @@ emit_diagnostic_valist (diagnostic_t kind,
 /* As above, but with rich_location and metadata.  */
 
 bool
-emit_diagnostic_valist_meta (diagnostic_t kind,
+emit_diagnostic_valist_meta (enum diagnostics::kind kind,
 			     rich_location *richloc,
-			     const diagnostic_metadata *metadata,
-			     diagnostic_option_id option_id,
+			     const diagnostics::metadata *metadata,
+			     diagnostics::option_id option_id,
 			     const char *gmsgid, va_list *ap)
 {
   return global_dc->diagnostic_impl (richloc, metadata, option_id,
@@ -120,7 +120,8 @@ inform (location_t location, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_NOTE);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::note);
   va_end (ap);
 }
 
@@ -133,7 +134,8 @@ inform (rich_location *richloc, const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap, DK_NOTE);
+  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::note);
   va_end (ap);
 }
 
@@ -149,7 +151,7 @@ inform_n (location_t location, unsigned HOST_WIDE_INT n,
   rich_location richloc (line_table, location);
   global_dc->diagnostic_n_impl (&richloc, nullptr, -1, n,
 				singular_gmsgid, plural_gmsgid,
-				&ap, DK_NOTE);
+				&ap, diagnostics::kind::note);
   va_end (ap);
 }
 
@@ -157,14 +159,15 @@ inform_n (location_t location, unsigned HOST_WIDE_INT n,
    to the relevant language specification but is likely to be buggy anyway.
    Returns true if the warning was printed, false if it was inhibited.  */
 bool
-warning (diagnostic_option_id option_id, const char *gmsgid, ...)
+warning (diagnostics::option_id option_id, const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -175,7 +178,7 @@ warning (diagnostic_option_id option_id, const char *gmsgid, ...)
 
 bool
 warning_at (location_t location,
-	    diagnostic_option_id option_id,
+	    diagnostics::option_id option_id,
 	    const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -183,7 +186,8 @@ warning_at (location_t location,
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -192,7 +196,7 @@ warning_at (location_t location,
 
 bool
 warning_at (rich_location *richloc,
-	    diagnostic_option_id option_id,
+	    diagnostics::option_id option_id,
 	    const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -201,7 +205,8 @@ warning_at (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -210,8 +215,8 @@ warning_at (rich_location *richloc,
 
 bool
 warning_meta (rich_location *richloc,
-	      const diagnostic_metadata &metadata,
-	      diagnostic_option_id option_id,
+	      const diagnostics::metadata &metadata,
+	      diagnostics::option_id option_id,
 	      const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -220,7 +225,8 @@ warning_meta (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, &metadata, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -229,7 +235,7 @@ warning_meta (rich_location *richloc,
 
 bool
 warning_n (rich_location *richloc,
-	   diagnostic_option_id option_id,
+	   diagnostics::option_id option_id,
 	   unsigned HOST_WIDE_INT n,
 	   const char *singular_gmsgid, const char *plural_gmsgid, ...)
 {
@@ -240,7 +246,7 @@ warning_n (rich_location *richloc,
   va_start (ap, plural_gmsgid);
   bool ret = global_dc->diagnostic_n_impl (richloc, nullptr, option_id, n,
 					   singular_gmsgid, plural_gmsgid,
-					   &ap, DK_WARNING);
+					   &ap, diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -251,7 +257,7 @@ warning_n (rich_location *richloc,
 
 bool
 warning_n (location_t location,
-	   diagnostic_option_id option_id,
+	   diagnostics::option_id option_id,
 	   unsigned HOST_WIDE_INT n,
 	   const char *singular_gmsgid, const char *plural_gmsgid, ...)
 {
@@ -261,7 +267,7 @@ warning_n (location_t location,
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_n_impl (&richloc, nullptr, option_id, n,
 					   singular_gmsgid, plural_gmsgid,
-					   &ap, DK_WARNING);
+					   &ap, diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -281,7 +287,7 @@ warning_n (location_t location,
 
 bool
 pedwarn (location_t location,
-	 diagnostic_option_id option_id,
+	 diagnostics::option_id option_id,
 	 const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -289,7 +295,8 @@ pedwarn (location_t location,
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PEDWARN);
+					 gmsgid, &ap,
+					 diagnostics::kind::pedwarn);
   va_end (ap);
   return ret;
 }
@@ -298,7 +305,7 @@ pedwarn (location_t location,
 
 bool
 pedwarn (rich_location *richloc,
-	 diagnostic_option_id option_id,
+	 diagnostics::option_id option_id,
 	 const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -307,7 +314,8 @@ pedwarn (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PEDWARN);
+					 gmsgid, &ap,
+					 diagnostics::kind::pedwarn);
   va_end (ap);
   return ret;
 }
@@ -327,7 +335,7 @@ permerror (location_t location, const char *gmsgid, ...)
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
-					 DK_PERMERROR);
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -343,7 +351,7 @@ permerror (rich_location *richloc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap,
-					 DK_PERMERROR);
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -354,7 +362,7 @@ permerror (rich_location *richloc, const char *gmsgid, ...)
 
 bool
 permerror_opt (location_t location,
-	       diagnostic_option_id option_id,
+	       diagnostics::option_id option_id,
 	       const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -362,7 +370,8 @@ permerror_opt (location_t location,
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PERMERROR);
+					 gmsgid, &ap,
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -371,7 +380,7 @@ permerror_opt (location_t location,
 
 bool
 permerror_opt (rich_location *richloc,
-	       diagnostic_option_id option_id,
+	       diagnostics::option_id option_id,
 	       const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -380,7 +389,8 @@ permerror_opt (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PERMERROR);
+					 gmsgid, &ap,
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -394,7 +404,8 @@ error (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -410,7 +421,7 @@ error_n (location_t location, unsigned HOST_WIDE_INT n,
   rich_location richloc (line_table, location);
   global_dc->diagnostic_n_impl (&richloc, nullptr, -1, n,
 				singular_gmsgid, plural_gmsgid,
-				&ap, DK_ERROR);
+				&ap, diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -422,7 +433,8 @@ error_at (location_t loc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, loc);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -436,14 +448,15 @@ error_at (rich_location *richloc, const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
 /* Same as above, but with metadata.  */
 
 void
-error_meta (rich_location *richloc, const diagnostic_metadata &metadata,
+error_meta (rich_location *richloc, const diagnostics::metadata &metadata,
 	    const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -451,7 +464,8 @@ error_meta (rich_location *richloc, const diagnostic_metadata &metadata,
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  global_dc->diagnostic_impl (richloc, &metadata, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (richloc, &metadata, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -465,7 +479,8 @@ sorry (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_SORRY);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::sorry);
   va_end (ap);
 }
 
@@ -477,7 +492,8 @@ sorry_at (location_t loc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, loc);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_SORRY);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::sorry);
   va_end (ap);
 }
 
@@ -499,7 +515,8 @@ fatal_error (location_t loc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, loc);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_FATAL);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::fatal);
   va_end (ap);
 
   gcc_unreachable ();
@@ -514,7 +531,8 @@ internal_error (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ICE);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::ice);
   va_end (ap);
 
   gcc_unreachable ();
@@ -530,7 +548,8 @@ internal_error_no_backtrace (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ICE_NOBT);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::ice_nobt);
   va_end (ap);
 
   gcc_unreachable ();
diff --git a/gcc/diagnostic-state-to-dot.cc b/gcc/diagnostic-state-to-dot.cc
deleted file mode 100644
index ddae83b..0000000
--- a/gcc/diagnostic-state-to-dot.cc
+++ /dev/null
@@ -1,537 +0,0 @@
-/* Creating GraphViz .dot files from XML state documents.
-   Copyright (C) 2025 Free Software Foundation, Inc.
-   Contributed by David Malcolm <dmalcolm@redhat.com>.
-
-This file is part of GCC.
-
-GCC is free software; you can redistribute it and/or modify it
-under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 3, or (at your option)
-any later version.
-
-GCC is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with GCC; see the file COPYING3.  If not see
-<http://www.gnu.org/licenses/>.  */
-
-#define INCLUDE_ALGORITHM
-#define INCLUDE_MAP
-#define INCLUDE_SET
-#define INCLUDE_STRING
-#define INCLUDE_VECTOR
-#include "config.h"
-#include "system.h"
-#include "coretypes.h"
-
-#include "xml.h"
-#include "xml-printer.h"
-#include "graphviz.h"
-
-static int
-get_depth (const xml::element &e)
-{
-  int deepest_child = 0;
-  for (auto &iter : e.m_children)
-    if (xml::element *child_element = iter->dyn_cast_element ())
-      deepest_child = std::max (deepest_child,
-				get_depth (*child_element));
-  return deepest_child + 1;
-}
-
-enum class dynalloc_state
-{
-  unknown,
-  nonnull,
-  unchecked,
-  freed
-};
-
-static const char *
-get_color_for_dynalloc_state (enum dynalloc_state dynalloc_st)
-{
-  switch (dynalloc_st)
-    {
-    default:
-      gcc_unreachable ();
-      break;
-    case dynalloc_state::unknown:
-    case dynalloc_state::nonnull:
-      return nullptr;
-
-    case dynalloc_state::unchecked:
-      return "#ec7a08"; // pf-orange-400
-
-    case dynalloc_state::freed:
-      return "#cc0000"; // pf-red-100
-    }
-}
-
-static void
-set_color_for_dynalloc_state (dot::attr_list &attrs,
-			      enum dynalloc_state dynalloc_state)
-{
-  if (const char *color = get_color_for_dynalloc_state (dynalloc_state))
-    attrs.add (dot::id ("color"), dot::id (color));
-}
-
-static enum dynalloc_state
-get_dynalloc_state (const xml::element &input_element)
-{
-  const char *dyn_alloc_state = input_element.get_attr ("dynamic-alloc-state");
-  if (!dyn_alloc_state)
-    return dynalloc_state::unknown;
-
-  if (dyn_alloc_state == std::string ("unchecked"))
-    return dynalloc_state::unchecked;
-
-  if (dyn_alloc_state == std::string ("nonnull"))
-    return dynalloc_state::nonnull;
-
-  if (dyn_alloc_state == std::string ("freed"))
-    return dynalloc_state::freed;
-
-  return dynalloc_state::unknown;
-}
-
-class state_diagram : public dot::graph
-{
-public:
-  state_diagram (const xml::document &input_state_doc)
-  : m_next_id (0),
-    m_show_tags (false)
-  {
-    // "node [shape=plaintext]\n"
-    {
-      auto attr_stmt
-	= std::make_unique<dot::attr_stmt> (dot::attr_stmt::kind::node);
-      attr_stmt->m_attrs.add (dot::id ("shape"), dot::id ("plaintext"));
-      add_stmt (std::move (attr_stmt));
-    }
-
-    /* Recurse down the XML state diagram, creating subgraphs
-       and then eventually creating nodes, and recursively
-       creating XML tables, adding ports for the endpoints of edges,
-       and recording edges we'll want to create (into m_pending_edges).  */
-    xml::element *input_elmt_state_diagram
-      = input_state_doc.find_child_element ("state-diagram");
-    gcc_assert (input_elmt_state_diagram);
-    xml::element *input_elmt_mem_regions
-      = input_elmt_state_diagram->find_child_element ("memory-regions");
-    if (!input_elmt_mem_regions)
-      return;
-    auto root_cluster
-      = std::make_unique<dot::subgraph> (dot::id ("cluster_memory_regions"));
-    for (auto &iter : input_elmt_mem_regions->m_children)
-      on_input_xml_node (*root_cluster, *iter);
-    add_stmt (std::move (root_cluster));
-
-    /* We should now have ports for edge endpoints for all region ids.
-       Use them now to create edges.  */
-    for (auto &pe : m_pending_edges)
-      {
-	auto search = m_region_id_to_dst_node_id.find (pe.m_dst_region_id);
-	if (search != m_region_id_to_dst_node_id.end ())
-	  {
-	    auto &dst_node_id = search->second;
-	    auto e = std::make_unique<dot::edge_stmt> (pe.m_src_node_id,
-						       dst_node_id);
-
-	    auto dynalloc_state = m_region_id_to_dynalloc_state.find (pe.m_dst_region_id);
-	    if (dynalloc_state != m_region_id_to_dynalloc_state.end ())
-	      set_color_for_dynalloc_state (e->m_attrs,
-					    dynalloc_state->second);
-
-	    add_stmt (std::move (e));
-	  }
-      }
-  }
-
-private:
-  struct pending_edge
-  {
-    dot::node_id m_src_node_id;
-    std::string m_dst_region_id;
-  };
-
-  dot::id
-  get_id_for_region (const char *region_id)
-  {
-    gcc_assert (region_id);
-    return std::string ("cluster_region_") + region_id;
-  }
-
-  dot::id
-  make_id (bool cluster = false)
-  {
-    if (cluster)
-      return std::string ("cluster_") + std::to_string (m_next_id++);
-    else
-      return std::string ("id_") + std::to_string (m_next_id++);
-  }
-
-  bool
-  starts_node_p (const xml::element &e)
-  {
-    if (e.m_kind == "stack"
-	|| e.m_kind == "heap-buffer"
-	|| e.m_kind == "variable") // e.g. within globals
-      return true;
-    return false;
-  }
-
-  void
-  on_input_xml_node (dot::subgraph &parent_subgraph,
-		     xml::node &input_node)
-  {
-    xml::element *input_element = input_node.dyn_cast_element ();
-    if (!input_element)
-      return;
-
-    dot::id sg_id = make_id (true);
-
-    if (starts_node_p (*input_element))
-      {
-	// Create node with table
-	xml::element table ("table", false);
-	xml::printer xp (table);
-	xp.set_attr ("border", "0");
-	xp.set_attr ("cellborder", "1");
-	xp.set_attr ("cellspacing", "0");
-
-	const int max_depth = get_depth (*input_element);
-	const int num_columns = max_depth + 2;
-
-	dot::id id_of_node = make_id ();
-	on_xml_node (id_of_node, xp, *input_element,
-		     max_depth, 0, num_columns);
-
-	auto node = std::make_unique<dot::node_stmt> (std::move (id_of_node));
-	node->m_attrs.add (dot::id ("shape"),
-			   dot::id ("plaintext"));
-
-	// xml must be done by now
-
-	node->m_attrs.add (dot::id ("label"),
-			   dot::id (table));
-
-	parent_subgraph.m_stmt_list.add_stmt (std::move (node));
-      }
-    else
-      {
-	auto child_subgraph = std::make_unique<dot::subgraph> (std::move (sg_id));
-
-	if (const char *label = input_element->get_attr ("label"))
-	  child_subgraph->add_attr (dot::id ("label"), dot::id (label));
-
-	// recurse:
-	for (auto &iter : input_element->m_children)
-	  on_input_xml_node (*child_subgraph, *iter);
-	parent_subgraph.m_stmt_list.add_stmt (std::move (child_subgraph));
-      }
-  }
-
-  enum class style { h1, h2 };
-
-  void
-  add_title_tr (const dot::id &id_of_node,
-		xml::printer &xp,
-		int num_columns,
-		const xml::element &input_element,
-		std::string heading,
-		enum style styl,
-		enum dynalloc_state dynalloc_state)
-  {
-    xp.push_tag ("tr", true);
-    xp.push_tag ("td", false);
-    xp.set_attr ("colspan", std::to_string (num_columns));
-    xp.set_attr ("cellpadding", "5");
-
-    const char *bgcolor;
-    const char *color;
-    if (const char *c = get_color_for_dynalloc_state (dynalloc_state))
-      {
-	bgcolor = c;
-	color = "white";
-      }
-    else
-      switch (styl)
-	{
-	default:
-	  gcc_unreachable ();
-	case style::h1:
-	  // from diagnostic-format-html.cc: HTML_STYLE .linenum
-	  bgcolor = "#0088ce";
-	  color = "white";
-	  break;
-	case style::h2:
-	  // from diagnostic-format-html.cc: HTML_STYLE .events-hdr
-	  bgcolor = "#393f44"; // pf-black-800
-	  color = "white";
-	  break;
-	}
-
-    xp.set_attr ("bgcolor", bgcolor);
-    xp.push_tag ("font", false);
-    xp.set_attr ("color", color);
-    if (heading == "")
-      heading = " ";
-    xp.add_text (std::move (heading));
-    xp.pop_tag ("font");
-
-    maybe_add_dst_port (id_of_node, xp, input_element);
-
-    xp.pop_tag ("td");
-    xp.pop_tag ("tr");
-  }
-
-  /* Recursively add <TR> to XP for INPUT_NODE and its descendents.  */
-  void
-  on_xml_node (const dot::id &id_of_node,
-	       xml::printer &xp,
-	       xml::node &input_node,
-	       int max_depth,
-	       int depth,
-	       int num_columns)
-  {
-    bool recurse = true;
-
-    xml::element *input_element = input_node.dyn_cast_element ();
-    if (!input_element)
-      return;
-
-    if (input_element->m_kind == "concrete-bindings")
-      return;
-    if (input_element->m_kind == "padding")
-      return;
-
-    if (input_element->m_kind == "stack")
-      {
-	add_title_tr (id_of_node, xp, num_columns, *input_element, "Stack",
-		      style::h1, dynalloc_state::unknown);
-      }
-    else if (input_element->m_kind == "stack-frame")
-      {
-	if (const char *function = input_element->get_attr ("function"))
-	  add_title_tr (id_of_node, xp, num_columns, *input_element,
-			std::string ("Frame: ") + function,
-			style::h2, dynalloc_state::unknown);
-      }
-    else if (input_element->m_kind == "heap-buffer")
-      {
-	const char *extents = input_element->get_attr ("dynamic-extents");
-	enum dynalloc_state dynalloc_st = get_dynalloc_state (*input_element);
-	if (auto region_id = input_element->get_attr ("region_id"))
-	    m_region_id_to_dynalloc_state[region_id] = dynalloc_st;
-	const char *type = input_element->get_attr ("type");
-	pretty_printer pp;
-	switch (dynalloc_st)
-	  {
-	  default:
-	    gcc_unreachable ();
-
-	  case dynalloc_state::unknown:
-	  case dynalloc_state::nonnull:
-	    if (type)
-	      {
-		if (extents)
-		  pp_printf (&pp, "%s (%s byte allocation)",
-			     type, extents);
-		else
-		  pp_printf (&pp, "%s", type);
-	      }
-	    else
-	      {
-		if (extents)
-		  pp_printf (&pp, "%s byte allocation",
-			     extents);
-	      }
-	    break;
-
-	  case dynalloc_state::unchecked:
-	    if (type)
-	      {
-		if (extents)
-		  pp_printf (&pp, "%s (unchecked %s byte allocation)",
-			     type, extents);
-	      }
-	    else
-	      {
-		if (extents)
-		  pp_printf (&pp, "Unchecked %s byte allocation",
-			     extents);
-	      }
-	    break;
-
-	  case dynalloc_state::freed:
-	    // TODO: show deallocator
-	    // TODO: show deallocation event
-	    pp_printf (&pp, "Freed buffer");
-	    break;
-	  }
-	add_title_tr (id_of_node, xp, num_columns, *input_element,
-		      pp_formatted_text (&pp),
-		      style::h2,
-		      dynalloc_st);
-      }
-    else
-      {
-	xp.push_tag ("tr", true);
-	if (depth > 0)
-	  {
-	    /* Indent, by create a <td> spanning "depth" columns.  */
-	    xp.push_tag ("td", false);
-	    xp.set_attr ("colspan", std::to_string (depth));
-	    xp.add_text (" "); // graphviz doesn't like <td/>
-	    xp.pop_tag ("td");
-	  }
-	if (m_show_tags)
-	  {
-	    // Debug: show XML tag
-	    xp.push_tag ("td", false);
-	    xp.add_text ("<");
-	    xp.add_text (input_element->m_kind);
-	    xp.add_text (">");
-	    xp.pop_tag ("td");
-	  }
-	if (input_element->m_kind == "variable")
-	  {
-	    const char *name = input_element->get_attr ("name");
-	    gcc_assert (name);
-	    xp.push_tag ("td", false);
-	    maybe_add_dst_port (id_of_node, xp, *input_element);
-	    push_src_text (xp);
-	    xp.add_text (name);
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	else if (input_element->m_kind == "element")
-	  {
-	    const char *index = input_element->get_attr ("index");
-	    gcc_assert (index);
-	    xp.push_tag ("td", false);
-	    maybe_add_dst_port (id_of_node, xp, *input_element);
-	    push_src_text (xp);
-	    xp.add_text ("[");
-	    xp.add_text (index);
-	    xp.add_text ("]");
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	else if (input_element->m_kind == "field")
-	  {
-	    const char *name = input_element->get_attr ("name");
-	    gcc_assert (name);
-	    xp.push_tag ("td", false);
-	    maybe_add_dst_port (id_of_node, xp, *input_element);
-	    push_src_text (xp);
-	    xp.add_text (".");
-	    xp.add_text (name);
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	if (const char *type = input_element->get_attr ("type"))
-	  {
-	    xp.push_tag ("td", false);
-	    if (max_depth > depth)
-	      xp.set_attr ("colspan", std::to_string (max_depth - depth));
-	    xp.set_attr ("align", "right");
-	    push_src_text (xp);
-	    xp.add_text (type);
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	if (auto value = input_element->find_child_element ("value-of-region"))
-	  {
-	    xp.push_tag ("td", false);
-	    for (auto &iter : value->m_children)
-	      if (auto child_element = iter->dyn_cast_element ())
-		print_value (id_of_node, xp, *child_element);
-	    xp.pop_tag ("td");
-	    recurse = false;
-	  }
-	xp.pop_tag ("tr");
-      }
-
-    if (recurse)
-      for (auto &iter : input_element->m_children)
-	on_xml_node (id_of_node, xp, *iter, max_depth, depth + 1, num_columns);
-  }
-
-  void
-  push_src_text (xml::printer &xp)
-  {
-    xp.push_tag ("font");
-    xp.set_attr ("color", "blue");
-  }
-
-  void
-  pop_src_text (xml::printer &xp)
-  {
-    xp.pop_tag ("font");
-  }
-
-  void
-  print_value (const dot::id &id_of_node,
-	       xml::printer &xp,
-	       xml::element &input_element)
-  {
-    if (input_element.m_kind == "pointer-to-region")
-      if (const char *dst_region_id = input_element.get_attr ("region_id"))
-	{
-	  dot::id src_port_id = make_id ();
-	  xp.set_attr ("port", src_port_id.m_str);
-	  m_pending_edges.push_back
-	    ({dot::node_id (id_of_node,
-			    dot::port (src_port_id,
-				       dot::compass_pt::e)),
-	       dst_region_id});
-      }
-
-    if (input_element.m_kind == "uninitialized")
-      {
-	xp.add_text ("(uninitialized)");
-	return;
-      }
-
-    if (auto dump_text = input_element.get_attr ("dump-text"))
-      xp.add_text (dump_text);
-  }
-
-  /* If INPUT_ELEMENT has a "region_id", add a port to XP for possible
-     incoming edges to use.  */
-
-  void
-  maybe_add_dst_port (const dot::id &id_of_node,
-		      xml::printer &xp,
-		      const xml::element &input_element)
-  {
-    if (const char *region_id = input_element.get_attr ("region_id"))
-      {
-	dot::id dst_id = make_id ();
-	dot::node_id node_id (id_of_node,
-			      dot::port (dst_id/*,
-						 dot::compass_pt::w*/));
-	xp.set_attr ("port", dst_id.m_str);
-	m_region_id_to_dst_node_id.emplace (std::string (region_id),
-					    std::move (node_id));
-      }
-  }
-
-
-private:
-  int m_next_id;
-  std::vector<pending_edge> m_pending_edges;
-  std::map<std::string, dot::node_id> m_region_id_to_dst_node_id;
-  std::map<std::string, enum dynalloc_state> m_region_id_to_dynalloc_state;
-  bool m_show_tags;
-};
-
-std::unique_ptr<dot::graph>
-make_dot_graph_from_xml_state (const xml::document &xml_state)
-{
-  return std::make_unique<state_diagram> (xml_state);
-}
diff --git a/gcc/diagnostic-state.h b/gcc/diagnostic-state.h
deleted file mode 100644
index 68c3e00..0000000
--- a/gcc/diagnostic-state.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Capturing changing state in diagnostic paths.
-   Copyright (C) 2025 Free Software Foundation, Inc.
-   Contributed by David Malcolm <dmalcolm@redhat.com>.
-
-This file is part of GCC.
-
-GCC is free software; you can redistribute it and/or modify it
-under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 3, or (at your option)
-any later version.
-
-GCC is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with GCC; see the file COPYING3.  If not see
-<http://www.gnu.org/licenses/>.  */
-
-#ifndef GCC_DIAGNOSTIC_STATE_H
-#define GCC_DIAGNOSTIC_STATE_H
-
-/* We want to be able to express changing program states in diagnostic paths,
-   so that we can emit this in HTML and SARIF output, and to keep this
-   separate from implementation details of -fanalyzer.
-
-   For now, we use xml::document as the type in the diagnostic subsystem
-   for (optionally) tracking the state at a diagnostic_event.  */
-
-namespace xml { class document; }
-namespace dot { class graph; }
-
-extern std::unique_ptr<dot::graph>
-make_dot_graph_from_xml_state (const xml::document &xml_state);
-
-#endif /* GCC_DIAGNOSTIC_STATE_H */
diff --git a/gcc/diagnostic.h b/gcc/diagnostic.h
index 00f6e35..7572e04 100644
--- a/gcc/diagnostic.h
+++ b/gcc/diagnostic.h
@@ -21,1135 +21,31 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_DIAGNOSTIC_H
 #define GCC_DIAGNOSTIC_H
 
-#include "unique-argv.h"
 #include "rich-location.h"
 #include "pretty-print.h"
 #include "diagnostic-core.h"
 
-namespace text_art
-{
-  class theme;
-} // namespace text_art
-
-namespace xml
-{
-  class printer;
-} // namespace xml
-
-/* An enum for controlling what units to use for the column number
-   when diagnostics are output, used by the -fdiagnostics-column-unit option.
-   Tabs will be expanded or not according to the value of -ftabstop.  The origin
-   (default 1) is controlled by -fdiagnostics-column-origin.  */
-
-enum diagnostics_column_unit
-{
-  /* The default from GCC 11 onwards: display columns.  */
-  DIAGNOSTICS_COLUMN_UNIT_DISPLAY,
-
-  /* The behavior in GCC 10 and earlier: simple bytes.  */
-  DIAGNOSTICS_COLUMN_UNIT_BYTE
-};
-
-/* An enum for controlling how to print non-ASCII characters/bytes when
-   a diagnostic suggests escaping the source code on output.  */
-
-enum diagnostics_escape_format
-{
-  /* Escape non-ASCII Unicode characters in the form <U+XXXX> and
-     non-UTF-8 bytes in the form <XX>.  */
-  DIAGNOSTICS_ESCAPE_FORMAT_UNICODE,
-
-  /* Escape non-ASCII bytes in the form <XX> (thus showing the underlying
-     encoding of non-ASCII Unicode characters).  */
-  DIAGNOSTICS_ESCAPE_FORMAT_BYTES
-};
-
-/* Enum for overriding the standard output format.  */
-
-enum diagnostics_output_format
-{
-  /* The default: textual output.  */
-  DIAGNOSTICS_OUTPUT_FORMAT_TEXT,
-
-  /* SARIF-based output, as JSON to stderr.  */
-  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_STDERR,
-
-  /* SARIF-based output, to a JSON file.  */
-  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_FILE
-};
-
-/* An enum for controlling how diagnostic_paths should be printed.  */
-enum diagnostic_path_format
-{
-  /* Don't print diagnostic_paths.  */
-  DPF_NONE,
-
-  /* Print diagnostic_paths by emitting a separate "note" for every event
-     in the path.  */
-  DPF_SEPARATE_EVENTS,
-
-  /* Print diagnostic_paths by consolidating events together where they
-     are close enough, and printing such runs of events with multiple
-     calls to diagnostic_show_locus, showing the individual events in
-     each run via labels in the source.  */
-  DPF_INLINE_EVENTS
-};
-
-/* An enum for capturing values of GCC_EXTRA_DIAGNOSTIC_OUTPUT,
-   and for -fdiagnostics-parseable-fixits.  */
-
-enum diagnostics_extra_output_kind
-{
-  /* No extra output, or an unrecognized value.  */
-  EXTRA_DIAGNOSTIC_OUTPUT_none,
-
-  /* Emit fix-it hints using the "fixits-v1" format, equivalent to
-     -fdiagnostics-parseable-fixits.  */
-  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v1,
-
-  /* Emit fix-it hints using the "fixits-v2" format.  */
-  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v2
-};
-
-/* Values for -fdiagnostics-text-art-charset=.  */
-
-enum diagnostic_text_art_charset
-{
-  /* No text art diagrams shall be emitted.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_NONE,
-
-  /* Use pure ASCII for text art diagrams.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_ASCII,
-
-  /* Use ASCII + conservative use of other unicode characters
-     in text art diagrams.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE,
-
-  /* Use Emoji.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_EMOJI
-};
-
-/* A diagnostic is described by the MESSAGE to send, the FILE and LINE of
-   its context and its KIND (ice, error, warning, note, ...)  See complete
-   list in diagnostic.def.  */
-struct diagnostic_info
-{
-  diagnostic_info ()
-    : message (), richloc (), metadata (), x_data (), kind (), option_id (),
-      m_iinfo ()
-  { }
-
-  /* Text to be formatted.  */
-  text_info message;
-
-  /* The location at which the diagnostic is to be reported.  */
-  rich_location *richloc;
-
-  /* An optional bundle of metadata associated with the diagnostic
-     (or NULL).  */
-  const diagnostic_metadata *metadata;
-
-  /* Auxiliary data for client.  */
-  void *x_data;
-  /* The kind of diagnostic it is about.  */
-  diagnostic_t kind;
-  /* Which OPT_* directly controls this diagnostic.  */
-  diagnostic_option_id option_id;
-
-  /* Inlining context containing locations for each call site along
-     the inlining stack.  */
-  struct inlining_info
-  {
-    /* Locations along the inlining stack.  */
-    auto_vec<location_t, 8> m_ilocs;
-    /* The abstract origin of the location.  */
-    void *m_ao;
-    /* Set if every M_ILOCS element is in a system header.  */
-    bool m_allsyslocs;
-  } m_iinfo;
-};
-
-/*  Forward declarations.  */
-class diagnostic_location_print_policy;
-class diagnostic_source_print_policy;
-
-typedef void (*diagnostic_text_starter_fn) (diagnostic_text_output_format &,
-					    const diagnostic_info *);
-
-struct to_text;
-struct to_html;
-
-extern pretty_printer *get_printer (to_text &);
-
-template <typename Sink>
-using diagnostic_start_span_fn = void (*) (const diagnostic_location_print_policy &,
-					   Sink &sink,
-					   expanded_location);
-
-typedef void (*diagnostic_text_finalizer_fn) (diagnostic_text_output_format &,
-					      const diagnostic_info *,
-					      diagnostic_t);
-
-/* Abstract base class for the diagnostic subsystem to make queries
-   about command-line options.  */
-
-class diagnostic_option_manager
-{
-public:
-  virtual ~diagnostic_option_manager () {}
-
-  /* Return 1 if option OPTION_ID is enabled, 0 if it is disabled,
-     or -1 if it isn't a simple on-off switch
-     (or if the value is unknown, typically set later in target).  */
-  virtual int option_enabled_p (diagnostic_option_id option_id) const = 0;
-
-  /* Return malloced memory for the name of the option OPTION_ID
-     which enabled a diagnostic, originally of type ORIG_DIAG_KIND but
-     possibly converted to DIAG_KIND by options such as -Werror.
-     May return NULL if no name is to be printed.
-     May be passed 0 as well as the index of a particular option.  */
-  virtual char *make_option_name (diagnostic_option_id option_id,
-				  diagnostic_t orig_diag_kind,
-				  diagnostic_t diag_kind) const = 0;
-
-  /* Return malloced memory for a URL describing the option that controls
-     a diagnostic.
-     May return NULL if no URL is available.
-     May be passed 0 as well as the index of a particular option.  */
-  virtual char *make_option_url (diagnostic_option_id option_id) const = 0;
-};
-
-class edit_context;
-class diagnostic_client_data_hooks;
-class logical_location_manager;
-class diagnostic_diagram;
-class diagnostic_source_effect_info;
-class diagnostic_output_format;
-  class diagnostic_text_output_format;
-class diagnostic_buffer;
-
-/* A stack of sets of classifications: each entry in the stack is
-   a mapping from option index to diagnostic severity that can be changed
-   via pragmas.  The stack can be pushed and popped.  */
-
-class diagnostic_option_classifier
-{
-public:
-  void init (int n_opts);
-  void fini ();
-
-  /* Save all diagnostic classifications in a stack.  */
-  void push ();
-
-  /* Restore the topmost classification set off the stack.  If the stack
-     is empty, revert to the state based on command line parameters.  */
-  void pop (location_t where);
-
-  bool option_unspecified_p (diagnostic_option_id option_id) const
-  {
-    return get_current_override (option_id) == DK_UNSPECIFIED;
-  }
-
-  diagnostic_t get_current_override (diagnostic_option_id option_id) const
-  {
-    gcc_assert (option_id.m_idx < m_n_opts);
-    return m_classify_diagnostic[option_id.m_idx];
-  }
-
-  diagnostic_t
-  classify_diagnostic (const diagnostic_context *context,
-		       diagnostic_option_id option_id,
-		       diagnostic_t new_kind,
-		       location_t where);
-
-  diagnostic_t
-  update_effective_level_from_pragmas (diagnostic_info *diagnostic) const;
-
-  int pch_save (FILE *);
-  int pch_restore (FILE *);
-
-private:
-  /* Each time a diagnostic's classification is changed with a pragma,
-     we record the change and the location of the change in an array of
-     these structs.  */
-  struct diagnostic_classification_change_t
-  {
-    location_t location;
-
-    /* For DK_POP, this is the index of the corresponding push (as stored
-       in m_push_list).
-       Otherwise, this is an option index.  */
-    int option;
-
-    diagnostic_t kind;
-  };
-
-  int m_n_opts;
-
-  /* For each option index that can be passed to warning() et al
-     (OPT_* from options.h when using this code with the core GCC
-     options), this array may contain a new kind that the diagnostic
-     should be changed to before reporting, or DK_UNSPECIFIED to leave
-     it as the reported kind, or DK_IGNORED to not report it at
-     all.  */
-  diagnostic_t *m_classify_diagnostic;
-
-  /* History of all changes to the classifications above.  This list
-     is stored in location-order, so we can search it, either
-     binary-wise or end-to-front, to find the most recent
-     classification for a given diagnostic, given the location of the
-     diagnostic.  */
-  vec<diagnostic_classification_change_t> m_classification_history;
-
-  /* For diagnostic_context::get_classification_history, declared later.  */
-  friend class diagnostic_context;
-
-  /* For pragma push/pop.  */
-  vec<int> m_push_list;
-};
-
-/* A bundle of options relating to printing the user's source code
-   (potentially with a margin, underlining, labels, etc).  */
-
-struct diagnostic_source_printing_options
-{
-  /* True if we should print the source line with a caret indicating
-     the location.
-     Corresponds to -fdiagnostics-show-caret.  */
-  bool enabled;
-
-  /* Maximum width of the source line printed.  */
-  int max_width;
-
-  /* Character used at the caret when printing source locations.  */
-  char caret_chars[rich_location::STATICALLY_ALLOCATED_RANGES];
-
-  /* When printing source code, should the characters at carets and ranges
-     be colorized? (assuming colorization is on at all).
-     This should be true for frontends that generate range information
-     (so that the ranges of code are colorized),
-     and false for frontends that merely specify points within the
-     source code (to avoid e.g. colorizing just the first character in
-     a token, which would look strange).  */
-  bool colorize_source_p;
-
-  /* When printing source code, should labelled ranges be printed?
-     Corresponds to -fdiagnostics-show-labels.  */
-  bool show_labels_p;
-
-  /* When printing source code, should there be a left-hand margin
-     showing line numbers?
-     Corresponds to -fdiagnostics-show-line-numbers.  */
-  bool show_line_numbers_p;
-
-  /* If printing source code, what should the minimum width of the margin
-     be?  Line numbers will be right-aligned, and padded to this width.
-     Corresponds to -fdiagnostics-minimum-margin-width=VALUE.  */
-  int min_margin_width;
-
-  /* Usable by plugins; if true, print a debugging ruler above the
-     source output.  */
-  bool show_ruler_p;
-
-  /* When printing events in an inline path, should we print lines
-     visualizing links between related events (e.g. for CFG paths)?
-     Corresponds to -fdiagnostics-show-event-links.  */
-  bool show_event_links_p;
-};
-
-/* A bundle of state for determining column numbers in diagnostics
-   (tab stops, whether to start at 0 or 1, etc).
-   Uses a file_cache to handle tabs.  */
-
-class diagnostic_column_policy
-{
-public:
-  diagnostic_column_policy (const diagnostic_context &dc);
-
-  int converted_column (expanded_location s) const;
-
-  label_text get_location_text (const expanded_location &s,
-				bool show_column,
-				bool colorize) const;
-
-  int get_tabstop () const { return m_tabstop; }
-
-private:
-  file_cache &m_file_cache;
-  enum diagnostics_column_unit m_column_unit;
-  int m_column_origin;
-  int m_tabstop;
-};
-
-/* A bundle of state for printing locations within diagnostics
-   (e.g. "FILENAME:LINE:COLUMN"), to isolate the interactions between
-   diagnostic_context and the start_span callbacks.  */
-
-class diagnostic_location_print_policy
-{
-public:
-  diagnostic_location_print_policy (const diagnostic_context &dc);
-  diagnostic_location_print_policy (const diagnostic_text_output_format &);
-
-  bool show_column_p () const { return m_show_column; }
-
-  const diagnostic_column_policy &
-  get_column_policy () const { return m_column_policy; }
-
-  void
-  print_text_span_start (const diagnostic_context &dc,
-			 pretty_printer &pp,
-			 const expanded_location &exploc);
-
-  void
-  print_html_span_start (const diagnostic_context &dc,
-			 xml::printer &xp,
-			 const expanded_location &exploc);
-
-private:
-  diagnostic_column_policy m_column_policy;
-  bool m_show_column;
-};
-
-/* Abstract base class for optionally supplying extra tags when writing
-   out annotation labels in HTML output.  */
-
-class html_label_writer
-{
-public:
-  virtual ~html_label_writer () {}
-  virtual void begin_label () = 0;
-  virtual void end_label () = 0;
-};
-
-/* A bundle of state for printing source within a diagnostic,
-   to isolate the interactions between diagnostic_context and the
-   implementation of diagnostic_show_locus.  */
-
-class diagnostic_source_print_policy
-{
-public:
-  diagnostic_source_print_policy (const diagnostic_context &);
-  diagnostic_source_print_policy (const diagnostic_context &,
-				  const diagnostic_source_printing_options &);
-
-  void
-  print (pretty_printer &pp,
-	 const rich_location &richloc,
-	 diagnostic_t diagnostic_kind,
-	 diagnostic_source_effect_info *effect_info) const;
-
-  void
-  print_as_html (xml::printer &xp,
-		 const rich_location &richloc,
-		 diagnostic_t diagnostic_kind,
-		 diagnostic_source_effect_info *effect_info,
-		 html_label_writer *label_writer) const;
-
-  const diagnostic_source_printing_options &
-  get_options () const { return m_options; }
-
-  diagnostic_start_span_fn<to_text>
-  get_text_start_span_fn () const { return m_text_start_span_cb; }
-
-  diagnostic_start_span_fn<to_html>
-  get_html_start_span_fn () const { return m_html_start_span_cb; }
-
-  file_cache &
-  get_file_cache () const { return m_file_cache; }
-
-  enum diagnostics_escape_format
-  get_escape_format () const
-  {
-    return m_escape_format;
-  }
-
-  text_art::theme *
-  get_diagram_theme () const { return m_diagram_theme; }
-
-  const diagnostic_column_policy &get_column_policy () const
-  {
-    return m_location_policy.get_column_policy ();
-  }
-
-  const diagnostic_location_print_policy &get_location_policy () const
-  {
-    return m_location_policy;
-  }
-
-private:
-  const diagnostic_source_printing_options &m_options;
-  class diagnostic_location_print_policy m_location_policy;
-  diagnostic_start_span_fn<to_text> m_text_start_span_cb;
-  diagnostic_start_span_fn<to_html> m_html_start_span_cb;
-  file_cache &m_file_cache;
-
-  /* Other data copied from diagnostic_context.  */
-  text_art::theme *m_diagram_theme;
-  enum diagnostics_escape_format m_escape_format;
-};
-
-/* A collection of counters of diagnostics, per-kind
-   (e.g. "3 errors and 1 warning"), for use by both diagnostic_context
-   and by diagnostic_buffer.  */
-
-struct diagnostic_counters
-{
-  diagnostic_counters ();
-
-  void dump (FILE *out, int indent) const;
-  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
-
-  int get_count (diagnostic_t kind) const { return m_count_for_kind[kind]; }
-
-  void move_to (diagnostic_counters &dest);
-  void clear ();
-
-  int m_count_for_kind[DK_LAST_DIAGNOSTIC_KIND];
-};
-
-/* This class encapsulates the state of the diagnostics subsystem
-   as a whole (either directly, or via owned objects of other classes, to
-   avoid global variables).
-
-   It has responsibility for:
-   - being a central place for clients to report diagnostics
-   - reporting those diagnostics to zero or more output sinks
-     (e.g. text vs SARIF)
-   - providing a "dump" member function for a debug dump of the state of
-     the diagnostics subsytem
-   - direct vs buffered diagnostics (see class diagnostic_buffer)
-   - tracking the original argv of the program (for SARIF output)
-   - crash-handling
-
-   It delegates responsibility to various other classes:
-   - the various output sinks (instances of diagnostic_output_format
-     subclasses)
-   - formatting of messages (class pretty_printer)
-   - an optional urlifier to inject URLs into formatted messages
-   - counting the number of diagnostics reported of each kind
-     (class diagnostic_counters)
-   - calling out to a diagnostic_option_manager to determine if
-     a particular warning is enabled or disabled
-   - tracking pragmas that enable/disable warnings in a range of
-     source code
-   - a cache for use when quoting the user's source code (class file_cache)
-   - a text_art::theme
-   - an edit_context for generating patches from fix-it hints
-   - diagnostic_client_data_hooks for metadata.
-
-   Try to avoid adding new responsibilities to this class itself, to avoid
-   the "blob" anti-pattern.  */
-
-class diagnostic_context
-{
-public:
-  /* Give access to m_text_callbacks.  */
-  friend diagnostic_text_starter_fn &
-  diagnostic_text_starter (diagnostic_context *context);
-  friend diagnostic_start_span_fn<to_text> &
-  diagnostic_start_span (diagnostic_context *context);
-  friend diagnostic_text_finalizer_fn &
-  diagnostic_text_finalizer (diagnostic_context *context);
-
-  friend class diagnostic_source_print_policy;
-  friend class diagnostic_text_output_format;
-  friend class diagnostic_buffer;
-
-  typedef void (*set_locations_callback_t) (diagnostic_context *,
-					    diagnostic_info *);
-
-  void initialize (int n_opts);
-  void color_init (int value);
-  void urls_init (int value);
-  void set_pretty_printer (std::unique_ptr<pretty_printer> pp);
-  void refresh_output_sinks ();
-
-  void finish ();
-
-  void dump (FILE *out) const;
-  void DEBUG_FUNCTION dump () const { dump (stderr); }
-
-  bool execution_failed_p () const;
-
-  void set_original_argv (unique_argv original_argv);
-  const char * const *get_original_argv ()
-  {
-    return const_cast<const char * const *> (m_original_argv);
-  }
-
-  void set_set_locations_callback (set_locations_callback_t cb)
-  {
-    m_set_locations_cb = cb;
-  }
-
-  void
-  initialize_input_context (diagnostic_input_charset_callback ccb,
-			    bool should_skip_bom);
-
-  void begin_group ();
-  void end_group ();
-
-  void push_nesting_level ();
-  void pop_nesting_level ();
-
-  bool warning_enabled_at (location_t loc, diagnostic_option_id option_id);
-
-  bool option_unspecified_p (diagnostic_option_id option_id) const
-  {
-    return m_option_classifier.option_unspecified_p (option_id);
-  }
-
-  bool emit_diagnostic_with_group (diagnostic_t kind,
-				   rich_location &richloc,
-				   const diagnostic_metadata *metadata,
-				   diagnostic_option_id option_id,
-				   const char *gmsgid, ...)
-    ATTRIBUTE_GCC_DIAG(6,7);
-  bool emit_diagnostic_with_group_va (diagnostic_t kind,
-				      rich_location &richloc,
-				      const diagnostic_metadata *metadata,
-				      diagnostic_option_id option_id,
-				      const char *gmsgid, va_list *ap)
-    ATTRIBUTE_GCC_DIAG(6,0);
-
-  bool report_diagnostic (diagnostic_info *);
-  void report_verbatim (text_info &);
-
-  diagnostic_t
-  classify_diagnostic (diagnostic_option_id option_id,
-		       diagnostic_t new_kind,
-		       location_t where)
-  {
-    return m_option_classifier.classify_diagnostic (this,
-						    option_id,
-						    new_kind,
-						    where);
-  }
-
-  void push_diagnostics (location_t where ATTRIBUTE_UNUSED)
-  {
-    m_option_classifier.push ();
-  }
-  void pop_diagnostics (location_t where)
-  {
-    m_option_classifier.pop (where);
-  }
-
-  void maybe_show_locus (const rich_location &richloc,
-			 const diagnostic_source_printing_options &opts,
-			 diagnostic_t diagnostic_kind,
-			 pretty_printer &pp,
-			 diagnostic_source_effect_info *effect_info);
-  void maybe_show_locus_as_html (const rich_location &richloc,
-				 const diagnostic_source_printing_options &opts,
-				 diagnostic_t diagnostic_kind,
-				 xml::printer &xp,
-				 diagnostic_source_effect_info *effect_info,
-				 html_label_writer *label_writer);
-
-  void emit_diagram (const diagnostic_diagram &diagram);
-
-  /* Various setters for use by option-handling logic.  */
-  void set_output_format (std::unique_ptr<diagnostic_output_format> output_format);
-  void set_text_art_charset (enum diagnostic_text_art_charset charset);
-  void set_client_data_hooks (std::unique_ptr<diagnostic_client_data_hooks> hooks);
-
-  void push_owned_urlifier (std::unique_ptr<urlifier>);
-  void push_borrowed_urlifier (const urlifier &);
-  void pop_urlifier ();
-
-  void create_edit_context ();
-  void set_warning_as_error_requested (bool val)
-  {
-    m_warning_as_error_requested = val;
-  }
-  void set_report_bug (bool val) { m_report_bug = val; }
-  void set_extra_output_kind (enum diagnostics_extra_output_kind kind)
-  {
-    m_extra_output_kind = kind;
-  }
-  void set_show_cwe (bool val) { m_show_cwe = val;  }
-  void set_show_rules (bool val) { m_show_rules = val; }
-  void set_show_highlight_colors (bool val);
-  void set_path_format (enum diagnostic_path_format val)
-  {
-    m_path_format = val;
-  }
-  void set_show_path_depths (bool val) { m_show_path_depths = val; }
-  void set_show_option_requested (bool val) { m_show_option_requested = val; }
-  void set_max_errors (int val) { m_max_errors = val; }
-  void set_escape_format (enum diagnostics_escape_format val)
-  {
-    m_escape_format = val;
-  }
-
-  void set_format_decoder (printer_fn format_decoder);
-  void set_prefixing_rule (diagnostic_prefixing_rule_t rule);
-
-  /* Various accessors.  */
-  bool warning_as_error_requested_p () const
-  {
-    return m_warning_as_error_requested;
-  }
-  bool show_path_depths_p () const { return m_show_path_depths; }
-  diagnostic_output_format &get_output_format (size_t idx) const;
-  enum diagnostic_path_format get_path_format () const { return m_path_format; }
-  enum diagnostics_escape_format get_escape_format () const
-  {
-    return m_escape_format;
-  }
-
-  file_cache &
-  get_file_cache () const
-  {
-    gcc_assert (m_file_cache);
-    return *m_file_cache;
-  }
-
-  edit_context *get_edit_context () const
-  {
-    return m_edit_context_ptr;
-  }
-  const diagnostic_client_data_hooks *get_client_data_hooks () const
-  {
-    return m_client_data_hooks;
-  }
-
-  const logical_location_manager *
-  get_logical_location_manager () const;
-
-  const urlifier *get_urlifier () const;
-
-  text_art::theme *get_diagram_theme () const { return m_diagrams.m_theme; }
-
-  int &diagnostic_count (diagnostic_t kind)
-  {
-    return m_diagnostic_counters.m_count_for_kind[kind];
-  }
-  int diagnostic_count (diagnostic_t kind) const
-  {
-    return m_diagnostic_counters.get_count (kind);
-  }
-
-  /* Option-related member functions.  */
-  inline bool option_enabled_p (diagnostic_option_id option_id) const
-  {
-    if (!m_option_mgr)
-      return true;
-    return m_option_mgr->option_enabled_p (option_id);
-  }
-
-  inline char *make_option_name (diagnostic_option_id option_id,
-				 diagnostic_t orig_diag_kind,
-				 diagnostic_t diag_kind) const
-  {
-    if (!m_option_mgr)
-      return nullptr;
-    return m_option_mgr->make_option_name (option_id,
-					   orig_diag_kind,
-					   diag_kind);
-  }
-
-  inline char *make_option_url (diagnostic_option_id option_id) const
-  {
-    if (!m_option_mgr)
-      return nullptr;
-    return m_option_mgr->make_option_url (option_id);
-  }
-
-  void
-  set_option_manager (std::unique_ptr<diagnostic_option_manager> mgr,
-		      unsigned lang_mask);
-
-  unsigned get_lang_mask () const
-  {
-    return m_lang_mask;
-  }
-
-  bool diagnostic_impl (rich_location *, const diagnostic_metadata *,
-			diagnostic_option_id, const char *,
-			va_list *, diagnostic_t) ATTRIBUTE_GCC_DIAG(5,0);
-  bool diagnostic_n_impl (rich_location *, const diagnostic_metadata *,
-			  diagnostic_option_id, unsigned HOST_WIDE_INT,
-			  const char *, const char *, va_list *,
-			  diagnostic_t) ATTRIBUTE_GCC_DIAG(7,0);
-
-  int get_diagnostic_nesting_level () const
-  {
-    return m_diagnostic_groups.m_diagnostic_nesting_level;
-  }
-
-  char *build_indent_prefix () const;
-
-  int
-  pch_save (FILE *f)
-  {
-    return m_option_classifier.pch_save (f);
-  }
-
-  int
-  pch_restore (FILE *f)
-  {
-    return m_option_classifier.pch_restore (f);
-  }
-
-
-  void set_diagnostic_buffer (diagnostic_buffer *);
-  diagnostic_buffer *get_diagnostic_buffer () const
-  {
-    return m_diagnostic_buffer;
-  }
-  void clear_diagnostic_buffer (diagnostic_buffer &);
-  void flush_diagnostic_buffer (diagnostic_buffer &);
-
-  std::unique_ptr<pretty_printer> clone_printer () const
-  {
-    return m_reference_printer->clone ();
-  }
-
-  pretty_printer *get_reference_printer () const
-  {
-    return m_reference_printer;
-  }
-
-  void
-  add_sink (std::unique_ptr<diagnostic_output_format>);
-
-  void remove_all_output_sinks ();
-
-  bool supports_fnotice_on_stderr_p () const;
-
-  /* Raise SIGABRT on any diagnostic of severity DK_ERROR or higher.  */
-  void
-  set_abort_on_error (bool val)
-  {
-    m_abort_on_error = val;
-  }
-
-  /* Accessor for use in serialization, e.g. by C++ modules.  */
-  auto &
-  get_classification_history ()
-  {
-    return m_option_classifier.m_classification_history;
-  }
-
-  void set_main_input_filename (const char *filename);
-
-  void
-  set_permissive_option (diagnostic_option_id opt_permissive)
-  {
-    m_opt_permissive = opt_permissive;
-  }
-
-  void
-  set_fatal_errors (bool fatal_errors)
-  {
-    m_fatal_errors = fatal_errors;
-  }
-
-  void
-  set_internal_error_callback (void (*cb) (diagnostic_context *,
-					   const char *,
-					   va_list *))
-  {
-    m_internal_error = cb;
-  }
-
-  void
-  set_adjust_diagnostic_info_callback (void (*cb) (diagnostic_context *,
-						   diagnostic_info *))
-  {
-    m_adjust_diagnostic_info = cb;
-  }
-
-  void
-  inhibit_notes () { m_inhibit_notes_p = true; }
-
-private:
-  void error_recursion () ATTRIBUTE_NORETURN;
-
-  bool diagnostic_enabled (diagnostic_info *diagnostic);
-
-  void get_any_inlining_info (diagnostic_info *diagnostic);
-
-  void check_max_errors (bool flush);
-  void action_after_output (diagnostic_t diag_kind);
-
-  /* Data members.
-     Ideally, all of these would be private.  */
-
-private:
-  /* A reference instance of pretty_printer created by the client
-     and owned by the context.  Used for cloning when creating/adding
-     output formats.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  pretty_printer *m_reference_printer;
-
-  /* Cache of source code.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  file_cache *m_file_cache;
-
-  /* The number of times we have issued diagnostics.  */
-  diagnostic_counters m_diagnostic_counters;
-
-  /* True if it has been requested that warnings be treated as errors.  */
-  bool m_warning_as_error_requested;
-
-  /* The number of option indexes that can be passed to warning() et
-     al.  */
-  int m_n_opts;
-
-  /* The stack of sets of overridden diagnostic option severities.  */
-  diagnostic_option_classifier m_option_classifier;
-
-  /* True if we should print any CWE identifiers associated with
-     diagnostics.  */
-  bool m_show_cwe;
-
-  /* True if we should print any rules associated with diagnostics.  */
-  bool m_show_rules;
-
-  /* How should diagnostic_path objects be printed.  */
-  enum diagnostic_path_format m_path_format;
-
-  /* True if we should print stack depths when printing diagnostic paths.  */
-  bool m_show_path_depths;
-
-  /* True if we should print the command line option which controls
-     each diagnostic, if known.  */
-  bool m_show_option_requested;
-
-  /* True if we should raise a SIGABRT on errors.  */
-  bool m_abort_on_error;
-
-public:
-  /* True if we should show the column number on diagnostics.  */
-  bool m_show_column;
-
-  /* True if pedwarns are errors.  */
-  bool m_pedantic_errors;
-
-  /* True if permerrors are warnings.  */
-  bool m_permissive;
-
-private:
-  /* The option to associate with turning permerrors into warnings,
-     if any.  */
-  diagnostic_option_id m_opt_permissive;
-
-  /* True if errors are fatal.  */
-  bool m_fatal_errors;
-
-public:
-  /* True if all warnings should be disabled.  */
-  bool m_inhibit_warnings;
-
-  /* True if warnings should be given in system headers.  */
-  bool m_warn_system_headers;
-
-private:
-  /* Maximum number of errors to report.  */
-  int m_max_errors;
-
-  /* Client-supplied callbacks for use in text output.  */
-  struct {
-    /* This function is called before any message is printed out.  It is
-       responsible for preparing message prefix and such.  For example, it
-       might say:
-       In file included from "/usr/local/include/curses.h:5:
-       from "/home/gdr/src/nifty_printer.h:56:
-       ...
-    */
-    diagnostic_text_starter_fn m_begin_diagnostic;
-
-    /* This function is called by diagnostic_show_locus in between
-       disjoint spans of source code, so that the context can print
-       something to indicate that a new span of source code has begun.  */
-    diagnostic_start_span_fn<to_text> m_text_start_span;
-    diagnostic_start_span_fn<to_html> m_html_start_span;
-
-    /* This function is called after the diagnostic message is printed.  */
-    diagnostic_text_finalizer_fn m_end_diagnostic;
-  } m_text_callbacks;
-
-  /* Client hook to report an internal error.  */
-  void (*m_internal_error) (diagnostic_context *, const char *, va_list *);
-
-  /* Client hook to adjust properties of the given diagnostic that we're
-     about to issue, such as its kind.  */
-  void (*m_adjust_diagnostic_info)(diagnostic_context *, diagnostic_info *);
-
-  /* Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  diagnostic_option_manager *m_option_mgr;
-  unsigned m_lang_mask;
-
-  /* A stack of optional hooks for adding URLs to quoted text strings in
-     diagnostics.  Only used for the main diagnostic message.
-     Typically a single one owner by the context, but can be temporarily
-     overridden by a borrowed urlifier (e.g. on-stack).  */
-  struct urlifier_stack_node
-  {
-    urlifier *m_urlifier;
-    bool m_owned;
-  };
-  auto_vec<urlifier_stack_node> *m_urlifier_stack;
-
-public:
-  /* Auxiliary data for client.  */
-  void *m_client_aux_data;
-
-  /* Used to detect that the last caret was printed at the same location.  */
-  location_t m_last_location;
-
-private:
-  int m_lock;
-
-  bool m_inhibit_notes_p;
-
-public:
-  diagnostic_source_printing_options m_source_printing;
-
-private:
-  /* True if -freport-bug option is used.  */
-  bool m_report_bug;
-
-  /* Used to specify additional diagnostic output to be emitted after the
-     rest of the diagnostic.  This is for implementing
-     -fdiagnostics-parseable-fixits and GCC_EXTRA_DIAGNOSTIC_OUTPUT.  */
-  enum diagnostics_extra_output_kind m_extra_output_kind;
-
-public:
-  /* What units to use when outputting the column number.  */
-  enum diagnostics_column_unit m_column_unit;
-
-  /* The origin for the column number (1-based or 0-based typically).  */
-  int m_column_origin;
-
-  /* The size of the tabstop for tab expansion.  */
-  int m_tabstop;
-
-private:
-  /* How should non-ASCII/non-printable bytes be escaped when
-     a diagnostic suggests escaping the source code on output.  */
-  enum diagnostics_escape_format m_escape_format;
-
-  /* If non-NULL, an edit_context to which fix-it hints should be
-     applied, for generating patches.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  edit_context *m_edit_context_ptr;
-
-  /* Fields relating to diagnostic groups.  */
-  struct {
-    /* How many diagnostic_group instances are currently alive.  */
-    int m_group_nesting_depth;
-
-    /* How many nesting levels have been pushed within this group.  */
-    int m_diagnostic_nesting_level;
-
-    /* How many diagnostics have been emitted since the bottommost
-       diagnostic_group was pushed.  */
-    int m_emission_count;
-
-    /* The "group+diagnostic" nesting depth from which to inhibit notes.  */
-    int m_inhibiting_notes_from;
-  } m_diagnostic_groups;
-
-  void inhibit_notes_in_group (bool inhibit = true);
-  bool notes_inhibited_in_group () const;
-
-  /* The various sinks to which diagnostics are to be outputted
-     (text vs structured formats such as SARIF).
-     The sinks are owned by the context; this would be a
-     std::vector<std::unique_ptr> if diagnostic_context had a
-     proper ctor.  */
-  auto_vec<diagnostic_output_format *> m_output_sinks;
-
-  /* Callback to set the locations of call sites along the inlining
-     stack corresponding to a diagnostic location.  Needed to traverse
-     the BLOCK_SUPERCONTEXT() chain hanging off the LOCATION_BLOCK()
-     of a diagnostic's location.  */
-  set_locations_callback_t m_set_locations_cb;
-
-  /* A bundle of hooks for providing data to the context about its client
-     e.g. version information, plugins, etc.
-     Used by SARIF output to give metadata about the client that's
-     producing diagnostics.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  diagnostic_client_data_hooks *m_client_data_hooks;
-
-  /* Support for diagrams.  */
-  struct
-  {
-    /* Theme to use when generating diagrams.
-       Can be NULL (if text art is disabled).
-       Owned by the context; this would be a std::unique_ptr if
-       diagnostic_context had a proper ctor.  */
-    text_art::theme *m_theme;
-
-  } m_diagrams;
-
-  /* Owned by the context.  */
-  char **m_original_argv;
-
-  /* Borrowed pointer to the active diagnostic_buffer, if any.
-     If null (the default), then diagnostics that are reported to the
-     context are immediately issued to the output format.
-     If non-null, then diagnostics that are reported to the context
-     are buffered in the buffer, and may be issued to the output format
-     later (if the buffer is flushed), moved to other buffers, or
-     discarded (if the buffer is cleared).  */
-  diagnostic_buffer *m_diagnostic_buffer;
-};
-
-/* Client supplied function to announce a diagnostic
-   (for text-based diagnostic output).  */
-inline diagnostic_text_starter_fn &
-diagnostic_text_starter (diagnostic_context *context)
-{
-  return context->m_text_callbacks.m_begin_diagnostic;
-}
-
-/* Client supplied function called between disjoint spans of source code,
-   so that the context can print
-   something to indicate that a new span of source code has begun.  */
-inline diagnostic_start_span_fn<to_text> &
-diagnostic_start_span (diagnostic_context *context)
-{
-  return context->m_text_callbacks.m_text_start_span;
-}
-
-/* Client supplied function called after a diagnostic message is
-   displayed (for text-based diagnostic output).  */
-inline diagnostic_text_finalizer_fn &
-diagnostic_text_finalizer (diagnostic_context *context)
-{
-  return context->m_text_callbacks.m_end_diagnostic;
-}
+#include "diagnostics/diagnostic-info.h"
+#include "diagnostics/context.h"
 
 /* Extension hooks for client.  */
 #define diagnostic_context_auxiliary_data(DC) (DC)->m_client_aux_data
-#define diagnostic_info_auxiliary_data(DI) (DI)->x_data
+#define diagnostic_info_auxiliary_data(DI) (DI)->m_x_data
 
-/* This diagnostic_context is used by front-ends that directly output
+/* This diagnostics::context is used by front-ends that directly output
    diagnostic messages without going through `error', `warning',
    and similar functions.  */
-extern diagnostic_context *global_dc;
+extern diagnostics::context *global_dc;
 
 /* The number of errors that have been issued so far.  Ideally, these
-   would take a diagnostic_context as an argument.  */
-#define errorcount global_dc->diagnostic_count (DK_ERROR)
+   would take a diagnostics::context as an argument.  */
+#define errorcount global_dc->diagnostic_count (diagnostics::kind::error)
 /* Similarly, but for warnings.  */
-#define warningcount global_dc->diagnostic_count (DK_WARNING)
+#define warningcount global_dc->diagnostic_count (diagnostics::kind::warning)
 /* Similarly, but for warnings promoted to errors.  */
-#define werrorcount global_dc->diagnostic_count (DK_WERROR)
+#define werrorcount global_dc->diagnostic_count (diagnostics::kind::werror)
 /* Similarly, but for sorrys.  */
-#define sorrycount global_dc->diagnostic_count (DK_SORRY)
+#define sorrycount global_dc->diagnostic_count (diagnostics::kind::sorry)
 
 /* Returns nonzero if warnings should be emitted.  */
 #define diagnostic_report_warnings_p(DC, LOC)				\
@@ -1160,45 +56,45 @@ extern diagnostic_context *global_dc;
    diagnostic.  */
 
 inline void
-diagnostic_set_option_id (diagnostic_info *info,
-			  diagnostic_option_id option_id)
+diagnostic_set_option_id (diagnostics::diagnostic_info *info,
+			  diagnostics::option_id opt_id)
 {
-  info->option_id = option_id;
+  info->m_option_id = opt_id;
 }
 
 /* Diagnostic related functions.  */
 
 inline void
-diagnostic_initialize (diagnostic_context *context, int n_opts)
+diagnostic_initialize (diagnostics::context *context, int n_opts)
 {
   context->initialize (n_opts);
 }
 
 inline void
-diagnostic_color_init (diagnostic_context *context, int value = -1)
+diagnostic_color_init (diagnostics::context *context, int value = -1)
 {
   context->color_init (value);
 }
 
 inline void
-diagnostic_urls_init (diagnostic_context *context, int value = -1)
+diagnostic_urls_init (diagnostics::context *context, int value = -1)
 {
   context->urls_init (value);
 }
 
 inline void
-diagnostic_finish (diagnostic_context *context)
+diagnostic_finish (diagnostics::context *context)
 {
   context->finish ();
 }
 
 inline void
-diagnostic_show_locus (diagnostic_context *context,
-		       const diagnostic_source_printing_options &opts,
+diagnostic_show_locus (diagnostics::context *context,
+		       const diagnostics::source_printing_options &opts,
 		       rich_location *richloc,
-		       diagnostic_t diagnostic_kind,
+		       enum diagnostics::kind diagnostic_kind,
 		       pretty_printer *pp,
-		       diagnostic_source_effect_info *effect_info = nullptr)
+		       diagnostics::source_effect_info *effect_info = nullptr)
 {
   gcc_assert (context);
   gcc_assert (richloc);
@@ -1207,13 +103,13 @@ diagnostic_show_locus (diagnostic_context *context,
 }
 
 inline void
-diagnostic_show_locus_as_html (diagnostic_context *context,
-			       const diagnostic_source_printing_options &opts,
+diagnostic_show_locus_as_html (diagnostics::context *context,
+			       const diagnostics::source_printing_options &opts,
 			       rich_location *richloc,
-			       diagnostic_t diagnostic_kind,
+			       enum diagnostics::kind diagnostic_kind,
 			       xml::printer &xp,
-			       diagnostic_source_effect_info *effect_info = nullptr,
-			       html_label_writer *label_writer = nullptr)
+			       diagnostics::source_effect_info *effect_info = nullptr,
+			       diagnostics::html_label_writer *label_writer = nullptr)
 {
   gcc_assert (context);
   gcc_assert (richloc);
@@ -1237,7 +133,7 @@ diagnostic_show_locus_as_html (diagnostic_context *context,
    rather skipped as part of the conversion process.)  */
 
 inline void
-diagnostic_initialize_input_context (diagnostic_context *context,
+diagnostic_initialize_input_context (diagnostics::context *context,
 				     diagnostic_input_charset_callback ccb,
 				     bool should_skip_bom)
 {
@@ -1245,23 +141,23 @@ diagnostic_initialize_input_context (diagnostic_context *context,
 }
 
 /* Force diagnostics controlled by OPTIDX to be kind KIND.  */
-inline diagnostic_t
-diagnostic_classify_diagnostic (diagnostic_context *context,
-				diagnostic_option_id option_id,
-				diagnostic_t kind,
+inline diagnostics::kind
+diagnostic_classify_diagnostic (diagnostics::context *context,
+				diagnostics::option_id opt_id,
+				enum diagnostics::kind kind,
 				location_t where)
 {
-  return context->classify_diagnostic (option_id, kind, where);
+  return context->classify_diagnostic (opt_id, kind, where);
 }
 
 inline void
-diagnostic_push_diagnostics (diagnostic_context *context,
+diagnostic_push_diagnostics (diagnostics::context *context,
 			     location_t where)
 {
   context->push_diagnostics (where);
 }
 inline void
-diagnostic_pop_diagnostics (diagnostic_context *context,
+diagnostic_pop_diagnostics (diagnostics::context *context,
 			    location_t where)
 {
   context->pop_diagnostics (where);
@@ -1275,8 +171,8 @@ diagnostic_pop_diagnostics (diagnostic_context *context,
    Return true if a diagnostic was printed, false otherwise.  */
 
 inline bool
-diagnostic_report_diagnostic (diagnostic_context *context,
-			      diagnostic_info *diagnostic)
+diagnostic_report_diagnostic (diagnostics::context *context,
+			      diagnostics::diagnostic_info *diagnostic)
 {
   context->begin_group ();
   bool warned = context->report_diagnostic (diagnostic);
@@ -1285,23 +181,30 @@ diagnostic_report_diagnostic (diagnostic_context *context,
 }
 
 #ifdef ATTRIBUTE_GCC_DIAG
-extern void diagnostic_set_info (diagnostic_info *, const char *, va_list *,
-				 rich_location *, diagnostic_t) ATTRIBUTE_GCC_DIAG(2,0);
-extern void diagnostic_set_info_translated (diagnostic_info *, const char *,
-					    va_list *, rich_location *,
-					    diagnostic_t)
+extern void diagnostic_set_info (diagnostics::diagnostic_info *,
+				 const char *, va_list *,
+				 rich_location *,
+				 enum diagnostics::kind)
+  ATTRIBUTE_GCC_DIAG(2,0);
+extern void diagnostic_set_info_translated (diagnostics::diagnostic_info *,
+					    const char *, va_list *,
+					    rich_location *,
+					    enum diagnostics::kind)
      ATTRIBUTE_GCC_DIAG(2,0);
 #endif
-void default_diagnostic_text_starter (diagnostic_text_output_format &,
-				      const diagnostic_info *);
-template <typename Sink>
-void default_diagnostic_start_span_fn (const diagnostic_location_print_policy &,
-				       Sink &sink,
-				       expanded_location);
-void default_diagnostic_text_finalizer (diagnostic_text_output_format &,
-					const diagnostic_info *,
-					diagnostic_t);
-void diagnostic_set_caret_max_width (diagnostic_context *context, int value);
+
+namespace diagnostics {
+
+void default_text_starter (diagnostics::text_sink &,
+			   const diagnostics::diagnostic_info *);
+template <typename TextOrHtml>
+void default_start_span_fn (const diagnostics::location_print_policy &,
+			    TextOrHtml &text_or_html,
+			    expanded_location);
+void default_text_finalizer (diagnostics::text_sink &,
+			     const diagnostics::diagnostic_info *,
+			     enum diagnostics::kind);
+} // namespace diagnostics
 
 int get_terminal_width (void);
 
@@ -1309,17 +212,18 @@ int get_terminal_width (void);
    specifies which location. By default, expand the first one.  */
 
 inline location_t
-diagnostic_location (const diagnostic_info * diagnostic, int which = 0)
+diagnostic_location (const diagnostics::diagnostic_info *diagnostic,
+		     int which = 0)
 {
-  return diagnostic->message.get_location (which);
+  return diagnostic->m_message.get_location (which);
 }
 
 /* Return the number of locations to be printed in DIAGNOSTIC.  */
 
 inline unsigned int
-diagnostic_num_locations (const diagnostic_info * diagnostic)
+diagnostic_num_locations (const diagnostics::diagnostic_info *diagnostic)
 {
-  return diagnostic->message.m_richloc->get_num_locations ();
+  return diagnostic->m_message.m_richloc->get_num_locations ();
 }
 
 /* Expand the location of this diagnostic. Use this function for
@@ -1327,9 +231,10 @@ diagnostic_num_locations (const diagnostic_info * diagnostic)
    expand the first one.  */
 
 inline expanded_location
-diagnostic_expand_location (const diagnostic_info * diagnostic, int which = 0)
+diagnostic_expand_location (const diagnostics::diagnostic_info *diagnostic,
+			    int which = 0)
 {
-  return diagnostic->richloc->get_expanded_location (which);
+  return diagnostic->m_richloc->get_expanded_location (which);
 }
 
 /* This is somehow the right-side margin of a caret line, that is, we
@@ -1342,39 +247,39 @@ const int CARET_LINE_MARGIN = 10;
    whether to print one or two caret lines.  */
 
 inline bool
-diagnostic_same_line (const diagnostic_context *context,
-		       expanded_location s1, expanded_location s2)
+diagnostic_same_line (const diagnostics::context *context,
+		      expanded_location s1, expanded_location s2)
 {
   return (s2.column && s1.line == s2.line
-	  && (context->m_source_printing.max_width - CARET_LINE_MARGIN
+	  && ((context->get_source_printing_options ().max_width
+	       - CARET_LINE_MARGIN)
 	      > abs (s1.column - s2.column)));
 }
 
-extern const char *diagnostic_get_color_for_kind (diagnostic_t kind);
-
 /* Pure text formatting support functions.  */
 
 extern char *build_message_string (const char *, ...) ATTRIBUTE_PRINTF_1;
 
-/* Compute the number of digits in the decimal representation of an integer.  */
-extern int num_digits (int);
-
 inline bool
-warning_enabled_at (location_t loc, diagnostic_option_id option_id)
+warning_enabled_at (location_t loc, diagnostics::option_id opt_id)
 {
-  return global_dc->warning_enabled_at (loc, option_id);
+  return global_dc->warning_enabled_at (loc, opt_id);
 }
 
 inline bool
-option_unspecified_p (diagnostic_option_id option_id)
+option_unspecified_p (diagnostics::option_id opt_id)
 {
-  return global_dc->option_unspecified_p (option_id);
+  return global_dc->option_unspecified_p (opt_id);
 }
 
-extern char *get_cwe_url (int cwe);
+namespace diagnostics {
 
-extern const char *get_diagnostic_kind_text (diagnostic_t kind);
+/* Compute the number of digits in the decimal representation of an integer.  */
+extern int num_digits (int);
+
+extern char *get_cwe_url (int cwe);
+extern const char *maybe_line_and_column (int line, int col);
 
-const char *maybe_line_and_column (int line, int col);
+} // namespace diagnostics
 
 #endif /* ! GCC_DIAGNOSTIC_H */
diff --git a/gcc/diagnostics/buffering.cc b/gcc/diagnostics/buffering.cc
new file mode 100644
index 0000000..29f039f
--- /dev/null
+++ b/gcc/diagnostics/buffering.cc
@@ -0,0 +1,199 @@
+/* Support for buffering diagnostics before flushing them to output sinks.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "diagnostic.h"
+#include "diagnostics/buffering.h"
+#include "diagnostics/sink.h"
+
+namespace diagnostics {
+
+/* Methods fns of diagnostics::context relating to buffering.  */
+
+/* If BUFFER_ is non-null, use BUFFER as the active diagnostics::buffer on
+   this context.  BUFFER is borrowed.
+
+   If BUFFER_ is null, stop any buffering on this context until the next call
+   to this function.  */
+
+void
+context::set_diagnostic_buffer (buffer *buffer_)
+{
+  /* We don't allow changing buffering within a diagnostic group
+     (to simplify handling of buffered diagnostics within the
+     diagnostic_format implementations).  */
+  gcc_assert (m_diagnostic_groups.m_group_nesting_depth == 0);
+
+  /* Likewise, for simplicity, we only allow changing buffers
+     at nesting level 0.  */
+  gcc_assert (m_diagnostic_groups.m_diagnostic_nesting_level == 0);
+
+  m_diagnostic_buffer = buffer_;
+
+  if (buffer_)
+    {
+      buffer_->ensure_per_sink_buffers ();
+      gcc_assert (buffer_->m_per_sink_buffers);
+      gcc_assert (buffer_->m_per_sink_buffers->length ()
+		  == m_sinks.length ());
+      for (unsigned idx = 0; idx < m_sinks.length (); ++idx)
+	{
+	  auto sink_ = m_sinks[idx];
+	  auto per_sink_buffer = (*buffer_->m_per_sink_buffers)[idx];
+	  sink_->set_buffer (per_sink_buffer);
+	}
+    }
+  else
+    for (auto sink_ : m_sinks)
+      sink_->set_buffer (nullptr);
+}
+
+/* Clear BUFFER_ without flushing it.  */
+
+void
+context::clear_diagnostic_buffer (buffer &buffer_)
+{
+  if (buffer_.m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *buffer_.m_per_sink_buffers)
+      per_sink_buffer_->clear ();
+
+  buffer_.m_diagnostic_counters.clear ();
+
+  /* We need to reset last_location, otherwise we may skip caret lines
+     when we actually give a diagnostic.  */
+  m_last_location = UNKNOWN_LOCATION;
+}
+
+/* Flush the diagnostics in BUFFER_ to this context, clearing BUFFER_.  */
+
+void
+context::flush_diagnostic_buffer (buffer &buffer_)
+{
+  bool had_errors
+    = (buffer_.diagnostic_count (kind::error) > 0
+       || buffer_.diagnostic_count (kind::werror) > 0);
+  if (buffer_.m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *buffer_.m_per_sink_buffers)
+      per_sink_buffer_->flush ();
+  buffer_.m_diagnostic_counters.move_to (m_diagnostic_counters);
+
+  action_after_output (had_errors ? kind::error : kind::warning);
+  check_max_errors (true);
+}
+
+/* class diagnostics::buffer.  */
+
+buffer::buffer (context &ctxt)
+: m_ctxt (ctxt),
+  m_per_sink_buffers (nullptr)
+{
+}
+
+buffer::~buffer ()
+{
+  if (m_per_sink_buffers)
+    {
+      for (auto iter : *m_per_sink_buffers)
+	delete iter;
+      delete m_per_sink_buffers;
+    }
+}
+
+void
+buffer::dump (FILE *out, int indent) const
+{
+  m_diagnostic_counters.dump (out, indent + 2);
+  fprintf (out, "%*sm_per_sink_buffers:\n", indent, "");
+  if (m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *m_per_sink_buffers)
+      per_sink_buffer_->dump (out, indent + 2);
+  else
+    fprintf (out, "%*s(none)\n", indent + 2, "");
+}
+
+bool
+buffer::empty_p () const
+{
+  if (m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *m_per_sink_buffers)
+      /* Query initial buffer.  */
+      return per_sink_buffer_->empty_p ();
+  return true;
+}
+
+void
+buffer::move_to (buffer &dest)
+{
+  /* Bail if there's nothing to move.  */
+  if (!m_per_sink_buffers)
+    return;
+
+  m_diagnostic_counters.move_to (dest.m_diagnostic_counters);
+
+  if (!dest.m_per_sink_buffers)
+    {
+      /* Optimization for the "move to empty" case:
+	 simply move the vec to the dest.  */
+      dest.m_per_sink_buffers = m_per_sink_buffers;
+      m_per_sink_buffers = nullptr;
+      return;
+    }
+
+  dest.ensure_per_sink_buffers ();
+  gcc_assert (m_per_sink_buffers);
+  gcc_assert (m_per_sink_buffers->length ()
+	      == m_ctxt.m_sinks.length ());
+  gcc_assert (dest.m_per_sink_buffers);
+  gcc_assert (dest.m_per_sink_buffers->length ()
+	      == m_ctxt.m_sinks.length ());
+  for (unsigned idx = 0; idx < m_ctxt.m_sinks.length (); ++idx)
+    {
+      auto per_sink_buffer_src = (*m_per_sink_buffers)[idx];
+      auto per_sink_buffer_dest = (*dest.m_per_sink_buffers)[idx];
+      per_sink_buffer_src->move_to (*per_sink_buffer_dest);
+    }
+}
+
+/* Lazily get the output formats to create their own kind of buffers.
+   We can't change the output sinks on a context once this has been called
+   on any diagnostics::buffer instances for that context, since there's no
+   way to update all diagnostics::buffer instances for that context.  */
+
+void
+buffer::ensure_per_sink_buffers ()
+{
+  if (!m_per_sink_buffers)
+    {
+      m_per_sink_buffers = new auto_vec<per_sink_buffer *> ();
+      for (unsigned idx = 0; idx < m_ctxt.m_sinks.length (); ++idx)
+	{
+	  auto sink_ = m_ctxt.m_sinks[idx];
+	  auto per_sink_buffer = sink_->make_per_sink_buffer ();
+	  m_per_sink_buffers->safe_push (per_sink_buffer.release ());
+	}
+    }
+  gcc_assert (m_per_sink_buffers);
+  gcc_assert (m_per_sink_buffers->length ()
+	      == m_ctxt.m_sinks.length ());
+}
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-buffer.h b/gcc/diagnostics/buffering.h
index e766a3b..9b86fee 100644
--- a/gcc/diagnostic-buffer.h
+++ b/gcc/diagnostics/buffering.h
@@ -1,4 +1,4 @@
-/* Support for buffering diagnostics before flushing them to output format.
+/* Support for buffering diagnostics before flushing them to output sinks.
    Copyright (C) 2024-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -18,92 +18,96 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_BUFFER_H
-#define GCC_DIAGNOSTIC_BUFFER_H
+#ifndef GCC_DIAGNOSTICS_BUFFERING_H
+#define GCC_DIAGNOSTICS_BUFFERING_H
 
 #include "diagnostic.h"
 
-class diagnostic_per_format_buffer;
-class diagnostic_output_format;
-  class diagnostic_text_output_format;
+namespace diagnostics {
+
+class per_sink_buffer;
+class sink;
+  class text_sink;
 
 /* Class representing a buffer of zero or more diagnostics that
-   have been reported to a diagnostic_context, but which haven't
+   have been reported to a diagnostics::context, but which haven't
    yet been flushed.
 
-   A diagnostic_buffer can be:
+   A diagnostics::buffer can be:
 
-   * flushed to the diagnostic_context, which issues
-   the diagnostics within the buffer to the output format
+   * flushed to the diagnostics::context, which issues
+   the diagnostics within the buffer to the output sinks
    and checks for limits such as -fmax-errors=, or
 
-   * moved to another diagnostic_buffer, which moves the diagnostics
+   * moved to another diagnostics::buffer, which moves the diagnostics
    within the first buffer to the other buffer, appending them after any
    existing diagnostics within the destination buffer, emptying the
    source buffer, or
 
    * cleared, which discards any diagnostics within the buffer
-   without issuing them to the output format.
+   without issuing them to the output sinks.
 
-   Since a buffer needs to contain output-format-specific data,
-   it's not possible to change the output format of the
-   diagnostic_context once any buffers are non-empty.
+   Since a buffer needs to contain sink-specific data,
+   it's not possible to change the output sink(s) of the
+   diagnostics::context once any buffers are non-empty.
 
-   To simplify implementing output formats, it's not possible
-   to change buffering on a diagnostic_context whilst within a
+   To simplify implementing output sinks, it's not possible
+   to change buffering on a diagnostics::context whilst within a
    diagnostic group.  */
 
-class diagnostic_buffer
+class buffer
 {
  public:
-  friend class diagnostic_context;
+  friend class context;
 
-  diagnostic_buffer (diagnostic_context &ctxt);
-  ~diagnostic_buffer ();
+  buffer (context &ctxt);
+  ~buffer ();
 
   void dump (FILE *out, int indent) const;
   void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
 
-  int diagnostic_count (diagnostic_t kind) const
+  int diagnostic_count (enum kind kind) const
   {
     return m_diagnostic_counters.get_count (kind);
   }
 
   bool empty_p () const;
 
-  void move_to (diagnostic_buffer &dest);
+  void move_to (buffer &dest);
 
  private:
-  void ensure_per_format_buffers ();
+  void ensure_per_sink_buffers ();
 
-  diagnostic_context &m_ctxt;
-  auto_vec<diagnostic_per_format_buffer *> *m_per_format_buffers;
+  context &m_ctxt;
+  auto_vec<per_sink_buffer *> *m_per_sink_buffers;
 
   /* The number of buffered diagnostics of each kind.  */
-  diagnostic_counters m_diagnostic_counters;
+  counters m_diagnostic_counters;
 };
 
-/* Implementation detail of diagnostic_buffer.
+/* Implementation detail of diagnostics::buffer.
 
    Abstract base class describing how to represent zero of more
-   buffered diagnostics for a particular diagnostic_output_format
+   buffered diagnostics for a particular diagnostics::sink
    (e.g. text vs SARIF).
 
-   Each diagnostic_output_format subclass should implement its own
-   subclass for handling diagnostic_buffer.  */
+   Each diagnostics::sink subclass should implement its own
+   subclass for handling diagnostics::buffer.  */
 
-class diagnostic_per_format_buffer
+class per_sink_buffer
 {
 public:
-  virtual ~diagnostic_per_format_buffer () {}
+  virtual ~per_sink_buffer () {}
 
   virtual void dump (FILE *out, int indent) const = 0;
   void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
 
   virtual bool empty_p () const = 0;
-  virtual void move_to (diagnostic_per_format_buffer &dest) = 0;
+  virtual void move_to (per_sink_buffer &dest) = 0;
   virtual void clear () = 0;
   virtual void flush () = 0;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_BUFFER_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_BUFFERING_H */
diff --git a/gcc/edit-context.cc b/gcc/diagnostics/changes.cc
index c767394..e1caab0 100644
--- a/gcc/edit-context.cc
+++ b/gcc/diagnostics/changes.cc
@@ -21,14 +21,18 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "line-map.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
 #include "pretty-print.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
+#include "diagnostics/file-cache.h"
 #include "selftest.h"
 
+namespace diagnostics {
+namespace changes {
+
 /* This file implements a way to track the effect of fix-its,
-   via a class edit_context; the other classes are support classes for
-   edit_context.
+   via a class change_set; the other classes are support classes for
+   change_set.
 
    A complication here is that fix-its are expressed relative to coordinates
    in the file when it was parsed, before any changes have been made, and
@@ -36,14 +40,14 @@ along with GCC; see the file COPYING3.  If not see
    later fix-its to allow for the changes made by earlier ones.  This
    is done by the various "get_effective_column" methods.
 
-   The "filename" params are required to outlive the edit_context (no
+   The "filename" params are required to outlive the change_set (no
    copy of the underlying str is taken, just the ptr).  */
 
-/* Forward decls.  class edit_context is declared within edit-context.h.
+/* Forward decls.  class change_set is declared within changes.h.
    The other types are declared here.  */
-class edit_context;
-class edited_file;
-class edited_line;
+class change_set;
+class changed_file;
+class changed_line;
 class line_event;
 
 /* A struct to hold the params of a print_diff call.  */
@@ -58,14 +62,14 @@ public:
   bool m_show_filenames;
 };
 
-/* The state of one named file within an edit_context: the filename,
+/* The state of one named file within an change_set: the filename,
    and the lines that have been edited so far.  */
 
-class edited_file
+class changed_file
 {
  public:
-  edited_file (edit_context &ec, const char *filename);
-  static void delete_cb (edited_file *file);
+  changed_file (change_set &ec, const char *filename);
+  static void delete_cb (changed_file *file);
 
   const char *get_filename () const { return m_filename; }
   char *get_content ();
@@ -76,7 +80,7 @@ class edited_file
 		    int replacement_len);
   int get_effective_column (int line, int column);
 
-  static int call_print_diff (const char *, edited_file *file,
+  static int call_print_diff (const char *, changed_file *file,
 			      void *user_data)
   {
     diff *d = (diff *)user_data;
@@ -86,7 +90,7 @@ class edited_file
 
   file_cache &get_file_cache () const
   {
-    return m_edit_context.get_file_cache ();
+    return m_change_set.get_file_cache ();
   }
 
  private:
@@ -94,8 +98,8 @@ class edited_file
   void print_diff (pretty_printer *pp, bool show_filenames);
   int print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
 		       int old_end_of_hunk, int new_start_of_hunk);
-  edited_line *get_line (int line);
-  edited_line *get_or_insert_line (int line);
+  changed_line *get_line (int line);
+  changed_line *get_or_insert_line (int line);
   int get_num_lines (bool *missing_trailing_newline);
 
   int get_effective_line_count (int old_start_of_hunk,
@@ -105,13 +109,13 @@ class edited_file
 				   int start_of_run,
 				   int end_of_run);
 
-  edit_context &m_edit_context;
+  change_set &m_change_set;
   const char *m_filename;
-  typed_splay_tree<int, edited_line *> m_edited_lines;
+  typed_splay_tree<int, changed_line *> m_changed_lines;
   int m_num_lines;
 };
 
-/* A line added before an edited_line.  */
+/* A line added before an changed_line.  */
 
 class added_line
 {
@@ -154,24 +158,24 @@ class line_event
   int m_delta;
 };
 
-/* The state of one edited line within an edited_file.
+/* The state of one edited line within an changed_file.
    As well as the current content of the line, it contains a record of
    the changes, so that further changes can be applied in the correct
    place.
 
    When handling fix-it hints containing newlines, new lines are added
-   as added_line predecessors to an edited_line.  Hence it's possible
-   for an "edited_line" to not actually have been changed, but to merely
+   as added_line predecessors to an changed_line.  Hence it's possible
+   for an "changed_line" to not actually have been changed, but to merely
    be a placeholder for the lines added before it.  This can be tested
    for with actuall_edited_p, and has a slight effect on how diff hunks
    are generated.  */
 
-class edited_line
+class changed_line
 {
  public:
-  edited_line (file_cache &fc, const char *filename, int line_num);
-  ~edited_line ();
-  static void delete_cb (edited_line *el);
+  changed_line (file_cache &fc, const char *filename, int line_num);
+  ~changed_line ();
+  static void delete_cb (changed_line *el);
 
   int get_line_num () const { return m_line_num; }
   const char *get_content () const { return m_content; }
@@ -210,21 +214,21 @@ static void
 print_diff_line (pretty_printer *pp, char prefix_char,
 		 const char *line, int line_size);
 
-/* Implementation of class edit_context.  */
+/* Implementation of class change_set.  */
 
-/* edit_context's ctor.  */
+/* change_set's ctor.  */
 
-edit_context::edit_context (file_cache &fc)
+change_set::change_set (file_cache &fc)
 : m_file_cache (fc),
   m_valid (true),
-  m_files (strcmp, NULL, edited_file::delete_cb)
+  m_files (strcmp, NULL, changed_file::delete_cb)
 {}
 
 /* Add any fixits within RICHLOC to this context, recording the
    changes that they make.  */
 
 void
-edit_context::add_fixits (rich_location *richloc)
+change_set::add_fixits (rich_location *richloc)
 {
   if (!m_valid)
     return;
@@ -242,15 +246,15 @@ edit_context::add_fixits (rich_location *richloc)
 }
 
 /* Get the content of the given file, with fix-its applied.
-   If any errors occurred in this edit_context, return NULL.
+   If any errors occurred in this change_set, return NULL.
    The ptr should be freed by the caller.  */
 
 char *
-edit_context::get_content (const char *filename)
+change_set::get_content (const char *filename)
 {
   if (!m_valid)
     return NULL;
-  edited_file &file = get_or_insert_file (filename);
+  changed_file &file = get_or_insert_file (filename);
   return file.get_content ();
 }
 
@@ -258,10 +262,10 @@ edit_context::get_content (const char *filename)
    This method is for the selftests.  */
 
 int
-edit_context::get_effective_column (const char *filename, int line,
+change_set::get_effective_column (const char *filename, int line,
 				    int column)
 {
-  edited_file *file = get_file (filename);
+  changed_file *file = get_file (filename);
   if (!file)
     return column;
   return file->get_effective_column (line, column);
@@ -269,10 +273,10 @@ edit_context::get_effective_column (const char *filename, int line,
 
 /* Generate a unified diff.  The resulting string should be freed by the
    caller.  Primarily for selftests.
-   If any errors occurred in this edit_context, return NULL.  */
+   If any errors occurred in this change_set, return NULL.  */
 
 char *
-edit_context::generate_diff (bool show_filenames)
+change_set::generate_diff (bool show_filenames)
 {
   if (!m_valid)
     return NULL;
@@ -286,19 +290,19 @@ edit_context::generate_diff (bool show_filenames)
    context.  */
 
 void
-edit_context::print_diff (pretty_printer *pp, bool show_filenames)
+change_set::print_diff (pretty_printer *pp, bool show_filenames)
 {
   if (!m_valid)
     return;
   diff d (pp, show_filenames);
-  m_files.foreach (edited_file::call_print_diff, &d);
+  m_files.foreach (changed_file::call_print_diff, &d);
 }
 
 /* Attempt to apply the given fixit.  Return true if it can be
    applied, or false otherwise.  */
 
 bool
-edit_context::apply_fixit (const fixit_hint *hint)
+change_set::apply_fixit (const fixit_hint *hint)
 {
   expanded_location start = expand_location (hint->get_start_loc ());
   expanded_location next_loc = expand_location (hint->get_next_loc ());
@@ -311,7 +315,7 @@ edit_context::apply_fixit (const fixit_hint *hint)
   if (next_loc.column == 0)
     return false;
 
-  edited_file &file = get_or_insert_file (start.file);
+  changed_file &file = get_or_insert_file (start.file);
   if (!m_valid)
     return false;
   return file.apply_fixit (start.line, start.column, next_loc.column,
@@ -319,57 +323,57 @@ edit_context::apply_fixit (const fixit_hint *hint)
 			   hint->get_length ());
 }
 
-/* Locate the edited_file * for FILENAME, if any
+/* Locate the changed_file * for FILENAME, if any
    Return NULL if there isn't one.  */
 
-edited_file *
-edit_context::get_file (const char *filename)
+changed_file *
+change_set::get_file (const char *filename)
 {
   gcc_assert (filename);
   return m_files.lookup (filename);
 }
 
-/* Locate the edited_file for FILENAME, adding one if there isn't one.  */
+/* Locate the changed_file for FILENAME, adding one if there isn't one.  */
 
-edited_file &
-edit_context::get_or_insert_file (const char *filename)
+changed_file &
+change_set::get_or_insert_file (const char *filename)
 {
   gcc_assert (filename);
 
-  edited_file *file = get_file (filename);
+  changed_file *file = get_file (filename);
   if (file)
     return *file;
 
   /* Not found.  */
-  file = new edited_file (*this, filename);
+  file = new changed_file (*this, filename);
   m_files.insert (filename, file);
   return *file;
 }
 
-/* Implementation of class edited_file.  */
+/* Implementation of class changed_file.  */
 
-/* Callback for m_edited_lines, for comparing line numbers.  */
+/* Callback for m_changed_lines, for comparing line numbers.  */
 
 static int line_comparator (int a, int b)
 {
   return a - b;
 }
 
-/* edited_file's constructor.  */
+/* changed_file's constructor.  */
 
-edited_file::edited_file (edit_context &ec, const char *filename)
-: m_edit_context (ec),
+changed_file::changed_file (change_set &ec, const char *filename)
+: m_change_set (ec),
   m_filename (filename),
-  m_edited_lines (line_comparator, NULL, edited_line::delete_cb),
+  m_changed_lines (line_comparator, NULL, changed_line::delete_cb),
   m_num_lines (-1)
 {
 }
 
-/* A callback for deleting edited_file *, for use as a
-   delete_value_fn for edit_context::m_files.  */
+/* A callback for deleting changed_file *, for use as a
+   delete_value_fn for change_set::m_files.  */
 
 void
-edited_file::delete_cb (edited_file *file)
+changed_file::delete_cb (changed_file *file)
 {
   delete file;
 }
@@ -378,7 +382,7 @@ edited_file::delete_cb (edited_file *file)
    The ptr should be freed by the caller.  */
 
 char *
-edited_file::get_content ()
+changed_file::get_content ()
 {
   pretty_printer pp;
   if (!print_content (&pp))
@@ -392,11 +396,11 @@ edited_file::get_content ()
    the line.  */
 
 bool
-edited_file::apply_fixit (int line, int start_column, int next_column,
+changed_file::apply_fixit (int line, int start_column, int next_column,
 			  const char *replacement_str,
 			  int replacement_len)
 {
-  edited_line *el = get_or_insert_line (line);
+  changed_line *el = get_or_insert_line (line);
   if (!el)
     return false;
   return el->apply_fixit (start_column, next_column, replacement_str,
@@ -407,9 +411,9 @@ edited_file::apply_fixit (int line, int start_column, int next_column,
    column after edits have been applied.  */
 
 int
-edited_file::get_effective_column (int line, int column)
+changed_file::get_effective_column (int line, int column)
 {
-  const edited_line *el = get_line (line);
+  const changed_line *el = get_line (line);
   if (!el)
     return column;
   return el->get_effective_column (column);
@@ -419,13 +423,13 @@ edited_file::get_effective_column (int line, int column)
    Return true if successful, false otherwise.  */
 
 bool
-edited_file::print_content (pretty_printer *pp)
+changed_file::print_content (pretty_printer *pp)
 {
   bool missing_trailing_newline;
   int line_count = get_num_lines (&missing_trailing_newline);
   for (int line_num = 1; line_num <= line_count; line_num++)
     {
-      edited_line *el = get_line (line_num);
+      changed_line *el = get_line (line_num);
       if (el)
 	el->print_content (pp);
       else
@@ -451,7 +455,7 @@ edited_file::print_content (pretty_printer *pp)
    to this file.  */
 
 void
-edited_file::print_diff (pretty_printer *pp, bool show_filenames)
+changed_file::print_diff (pretty_printer *pp, bool show_filenames)
 {
   if (show_filenames)
     {
@@ -466,7 +470,7 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
       pp_string (pp, colorize_stop (pp_show_color (pp)));
     }
 
-  edited_line *el = m_edited_lines.min ();
+  changed_line *el = m_changed_lines.min ();
 
   bool missing_trailing_newline;
   int line_count = get_num_lines (&missing_trailing_newline);
@@ -488,8 +492,8 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
 	 that are sufficiently close.  */
       while (true)
 	{
-	  edited_line *next_el
-	    = m_edited_lines.successor (el->get_line_num ());
+	  changed_line *next_el
+	    = m_changed_lines.successor (el->get_line_num ());
 	  if (!next_el)
 	    break;
 
@@ -514,7 +518,7 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
       int new_start_of_hunk = start_of_hunk + line_delta;
       line_delta += print_diff_hunk (pp, start_of_hunk, end_of_hunk,
 				     new_start_of_hunk);
-      el = m_edited_lines.successor (el->get_line_num ());
+      el = m_changed_lines.successor (el->get_line_num ());
     }
 }
 
@@ -525,7 +529,7 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
    Return the change in the line count within the hunk.  */
 
 int
-edited_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
+changed_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
 			      int old_end_of_hunk, int new_start_of_hunk)
 {
   int old_num_lines = old_end_of_hunk - old_start_of_hunk + 1;
@@ -541,7 +545,7 @@ edited_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
   int line_num = old_start_of_hunk;
   while (line_num <= old_end_of_hunk)
     {
-      edited_line *el = get_line (line_num);
+      changed_line *el = get_line (line_num);
       if (el)
 	{
 	  /* We have an edited line.
@@ -566,12 +570,12 @@ edited_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
   return new_num_lines - old_num_lines;
 }
 
-/* Subroutine of edited_file::print_diff_hunk: given a run of lines
-   from START_OF_RUN to END_OF_RUN that all have edited_line instances,
+/* Subroutine of changed_file::print_diff_hunk: given a run of lines
+   from START_OF_RUN to END_OF_RUN that all have changed_line instances,
    print the diff to PP.  */
 
 void
-edited_file::print_run_of_changed_lines (pretty_printer *pp,
+changed_file::print_run_of_changed_lines (pretty_printer *pp,
 					 int start_of_run,
 					 int end_of_run)
 {
@@ -582,7 +586,7 @@ edited_file::print_run_of_changed_lines (pretty_printer *pp,
        line_num <= end_of_run;
        line_num++)
     {
-      edited_line *el_in_run = get_line (line_num);
+      changed_line *el_in_run = get_line (line_num);
       gcc_assert (el_in_run);
       if (el_in_run->actually_edited_p ())
 	{
@@ -601,7 +605,7 @@ edited_file::print_run_of_changed_lines (pretty_printer *pp,
        line_num <= end_of_run;
        line_num++)
     {
-      edited_line *el_in_run = get_line (line_num);
+      changed_line *el_in_run = get_line (line_num);
       gcc_assert (el_in_run);
       el_in_run->print_diff_lines (pp);
     }
@@ -627,14 +631,14 @@ print_diff_line (pretty_printer *pp, char prefix_char,
    OLD_END_OF_HUNK inclusive.  */
 
 int
-edited_file::get_effective_line_count (int old_start_of_hunk,
+changed_file::get_effective_line_count (int old_start_of_hunk,
 				       int old_end_of_hunk)
 {
   int line_count = 0;
   for (int old_line_num = old_start_of_hunk; old_line_num <= old_end_of_hunk;
        old_line_num++)
     {
-      edited_line *el = get_line (old_line_num);
+      changed_line *el = get_line (old_line_num);
       if (el)
 	line_count += el->get_effective_line_count ();
       else
@@ -645,28 +649,28 @@ edited_file::get_effective_line_count (int old_start_of_hunk,
 
 /* Get the state of LINE within the file, or NULL if it is untouched.  */
 
-edited_line *
-edited_file::get_line (int line)
+changed_line *
+changed_file::get_line (int line)
 {
-  return m_edited_lines.lookup (line);
+  return m_changed_lines.lookup (line);
 }
 
 /* Get the state of LINE within the file, creating a state for it
    if necessary.  Return NULL if an error occurs.  */
 
-edited_line *
-edited_file::get_or_insert_line (int line)
+changed_line *
+changed_file::get_or_insert_line (int line)
 {
-  edited_line *el = get_line (line);
+  changed_line *el = get_line (line);
   if (el)
     return el;
-  el = new edited_line (get_file_cache (), m_filename, line);
+  el = new changed_line (get_file_cache (), m_filename, line);
   if (el->get_content () == NULL)
     {
       delete el;
       return NULL;
     }
-  m_edited_lines.insert (line, el);
+  m_changed_lines.insert (line, el);
   return el;
 }
 
@@ -675,7 +679,7 @@ edited_file::get_or_insert_line (int line)
    if missing a newline, false otherwise.  */
 
 int
-edited_file::get_num_lines (bool *missing_trailing_newline)
+changed_file::get_num_lines (bool *missing_trailing_newline)
 {
   gcc_assert (missing_trailing_newline);
   if (m_num_lines == -1)
@@ -696,11 +700,11 @@ edited_file::get_num_lines (bool *missing_trailing_newline)
   return m_num_lines;
 }
 
-/* Implementation of class edited_line.  */
+/* Implementation of class changed_line.  */
 
-/* edited_line's ctor.  */
+/* changed_line's ctor.  */
 
-edited_line::edited_line (file_cache &fc, const char *filename, int line_num)
+changed_line::changed_line (file_cache &fc, const char *filename, int line_num)
 : m_line_num (line_num),
   m_content (NULL), m_len (0), m_alloc_sz (0),
   m_line_events (),
@@ -715,9 +719,9 @@ edited_line::edited_line (file_cache &fc, const char *filename, int line_num)
   ensure_terminated ();
 }
 
-/* edited_line's dtor.  */
+/* changed_line's dtor.  */
 
-edited_line::~edited_line ()
+changed_line::~changed_line ()
 {
   unsigned i;
   added_line *pred;
@@ -727,11 +731,11 @@ edited_line::~edited_line ()
     delete pred;
 }
 
-/* A callback for deleting edited_line *, for use as a
-   delete_value_fn for edited_file::m_edited_lines.  */
+/* A callback for deleting changed_line *, for use as a
+   delete_value_fn for changed_file::m_changed_lines.  */
 
 void
-edited_line::delete_cb (edited_line *el)
+changed_line::delete_cb (changed_line *el)
 {
   delete el;
 }
@@ -740,7 +744,7 @@ edited_line::delete_cb (edited_line *el)
    within a specific line.  */
 
 int
-edited_line::get_effective_column (int orig_column) const
+changed_line::get_effective_column (int orig_column) const
 {
   int i;
   line_event *event;
@@ -756,7 +760,7 @@ edited_line::get_effective_column (int orig_column) const
    Return true if successful; false if an error occurred.  */
 
 bool
-edited_line::apply_fixit (int start_column,
+changed_line::apply_fixit (int start_column,
 			  int next_column,
 			  const char *replacement_str,
 			  int replacement_len)
@@ -824,16 +828,16 @@ edited_line::apply_fixit (int start_column,
    also be counted.  */
 
 int
-edited_line::get_effective_line_count () const
+changed_line::get_effective_line_count () const
 {
   return m_predecessors.length () + 1;
 }
 
-/* Subroutine of edited_file::print_content.
+/* Subroutine of changed_file::print_content.
    Print this line and any new lines added before it, to PP.  */
 
 void
-edited_line::print_content (pretty_printer *pp) const
+changed_line::print_content (pretty_printer *pp) const
 {
   unsigned i;
   added_line *pred;
@@ -845,18 +849,18 @@ edited_line::print_content (pretty_printer *pp) const
   pp_string (pp, m_content);
 }
 
-/* Subroutine of edited_file::print_run_of_changed_lines for
+/* Subroutine of changed_file::print_run_of_changed_lines for
    printing diff hunks to PP.
    Print the '+' line for this line, and any newlines added
    before it.
-   Note that if this edited_line was actually edited, the '-'
+   Note that if this changed_line was actually edited, the '-'
    line has already been printed.  If it wasn't, then we merely
-   have a placeholder edited_line for adding newlines to, and
-   we need to print a ' ' line for the edited_line as we haven't
+   have a placeholder changed_line for adding newlines to, and
+   we need to print a ' ' line for the changed_line as we haven't
    printed it yet.  */
 
 void
-edited_line::print_diff_lines (pretty_printer *pp) const
+changed_line::print_diff_lines (pretty_printer *pp) const
 {
   unsigned i;
   added_line *pred;
@@ -874,7 +878,7 @@ edited_line::print_diff_lines (pretty_printer *pp) const
    allocations.  */
 
 void
-edited_line::ensure_capacity (int len)
+changed_line::ensure_capacity (int len)
 {
   /* Allow 1 extra byte for 0-termination.  */
   if (m_alloc_sz < (len + 1))
@@ -888,7 +892,7 @@ edited_line::ensure_capacity (int len)
 /* Ensure that m_content is 0-terminated.  */
 
 void
-edited_line::ensure_terminated ()
+changed_line::ensure_terminated ()
 {
   /* 0-terminate the buffer.  */
   gcc_assert (m_len < m_alloc_sz);
@@ -901,6 +905,11 @@ edited_line::ensure_terminated ()
 
 namespace selftest {
 
+using line_table_case = ::selftest::line_table_case;
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+using named_temp_file = ::selftest::named_temp_file;
+
 /* A wrapper class for ensuring that the underlying pointer is freed.  */
 
 template <typename POINTER_T>
@@ -916,7 +925,7 @@ class auto_free
   POINTER_T m_ptr;
 };
 
-/* Verify that edit_context::get_content works for unedited files.  */
+/* Verify that change_set::get_content works for unedited files.  */
 
 static void
 test_get_content ()
@@ -926,7 +935,7 @@ test_get_content ()
     const char *content = ("");
     temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     auto_free <char *> result = edit.get_content (tmp.get_filename ());
     ASSERT_STREQ ("", result);
   }
@@ -938,7 +947,7 @@ test_get_content ()
 			   "/* after */\n");
     temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     auto_free <char *> result = edit.get_content (tmp.get_filename ());
     ASSERT_STREQ ("/* before */\n"
 		  "foo = bar.field;\n"
@@ -952,7 +961,7 @@ test_get_content ()
 			   "/* after */");
     temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     auto_free <char *> result = edit.get_content (tmp.get_filename ());
     /* We should respect the omitted trailing newline.  */
     ASSERT_STREQ ("/* before */\n"
@@ -986,7 +995,7 @@ test_applying_fixits_insert_before (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (start <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1045,7 +1054,7 @@ test_applying_fixits_insert_after (const line_table_case &case_)
 
   /* Verify that the text was inserted after the end of "field". */
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   ASSERT_STREQ ("/* before */\n"
@@ -1088,7 +1097,7 @@ test_applying_fixits_insert_after_at_line_end (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   ASSERT_STREQ ("/* before */\n"
@@ -1147,7 +1156,7 @@ test_applying_fixits_insert_after_failure (const line_table_case &case_)
   ASSERT_TRUE (richloc.seen_impossible_fixit_p ());
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   ASSERT_FALSE (edit.valid_p ());
   ASSERT_EQ (NULL, edit.get_content (filename));
@@ -1185,7 +1194,7 @@ test_applying_fixits_insert_containing_newline (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   ASSERT_STREQ (("    case 'a':\n"
@@ -1230,7 +1239,7 @@ test_applying_fixits_growing_replace (const line_table_case &case_)
   richloc.add_fixit_replace ("m_field");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (finish <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1276,7 +1285,7 @@ test_applying_fixits_shrinking_replace (const line_table_case &case_)
   richloc.add_fixit_replace ("field");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (finish <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1331,7 +1340,7 @@ test_applying_fixits_replace_containing_newline (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   //ASSERT_STREQ ("foo\n  = bar ();\n", new_content);
@@ -1363,7 +1372,7 @@ test_applying_fixits_remove (const line_table_case &case_)
   richloc.add_fixit_remove (range);
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (finish <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1429,7 +1438,7 @@ test_applying_fixits_multiple (const line_table_case &case_)
 			       "meadow");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&insert_a);
   ASSERT_EQ (100, edit.get_effective_column (filename, 1, 100));
   ASSERT_EQ (1, edit.get_effective_column (filename, 2, 1));
@@ -1464,7 +1473,7 @@ test_applying_fixits_multiple (const line_table_case &case_)
    Add the text "CHANGED: " to the front of the given line.  */
 
 static location_t
-change_line (edit_context &edit, int line_num)
+change_line (change_set &edit, int line_num)
 {
   const line_map_ordinary *ord_map
     = LINEMAPS_LAST_ORDINARY_MAP (line_table);
@@ -1490,7 +1499,7 @@ change_line (edit_context &edit, int line_num)
    Add the text "INSERTED\n" in front of the given line.  */
 
 static location_t
-insert_line (edit_context &edit, int line_num)
+insert_line (change_set &edit, int line_num)
 {
   const line_map_ordinary *ord_map
     = LINEMAPS_LAST_ORDINARY_MAP (line_table);
@@ -1532,7 +1541,7 @@ test_applying_fixits_multiple_lines (const line_table_case &case_)
   linemap_position_for_column (line_table, 127);
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
 
   /* A run of consecutive lines.  */
   change_line (edit, 2);
@@ -1621,7 +1630,7 @@ test_applying_fixits_modernize_named_init (const line_table_case &case_)
   /* The order should not matter.  Do r1 then r2. */
   {
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&r1);
 
     /* Verify state after first replacement.  */
@@ -1655,7 +1664,7 @@ test_applying_fixits_modernize_named_init (const line_table_case &case_)
   /* Try again, doing r2 then r1; the new_content should be the same.  */
   {
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&r2);
     edit.add_fixits (&r1);
     auto_free <char *> new_content = edit.get_content (tmp.get_filename ());
@@ -1684,7 +1693,7 @@ test_applying_fixits_unreadable_file ()
   insert.add_fixit_insert_before ("change 2");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   /* Attempting to add the fixits affecting the unreadable file
      should transition the edit from valid to invalid.  */
   ASSERT_TRUE (edit.valid_p ());
@@ -1715,10 +1724,10 @@ test_applying_fixits_line_out_of_range ()
   rich_location insert (line_table, loc);
   insert.add_fixit_insert_before ("change");
 
-  /* Verify that attempting the insertion puts an edit_context
+  /* Verify that attempting the insertion puts an change_set
      into an invalid state.  */
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   ASSERT_TRUE (edit.valid_p ());
   edit.add_fixits (&insert);
   ASSERT_FALSE (edit.valid_p ());
@@ -1727,7 +1736,7 @@ test_applying_fixits_line_out_of_range ()
 }
 
 /* Verify the boundary conditions of column values in fix-it
-   hints applied to edit_context instances.  */
+   hints applied to change_set instances.  */
 
 static void
 test_applying_fixits_column_validation (const line_table_case &case_)
@@ -1756,7 +1765,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 15 is at the end of the line, so the insertion
        should succeed.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&richloc);
     auto_free <char *> new_content = edit.get_content (tmp.get_filename ());
     if (c15 <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1773,7 +1782,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 16 is beyond the end of the line, so the insertion
        should fail gracefully.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     ASSERT_TRUE (edit.valid_p ());
     edit.add_fixits (&richloc);
     ASSERT_FALSE (edit.valid_p ());
@@ -1792,7 +1801,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 14 is at the end of the line, so the replacement
        should succeed.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&richloc);
     auto_free <char *> new_content = edit.get_content (tmp.get_filename ());
     if (c14 <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1810,7 +1819,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 15 is after the end of the line, so the replacement
        should fail; verify that the attempt fails gracefully.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     ASSERT_TRUE (edit.valid_p ());
     edit.add_fixits (&richloc);
     ASSERT_FALSE (edit.valid_p ());
@@ -1819,10 +1828,8 @@ test_applying_fixits_column_validation (const line_table_case &case_)
   }
 }
 
-/* Run all of the selftests within this file.  */
-
-void
-edit_context_cc_tests ()
+static void
+run_all_tests ()
 {
   test_get_content ();
   for_each_line_table_case (test_applying_fixits_insert_before);
@@ -1842,6 +1849,26 @@ edit_context_cc_tests ()
   for_each_line_table_case (test_applying_fixits_column_validation);
 }
 
+} // namespace diagnostics::changes::selftest
+
+#endif /* CHECKING_P */
+
+} // namespace diagnostics::changes
+
+#if CHECKING_P
+
+namespace selftest { // diagnostics::selftest
+
+/* Run all of the selftests within this file.  */
+
+void
+changes_cc_tests ()
+{
+  diagnostics::changes::selftest::run_all_tests ();
+}
+
 } // namespace selftest
 
 #endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/edit-context.h b/gcc/diagnostics/changes.h
index 9bd2aba..a5fe4b0 100644
--- a/gcc/edit-context.h
+++ b/gcc/diagnostics/changes.h
@@ -17,21 +17,25 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_EDIT_CONTEXT_H
-#define GCC_EDIT_CONTEXT_H
+#ifndef GCC_DIAGNOSTICS_CHANGES_H
+#define GCC_DIAGNOSTICS_CHANGES_H
 
 #include "typed-splay-tree.h"
 
 class fixit_hint;
-class edit_context;
-class edited_file;
+
+namespace diagnostics {
+namespace changes {
+
+class change_set;
+class changed_file;
 
 /* A set of changes to the source code.
 
    The changes are "atomic" - if any changes can't be applied,
    none of them can be (tracked by the m_valid flag).
    Similarly, attempts to add the changes from a rich_location flagged
-   as containing invalid changes mean that the whole of the edit_context
+   as containing invalid changes mean that the whole of the change_set
    is flagged as invalid.
 
    A complication here is that fix-its are expressed relative to coordinates
@@ -40,10 +44,10 @@ class edited_file;
    later fix-its to allow for the changes made by earlier ones.  This
    is done by the various "get_effective_column" methods.  */
 
-class edit_context
+class change_set
 {
  public:
-  edit_context (file_cache &);
+  change_set (file_cache &);
 
   bool valid_p () const { return m_valid; }
 
@@ -60,12 +64,15 @@ class edit_context
 
  private:
   bool apply_fixit (const fixit_hint *hint);
-  edited_file *get_file (const char *filename);
-  edited_file &get_or_insert_file (const char *filename);
+  changed_file *get_file (const char *filename);
+  changed_file &get_or_insert_file (const char *filename);
 
   file_cache &m_file_cache;
   bool m_valid;
-  typed_splay_tree<const char *, edited_file *> m_files;
+  typed_splay_tree<const char *, changed_file *> m_files;
 };
 
-#endif /* GCC_EDIT_CONTEXT_H.  */
+} // namespace diagnostics::changes
+} // namespace diagnostics
+
+#endif /* GCC_DIAGNOSTICS_CHANGES_H.  */
diff --git a/gcc/diagnostic-client-data-hooks.h b/gcc/diagnostics/client-data-hooks.h
index 9909172..94c51b2 100644
--- a/gcc/diagnostic-client-data-hooks.h
+++ b/gcc/diagnostics/client-data-hooks.h
@@ -18,32 +18,36 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_CLIENT_DATA_HOOKS_H
-#define GCC_DIAGNOSTIC_CLIENT_DATA_HOOKS_H
+#ifndef GCC_DIAGNOSTICS_CLIENT_DATA_HOOKS_H
+#define GCC_DIAGNOSTICS_CLIENT_DATA_HOOKS_H
 
-#include "logical-location.h"
+#include "diagnostics/logical-locations.h"
+
+namespace diagnostics {
 
 class sarif_object;
 class client_version_info;
 
-/* A bundle of additional metadata, owned by the diagnostic_context,
+/* A bundle of additional metadata, owned by the diagnostics::context,
    for querying things about the client, like version data.  */
 
-class diagnostic_client_data_hooks
+class client_data_hooks
 {
  public:
-  virtual ~diagnostic_client_data_hooks () {}
+  virtual ~client_data_hooks () {}
 
   /* Get version info for this client, or NULL.  */
   virtual const client_version_info *get_any_version_info () const = 0;
 
-  /* Get the current logical_location_manager for this client, or NULL.  */
-  virtual const logical_location_manager *get_logical_location_manager () const = 0;
+  /* Get the current logical_locations::manager for this client, or null.  */
+  virtual const logical_locations::manager *
+  get_logical_location_manager () const = 0;
 
-  /* Get the current logical_location, or null.
-     If this returns a non-null logical_location, then
-     get_logical_location_manager must return non-NULL.  */
-  virtual logical_location get_current_logical_location () const = 0;
+  /* Get the current logical location, or null.
+     If this returns a non-null logical location, then
+     get_logical_location_manager must return non-null.  */
+  virtual logical_locations::key
+  get_current_logical_location () const = 0;
 
   /* Get a sourceLanguage value for FILENAME, or return NULL.
      See SARIF v2.1.0 Appendix J for suggested values.  */
@@ -56,15 +60,9 @@ class diagnostic_client_data_hooks
   add_sarif_invocation_properties (sarif_object &invocation_obj) const = 0;
 };
 
-/* Factory function for making an instance of diagnostic_client_data_hooks
-   for use in the compiler (i.e. with knowledge of "tree", access to
-   langhooks, etc).  */
-
-extern std::unique_ptr<diagnostic_client_data_hooks> make_compiler_data_hooks ();
-
-class diagnostic_client_plugin_info;
+class client_plugin_info;
 
-/* Abstract base class for a diagnostic_context to get at
+/* Abstract base class for a diagnostics::context to get at
    version information about the client.  */
 
 class client_version_info
@@ -73,7 +71,7 @@ public:
   class plugin_visitor
   {
   public:
-    virtual void on_plugin (const diagnostic_client_plugin_info &) = 0;
+    virtual void on_plugin (const client_plugin_info &) = 0;
   };
 
   virtual ~client_version_info () {}
@@ -97,10 +95,10 @@ public:
   virtual void for_each_plugin (plugin_visitor &v) const = 0;
 };
 
-/* Abstract base class for a diagnostic_context to get at
+/* Abstract base class for a diagnostics::context to get at
    information about a specific plugin within a client.  */
 
-class diagnostic_client_plugin_info
+class client_plugin_info
 {
 public:
   /* For use e.g. by SARIF "name" property (SARIF v2.1.0 section 3.19.8).  */
@@ -115,4 +113,13 @@ public:
   virtual const char *get_version () const = 0;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_CLIENT_DATA_HOOKS_H */
+} // namespace diagnostics
+
+/* Factory function for making an instance of client_data_hooks
+   for use in the compiler (i.e. with knowledge of "tree", access to
+   langhooks, etc).  */
+
+extern std::unique_ptr<diagnostics::client_data_hooks>
+make_compiler_data_hooks ();
+
+#endif /* ! GCC_DIAGNOSTICS_CLIENT_DATA_HOOKS_H */
diff --git a/gcc/diagnostic-color.cc b/gcc/diagnostics/color.cc
index e95aaeb..7b499fe 100644
--- a/gcc/diagnostic-color.cc
+++ b/gcc/diagnostics/color.cc
@@ -19,8 +19,8 @@
 #include "config.h"
 #define INCLUDE_VECTOR
 #include "system.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
+#include "diagnostics/color.h"
+#include "diagnostics/url.h"
 #include "label-text.h"
 
 #ifdef __MINGW32__
@@ -475,6 +475,7 @@ determine_url_format (diagnostic_url_rule_t rule)
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
 /* Test of an empty diagnostic_color_dict.  */
@@ -522,7 +523,7 @@ test_color_dict_envvar_parsing ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_color_cc_tests ()
+color_cc_tests ()
 {
   test_empty_color_dict ();
   test_default_color_dict ();
@@ -530,5 +531,6 @@ diagnostic_color_cc_tests ()
 }
 
 } // namespace selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/diagnostic-color.h b/gcc/diagnostics/color.h
index 8aeaa57..42b67eb 100644
--- a/gcc/diagnostic-color.h
+++ b/gcc/diagnostics/color.h
@@ -38,8 +38,8 @@ along with GCC; see the file COPYING3.  If not see
 
    Written July 1992 by Mike Haertel.  */
 
-#ifndef GCC_DIAGNOSTIC_COLOR_H
-#define GCC_DIAGNOSTIC_COLOR_H
+#ifndef GCC_DIAGNOSTICS_COLOR_H
+#define GCC_DIAGNOSTICS_COLOR_H
 
 /* Whether to add color to diagnostics:
    o DIAGNOSTICS_COLOR_NO: never
@@ -62,4 +62,4 @@ colorize_start (bool show_color, const char *name)
   return colorize_start (show_color, name, strlen (name));
 }
 
-#endif /* ! GCC_DIAGNOSTIC_COLOR_H */
+#endif /* ! GCC_DIAGNOSTICS_COLOR_H */
diff --git a/gcc/diagnostics/context-options.h b/gcc/diagnostics/context-options.h
new file mode 100644
index 0000000..c4e92eb
--- /dev/null
+++ b/gcc/diagnostics/context-options.h
@@ -0,0 +1,116 @@
+/* Declare enums for diagnostics::context and related types.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_CONTEXT_OPTIONS_H
+#define GCC_DIAGNOSTICS_CONTEXT_OPTIONS_H
+
+/* An enum for controlling what units to use for the column number
+   when diagnostics are output, used by the -fdiagnostics-column-unit option.
+   Tabs will be expanded or not according to the value of -ftabstop.  The origin
+   (default 1) is controlled by -fdiagnostics-column-origin.  */
+
+enum diagnostics_column_unit
+{
+  /* The default from GCC 11 onwards: display columns.  */
+  DIAGNOSTICS_COLUMN_UNIT_DISPLAY,
+
+  /* The behavior in GCC 10 and earlier: simple bytes.  */
+  DIAGNOSTICS_COLUMN_UNIT_BYTE
+};
+
+/* An enum for controlling how to print non-ASCII characters/bytes when
+   a diagnostic suggests escaping the source code on output.  */
+
+enum diagnostics_escape_format
+{
+  /* Escape non-ASCII Unicode characters in the form <U+XXXX> and
+     non-UTF-8 bytes in the form <XX>.  */
+  DIAGNOSTICS_ESCAPE_FORMAT_UNICODE,
+
+  /* Escape non-ASCII bytes in the form <XX> (thus showing the underlying
+     encoding of non-ASCII Unicode characters).  */
+  DIAGNOSTICS_ESCAPE_FORMAT_BYTES
+};
+
+/* Enum for overriding the standard output format.  */
+
+enum diagnostics_output_format
+{
+  /* The default: textual output.  */
+  DIAGNOSTICS_OUTPUT_FORMAT_TEXT,
+
+  /* SARIF-based output, as JSON to stderr.  */
+  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_STDERR,
+
+  /* SARIF-based output, to a JSON file.  */
+  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_FILE
+};
+
+/* An enum for controlling how diagnostic paths should be printed.  */
+enum diagnostic_path_format
+{
+  /* Don't print diagnostic paths.  */
+  DPF_NONE,
+
+  /* Print diagnostic paths by emitting a separate "note" for every event
+     in the path.  */
+  DPF_SEPARATE_EVENTS,
+
+  /* Print diagnostic paths by consolidating events together where they
+     are close enough, and printing such runs of events with multiple
+     calls to diagnostic_show_locus, showing the individual events in
+     each run via labels in the source.  */
+  DPF_INLINE_EVENTS
+};
+
+/* An enum for capturing values of GCC_EXTRA_DIAGNOSTIC_OUTPUT,
+   and for -fdiagnostics-parseable-fixits.  */
+
+enum diagnostics_extra_output_kind
+{
+  /* No extra output, or an unrecognized value.  */
+  EXTRA_DIAGNOSTIC_OUTPUT_none,
+
+  /* Emit fix-it hints using the "fixits-v1" format, equivalent to
+     -fdiagnostics-parseable-fixits.  */
+  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v1,
+
+  /* Emit fix-it hints using the "fixits-v2" format.  */
+  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v2
+};
+
+/* Values for -fdiagnostics-text-art-charset=.  */
+
+enum diagnostic_text_art_charset
+{
+  /* No text art diagrams shall be emitted.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_NONE,
+
+  /* Use pure ASCII for text art diagrams.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_ASCII,
+
+  /* Use ASCII + conservative use of other unicode characters
+     in text art diagrams.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE,
+
+  /* Use Emoji.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_EMOJI
+};
+
+#endif /* ! GCC_DIAGNOSTICS_CONTEXT_OPTIONS_H */
diff --git a/gcc/diagnostic.cc b/gcc/diagnostics/context.cc
index 088390b..a1441ca 100644
--- a/gcc/diagnostic.cc
+++ b/gcc/diagnostics/context.cc
@@ -31,24 +31,25 @@ along with GCC; see the file COPYING3.  If not see
 #include "intl.h"
 #include "backtrace.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-diagram.h"
-#include "diagnostic-format.h"
-#include "diagnostic-format-sarif.h"
-#include "diagnostic-format-text.h"
-#include "edit-context.h"
+#include "diagnostics/color.h"
+#include "diagnostics/url.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/diagram.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/changes.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 #include "opts.h"
 #include "cpplib.h"
 #include "text-art/theme.h"
 #include "pretty-print-urlifier.h"
-#include "logical-location.h"
-#include "diagnostic-buffer.h"
+#include "diagnostics/logical-locations.h"
+#include "diagnostics/buffering.h"
+#include "diagnostics/file-cache.h"
 
 #ifdef HAVE_TERMIOS_H
 # include <termios.h>
@@ -111,114 +112,28 @@ get_terminal_width (void)
   return INT_MAX;
 }
 
+namespace diagnostics {
+
 /* Set caret_max_width to value.  */
+
 void
-diagnostic_set_caret_max_width (diagnostic_context *context, int value)
+context::set_caret_max_width (int value)
 {
   /* One minus to account for the leading empty space.  */
   value = value ? value - 1
-    : (isatty (fileno (pp_buffer (context->get_reference_printer ())->m_stream))
+    : (isatty (fileno (pp_buffer (get_reference_printer ())->m_stream))
        ? get_terminal_width () - 1 : INT_MAX);
 
   if (value <= 0)
     value = INT_MAX;
 
-  context->m_source_printing.max_width = value;
-}
-
-void
-diagnostic_option_classifier::init (int n_opts)
-{
-  m_n_opts = n_opts;
-  m_classify_diagnostic = XNEWVEC (diagnostic_t, n_opts);
-  for (int i = 0; i < n_opts; i++)
-    m_classify_diagnostic[i] = DK_UNSPECIFIED;
-  m_push_list = vNULL;
-  m_classification_history = vNULL;
-}
-
-void
-diagnostic_option_classifier::fini ()
-{
-  XDELETEVEC (m_classify_diagnostic);
-  m_classify_diagnostic = nullptr;
-  m_classification_history.release ();
-  m_push_list.release ();
-}
-
-/* Save the diagnostic_option_classifier state to F for PCH
-   output.  Returns 0 on success, -1 on error.  */
-
-int
-diagnostic_option_classifier::pch_save (FILE *f)
-{
-  unsigned int lengths[2] = { m_classification_history.length (),
-			      m_push_list.length () };
-  if (fwrite (lengths, sizeof (lengths), 1, f) != 1
-      || (lengths[0]
-	  && fwrite (m_classification_history.address (),
-		     sizeof (diagnostic_classification_change_t),
-		     lengths[0], f) != lengths[0])
-      || (lengths[1]
-	  && fwrite (m_push_list.address (), sizeof (int),
-		     lengths[1], f) != lengths[1]))
-    return -1;
-  return 0;
-}
-
-/* Read the diagnostic_option_classifier state from F for PCH
-   read.  Returns 0 on success, -1 on error.  */
-
-int
-diagnostic_option_classifier::pch_restore (FILE *f)
-{
-  unsigned int lengths[2];
-  if (fread (lengths, sizeof (lengths), 1, f) != 1)
-    return -1;
-  gcc_checking_assert (m_classification_history.is_empty ());
-  gcc_checking_assert (m_push_list.is_empty ());
-  m_classification_history.safe_grow (lengths[0]);
-  m_push_list.safe_grow (lengths[1]);
-  if ((lengths[0]
-       && fread (m_classification_history.address (),
-		 sizeof (diagnostic_classification_change_t),
-		 lengths[0], f) != lengths[0])
-      || (lengths[1]
-	  && fread (m_push_list.address (), sizeof (int),
-		    lengths[1], f) != lengths[1]))
-    return -1;
-  return 0;
-}
-
-/* Save all diagnostic classifications in a stack.  */
-
-void
-diagnostic_option_classifier::push ()
-{
-  m_push_list.safe_push (m_classification_history.length ());
-}
-
-/* Restore the topmost classification set off the stack.  If the stack
-   is empty, revert to the state based on command line parameters.  */
-
-void
-diagnostic_option_classifier::pop (location_t where)
-{
-  int jump_to;
-
-  if (!m_push_list.is_empty ())
-    jump_to = m_push_list.pop ();
-  else
-    jump_to = 0;
-
-  diagnostic_classification_change_t v = { where, jump_to, DK_POP };
-  m_classification_history.safe_push (v);
+  m_source_printing.max_width = value;
 }
 
 /* Initialize the diagnostic message outputting machinery.  */
 
 void
-diagnostic_context::initialize (int n_opts)
+context::initialize (int n_opts)
 {
   /* Allocate a basic pretty-printer.  Clients will replace this a
      much more elaborated pretty-printer if they wish.  */
@@ -230,7 +145,7 @@ diagnostic_context::initialize (int n_opts)
   m_n_opts = n_opts;
   m_option_classifier.init (n_opts);
   m_source_printing.enabled = false;
-  diagnostic_set_caret_max_width (this, pp_line_cutoff (get_reference_printer ()));
+  set_caret_max_width (pp_line_cutoff (get_reference_printer ()));
   for (int i = 0; i < rich_location::STATICALLY_ALLOCATED_RANGES; i++)
     m_source_printing.caret_chars[i] = '^';
   m_show_cwe = false;
@@ -249,13 +164,13 @@ diagnostic_context::initialize (int n_opts)
   m_max_errors = 0;
   m_internal_error = nullptr;
   m_adjust_diagnostic_info = nullptr;
-  m_text_callbacks.m_begin_diagnostic = default_diagnostic_text_starter;
+  m_text_callbacks.m_begin_diagnostic = default_text_starter;
   m_text_callbacks.m_text_start_span
-    = default_diagnostic_start_span_fn<to_text>;
+    = default_start_span_fn<to_text>;
   m_text_callbacks.m_html_start_span
-    = default_diagnostic_start_span_fn<to_html>;
-  m_text_callbacks.m_end_diagnostic = default_diagnostic_text_finalizer;
-  m_option_mgr = nullptr;
+    = default_start_span_fn<to_html>;
+  m_text_callbacks.m_end_diagnostic = default_text_finalizer;
+  m_option_id_mgr = nullptr;
   m_urlifier_stack = new auto_vec<urlifier_stack_node> ();
   m_last_location = UNKNOWN_LOCATION;
   m_client_aux_data = nullptr;
@@ -281,13 +196,12 @@ diagnostic_context::initialize (int n_opts)
   m_column_origin = 1;
   m_tabstop = 8;
   m_escape_format = DIAGNOSTICS_ESCAPE_FORMAT_UNICODE;
-  m_edit_context_ptr = nullptr;
+  m_fixits_change_set = nullptr;
   m_diagnostic_groups.m_group_nesting_depth = 0;
   m_diagnostic_groups.m_diagnostic_nesting_level = 0;
   m_diagnostic_groups.m_emission_count = 0;
   m_diagnostic_groups.m_inhibiting_notes_from = 0;
-  m_output_sinks.safe_push
-    (new diagnostic_text_output_format (*this, nullptr, true));
+  m_sinks.safe_push (new text_sink (*this, nullptr, true));
   m_set_locations_cb = nullptr;
   m_client_data_hooks = nullptr;
   m_diagrams.m_theme = nullptr;
@@ -311,7 +225,7 @@ diagnostic_context::initialize (int n_opts)
    without a VALUE, it initializes with DIAGNOSTICS_COLOR_DEFAULT.  */
 
 void
-diagnostic_context::color_init (int value)
+context::color_init (int value)
 {
   /* value == -1 is the default value.  */
   if (value < 0)
@@ -332,9 +246,9 @@ diagnostic_context::color_init (int value)
     }
   pp_show_color (m_reference_printer)
     = colorize_init ((diagnostic_color_rule_t) value);
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      pp_show_color (sink->get_printer ())
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      pp_show_color (sink_->get_printer ())
 	= pp_show_color (m_reference_printer);
 }
 
@@ -342,7 +256,7 @@ diagnostic_context::color_init (int value)
    handling "auto".  */
 
 void
-diagnostic_context::urls_init (int value)
+context::urls_init (int value)
 {
   /* value == -1 is the default value.  */
   if (value < 0)
@@ -364,18 +278,17 @@ diagnostic_context::urls_init (int value)
 
   m_reference_printer->set_url_format
     (determine_url_format ((diagnostic_url_rule_t) value));
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      sink->get_printer ()->set_url_format
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      sink_->get_printer ()->set_url_format
 	(m_reference_printer->get_url_format ());
 }
 
 /* Create the file_cache, if not already created, and tell it how to
    translate files on input.  */
 void
-diagnostic_context::
-initialize_input_context (diagnostic_input_charset_callback ccb,
-			  bool should_skip_bom)
+context::initialize_input_context (diagnostic_input_charset_callback ccb,
+				   bool should_skip_bom)
 {
   m_file_cache->initialize_input_context (ccb, should_skip_bom);
 }
@@ -383,7 +296,7 @@ initialize_input_context (diagnostic_input_charset_callback ccb,
 /* Do any cleaning up required after the last diagnostic is emitted.  */
 
 void
-diagnostic_context::finish ()
+context::finish ()
 {
   /* We might be handling a fatal error.
      Close any active diagnostic groups, which may trigger flushing
@@ -395,8 +308,8 @@ diagnostic_context::finish ()
 
   /* Clean ups.  */
 
-  while (!m_output_sinks.is_empty ())
-    delete m_output_sinks.pop ();
+  while (!m_sinks.is_empty ())
+    delete m_sinks.pop ();
 
   if (m_diagrams.m_theme)
     {
@@ -412,10 +325,10 @@ diagnostic_context::finish ()
   delete m_reference_printer;
   m_reference_printer = nullptr;
 
-  if (m_edit_context_ptr)
+  if (m_fixits_change_set)
     {
-      delete m_edit_context_ptr;
-      m_edit_context_ptr = nullptr;
+      delete m_fixits_change_set;
+      m_fixits_change_set = nullptr;
     }
 
   if (m_client_data_hooks)
@@ -424,8 +337,8 @@ diagnostic_context::finish ()
       m_client_data_hooks = nullptr;
     }
 
-  delete m_option_mgr;
-  m_option_mgr = nullptr;
+  delete m_option_id_mgr;
+  m_option_id_mgr = nullptr;
 
   if (m_urlifier_stack)
     {
@@ -439,22 +352,22 @@ diagnostic_context::finish ()
   m_original_argv = nullptr;
 }
 
-/* Dump state of this diagnostic_context to OUT, for debugging.  */
+/* Dump state of this diagnostics::context to OUT, for debugging.  */
 
 void
-diagnostic_context::dump (FILE *out) const
+context::dump (FILE *out) const
 {
-  fprintf (out, "diagnostic_context:\n");
+  fprintf (out, "diagnostics::context:\n");
   m_diagnostic_counters.dump (out, 2);
   fprintf (out, "  reference printer:\n");
   m_reference_printer->dump (out, 4);
   fprintf (out, "  output sinks:\n");
-  if (m_output_sinks.length () > 0)
+  if (m_sinks.length () > 0)
     {
-      for (unsigned i = 0; i < m_output_sinks.length (); ++i)
+      for (unsigned i = 0; i < m_sinks.length (); ++i)
 	{
 	  fprintf (out, "  sink %i:\n", i);
-	  m_output_sinks[i]->dump (out, 4);
+	  m_sinks[i]->dump (out, 4);
 	}
     }
   else
@@ -475,65 +388,63 @@ diagnostic_context::dump (FILE *out) const
    we ought to exit with a non-zero exit code.  */
 
 bool
-diagnostic_context::execution_failed_p () const
+context::execution_failed_p () const
 {
   /* Equivalent to (seen_error () || werrorcount), but on
      this context, rather than global_dc.  */
-  return (diagnostic_count (DK_ERROR)
-	  || diagnostic_count (DK_SORRY)
-	  || diagnostic_count (DK_WERROR));
+  return (diagnostic_count (kind::error)
+	  || diagnostic_count (kind::sorry)
+	  || diagnostic_count (kind::werror));
 }
 
 void
-diagnostic_context::remove_all_output_sinks ()
+context::remove_all_output_sinks ()
 {
-  while (!m_output_sinks.is_empty ())
-    delete m_output_sinks.pop ();
+  while (!m_sinks.is_empty ())
+    delete m_sinks.pop ();
 }
 
 void
-diagnostic_context::
-set_output_format (std::unique_ptr<diagnostic_output_format> output_format)
+context::set_sink (std::unique_ptr<sink> sink_)
 {
   remove_all_output_sinks ();
-  m_output_sinks.safe_push (output_format.release ());
+  m_sinks.safe_push (sink_.release ());
 }
 
-diagnostic_output_format &
-diagnostic_context::get_output_format (size_t idx) const
+sink &
+context::get_sink (size_t idx) const
 {
-  gcc_assert (idx < m_output_sinks.length ());
-  gcc_assert (m_output_sinks[idx]);
-  return *m_output_sinks[idx];
+  gcc_assert (idx < m_sinks.length ());
+  gcc_assert (m_sinks[idx]);
+  return *m_sinks[idx];
 }
 
 void
-diagnostic_context::add_sink (std::unique_ptr<diagnostic_output_format> sink)
+context::add_sink (std::unique_ptr<sink> sink_)
 {
-  m_output_sinks.safe_push (sink.release ());
+  m_sinks.safe_push (sink_.release ());
 }
 
 /* Return true if there are no machine-readable formats writing to stderr.  */
 
 bool
-diagnostic_context::supports_fnotice_on_stderr_p () const
+context::supports_fnotice_on_stderr_p () const
 {
-  for (auto sink : m_output_sinks)
-    if (sink->machine_readable_stderr_p ())
+  for (auto sink_ : m_sinks)
+    if (sink_->machine_readable_stderr_p ())
       return false;
   return true;
 }
 
 void
-diagnostic_context::set_main_input_filename (const char *filename)
+context::set_main_input_filename (const char *filename)
 {
-  for (auto sink : m_output_sinks)
-    sink->set_main_input_filename (filename);
+  for (auto sink_ : m_sinks)
+    sink_->set_main_input_filename (filename);
 }
 
 void
-diagnostic_context::
-set_client_data_hooks (std::unique_ptr<diagnostic_client_data_hooks> hooks)
+context::set_client_data_hooks (std::unique_ptr<client_data_hooks> hooks)
 {
   delete m_client_data_hooks;
   /* Ideally the field would be a std::unique_ptr here.  */
@@ -541,10 +452,10 @@ set_client_data_hooks (std::unique_ptr<diagnostic_client_data_hooks> hooks)
 }
 
 void
-diagnostic_context::set_original_argv (unique_argv original_argv)
+context::set_original_argv (unique_argv original_argv)
 {
   /* Ideally we'd use a unique_argv for m_original_argv, but
-     diagnostic_context doesn't yet have a ctor/dtor pair.  */
+     diagnostics::context doesn't yet have a ctor/dtor pair.  */
 
   // Ensure any old value is freed
   freeargv (m_original_argv);
@@ -554,17 +465,16 @@ diagnostic_context::set_original_argv (unique_argv original_argv)
 }
 
 void
-diagnostic_context::
-set_option_manager (std::unique_ptr<diagnostic_option_manager> mgr,
-		    unsigned lang_mask)
+context::set_option_id_manager (std::unique_ptr<option_id_manager> mgr,
+				unsigned lang_mask)
 {
-  delete m_option_mgr;
-  m_option_mgr = mgr.release ();
+  delete m_option_id_mgr;
+  m_option_id_mgr = mgr.release ();
   m_lang_mask = lang_mask;
 }
 
 void
-diagnostic_context::push_owned_urlifier (std::unique_ptr<urlifier> ptr)
+context::push_owned_urlifier (std::unique_ptr<urlifier> ptr)
 {
   gcc_assert (m_urlifier_stack);
   const urlifier_stack_node node = { ptr.release (), true };
@@ -572,7 +482,7 @@ diagnostic_context::push_owned_urlifier (std::unique_ptr<urlifier> ptr)
 }
 
 void
-diagnostic_context::push_borrowed_urlifier (const urlifier &loan)
+context::push_borrowed_urlifier (const urlifier &loan)
 {
   gcc_assert (m_urlifier_stack);
   const urlifier_stack_node node = { const_cast <urlifier *> (&loan), false };
@@ -580,7 +490,7 @@ diagnostic_context::push_borrowed_urlifier (const urlifier &loan)
 }
 
 void
-diagnostic_context::pop_urlifier ()
+context::pop_urlifier ()
 {
   gcc_assert (m_urlifier_stack);
   gcc_assert (m_urlifier_stack->length () > 0);
@@ -590,8 +500,8 @@ diagnostic_context::pop_urlifier ()
     delete node.m_urlifier;
 }
 
-const logical_location_manager *
-diagnostic_context::get_logical_location_manager () const
+const logical_locations::manager *
+context::get_logical_location_manager () const
 {
   if (!m_client_data_hooks)
     return nullptr;
@@ -599,7 +509,7 @@ diagnostic_context::get_logical_location_manager () const
 }
 
 const urlifier *
-diagnostic_context::get_urlifier () const
+context::get_urlifier () const
 {
   if (!m_urlifier_stack)
     return nullptr;
@@ -613,7 +523,7 @@ diagnostic_context::get_urlifier () const
    Refresh all output sinks.  */
 
 void
-diagnostic_context::set_pretty_printer (std::unique_ptr<pretty_printer> pp)
+context::set_pretty_printer (std::unique_ptr<pretty_printer> pp)
 {
   delete m_reference_printer;
   m_reference_printer = pp.release ();
@@ -623,81 +533,102 @@ diagnostic_context::set_pretty_printer (std::unique_ptr<pretty_printer> pp)
 /* Give all output sinks a chance to rebuild their pretty_printer.  */
 
 void
-diagnostic_context::refresh_output_sinks ()
+context::refresh_output_sinks ()
 {
-  for (auto sink : m_output_sinks)
-    sink->update_printer ();
+  for (auto sink_ : m_sinks)
+    sink_->update_printer ();
 }
 
 /* Set FORMAT_DECODER on the reference printer and on the pretty_printer
    of all output sinks.  */
 
 void
-diagnostic_context::set_format_decoder (printer_fn format_decoder)
+context::set_format_decoder (printer_fn format_decoder)
 {
   pp_format_decoder (m_reference_printer) = format_decoder;
-  for (auto sink : m_output_sinks)
-    pp_format_decoder (sink->get_printer ()) = format_decoder;
+  for (auto sink_ : m_sinks)
+    pp_format_decoder (sink_->get_printer ()) = format_decoder;
 }
 
 void
-diagnostic_context::set_show_highlight_colors (bool val)
+context::set_show_highlight_colors (bool val)
 {
   pp_show_highlight_colors (m_reference_printer) = val;
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      pp_show_highlight_colors (sink->get_printer ()) = val;
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      pp_show_highlight_colors (sink_->get_printer ()) = val;
 }
 
 void
-diagnostic_context::set_prefixing_rule (diagnostic_prefixing_rule_t rule)
+context::set_prefixing_rule (diagnostic_prefixing_rule_t rule)
 {
   pp_prefixing_rule (m_reference_printer) = rule;
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      pp_prefixing_rule (sink->get_printer ()) = rule;
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      pp_prefixing_rule (sink_->get_printer ()) = rule;
 }
 
 void
-diagnostic_context::create_edit_context ()
+context::initialize_fixits_change_set ()
 {
-  delete m_edit_context_ptr;
+  delete m_fixits_change_set;
   gcc_assert (m_file_cache);
-  m_edit_context_ptr = new edit_context (*m_file_cache);
+  m_fixits_change_set = new changes::change_set (*m_file_cache);
 }
 
+} // namespace diagnostics
+
 /* Initialize DIAGNOSTIC, where the message MSG has already been
    translated.  */
 void
-diagnostic_set_info_translated (diagnostic_info *diagnostic, const char *msg,
-				va_list *args, rich_location *richloc,
-				diagnostic_t kind)
+diagnostic_set_info_translated (diagnostics::diagnostic_info *diagnostic,
+				const char *msg, va_list *args,
+				rich_location *richloc,
+				enum diagnostics::kind kind)
 {
   gcc_assert (richloc);
-  diagnostic->message.m_err_no = errno;
-  diagnostic->message.m_args_ptr = args;
-  diagnostic->message.m_format_spec = msg;
-  diagnostic->message.m_richloc = richloc;
-  diagnostic->richloc = richloc;
-  diagnostic->metadata = nullptr;
-  diagnostic->kind = kind;
-  diagnostic->option_id = 0;
+  diagnostic->m_message.m_err_no = errno;
+  diagnostic->m_message.m_args_ptr = args;
+  diagnostic->m_message.m_format_spec = msg;
+  diagnostic->m_message.m_richloc = richloc;
+  diagnostic->m_richloc = richloc;
+  diagnostic->m_metadata = nullptr;
+  diagnostic->m_kind = kind;
+  diagnostic->m_option_id = 0;
 }
 
 /* Initialize DIAGNOSTIC, where the message GMSGID has not yet been
    translated.  */
 void
-diagnostic_set_info (diagnostic_info *diagnostic, const char *gmsgid,
-		     va_list *args, rich_location *richloc,
-		     diagnostic_t kind)
+diagnostic_set_info (diagnostics::diagnostic_info *diagnostic,
+		     const char *gmsgid, va_list *args,
+		     rich_location *richloc,
+		     enum diagnostics::kind kind)
 {
   gcc_assert (richloc);
   diagnostic_set_info_translated (diagnostic, _(gmsgid), args, richloc, kind);
 }
 
+namespace diagnostics {
+
+static const char *const diagnostic_kind_text[] = {
+#define DEFINE_DIAGNOSTIC_KIND(K, T, C) (T),
+#include "diagnostics/kinds.def"
+#undef DEFINE_DIAGNOSTIC_KIND
+  "must-not-happen"
+};
+
+/* Get unlocalized string describing KIND.  */
+
+const char *
+get_text_for_kind (enum kind kind)
+{
+  return diagnostic_kind_text[static_cast<int> (kind)];
+}
+
 static const char *const diagnostic_kind_color[] = {
 #define DEFINE_DIAGNOSTIC_KIND(K, T, C) (C),
-#include "diagnostic.def"
+#include "diagnostics/kinds.def"
 #undef DEFINE_DIAGNOSTIC_KIND
   nullptr
 };
@@ -706,9 +637,9 @@ static const char *const diagnostic_kind_color[] = {
    Result could be nullptr.  */
 
 const char *
-diagnostic_get_color_for_kind (diagnostic_t kind)
+get_color_for_kind (enum kind kind)
 {
-  return diagnostic_kind_color[kind];
+  return diagnostic_kind_color[static_cast<int> (kind)];
 }
 
 /* Given an expanded_location, convert the column (which is in 1-based bytes)
@@ -740,8 +671,7 @@ convert_column_unit (file_cache &fc,
     }
 }
 
-diagnostic_column_policy::
-diagnostic_column_policy (const diagnostic_context &dc)
+column_policy::column_policy (const context &dc)
 : m_file_cache (dc.get_file_cache ()),
   m_column_unit (dc.m_column_unit),
   m_column_origin (dc.m_column_origin),
@@ -753,7 +683,7 @@ diagnostic_column_policy (const diagnostic_context &dc)
    to the requested units and origin.  Return -1 if the column is
    invalid (<= 0).  */
 int
-diagnostic_column_policy::converted_column (expanded_location s) const
+column_policy::converted_column (expanded_location s) const
 {
   int one_based_col = convert_column_unit (m_file_cache,
 					   m_column_unit, m_tabstop, s);
@@ -765,9 +695,9 @@ diagnostic_column_policy::converted_column (expanded_location s) const
 /* Return a string describing a location e.g. "foo.c:42:10".  */
 
 label_text
-diagnostic_column_policy::get_location_text (const expanded_location &s,
-					     bool show_column,
-					     bool colorize) const
+column_policy::get_location_text (const expanded_location &s,
+				  bool show_column,
+				  bool colorize) const
 {
   const char *locus_cs = colorize_start (colorize, "locus");
   const char *locus_ce = colorize_stop (colorize);
@@ -786,35 +716,22 @@ diagnostic_column_policy::get_location_text (const expanded_location &s,
 						 line_col, locus_ce));
 }
 
-diagnostic_location_print_policy::
-diagnostic_location_print_policy (const diagnostic_context &dc)
+location_print_policy::
+location_print_policy (const context &dc)
 : m_column_policy (dc),
   m_show_column (dc.m_show_column)
 {
 }
 
-diagnostic_location_print_policy::
-diagnostic_location_print_policy (const diagnostic_text_output_format &text_output)
+location_print_policy::
+location_print_policy (const text_sink &text_output)
 :
   m_column_policy (text_output.get_context ()),
   m_show_column (text_output.get_context ().m_show_column)
 {
 }
 
-static const char *const diagnostic_kind_text[] = {
-#define DEFINE_DIAGNOSTIC_KIND(K, T, C) (T),
-#include "diagnostic.def"
-#undef DEFINE_DIAGNOSTIC_KIND
-  "must-not-happen"
-};
-
-/* Get unlocalized string describing KIND.  */
-
-const char *
-get_diagnostic_kind_text (diagnostic_t kind)
-{
-  return diagnostic_kind_text[kind];
-}
+} // namespace diagnostics
 
 /* Functions at which to stop the backtrace print.  It's not
    particularly helpful to print the callers of these functions.  */
@@ -840,10 +757,10 @@ bt_callback (void *data, uintptr_t pc, const char *filename, int lineno,
   if (filename == nullptr && function == nullptr)
     return 0;
 
-  /* Skip functions in diagnostic.cc.  */
+  /* Skip functions in context.cc.  */
   if (*pcount == 0
       && filename != nullptr
-      && strcmp (lbasename (filename), "diagnostic.cc") == 0)
+      && strcmp (lbasename (filename), "context.cc") == 0)
     return 0;
 
   /* Print up to 20 functions.  We could make this a --param, but
@@ -909,19 +826,21 @@ bt_err_callback (void *data ATTRIBUTE_UNUSED, const char *msg, int errnum)
 	   errnum == 0 ? "" : xstrerror (errnum));
 }
 
+namespace diagnostics {
+
 /* Check if we've met the maximum error limit, and if so fatally exit
    with a message.
-   FLUSH indicates whether a diagnostic_context::finish call is needed.  */
+   FLUSH indicates whether a diagnostics::context::finish call is needed.  */
 
 void
-diagnostic_context::check_max_errors (bool flush)
+context::check_max_errors (bool flush)
 {
   if (!m_max_errors)
     return;
 
-  int count = (diagnostic_count (DK_ERROR)
-	       + diagnostic_count (DK_SORRY)
-	       + diagnostic_count (DK_WERROR));
+  int count = (diagnostic_count (kind::error)
+	       + diagnostic_count (kind::sorry)
+	       + diagnostic_count (kind::werror));
 
   if (count >= m_max_errors)
     {
@@ -938,18 +857,18 @@ diagnostic_context::check_max_errors (bool flush)
    is written out.  This function does not always return.  */
 
 void
-diagnostic_context::action_after_output (diagnostic_t diag_kind)
+context::action_after_output (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_DEBUG:
-    case DK_NOTE:
-    case DK_ANACHRONISM:
-    case DK_WARNING:
+    case kind::debug:
+    case kind::note:
+    case kind::anachronism:
+    case kind::warning:
       break;
 
-    case DK_ERROR:
-    case DK_SORRY:
+    case kind::error:
+    case kind::sorry:
       if (m_abort_on_error)
 	real_abort ();
       if (m_fatal_errors)
@@ -960,8 +879,8 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
 	}
       break;
 
-    case DK_ICE:
-    case DK_ICE_NOBT:
+    case kind::ice:
+    case kind::ice_nobt:
       {
 	/* Attempt to ensure that any outputs are flushed e.g. that .sarif
 	   files are written out.
@@ -974,7 +893,7 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
 	  }
 
 	struct backtrace_state *state = nullptr;
-	if (diag_kind == DK_ICE)
+	if (diag_kind == kind::ice)
 	  state = backtrace_create_state (nullptr, 0, bt_err_callback, nullptr);
 	int count = 0;
 	if (state != nullptr)
@@ -999,7 +918,7 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
 	exit (ICE_EXIT_CODE);
       }
 
-    case DK_FATAL:
+    case kind::fatal:
       if (m_abort_on_error)
 	real_abort ();
       fnotice (stderr, "compilation terminated.\n");
@@ -1015,7 +934,7 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
    its future children if any.  */
 
 void
-diagnostic_context::inhibit_notes_in_group (bool inhibit)
+context::inhibit_notes_in_group (bool inhibit)
 {
   int curr_depth = (m_diagnostic_groups.m_group_nesting_depth
 		    + m_diagnostic_groups.m_diagnostic_nesting_level);
@@ -1045,7 +964,7 @@ diagnostic_context::inhibit_notes_in_group (bool inhibit)
 /* Return whether notes must be inhibited in the current diagnostic_group.  */
 
 bool
-diagnostic_context::notes_inhibited_in_group () const
+context::notes_inhibited_in_group () const
 {
   if (m_diagnostic_groups.m_inhibiting_notes_from
       && (m_diagnostic_groups.m_group_nesting_depth
@@ -1055,84 +974,32 @@ diagnostic_context::notes_inhibited_in_group () const
   return false;
 }
 
-/* class logical_location_manager.  */
+/* class diagnostics::logical_locations::manager.  */
 
 /* Return true iff this is a function or method.  */
 
 bool
-logical_location_manager::function_p (key k) const
+logical_locations::manager::function_p (key k) const
 {
   switch (get_kind (k))
     {
     default:
       gcc_unreachable ();
-    case logical_location_kind::unknown:
-    case logical_location_kind::module_:
-    case logical_location_kind::namespace_:
-    case logical_location_kind::type:
-    case logical_location_kind::return_type:
-    case logical_location_kind::parameter:
-    case logical_location_kind::variable:
+    case kind::unknown:
+    case kind::module_:
+    case kind::namespace_:
+    case kind::type:
+    case kind::return_type:
+    case kind::parameter:
+    case kind::variable:
       return false;
 
-    case logical_location_kind::function:
-    case logical_location_kind::member:
+    case kind::function:
+    case kind::member:
       return true;
     }
 }
 
-/* Interface to specify diagnostic kind overrides.  Returns the
-   previous setting, or DK_UNSPECIFIED if the parameters are out of
-   range.  If OPTION_ID is zero, the new setting is for all the
-   diagnostics.  */
-diagnostic_t
-diagnostic_option_classifier::
-classify_diagnostic (const diagnostic_context *context,
-		     diagnostic_option_id option_id,
-		     diagnostic_t new_kind,
-		     location_t where)
-{
-  diagnostic_t old_kind;
-
-  if (option_id.m_idx < 0
-      || option_id.m_idx >= m_n_opts
-      || new_kind >= DK_LAST_DIAGNOSTIC_KIND)
-    return DK_UNSPECIFIED;
-
-  old_kind = m_classify_diagnostic[option_id.m_idx];
-
-  /* Handle pragmas separately, since we need to keep track of *where*
-     the pragmas were.  */
-  if (where != UNKNOWN_LOCATION)
-    {
-      unsigned i;
-
-      /* Record the command-line status, so we can reset it back on DK_POP. */
-      if (old_kind == DK_UNSPECIFIED)
-	{
-	  old_kind = (!context->option_enabled_p (option_id)
-		      ? DK_IGNORED : DK_ANY);
-	  m_classify_diagnostic[option_id.m_idx] = old_kind;
-	}
-
-      diagnostic_classification_change_t *p;
-      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
-	if (p->option == option_id.m_idx)
-	  {
-	    old_kind = p->kind;
-	    break;
-	  }
-
-      diagnostic_classification_change_t v
-	= { where, option_id.m_idx, new_kind };
-      m_classification_history.safe_push (v);
-    }
-  else
-    m_classify_diagnostic[option_id.m_idx] = new_kind;
-
-  return old_kind;
-}
-
 /* Helper function for print_parseable_fixits.  Print TEXT to PP, obeying the
    escaping rules for -fdiagnostics-parseable-fixits.  */
 
@@ -1223,7 +1090,7 @@ print_parseable_fixits (file_cache &fc,
 /* Update the inlining info in this context for a DIAGNOSTIC.  */
 
 void
-diagnostic_context::get_any_inlining_info (diagnostic_info *diagnostic)
+context::get_any_inlining_info (diagnostic_info *diagnostic)
 {
   auto &ilocs = diagnostic->m_iinfo.m_ilocs;
 
@@ -1242,58 +1109,6 @@ diagnostic_context::get_any_inlining_info (diagnostic_info *diagnostic)
     }
 }
 
-/* Update the kind of DIAGNOSTIC based on its location(s), including
-   any of those in its inlining stack, relative to any
-     #pragma GCC diagnostic
-   directives recorded within this object.
-
-   Return the new kind of DIAGNOSTIC if it was updated, or DK_UNSPECIFIED
-   otherwise.  */
-
-diagnostic_t
-diagnostic_option_classifier::
-update_effective_level_from_pragmas (diagnostic_info *diagnostic) const
-{
-  if (m_classification_history.is_empty ())
-    return DK_UNSPECIFIED;
-
-  /* Iterate over the locations, checking the diagnostic disposition
-     for the diagnostic at each.  If it's explicitly set as opposed
-     to unspecified, update the disposition for this instance of
-     the diagnostic and return it.  */
-  for (location_t loc: diagnostic->m_iinfo.m_ilocs)
-    {
-      /* FIXME: Stupid search.  Optimize later. */
-      unsigned int i;
-      diagnostic_classification_change_t *p;
-      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
-	{
-	  location_t pragloc = p->location;
-	  if (!linemap_location_before_p (line_table, pragloc, loc))
-	    continue;
-
-	  if (p->kind == (int) DK_POP)
-	    {
-	      /* Move on to the next region.  */
-	      i = p->option;
-	      continue;
-	    }
-
-	  diagnostic_option_id option = p->option;
-	  /* The option 0 is for all the diagnostics.  */
-	  if (option == 0 || option == diagnostic->option_id)
-	    {
-	      diagnostic_t kind = p->kind;
-	      if (kind != DK_UNSPECIFIED)
-		diagnostic->kind = kind;
-	      return kind;
-	    }
-	}
-    }
-
-  return DK_UNSPECIFIED;
-}
-
 /* Generate a URL string describing CWE.  The caller is responsible for
    freeing the string.  */
 
@@ -1307,80 +1122,79 @@ get_cwe_url (int cwe)
    as appropriate for #pragma GCC diagnostic and -Werror=foo.  */
 
 bool
-diagnostic_context::diagnostic_enabled (diagnostic_info *diagnostic)
+context::diagnostic_enabled (diagnostic_info *diagnostic)
 {
   /* Update the inlining stack for this diagnostic.  */
   get_any_inlining_info (diagnostic);
 
   /* Diagnostics with no option or -fpermissive are always enabled.  */
-  if (!diagnostic->option_id.m_idx
-      || diagnostic->option_id == m_opt_permissive)
+  if (!diagnostic->m_option_id.m_idx
+      || diagnostic->m_option_id == m_opt_permissive)
     return true;
 
   /* This tests if the user provided the appropriate -Wfoo or
      -Wno-foo option.  */
-  if (!option_enabled_p (diagnostic->option_id))
+  if (!option_enabled_p (diagnostic->m_option_id))
     return false;
 
   /* This tests for #pragma diagnostic changes.  */
-  diagnostic_t diag_class
+  enum kind diag_class
     = m_option_classifier.update_effective_level_from_pragmas (diagnostic);
 
   /* This tests if the user provided the appropriate -Werror=foo
      option.  */
-  if (diag_class == DK_UNSPECIFIED
-      && !option_unspecified_p (diagnostic->option_id))
+  if (diag_class == kind::unspecified
+      && !option_unspecified_p (diagnostic->m_option_id))
     {
-      const diagnostic_t new_kind
-	= m_option_classifier.get_current_override (diagnostic->option_id);
-      if (new_kind != DK_ANY)
-	/* DK_ANY means the diagnostic is not to be ignored, but we don't want
-	   to change it specifically to DK_ERROR or DK_WARNING; we want to
+      const enum kind new_kind
+	= m_option_classifier.get_current_override (diagnostic->m_option_id);
+      if (new_kind != kind::any)
+	/* kind::any means the diagnostic is not to be ignored, but we don't want
+	   to change it specifically to kind::error or kind::warning; we want to
 	   preserve whatever the caller has specified.  */
-	diagnostic->kind = new_kind;
+	diagnostic->m_kind = new_kind;
     }
 
   /* This allows for future extensions, like temporarily disabling
      warnings for ranges of source code.  */
-  if (diagnostic->kind == DK_IGNORED)
+  if (diagnostic->m_kind == kind::ignored)
     return false;
 
   return true;
 }
 
-/* Returns whether warning OPTION_ID is enabled at LOC.  */
+/* Returns whether warning OPT_ID is enabled at LOC.  */
 
 bool
-diagnostic_context::warning_enabled_at (location_t loc,
-					diagnostic_option_id option_id)
+context::warning_enabled_at (location_t loc,
+			     option_id opt_id)
 {
   if (!diagnostic_report_warnings_p (this, loc))
     return false;
 
   rich_location richloc (line_table, loc);
   diagnostic_info diagnostic = {};
-  diagnostic.option_id = option_id;
-  diagnostic.richloc = &richloc;
-  diagnostic.message.m_richloc = &richloc;
-  diagnostic.kind = DK_WARNING;
+  diagnostic.m_option_id = opt_id;
+  diagnostic.m_richloc = &richloc;
+  diagnostic.m_message.m_richloc = &richloc;
+  diagnostic.m_kind = kind::warning;
   return diagnostic_enabled (&diagnostic);
 }
 
 /* Emit a diagnostic within a diagnostic group on this context.  */
 
 bool
-diagnostic_context::
-emit_diagnostic_with_group (diagnostic_t kind,
-			    rich_location &richloc,
-			    const diagnostic_metadata *metadata,
-			    diagnostic_option_id option_id,
-			    const char *gmsgid, ...)
+context::emit_diagnostic_with_group (enum kind kind,
+				     rich_location &richloc,
+				     const metadata *metadata,
+				     option_id opt_id,
+				     const char *gmsgid, ...)
 {
   begin_group ();
 
   va_list ap;
   va_start (ap, gmsgid);
-  bool ret = emit_diagnostic_with_group_va (kind, richloc, metadata, option_id,
+  bool ret = emit_diagnostic_with_group_va (kind, richloc, metadata, opt_id,
 					    gmsgid, &ap);
   va_end (ap);
 
@@ -1392,16 +1206,15 @@ emit_diagnostic_with_group (diagnostic_t kind,
 /* As above, but taking a va_list *.  */
 
 bool
-diagnostic_context::
-emit_diagnostic_with_group_va (diagnostic_t kind,
-			       rich_location &richloc,
-			       const diagnostic_metadata *metadata,
-			       diagnostic_option_id option_id,
-			       const char *gmsgid, va_list *ap)
+context::emit_diagnostic_with_group_va (enum kind kind,
+					rich_location &richloc,
+					const metadata *metadata,
+					option_id opt_id,
+					const char *gmsgid, va_list *ap)
 {
   begin_group ();
 
-  bool ret = diagnostic_impl (&richloc, metadata, option_id,
+  bool ret = diagnostic_impl (&richloc, metadata, opt_id,
 			      gmsgid, ap, kind);
 
   end_group ();
@@ -1410,15 +1223,15 @@ emit_diagnostic_with_group_va (diagnostic_t kind,
 }
 
 /* Report a diagnostic message (an error or a warning) as specified by
-   this diagnostic_context.
+   this diagnostics::context.
    front-end independent format specifiers are exactly those described
    in the documentation of output_format.
    Return true if a diagnostic was printed, false otherwise.  */
 
 bool
-diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
+context::report_diagnostic (diagnostic_info *diagnostic)
 {
-  diagnostic_t orig_diag_kind = diagnostic->kind;
+  enum kind orig_diag_kind = diagnostic->m_kind;
 
   /* Every call to report_diagnostic should be within a
      begin_group/end_group pair so that output formats can reliably
@@ -1427,8 +1240,8 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 
   /* Give preference to being able to inhibit warnings, before they
      get reclassified to something else.  */
-  bool was_warning = (diagnostic->kind == DK_WARNING
-		      || diagnostic->kind == DK_PEDWARN);
+  bool was_warning = (diagnostic->m_kind == kind::warning
+		      || diagnostic->m_kind == kind::pedwarn);
   if (was_warning && m_inhibit_warnings)
     {
       inhibit_notes_in_group ();
@@ -1438,15 +1251,15 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
   if (m_adjust_diagnostic_info)
     m_adjust_diagnostic_info (this, diagnostic);
 
-  if (diagnostic->kind == DK_PEDWARN)
+  if (diagnostic->m_kind == kind::pedwarn)
     {
-      diagnostic->kind = m_pedantic_errors ? DK_ERROR : DK_WARNING;
+      diagnostic->m_kind = m_pedantic_errors ? kind::error : kind::warning;
 
       /* We do this to avoid giving the message for -pedantic-errors.  */
-      orig_diag_kind = diagnostic->kind;
+      orig_diag_kind = diagnostic->m_kind;
     }
 
-  if (diagnostic->kind == DK_NOTE && m_inhibit_notes_p)
+  if (diagnostic->m_kind == kind::note && m_inhibit_notes_p)
     return false;
 
   /* If the user requested that warnings be treated as errors, so be
@@ -1454,10 +1267,10 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
      individual warnings can be overridden back to warnings with
      -Wno-error=*.  */
   if (m_warning_as_error_requested
-      && diagnostic->kind == DK_WARNING)
-    diagnostic->kind = DK_ERROR;
+      && diagnostic->m_kind == kind::warning)
+    diagnostic->m_kind = kind::error;
 
-  diagnostic->message.m_data = &diagnostic->x_data;
+  diagnostic->m_message.m_data = &diagnostic->m_x_data;
 
   /* Check to see if the diagnostic is enabled at the location and
      not disabled by #pragma GCC diagnostic anywhere along the inlining
@@ -1468,7 +1281,7 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
       return false;
     }
 
-  if ((was_warning || diagnostic->kind == DK_WARNING)
+  if ((was_warning || diagnostic->m_kind == kind::warning)
       && ((!m_warn_system_headers
 	   && diagnostic->m_iinfo.m_allsyslocs)
 	  || m_inhibit_warnings))
@@ -1476,11 +1289,11 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
        inlining stack (if there is one) are in system headers.  */
     return false;
 
-  if (diagnostic->kind == DK_NOTE && notes_inhibited_in_group ())
+  if (diagnostic->m_kind == kind::note && notes_inhibited_in_group ())
     /* Bail for all the notes in the diagnostic_group that started to inhibit notes.  */
     return false;
 
-  if (diagnostic->kind != DK_NOTE && diagnostic->kind != DK_ICE)
+  if (diagnostic->m_kind != kind::note && diagnostic->m_kind != kind::ice)
     check_max_errors (false);
 
   if (m_lock > 0)
@@ -1488,7 +1301,8 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
       /* If we're reporting an ICE in the middle of some other error,
 	 try to flush out the previous error, then let this one
 	 through.  Don't do this more than once.  */
-      if ((diagnostic->kind == DK_ICE || diagnostic->kind == DK_ICE_NOBT)
+      if ((diagnostic->m_kind == kind::ice
+	   || diagnostic->m_kind == kind::ice_nobt)
 	  && m_lock == 1)
 	pp_newline_and_flush (m_reference_printer);
       else
@@ -1500,14 +1314,14 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 
   m_lock++;
 
-  if (diagnostic->kind == DK_ICE || diagnostic->kind == DK_ICE_NOBT)
+  if (diagnostic->m_kind == kind::ice || diagnostic->m_kind == kind::ice_nobt)
     {
       /* When not checking, ICEs are converted to fatal errors when an
 	 error has already occurred.  This is counteracted by
 	 abort_on_error.  */
       if (!CHECKING_P
-	  && (diagnostic_count (DK_ERROR) > 0
-	      || diagnostic_count (DK_SORRY) > 0)
+	  && (diagnostic_count (kind::error) > 0
+	      || diagnostic_count (kind::sorry) > 0)
 	  && !m_abort_on_error)
 	{
 	  expanded_location s
@@ -1518,32 +1332,32 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 	}
       if (m_internal_error)
 	(*m_internal_error) (this,
-			     diagnostic->message.m_format_spec,
-			     diagnostic->message.m_args_ptr);
+			     diagnostic->m_message.m_format_spec,
+			     diagnostic->m_message.m_args_ptr);
     }
 
   /* Increment the counter for the appropriate diagnostic kind, either
      within this context, or within the diagnostic_buffer.  */
   {
-    const diagnostic_t kind_for_count =
-      ((diagnostic->kind == DK_ERROR && orig_diag_kind == DK_WARNING)
-       ? DK_WERROR
-       : diagnostic->kind);
-    diagnostic_counters &counters
+    const enum kind kind_for_count =
+      ((diagnostic->m_kind == kind::error && orig_diag_kind == kind::warning)
+       ? kind::werror
+       : diagnostic->m_kind);
+    counters &cs
       = (m_diagnostic_buffer
 	 ? m_diagnostic_buffer->m_diagnostic_counters
 	 : m_diagnostic_counters);
-    ++counters.m_count_for_kind[kind_for_count];
+    ++cs.m_count_for_kind[static_cast<size_t> (kind_for_count)];
   }
 
   /* Is this the initial diagnostic within the stack of groups?  */
   if (m_diagnostic_groups.m_emission_count == 0)
-    for (auto sink : m_output_sinks)
-      sink->on_begin_group ();
+    for (auto sink_ : m_sinks)
+      sink_->on_begin_group ();
   m_diagnostic_groups.m_emission_count++;
 
-  va_list *orig_args = diagnostic->message.m_args_ptr;
-  for (auto sink : m_output_sinks)
+  va_list *orig_args = diagnostic->m_message.m_args_ptr;
+  for (auto sink_ : m_sinks)
     {
       /* Formatting the message is done per-output-format,
 	 so that each output format gets its own set of pp_token_lists
@@ -1562,13 +1376,13 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 	 so that each has its own set to consume.  */
       va_list copied_args;
       va_copy (copied_args, *orig_args);
-      diagnostic->message.m_args_ptr = &copied_args;
-      pp_format (sink->get_printer (), &diagnostic->message);
+      diagnostic->m_message.m_args_ptr = &copied_args;
+      pp_format (sink_->get_printer (), &diagnostic->m_message);
       va_end (copied_args);
 
       /* Call vfunc in the output format.  This is responsible for
 	 phase 3 of formatting, and for printing the result.  */
-      sink->on_report_diagnostic (*diagnostic, orig_diag_kind);
+      sink_->on_report_diagnostic (*diagnostic, orig_diag_kind);
     }
 
   switch (m_extra_output_kind)
@@ -1577,53 +1391,60 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
       break;
     case EXTRA_DIAGNOSTIC_OUTPUT_fixits_v1:
       print_parseable_fixits (get_file_cache (),
-			      m_reference_printer, diagnostic->richloc,
+			      m_reference_printer, diagnostic->m_richloc,
 			      DIAGNOSTICS_COLUMN_UNIT_BYTE,
 			      m_tabstop);
       pp_flush (m_reference_printer);
       break;
     case EXTRA_DIAGNOSTIC_OUTPUT_fixits_v2:
       print_parseable_fixits (get_file_cache (),
-			      m_reference_printer, diagnostic->richloc,
+			      m_reference_printer, diagnostic->m_richloc,
 			      DIAGNOSTICS_COLUMN_UNIT_DISPLAY,
 			      m_tabstop);
       pp_flush (m_reference_printer);
       break;
     }
   if (m_diagnostic_buffer == nullptr
-      || diagnostic->kind == DK_ICE
-      || diagnostic->kind == DK_ICE_NOBT)
-    action_after_output (diagnostic->kind);
-  diagnostic->x_data = nullptr;
+      || diagnostic->m_kind == kind::ice
+      || diagnostic->m_kind == kind::ice_nobt)
+    action_after_output (diagnostic->m_kind);
+  diagnostic->m_x_data = nullptr;
 
-  if (m_edit_context_ptr)
-    if (diagnostic->richloc->fixits_can_be_auto_applied_p ())
+  if (m_fixits_change_set)
+    if (diagnostic->m_richloc->fixits_can_be_auto_applied_p ())
       if (!m_diagnostic_buffer)
-	m_edit_context_ptr->add_fixits (diagnostic->richloc);
+	m_fixits_change_set->add_fixits (diagnostic->m_richloc);
 
   m_lock--;
 
   if (!m_diagnostic_buffer)
-    for (auto sink : m_output_sinks)
-      sink->after_diagnostic (*diagnostic);
+    for (auto sink_ : m_sinks)
+      sink_->after_diagnostic (*diagnostic);
 
   return true;
 }
 
 void
-diagnostic_context::report_verbatim (text_info &text)
+context::report_verbatim (text_info &text)
 {
   va_list *orig_args = text.m_args_ptr;
-  for (auto sink : m_output_sinks)
+  for (auto sink_ : m_sinks)
     {
       va_list copied_args;
       va_copy (copied_args, *orig_args);
       text.m_args_ptr = &copied_args;
-      sink->on_report_verbatim (text);
+      sink_->on_report_verbatim (text);
       va_end (copied_args);
     }
 }
 
+void
+context::report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+{
+  for (auto sink_ : m_sinks)
+    sink_->report_global_digraph (ldg);
+}
+
 /* Get the number of digits in the decimal representation of VALUE.  */
 
 int
@@ -1645,6 +1466,8 @@ num_digits (int value)
   return digits;
 }
 
+} // namespace diagnostics
+
 /* Given a partial pathname as input, return another pathname that
    shares no directory elements with the pathname of __FILE__.  This
    is used by fancy_abort() to print `internal compiler error in expr.cc'
@@ -1675,43 +1498,48 @@ trim_filename (const char *name)
   return p;
 }
 
+namespace diagnostics {
+
 /* Implement emit_diagnostic, inform, warning, warning_at, pedwarn,
    permerror, error, error_at, error_at, sorry, fatal_error, internal_error,
    and internal_error_no_backtrace, as documented and defined below.  */
 bool
-diagnostic_context::diagnostic_impl (rich_location *richloc,
-				     const diagnostic_metadata *metadata,
-				     diagnostic_option_id option_id,
-				     const char *gmsgid,
-				     va_list *ap, diagnostic_t kind)
+context::diagnostic_impl (rich_location *richloc,
+			  const metadata *metadata,
+			  option_id opt_id,
+			  const char *gmsgid,
+			  va_list *ap, enum kind kind)
 {
   diagnostic_info diagnostic;
-  if (kind == DK_PERMERROR)
+  if (kind == diagnostics::kind::permerror)
     {
       diagnostic_set_info (&diagnostic, gmsgid, ap, richloc,
-			   m_permissive ? DK_WARNING : DK_ERROR);
-      diagnostic.option_id = (option_id.m_idx != -1 ? option_id : m_opt_permissive);
+			   (m_permissive
+			    ? diagnostics::kind::warning
+			    : diagnostics::kind::error));
+      diagnostic.m_option_id = (opt_id.m_idx != -1 ? opt_id : m_opt_permissive);
     }
   else
     {
       diagnostic_set_info (&diagnostic, gmsgid, ap, richloc, kind);
-      if (kind == DK_WARNING || kind == DK_PEDWARN)
-	diagnostic.option_id = option_id;
+      if (kind == diagnostics::kind::warning
+	  || kind == diagnostics::kind::pedwarn)
+	diagnostic.m_option_id = opt_id;
     }
-  diagnostic.metadata = metadata;
+  diagnostic.m_metadata = metadata;
   return report_diagnostic (&diagnostic);
 }
 
 /* Implement inform_n, warning_n, and error_n, as documented and
    defined below.  */
 bool
-diagnostic_context::diagnostic_n_impl (rich_location *richloc,
-				       const diagnostic_metadata *metadata,
-				       diagnostic_option_id option_id,
-				       unsigned HOST_WIDE_INT n,
-				       const char *singular_gmsgid,
-				       const char *plural_gmsgid,
-				       va_list *ap, diagnostic_t kind)
+context::diagnostic_n_impl (rich_location *richloc,
+			    const metadata *metadata,
+			    option_id opt_id,
+			    unsigned HOST_WIDE_INT n,
+			    const char *singular_gmsgid,
+			    const char *plural_gmsgid,
+			    va_list *ap, enum kind kind)
 {
   diagnostic_info diagnostic;
   unsigned long gtn;
@@ -1726,9 +1554,9 @@ diagnostic_context::diagnostic_n_impl (rich_location *richloc,
 
   const char *text = ngettext (singular_gmsgid, plural_gmsgid, gtn);
   diagnostic_set_info_translated (&diagnostic, text, ap, richloc, kind);
-  if (kind == DK_WARNING)
-    diagnostic.option_id = option_id;
-  diagnostic.metadata = metadata;
+  if (kind == diagnostics::kind::warning)
+    diagnostic.m_option_id = opt_id;
+  diagnostic.m_metadata = metadata;
   return report_diagnostic (&diagnostic);
 }
 
@@ -1736,13 +1564,13 @@ diagnostic_context::diagnostic_n_impl (rich_location *richloc,
 /* Emit DIAGRAM to this context, respecting the output format.  */
 
 void
-diagnostic_context::emit_diagram (const diagnostic_diagram &diagram)
+context::emit_diagram (const diagram &diag)
 {
   if (m_diagrams.m_theme == nullptr)
     return;
 
-  for (auto sink : m_output_sinks)
-    sink->on_diagram (diagram);
+  for (auto sink_ : m_sinks)
+    sink_->on_diagram (diag);
 }
 
 /* Inform the user that an error occurred while trying to report some
@@ -1751,7 +1579,7 @@ diagnostic_context::emit_diagram (const diagnostic_diagram &diagram)
    This mustn't use internal_error, that will cause infinite recursion.  */
 
 void
-diagnostic_context::error_recursion ()
+context::error_recursion ()
 {
   if (m_lock < 3)
     pp_newline_and_flush (m_reference_printer);
@@ -1761,13 +1589,15 @@ diagnostic_context::error_recursion ()
 
   /* Call action_after_output to get the "please submit a bug report"
      message.  */
-  action_after_output (DK_ICE);
+  action_after_output (kind::ice);
 
   /* Do not use gcc_unreachable here; that goes through internal_error
      and therefore would cause infinite recursion.  */
   real_abort ();
 }
 
+} // namespace diagnostics
+
 /* Report an internal compiler error in a friendly manner.  This is
    the function that gets called upon use of abort() in the source
    code generally, thanks to a special macro.  */
@@ -1787,7 +1617,7 @@ fancy_abort (const char *file, int line, const char *function)
   if (global_dc->get_reference_printer () == nullptr)
     {
       /* Print the error message.  */
-      fnotice (stderr, diagnostic_kind_text[DK_ICE]);
+      fnotice (stderr, diagnostics::get_text_for_kind (diagnostics::kind::ice));
       fnotice (stderr, "in %s, at %s:%d", function, trim_filename (file), line);
       fputc ('\n', stderr);
 
@@ -1810,16 +1640,18 @@ fancy_abort (const char *file, int line, const char *function)
   internal_error ("in %s, at %s:%d", function, trim_filename (file), line);
 }
 
-/* class diagnostic_context.  */
+namespace diagnostics {
+
+/* class diagnostics::context.  */
 
 void
-diagnostic_context::begin_group ()
+context::begin_group ()
 {
   m_diagnostic_groups.m_group_nesting_depth++;
 }
 
 void
-diagnostic_context::end_group ()
+context::end_group ()
 {
   if (--m_diagnostic_groups.m_group_nesting_depth == 0)
     {
@@ -1827,8 +1659,8 @@ diagnostic_context::end_group ()
 	 If any diagnostics were emitted, give the context a chance
 	 to do something.  */
       if (m_diagnostic_groups.m_emission_count > 0)
-	for (auto sink : m_output_sinks)
-	  sink->on_end_group ();
+	for (auto sink_ : m_sinks)
+	  sink_->on_end_group ();
       m_diagnostic_groups.m_emission_count = 0;
     }
   /* We're popping one level, so might need to stop inhibiting notes.  */
@@ -1836,13 +1668,13 @@ diagnostic_context::end_group ()
 }
 
 void
-diagnostic_context::push_nesting_level ()
+context::push_nesting_level ()
 {
   ++m_diagnostic_groups.m_diagnostic_nesting_level;
 }
 
 void
-diagnostic_context::pop_nesting_level ()
+context::pop_nesting_level ()
 {
   --m_diagnostic_groups.m_diagnostic_nesting_level;
   /* We're popping one level, so might need to stop inhibiting notes.  */
@@ -1850,29 +1682,35 @@ diagnostic_context::pop_nesting_level ()
 }
 
 void
-diagnostic_output_format::dump (FILE *out, int indent) const
+context::set_nesting_level (int new_level)
+{
+  m_diagnostic_groups.m_diagnostic_nesting_level = new_level;
+}
+
+void
+sink::dump (FILE *out, int indent) const
 {
   fprintf (out, "%*sprinter:\n", indent, "");
   m_printer->dump (out, indent + 2);
 }
 
 void
-diagnostic_output_format::on_report_verbatim (text_info &)
+sink::on_report_verbatim (text_info &)
 {
   /* No-op.  */
 }
 
-/* Set the output format for CONTEXT to FORMAT, using BASE_FILE_NAME for
+/* Set the output format for DC to FORMAT, using BASE_FILE_NAME for
    file-based output formats.  */
 
 void
-diagnostic_output_format_init (diagnostic_context &context,
-			       const char *main_input_filename_,
-			       const char *base_file_name,
-			       enum diagnostics_output_format format,
-			       bool json_formatting)
+output_format_init (context &dc,
+		    const char *main_input_filename_,
+		    const char *base_file_name,
+		    enum diagnostics_output_format format,
+		    bool json_formatting)
 {
-  diagnostic_output_format *new_sink = nullptr;
+  sink *new_sink = nullptr;
   switch (format)
     {
     default:
@@ -1882,16 +1720,16 @@ diagnostic_output_format_init (diagnostic_context &context,
       break;
 
     case DIAGNOSTICS_OUTPUT_FORMAT_SARIF_STDERR:
-      new_sink = &diagnostic_output_format_init_sarif_stderr (context,
-							      line_table,
-							      json_formatting);
+      new_sink = &init_sarif_stderr (dc,
+				     line_table,
+				     json_formatting);
       break;
 
     case DIAGNOSTICS_OUTPUT_FORMAT_SARIF_FILE:
-      new_sink = &diagnostic_output_format_init_sarif_file (context,
-							line_table,
-							    json_formatting,
-							    base_file_name);
+      new_sink = &init_sarif_file (dc,
+				   line_table,
+				   json_formatting,
+				   base_file_name);
       break;
     }
   if (new_sink)
@@ -1903,8 +1741,7 @@ diagnostic_output_format_init (diagnostic_context &context,
    (or nullptr for "no diagrams").  */
 
 void
-diagnostic_context::
-set_text_art_charset (enum diagnostic_text_art_charset charset)
+context::set_text_art_charset (enum diagnostic_text_art_charset charset)
 {
   delete m_diagrams.m_theme;
   switch (charset)
@@ -1930,95 +1767,24 @@ set_text_art_charset (enum diagnostic_text_art_charset charset)
     }
 }
 
-/* If BUFFER is non-null, use BUFFER as the active diagnostic_buffer on
-   this context.  BUFFER is borrowed.
-
-   If BUFFER is null, stop any buffering on this context until the next call
-   to this function.  */
-
-void
-diagnostic_context::set_diagnostic_buffer (diagnostic_buffer *buffer)
-{
-  /* We don't allow changing buffering within a diagnostic group
-     (to simplify handling of buffered diagnostics within the
-     diagnostic_format implementations).  */
-  gcc_assert (m_diagnostic_groups.m_group_nesting_depth == 0);
-
-  /* Likewise, for simplicity, we only allow changing buffers
-     at nesting level 0.  */
-  gcc_assert (m_diagnostic_groups.m_diagnostic_nesting_level == 0);
-
-  m_diagnostic_buffer = buffer;
-
-  if (buffer)
-    {
-      buffer->ensure_per_format_buffers ();
-      gcc_assert (buffer->m_per_format_buffers);
-      gcc_assert (buffer->m_per_format_buffers->length ()
-		  == m_output_sinks.length ());
-      for (unsigned idx = 0; idx < m_output_sinks.length (); ++idx)
-	{
-	  auto sink = m_output_sinks[idx];
-	  auto per_format_buffer = (*buffer->m_per_format_buffers)[idx];
-	  sink->set_buffer (per_format_buffer);
-	}
-    }
-  else
-    for (auto sink : m_output_sinks)
-      sink->set_buffer (nullptr);
-}
-
-/* Clear BUFFER without flushing it.  */
+/* struct diagnostics::counters.  */
 
-void
-diagnostic_context::clear_diagnostic_buffer (diagnostic_buffer &buffer)
-{
-  if (buffer.m_per_format_buffers)
-    for (auto per_format_buffer : *buffer.m_per_format_buffers)
-      per_format_buffer->clear ();
-
-  buffer.m_diagnostic_counters.clear ();
-
-  /* We need to reset last_location, otherwise we may skip caret lines
-     when we actually give a diagnostic.  */
-  m_last_location = UNKNOWN_LOCATION;
-}
-
-/* Flush the diagnostics in BUFFER to this context, clearing BUFFER.  */
-
-void
-diagnostic_context::flush_diagnostic_buffer (diagnostic_buffer &buffer)
-{
-  bool had_errors
-    = (buffer.m_diagnostic_counters.m_count_for_kind[DK_ERROR] > 0
-       || buffer.m_diagnostic_counters.m_count_for_kind[DK_WERROR] > 0);
-  if (buffer.m_per_format_buffers)
-    for (auto per_format_buffer : *buffer.m_per_format_buffers)
-      per_format_buffer->flush ();
-  buffer.m_diagnostic_counters.move_to (m_diagnostic_counters);
-
-  action_after_output (had_errors ? DK_ERROR : DK_WARNING);
-  check_max_errors (true);
-}
-
-/* struct diagnostic_counters.  */
-
-diagnostic_counters::diagnostic_counters ()
+counters::counters ()
 {
   clear ();
 }
 
 void
-diagnostic_counters::dump (FILE *out, int indent) const
+counters::dump (FILE *out, int indent) const
 {
   fprintf (out, "%*scounts:\n", indent, "");
   bool none = true;
-  for (int i = 0; i < DK_LAST_DIAGNOSTIC_KIND; i++)
+  for (int i = 0; i < static_cast<int> (kind::last_diagnostic_kind); i++)
     if (m_count_for_kind[i] > 0)
       {
 	fprintf (out, "%*s%s%i\n",
 		 indent + 2, "",
-		 get_diagnostic_kind_text (static_cast<diagnostic_t> (i)),
+		 get_text_for_kind (static_cast<enum kind> (i)),
 		 m_count_for_kind[i]);
 	none = false;
       }
@@ -2027,133 +1793,31 @@ diagnostic_counters::dump (FILE *out, int indent) const
 }
 
 void
-diagnostic_counters::move_to (diagnostic_counters &dest)
+counters::move_to (counters &dest)
 {
-  for (int i = 0; i < DK_LAST_DIAGNOSTIC_KIND; i++)
+  for (int i = 0; i < static_cast<int> (kind::last_diagnostic_kind); i++)
     dest.m_count_for_kind[i] += m_count_for_kind[i];
   clear ();
 }
 
 void
-diagnostic_counters::clear ()
+counters::clear ()
 {
   memset (&m_count_for_kind, 0, sizeof m_count_for_kind);
 }
 
-/* class diagnostic_buffer.  */
-
-diagnostic_buffer::diagnostic_buffer (diagnostic_context &ctxt)
-: m_ctxt (ctxt),
-  m_per_format_buffers (nullptr)
-{
-}
-
-diagnostic_buffer::~diagnostic_buffer ()
-{
-  if (m_per_format_buffers)
-    {
-      for (auto iter : *m_per_format_buffers)
-	delete iter;
-      delete m_per_format_buffers;
-    }
-}
-
-void
-diagnostic_buffer::dump (FILE *out, int indent) const
-{
-  m_diagnostic_counters.dump (out, indent + 2);
-  fprintf (out, "%*sm_per_format_buffers:\n", indent, "");
-  if (m_per_format_buffers)
-    for (auto per_format_buffer : *m_per_format_buffers)
-      per_format_buffer->dump (out, indent + 2);
-  else
-    fprintf (out, "%*s(none)\n", indent + 2, "");
-}
-
-bool
-diagnostic_buffer::empty_p () const
-{
-  if (m_per_format_buffers)
-    for (auto per_format_buffer : *m_per_format_buffers)
-      /* Query initial buffer.  */
-      return per_format_buffer->empty_p ();
-  return true;
-}
-
-void
-diagnostic_buffer::move_to (diagnostic_buffer &dest)
-{
-  /* Bail if there's nothing to move.  */
-  if (!m_per_format_buffers)
-    return;
-
-  m_diagnostic_counters.move_to (dest.m_diagnostic_counters);
-
-  if (!dest.m_per_format_buffers)
-    {
-      /* Optimization for the "move to empty" case:
-	 simply move the vec to the dest.  */
-      dest.m_per_format_buffers = m_per_format_buffers;
-      m_per_format_buffers = nullptr;
-      return;
-    }
-
-  dest.ensure_per_format_buffers ();
-  gcc_assert (m_per_format_buffers);
-  gcc_assert (m_per_format_buffers->length ()
-	      == m_ctxt.m_output_sinks.length ());
-  gcc_assert (dest.m_per_format_buffers);
-  gcc_assert (dest.m_per_format_buffers->length ()
-	      == m_ctxt.m_output_sinks.length ());
-  for (unsigned idx = 0; idx < m_ctxt.m_output_sinks.length (); ++idx)
-    {
-      auto per_format_buffer_src = (*m_per_format_buffers)[idx];
-      auto per_format_buffer_dest = (*dest.m_per_format_buffers)[idx];
-      per_format_buffer_src->move_to (*per_format_buffer_dest);
-    }
-}
-
-/* Lazily get the output formats to create their own kind of buffers.
-   We can't change the output sinks on a context once this has been called
-   on any diagnostic_buffer instances for that context, since there's no
-   way to update all diagnostic_buffer instances for that context.  */
-
-void
-diagnostic_buffer::ensure_per_format_buffers ()
-{
-  if (!m_per_format_buffers)
-    {
-      m_per_format_buffers = new auto_vec<diagnostic_per_format_buffer *> ();
-      for (unsigned idx = 0; idx < m_ctxt.m_output_sinks.length (); ++idx)
-	{
-	  auto sink = m_ctxt.m_output_sinks[idx];
-	  auto per_format_buffer = sink->make_per_format_buffer ();
-	  m_per_format_buffers->safe_push (per_format_buffer.release ());
-	}
-    }
-  gcc_assert (m_per_format_buffers);
-  gcc_assert (m_per_format_buffers->length ()
-	      == m_ctxt.m_output_sinks.length ());
-}
-
-/* Really call the system 'abort'.  This has to go right at the end of
-   this file, so that there are no functions after it that call abort
-   and get the system abort instead of our macro.  */
-#undef abort
-static void
-real_abort (void)
-{
-  abort ();
-}
-
 #if CHECKING_P
 
 namespace selftest {
 
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+
 /* Helper function for test_print_escaped_string.  */
 
 static void
-assert_print_escaped_string (const location &loc, const char *expected_output,
+assert_print_escaped_string (const ::selftest::location &loc,
+			     const char *expected_output,
 			     const char *input)
 {
   pretty_printer pp;
@@ -2338,7 +2002,7 @@ test_print_parseable_fixits_bytes_vs_display_columns ()
 }
 
 /* Verify that
-     diagnostic_column_policy::get_location_text (..., SHOW_COLUMN, ...)
+     diagnostics::column_policy::get_location_text (..., SHOW_COLUMN, ...)
    generates EXPECTED_LOC_TEXT, given FILENAME, LINE, COLUMN, with
    colorization disabled.  */
 
@@ -2350,7 +2014,7 @@ assert_location_text (const char *expected_loc_text,
 		      enum diagnostics_column_unit column_unit
 			= DIAGNOSTICS_COLUMN_UNIT_BYTE)
 {
-  test_diagnostic_context dc;
+  diagnostics::selftest::test_context dc;
   dc.m_column_unit = column_unit;
   dc.m_column_origin = origin;
 
@@ -2361,7 +2025,7 @@ assert_location_text (const char *expected_loc_text,
   xloc.data = nullptr;
   xloc.sysp = false;
 
-  diagnostic_column_policy column_policy (dc);
+  diagnostics::column_policy column_policy (dc);
   label_text actual_loc_text
     = column_policy.get_location_text (xloc, show_column, false);
   ASSERT_STREQ (expected_loc_text, actual_loc_text.get ());
@@ -2388,8 +2052,8 @@ test_get_location_text ()
   assert_location_text ("foo.c:42:", "foo.c", 42, 10, false);
   assert_location_text ("foo.c:", "foo.c", 0, 10, false);
 
-  maybe_line_and_column (INT_MAX, INT_MAX);
-  maybe_line_and_column (INT_MIN, INT_MIN);
+  diagnostics::maybe_line_and_column (INT_MAX, INT_MAX);
+  diagnostics::maybe_line_and_column (INT_MIN, INT_MIN);
 
   {
     /* In order to test display columns vs byte columns, we need to create a
@@ -2452,10 +2116,14 @@ test_num_digits ()
   ASSERT_EQ (8, num_digits (99999999));
 }
 
-/* Run all of the selftests within this file.  */
+/* Run all of the selftests within this file.
+
+   According to https://gcc.gnu.org/pipermail/gcc/2021-November/237703.html
+   there are some language-specific assumptions within these tests, so only
+   run them from C/C++.  */
 
 void
-c_diagnostic_cc_tests ()
+context_cc_tests ()
 {
   test_print_escaped_string ();
   test_print_parseable_fixits_none ();
@@ -2467,10 +2135,24 @@ c_diagnostic_cc_tests ()
   test_num_digits ();
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
 
 #endif /* #if CHECKING_P */
 
+} // namespace diagnostics
+
 #if __GNUC__ >= 10
 #  pragma GCC diagnostic pop
 #endif
+
+static void real_abort (void) ATTRIBUTE_NORETURN;
+
+/* Really call the system 'abort'.  This has to go right at the end of
+   this file, so that there are no functions after it that call abort
+   and get the system abort instead of our macro.  */
+#undef abort
+static void
+real_abort (void)
+{
+  abort ();
+}
diff --git a/gcc/diagnostics/context.h b/gcc/diagnostics/context.h
new file mode 100644
index 0000000..b6ec85c
--- /dev/null
+++ b/gcc/diagnostics/context.h
@@ -0,0 +1,863 @@
+/* Declare diagnostics::context and related types.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_CONTEXT_H
+#define GCC_DIAGNOSTICS_CONTEXT_H
+
+#include "lazily-created.h"
+#include "unique-argv.h"
+#include "diagnostics/option-classifier.h"
+#include "diagnostics/option-id-manager.h"
+#include "diagnostics/context-options.h"
+#include "diagnostics/source-printing-options.h"
+#include "diagnostics/counters.h"
+
+namespace diagnostics {
+
+  namespace changes {
+    class change_set;
+  }
+
+  namespace digraphs { class digraph; }
+
+  namespace logical_locations {
+    class manager;
+  }
+
+  class buffer;
+  class client_data_hooks;
+  class diagram;
+  class sink;
+    class text_sink;
+
+  class source_effect_info;
+
+} // namespace diagnostics
+
+namespace text_art
+{
+  class theme;
+} // namespace text_art
+
+namespace xml
+{
+  class printer;
+} // namespace xml
+
+namespace diagnostics {
+
+/*  Forward declarations.  */
+class context;
+class location_print_policy;
+class source_print_policy;
+
+typedef void (*text_starter_fn) (text_sink &,
+				 const diagnostic_info *);
+
+struct to_text;
+struct to_html;
+
+extern pretty_printer *get_printer (to_text &);
+
+template <typename TextOrHtml>
+using start_span_fn = void (*) (const location_print_policy &,
+				TextOrHtml &text_or_html,
+				expanded_location);
+
+typedef void (*text_finalizer_fn) (text_sink &,
+				   const diagnostic_info *,
+				   enum kind);
+
+/* A bundle of state for determining column numbers in diagnostics
+   (tab stops, whether to start at 0 or 1, etc).
+   Uses a file_cache to handle tabs.  */
+
+class column_policy
+{
+public:
+  column_policy (const context &dc);
+
+  int converted_column (expanded_location s) const;
+
+  label_text get_location_text (const expanded_location &s,
+				bool show_column,
+				bool colorize) const;
+
+  int get_tabstop () const { return m_tabstop; }
+
+private:
+  file_cache &m_file_cache;
+  enum diagnostics_column_unit m_column_unit;
+  int m_column_origin;
+  int m_tabstop;
+};
+
+/* A bundle of state for printing locations within diagnostics
+   (e.g. "FILENAME:LINE:COLUMN"), to isolate the interactions between
+   context and the start_span callbacks.  */
+
+class location_print_policy
+{
+public:
+  location_print_policy (const context &dc);
+  location_print_policy (const text_sink &);
+
+  bool show_column_p () const { return m_show_column; }
+
+  const column_policy &
+  get_column_policy () const { return m_column_policy; }
+
+  void
+  print_text_span_start (const context &dc,
+			 pretty_printer &pp,
+			 const expanded_location &exploc);
+
+  void
+  print_html_span_start (const context &dc,
+			 xml::printer &xp,
+			 const expanded_location &exploc);
+
+private:
+  column_policy m_column_policy;
+  bool m_show_column;
+};
+
+/* Abstract base class for optionally supplying extra tags when writing
+   out annotation labels in HTML output.  */
+
+class html_label_writer
+{
+public:
+  virtual ~html_label_writer () {}
+  virtual void begin_label () = 0;
+  virtual void end_label () = 0;
+};
+
+/* A bundle of state for printing source within a diagnostic,
+   to isolate the interactions between context and the
+   implementation of diagnostic_show_locus.  */
+
+class source_print_policy
+{
+public:
+  source_print_policy (const context &);
+  source_print_policy (const context &,
+		       const source_printing_options &);
+
+  void
+  print (pretty_printer &pp,
+	 const rich_location &richloc,
+	 enum kind diagnostic_kind,
+	 source_effect_info *effect_info) const;
+
+  void
+  print_as_html (xml::printer &xp,
+		 const rich_location &richloc,
+		 enum kind diagnostic_kind,
+		 source_effect_info *effect_info,
+		 html_label_writer *label_writer) const;
+
+  const source_printing_options &
+  get_options () const { return m_options; }
+
+  start_span_fn<to_text>
+  get_text_start_span_fn () const { return m_text_start_span_cb; }
+
+  start_span_fn<to_html>
+  get_html_start_span_fn () const { return m_html_start_span_cb; }
+
+  file_cache &
+  get_file_cache () const { return m_file_cache; }
+
+  enum diagnostics_escape_format
+  get_escape_format () const
+  {
+    return m_escape_format;
+  }
+
+  text_art::theme *
+  get_diagram_theme () const { return m_diagram_theme; }
+
+  const column_policy &get_column_policy () const
+  {
+    return m_location_policy.get_column_policy ();
+  }
+
+  const location_print_policy &get_location_policy () const
+  {
+    return m_location_policy;
+  }
+
+private:
+  const source_printing_options &m_options;
+  location_print_policy m_location_policy;
+  start_span_fn<to_text> m_text_start_span_cb;
+  start_span_fn<to_html> m_html_start_span_cb;
+  file_cache &m_file_cache;
+
+  /* Other data copied from context.  */
+  text_art::theme *m_diagram_theme;
+  enum diagnostics_escape_format m_escape_format;
+};
+
+/* This class encapsulates the state of the diagnostics subsystem
+   as a whole (either directly, or via owned objects of other classes, to
+   avoid global variables).
+
+   It has responsibility for:
+   - being a central place for clients to report diagnostics
+   - reporting those diagnostics to zero or more output sinks
+     (e.g. text vs SARIF)
+   - providing a "dump" member function for a debug dump of the state of
+     the diagnostics subsytem
+   - direct vs buffered diagnostics (see class diagnostics::buffer)
+   - tracking the original argv of the program (for SARIF output)
+   - crash-handling
+
+   It delegates responsibility to various other classes:
+   - the various output sinks (instances of diagnostics::sink
+     subclasses)
+   - formatting of messages (class pretty_printer)
+   - an optional urlifier to inject URLs into formatted messages
+   - counting the number of diagnostics reported of each kind
+     (class diagnostics::counters)
+   - calling out to a option_id_manager to determine if
+     a particular warning is enabled or disabled
+   - tracking pragmas that enable/disable warnings in a range of
+     source code
+   - a cache for use when quoting the user's source code (class file_cache)
+   - a text_art::theme
+   - a diagnostics::changes::change_set for generating patches from fix-it hints
+   - diagnostics::client_data_hooks for metadata.
+
+   Try to avoid adding new responsibilities to this class itself, to avoid
+   the "blob" anti-pattern.  */
+
+class context
+{
+public:
+  /* Give access to m_text_callbacks.  */
+  // FIXME: these need updating
+  friend text_starter_fn &
+  text_starter (context *dc);
+  friend start_span_fn<to_text> &
+  start_span (context *dc);
+  friend text_finalizer_fn &
+  text_finalizer (context *dc);
+
+  friend class source_print_policy;
+  friend class text_sink;
+  friend class buffer;
+
+  typedef void (*set_locations_callback_t) (context *,
+					    diagnostic_info *);
+
+  void initialize (int n_opts);
+  void color_init (int value);
+  void urls_init (int value);
+  void set_pretty_printer (std::unique_ptr<pretty_printer> pp);
+  void refresh_output_sinks ();
+
+  void finish ();
+
+  void dump (FILE *out) const;
+  void DEBUG_FUNCTION dump () const { dump (stderr); }
+
+  bool execution_failed_p () const;
+
+  void set_original_argv (unique_argv original_argv);
+  const char * const *get_original_argv ()
+  {
+    return const_cast<const char * const *> (m_original_argv);
+  }
+
+  void set_set_locations_callback (set_locations_callback_t cb)
+  {
+    m_set_locations_cb = cb;
+  }
+
+  void
+  initialize_input_context (diagnostic_input_charset_callback ccb,
+			    bool should_skip_bom);
+
+  void begin_group ();
+  void end_group ();
+
+  void push_nesting_level ();
+  void pop_nesting_level ();
+  void set_nesting_level (int new_level);
+
+  bool warning_enabled_at (location_t loc, option_id opt_id);
+
+  bool option_unspecified_p (option_id opt_id) const
+  {
+    return m_option_classifier.option_unspecified_p (opt_id);
+  }
+
+  bool emit_diagnostic_with_group (enum kind kind,
+				   rich_location &richloc,
+				   const metadata *metadata,
+				   option_id opt_id,
+				   const char *gmsgid, ...)
+    ATTRIBUTE_GCC_DIAG(6,7);
+  bool emit_diagnostic_with_group_va (enum kind kind,
+				      rich_location &richloc,
+				      const metadata *metadata,
+				      option_id opt_id,
+				      const char *gmsgid, va_list *ap)
+    ATTRIBUTE_GCC_DIAG(6,0);
+
+  bool report_diagnostic (diagnostic_info *);
+  void report_verbatim (text_info &);
+
+  /* Report a directed graph associated with the run as a whole
+     to any sinks that support directed graphs.  */
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &);
+
+  enum kind
+  classify_diagnostic (option_id opt_id,
+		       enum kind new_kind,
+		       location_t where)
+  {
+    return m_option_classifier.classify_diagnostic (this,
+						    opt_id,
+						    new_kind,
+						    where);
+  }
+
+  void push_diagnostics (location_t where ATTRIBUTE_UNUSED)
+  {
+    m_option_classifier.push ();
+  }
+  void pop_diagnostics (location_t where)
+  {
+    m_option_classifier.pop (where);
+  }
+
+  void maybe_show_locus (const rich_location &richloc,
+			 const source_printing_options &opts,
+			 enum kind diagnostic_kind,
+			 pretty_printer &pp,
+			 source_effect_info *effect_info);
+  void maybe_show_locus_as_html (const rich_location &richloc,
+				 const source_printing_options &opts,
+				 enum kind diagnostic_kind,
+				 xml::printer &xp,
+				 source_effect_info *effect_info,
+				 html_label_writer *label_writer);
+
+  void emit_diagram (const diagram &diag);
+
+  /* Various setters for use by option-handling logic.  */
+  void set_sink (std::unique_ptr<sink> sink_);
+  void set_text_art_charset (enum diagnostic_text_art_charset charset);
+  void set_client_data_hooks (std::unique_ptr<client_data_hooks> hooks);
+
+  void push_owned_urlifier (std::unique_ptr<urlifier>);
+  void push_borrowed_urlifier (const urlifier &);
+  void pop_urlifier ();
+
+  void initialize_fixits_change_set ();
+  void set_warning_as_error_requested (bool val)
+  {
+    m_warning_as_error_requested = val;
+  }
+  void set_report_bug (bool val) { m_report_bug = val; }
+  void set_extra_output_kind (enum diagnostics_extra_output_kind kind)
+  {
+    m_extra_output_kind = kind;
+  }
+  void set_show_cwe (bool val) { m_show_cwe = val;  }
+  void set_show_rules (bool val) { m_show_rules = val; }
+  void set_show_highlight_colors (bool val);
+  void set_path_format (enum diagnostic_path_format val)
+  {
+    m_path_format = val;
+  }
+  void set_show_path_depths (bool val) { m_show_path_depths = val; }
+  void set_show_option_requested (bool val) { m_show_option_requested = val; }
+  void set_max_errors (int val) { m_max_errors = val; }
+  void set_escape_format (enum diagnostics_escape_format val)
+  {
+    m_escape_format = val;
+  }
+
+  void set_format_decoder (printer_fn format_decoder);
+  void set_prefixing_rule (diagnostic_prefixing_rule_t rule);
+
+  /* Various accessors.  */
+  bool warning_as_error_requested_p () const
+  {
+    return m_warning_as_error_requested;
+  }
+  bool show_path_depths_p () const { return m_show_path_depths; }
+  sink &get_sink (size_t idx) const;
+  enum diagnostic_path_format get_path_format () const { return m_path_format; }
+  enum diagnostics_escape_format get_escape_format () const
+  {
+    return m_escape_format;
+  }
+
+  file_cache &
+  get_file_cache () const
+  {
+    gcc_assert (m_file_cache);
+    return *m_file_cache;
+  }
+
+  changes::change_set *get_fixits_change_set () const
+  {
+    return m_fixits_change_set;
+  }
+  const client_data_hooks *get_client_data_hooks () const
+  {
+    return m_client_data_hooks;
+  }
+
+  const logical_locations::manager *
+  get_logical_location_manager () const;
+
+  const urlifier *get_urlifier () const;
+
+  text_art::theme *get_diagram_theme () const { return m_diagrams.m_theme; }
+
+  int &diagnostic_count (enum kind kind)
+  {
+    return m_diagnostic_counters.m_count_for_kind[static_cast<size_t> (kind)];
+  }
+  int diagnostic_count (enum kind kind) const
+  {
+    return m_diagnostic_counters.get_count (kind);
+  }
+
+  /* Option-related member functions.  */
+  inline bool option_enabled_p (option_id opt_id) const
+  {
+    if (!m_option_id_mgr)
+      return true;
+    return m_option_id_mgr->option_enabled_p (opt_id);
+  }
+
+  inline char *make_option_name (option_id opt_id,
+				 enum kind orig_diag_kind,
+				 enum kind diag_kind) const
+  {
+    if (!m_option_id_mgr)
+      return nullptr;
+    return m_option_id_mgr->make_option_name (opt_id,
+					      orig_diag_kind,
+					      diag_kind);
+  }
+
+  inline char *make_option_url (option_id opt_id) const
+  {
+    if (!m_option_id_mgr)
+      return nullptr;
+    return m_option_id_mgr->make_option_url (opt_id);
+  }
+
+  void
+  set_option_id_manager (std::unique_ptr<option_id_manager> option_id_mgr,
+			 unsigned lang_mask);
+
+  unsigned get_lang_mask () const
+  {
+    return m_lang_mask;
+  }
+
+  bool diagnostic_impl (rich_location *, const metadata *,
+			option_id, const char *,
+			va_list *, enum kind) ATTRIBUTE_GCC_DIAG(5,0);
+  bool diagnostic_n_impl (rich_location *, const metadata *,
+			  option_id, unsigned HOST_WIDE_INT,
+			  const char *, const char *, va_list *,
+			  enum kind) ATTRIBUTE_GCC_DIAG(7,0);
+
+  int get_diagnostic_nesting_level () const
+  {
+    return m_diagnostic_groups.m_diagnostic_nesting_level;
+  }
+
+  char *build_indent_prefix () const;
+
+  int
+  pch_save (FILE *f)
+  {
+    return m_option_classifier.pch_save (f);
+  }
+
+  int
+  pch_restore (FILE *f)
+  {
+    return m_option_classifier.pch_restore (f);
+  }
+
+
+  void set_diagnostic_buffer (buffer *);
+  buffer *get_diagnostic_buffer () const
+  {
+    return m_diagnostic_buffer;
+  }
+  void clear_diagnostic_buffer (buffer &);
+  void flush_diagnostic_buffer (buffer &);
+
+  std::unique_ptr<pretty_printer> clone_printer () const
+  {
+    return m_reference_printer->clone ();
+  }
+
+  pretty_printer *get_reference_printer () const
+  {
+    return m_reference_printer;
+  }
+
+  void
+  add_sink (std::unique_ptr<sink>);
+
+  void remove_all_output_sinks ();
+
+  bool supports_fnotice_on_stderr_p () const;
+
+  /* Raise SIGABRT on any diagnostic of severity kind::error or higher.  */
+  void
+  set_abort_on_error (bool val)
+  {
+    m_abort_on_error = val;
+  }
+
+  /* Accessor for use in serialization, e.g. by C++ modules.  */
+  auto &
+  get_classification_history ()
+  {
+    return m_option_classifier.m_classification_history;
+  }
+
+  void set_main_input_filename (const char *filename);
+
+  void
+  set_permissive_option (option_id opt_permissive)
+  {
+    m_opt_permissive = opt_permissive;
+  }
+
+  void
+  set_fatal_errors (bool fatal_errors)
+  {
+    m_fatal_errors = fatal_errors;
+  }
+
+  void
+  set_internal_error_callback (void (*cb) (context *,
+					   const char *,
+					   va_list *))
+  {
+    m_internal_error = cb;
+  }
+
+  void
+  set_adjust_diagnostic_info_callback (void (*cb) (context *,
+						   diagnostic_info *))
+  {
+    m_adjust_diagnostic_info = cb;
+  }
+
+  void
+  inhibit_notes () { m_inhibit_notes_p = true; }
+
+  source_printing_options &
+  get_source_printing_options ()
+  {
+    return m_source_printing;
+  }
+  const source_printing_options &
+  get_source_printing_options () const
+  {
+    return m_source_printing;
+  }
+
+  void set_caret_max_width (int value);
+
+private:
+  void error_recursion () ATTRIBUTE_NORETURN;
+
+  bool diagnostic_enabled (diagnostic_info *diagnostic);
+
+  void get_any_inlining_info (diagnostic_info *diagnostic);
+
+  void check_max_errors (bool flush);
+  void action_after_output (enum kind diag_kind);
+
+  /* Data members.
+     Ideally, all of these would be private.  */
+
+private:
+  /* A reference instance of pretty_printer created by the client
+     and owned by the context.  Used for cloning when creating/adding
+     output formats.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  pretty_printer *m_reference_printer;
+
+  /* Cache of source code.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  file_cache *m_file_cache;
+
+  /* The number of times we have issued diagnostics.  */
+  counters m_diagnostic_counters;
+
+  /* True if it has been requested that warnings be treated as errors.  */
+  bool m_warning_as_error_requested;
+
+  /* The number of option indexes that can be passed to warning() et
+     al.  */
+  int m_n_opts;
+
+  /* The stack of sets of overridden diagnostic option severities.  */
+  option_classifier m_option_classifier;
+
+  /* True if we should print any CWE identifiers associated with
+     diagnostics.  */
+  bool m_show_cwe;
+
+  /* True if we should print any rules associated with diagnostics.  */
+  bool m_show_rules;
+
+  /* How should diagnostics::paths::path objects be printed.  */
+  enum diagnostic_path_format m_path_format;
+
+  /* True if we should print stack depths when printing diagnostic paths.  */
+  bool m_show_path_depths;
+
+  /* True if we should print the command line option which controls
+     each diagnostic, if known.  */
+  bool m_show_option_requested;
+
+  /* True if we should raise a SIGABRT on errors.  */
+  bool m_abort_on_error;
+
+public:
+  /* True if we should show the column number on diagnostics.  */
+  bool m_show_column;
+
+  /* True if pedwarns are errors.  */
+  bool m_pedantic_errors;
+
+  /* True if permerrors are warnings.  */
+  bool m_permissive;
+
+private:
+  /* The option to associate with turning permerrors into warnings,
+     if any.  */
+  option_id m_opt_permissive;
+
+  /* True if errors are fatal.  */
+  bool m_fatal_errors;
+
+public:
+  /* True if all warnings should be disabled.  */
+  bool m_inhibit_warnings;
+
+  /* True if warnings should be given in system headers.  */
+  bool m_warn_system_headers;
+
+private:
+  /* Maximum number of errors to report.  */
+  int m_max_errors;
+
+  /* Client-supplied callbacks for use in text output.  */
+  struct {
+    /* This function is called before any message is printed out.  It is
+       responsible for preparing message prefix and such.  For example, it
+       might say:
+       In file included from "/usr/local/include/curses.h:5:
+       from "/home/gdr/src/nifty_printer.h:56:
+       ...
+    */
+    text_starter_fn m_begin_diagnostic;
+
+    /* This function is called by diagnostic_show_locus in between
+       disjoint spans of source code, so that the context can print
+       something to indicate that a new span of source code has begun.  */
+    start_span_fn<to_text> m_text_start_span;
+    start_span_fn<to_html> m_html_start_span;
+
+    /* This function is called after the diagnostic message is printed.  */
+    text_finalizer_fn m_end_diagnostic;
+  } m_text_callbacks;
+
+  /* Client hook to report an internal error.  */
+  void (*m_internal_error) (context *, const char *, va_list *);
+
+  /* Client hook to adjust properties of the given diagnostic that we're
+     about to issue, such as its kind.  */
+  void (*m_adjust_diagnostic_info)(context *, diagnostic_info *);
+
+  /* Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  option_id_manager *m_option_id_mgr;
+  unsigned m_lang_mask;
+
+  /* A stack of optional hooks for adding URLs to quoted text strings in
+     diagnostics.  Only used for the main diagnostic message.
+     Typically a single one owner by the context, but can be temporarily
+     overridden by a borrowed urlifier (e.g. on-stack).  */
+  struct urlifier_stack_node
+  {
+    urlifier *m_urlifier;
+    bool m_owned;
+  };
+  auto_vec<urlifier_stack_node> *m_urlifier_stack;
+
+public:
+  /* Auxiliary data for client.  */
+  void *m_client_aux_data;
+
+  /* Used to detect that the last caret was printed at the same location.  */
+  location_t m_last_location;
+
+private:
+  int m_lock;
+
+  bool m_inhibit_notes_p;
+
+  source_printing_options m_source_printing;
+
+  /* True if -freport-bug option is used.  */
+  bool m_report_bug;
+
+  /* Used to specify additional diagnostic output to be emitted after the
+     rest of the diagnostic.  This is for implementing
+     -fdiagnostics-parseable-fixits and GCC_EXTRA_DIAGNOSTIC_OUTPUT.  */
+  enum diagnostics_extra_output_kind m_extra_output_kind;
+
+public:
+  /* What units to use when outputting the column number.  */
+  enum diagnostics_column_unit m_column_unit;
+
+  /* The origin for the column number (1-based or 0-based typically).  */
+  int m_column_origin;
+
+  /* The size of the tabstop for tab expansion.  */
+  int m_tabstop;
+
+private:
+  /* How should non-ASCII/non-printable bytes be escaped when
+     a diagnostic suggests escaping the source code on output.  */
+  enum diagnostics_escape_format m_escape_format;
+
+  /* If non-NULL, a diagnostics::changes::change_set to which fix-it hints
+     should be applied, for generating patches.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  changes::change_set *m_fixits_change_set;
+
+  /* Fields relating to diagnostic groups.  */
+  struct {
+    /* How many diagnostic_group instances are currently alive.  */
+    int m_group_nesting_depth;
+
+    /* How many nesting levels have been pushed within this group.  */
+    int m_diagnostic_nesting_level;
+
+    /* How many diagnostics have been emitted since the bottommost
+       diagnostic_group was pushed.  */
+    int m_emission_count;
+
+    /* The "group+diagnostic" nesting depth from which to inhibit notes.  */
+    int m_inhibiting_notes_from;
+  } m_diagnostic_groups;
+
+  void inhibit_notes_in_group (bool inhibit = true);
+  bool notes_inhibited_in_group () const;
+
+  /* The various sinks to which diagnostics are to be outputted
+     (text vs structured formats such as SARIF).
+     The sinks are owned by the context; this would be a
+     std::vector<std::unique_ptr> if context had a
+     proper ctor.  */
+  auto_vec<sink *> m_sinks;
+
+  /* Callback to set the locations of call sites along the inlining
+     stack corresponding to a diagnostic location.  Needed to traverse
+     the BLOCK_SUPERCONTEXT() chain hanging off the LOCATION_BLOCK()
+     of a diagnostic's location.  */
+  set_locations_callback_t m_set_locations_cb;
+
+  /* A bundle of hooks for providing data to the context about its client
+     e.g. version information, plugins, etc.
+     Used by SARIF output to give metadata about the client that's
+     producing diagnostics.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  client_data_hooks *m_client_data_hooks;
+
+  /* Support for diagrams.  */
+  struct
+  {
+    /* Theme to use when generating diagrams.
+       Can be NULL (if text art is disabled).
+       Owned by the context; this would be a std::unique_ptr if
+       context had a proper ctor.  */
+    text_art::theme *m_theme;
+
+  } m_diagrams;
+
+  /* Owned by the context.  */
+  char **m_original_argv;
+
+  /* Borrowed pointer to the active diagnostics::buffer, if any.
+     If null (the default), then diagnostics that are reported to the
+     context are immediately issued to the output format.
+     If non-null, then diagnostics that are reported to the context
+     are buffered in the buffer, and may be issued to the output format
+     later (if the buffer is flushed), moved to other buffers, or
+     discarded (if the buffer is cleared).  */
+  buffer *m_diagnostic_buffer;
+};
+
+/* Client supplied function to announce a diagnostic
+   (for text-based diagnostic output).  */
+inline text_starter_fn &
+text_starter (context *dc)
+{
+  return dc->m_text_callbacks.m_begin_diagnostic;
+}
+
+/* Client supplied function called between disjoint spans of source code,
+   so that the context can print
+   something to indicate that a new span of source code has begun.  */
+inline start_span_fn<to_text> &
+start_span (context *dc)
+{
+  return dc->m_text_callbacks.m_text_start_span;
+}
+
+/* Client supplied function called after a diagnostic message is
+   displayed (for text-based diagnostic output).  */
+inline text_finalizer_fn &
+text_finalizer (context *dc)
+{
+  return dc->m_text_callbacks.m_end_diagnostic;
+}
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_CONTEXT_H */
diff --git a/gcc/diagnostics/counters.h b/gcc/diagnostics/counters.h
new file mode 100644
index 0000000..42db3fe
--- /dev/null
+++ b/gcc/diagnostics/counters.h
@@ -0,0 +1,51 @@
+/* Counts of per-kind diagnostics.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_COUNTERS_H
+#define GCC_DIAGNOSTICS_COUNTERS_H
+
+#include "diagnostics/kinds.h"
+
+namespace diagnostics {
+
+/* A collection of counters of diagnostics, per-kind
+   (e.g. "3 errors and 1 warning"), for use by both diagnostics::context
+   and by diagnostics::buffer.  */
+
+struct counters
+{
+  counters ();
+
+  void dump (FILE *out, int indent) const;
+  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
+
+  int get_count (enum kind kind) const
+  {
+    return m_count_for_kind[static_cast<size_t> (kind)];
+  }
+
+  void move_to (counters &dest);
+  void clear ();
+
+  int m_count_for_kind[static_cast<size_t> (kind::last_diagnostic_kind)];
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_COUNTERS_H */
diff --git a/gcc/diagnostics/diagnostic-info.h b/gcc/diagnostics/diagnostic-info.h
new file mode 100644
index 0000000..052fef5
--- /dev/null
+++ b/gcc/diagnostics/diagnostic-info.h
@@ -0,0 +1,75 @@
+/* Various declarations for language-independent diagnostics subroutines.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_DIAGNOSTIC_INFO_H
+#define GCC_DIAGNOSTICS_DIAGNOSTIC_INFO_H
+
+namespace diagnostics {
+
+class metadata;
+
+/* A diagnostic is described by the MESSAGE to send, the FILE and LINE of
+   its context and its KIND (ice, error, warning, note, ...)  See complete
+   list in diagnostics/kinds.def.  */
+
+struct diagnostic_info
+{
+  diagnostic_info ()
+  : m_message (),
+    m_richloc (),
+    m_metadata (),
+    m_x_data (),
+    m_kind (),
+    m_option_id (),
+    m_iinfo ()
+  { }
+
+  /* Text to be formatted.  */
+  text_info m_message;
+
+  /* The location at which the diagnostic is to be reported.  */
+  rich_location *m_richloc;
+
+  /* An optional bundle of metadata associated with the diagnostic
+     (or NULL).  */
+  const metadata *m_metadata;
+
+  /* Auxiliary data for client.  */
+  void *m_x_data;
+  /* The kind of diagnostic it is about.  */
+  kind m_kind;
+  /* Which OPT_* directly controls this diagnostic.  */
+  option_id m_option_id;
+
+  /* Inlining context containing locations for each call site along
+     the inlining stack.  */
+  struct inlining_info
+  {
+    /* Locations along the inlining stack.  */
+    auto_vec<location_t, 8> m_ilocs;
+    /* The abstract origin of the location.  */
+    void *m_ao;
+    /* Set if every M_ILOCS element is in a system header.  */
+    bool m_allsyslocs;
+  } m_iinfo;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_DIAGNOSTIC_INFO_H */
diff --git a/gcc/diagnostics/diagnostics-selftests.cc b/gcc/diagnostics/diagnostics-selftests.cc
new file mode 100644
index 0000000..94a212a
--- /dev/null
+++ b/gcc/diagnostics/diagnostics-selftests.cc
@@ -0,0 +1,59 @@
+/* Selftest support for diagnostics.
+   Copyright (C) 2019-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "selftest.h"
+#include "diagnostics/diagnostics-selftests.h"
+
+#if CHECKING_P
+
+namespace diagnostics {
+
+namespace selftest {
+
+/* Run all diagnostics-specific selftests,
+   apart from context_cc_tests, which according to
+   https://gcc.gnu.org/pipermail/gcc/2021-November/237703.html
+   has some language-specific assumptions, and thus is run from
+   c_family_tests instead.  */
+
+void
+run_diagnostics_selftests ()
+{
+  color_cc_tests ();
+  file_cache_cc_tests ();
+  source_printing_cc_tests ();
+  html_sink_cc_tests ();
+  sarif_sink_cc_tests ();
+  digraphs_cc_tests ();
+  output_spec_cc_tests ();
+  state_graphs_cc_tests ();
+  lazy_paths_cc_tests ();
+  paths_output_cc_tests ();
+  changes_cc_tests ();
+}
+
+} /* end of namespace diagnostics::selftest.  */
+
+} // namespace diagnostics
+
+#endif /* #if CHECKING_P */
diff --git a/gcc/diagnostics/diagnostics-selftests.h b/gcc/diagnostics/diagnostics-selftests.h
new file mode 100644
index 0000000..994ebad
--- /dev/null
+++ b/gcc/diagnostics/diagnostics-selftests.h
@@ -0,0 +1,55 @@
+/* Selftests for diagnostics.
+   Copyright (C) 2019-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_SELFTESTS_H
+#define GCC_DIAGNOSTICS_SELFTESTS_H
+
+#if CHECKING_P
+
+namespace diagnostics {
+
+namespace selftest {
+
+extern void run_diagnostics_selftests ();
+
+/* Declarations for specific families of tests (by source file within
+   "diagnostics/"), in alphabetical order.  */
+
+extern void changes_cc_tests ();
+extern void color_cc_tests ();
+extern void context_cc_tests ();
+extern void digraphs_cc_tests ();
+extern void file_cache_cc_tests ();
+extern void html_sink_cc_tests ();
+extern void lazy_paths_cc_tests ();
+extern void output_spec_cc_tests ();
+extern void paths_output_cc_tests ();
+extern void sarif_sink_cc_tests ();
+extern void selftest_logical_locations_cc_tests ();
+extern void source_printing_cc_tests ();
+extern void state_graphs_cc_tests ();
+
+} /* end of namespace diagnostics::selftest.  */
+
+} // namespace diagnostics
+
+#endif /* #if CHECKING_P */
+
+#endif /* GCC_DIAGNOSTICS_SELFTESTS_H */
diff --git a/gcc/diagnostic-diagram.h b/gcc/diagnostics/diagram.h
index b1ff9a9..2d33b10 100644
--- a/gcc/diagnostic-diagram.h
+++ b/gcc/diagnostics/diagram.h
@@ -18,22 +18,24 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_DIAGRAM_H
-#define GCC_DIAGNOSTIC_DIAGRAM_H
+#ifndef GCC_DIAGNOSTICS_DIAGRAM_H
+#define GCC_DIAGNOSTICS_DIAGRAM_H
 
 namespace text_art
 {
   class canvas;
 } // namespace text_art
 
+namespace diagnostics {
+
 /* A text art diagram, along with an "alternative text" string
    describing it.  */
 
-class diagnostic_diagram
+class diagram
 {
  public:
-  diagnostic_diagram (const text_art::canvas &canvas,
-		      const char *alt_text)
+  diagram (const text_art::canvas &canvas,
+	   const char *alt_text)
   : m_canvas (canvas),
     m_alt_text (alt_text)
   {
@@ -48,4 +50,6 @@ class diagnostic_diagram
   const char *const m_alt_text;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_DIAGRAM_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_DIAGRAM_H */
diff --git a/gcc/diagnostics/digraphs.cc b/gcc/diagnostics/digraphs.cc
new file mode 100644
index 0000000..4a2ea4f
--- /dev/null
+++ b/gcc/diagnostics/digraphs.cc
@@ -0,0 +1,464 @@
+/* Directed graphs associated with a diagnostic.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#define INCLUDE_ALGORITHM
+#define INCLUDE_MAP
+#define INCLUDE_SET
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+
+#include "graphviz.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/sarif-sink.h"
+
+#include "selftest.h"
+
+using digraph = diagnostics::digraphs::digraph;
+using digraph_node = diagnostics::digraphs::node;
+using digraph_edge = diagnostics::digraphs::edge;
+
+namespace {
+
+class conversion_to_dot
+{
+public:
+  std::unique_ptr<dot::graph>
+  make_dot_graph_from_diagnostic_graph (const digraph &);
+
+  std::unique_ptr<dot::stmt>
+  make_dot_node_from_digraph_node (const digraph_node &);
+
+  std::unique_ptr<dot::edge_stmt>
+  make_dot_edge_from_digraph_edge (const digraph_edge &);
+
+  dot::id
+  get_dot_id_for_node (const digraph_node &);
+
+  bool
+  has_edges_p (const digraph_node &);
+
+private:
+  std::set<const digraph_node *> m_nodes_with_edges;
+  std::map<const digraph_node *, dot::stmt *> m_node_map;
+};
+
+} // anonymous namespace
+
+// class conversion_to_dot
+
+std::unique_ptr<dot::graph>
+conversion_to_dot::
+make_dot_graph_from_diagnostic_graph (const diagnostics::digraphs::digraph &input_graph)
+{
+  auto output_graph = std::make_unique<dot::graph> ();
+
+  if (const char *description = input_graph.get_description ())
+    output_graph->m_stmt_list.add_attr (dot::id ("label"),
+					dot::id (description));
+
+  const int num_nodes = input_graph.get_num_nodes ();
+  const int num_edges = input_graph.get_num_edges ();
+
+  /* Determine which nodes have in-edges and out-edges.  */
+  for (int i = 0; i < num_edges; ++i)
+    {
+      const digraph_edge &input_edge = input_graph.get_edge (i);
+      m_nodes_with_edges.insert (&input_edge.get_src_node ());
+      m_nodes_with_edges.insert (&input_edge.get_dst_node ());
+    }
+
+  for (int i = 0; i < num_nodes; ++i)
+    {
+      const digraph_node &input_node = input_graph.get_node (i);
+      auto dot_node_stmt = make_dot_node_from_digraph_node (input_node);
+      output_graph->m_stmt_list.add_stmt (std::move (dot_node_stmt));
+    }
+
+  for (int i = 0; i < num_edges; ++i)
+    {
+      const digraph_edge &input_edge = input_graph.get_edge (i);
+      auto dot_edge_stmt = make_dot_edge_from_digraph_edge (input_edge);
+      output_graph->m_stmt_list.add_stmt (std::move (dot_edge_stmt));
+    }
+
+  return output_graph;
+}
+
+std::unique_ptr<dot::stmt>
+conversion_to_dot::
+make_dot_node_from_digraph_node (const diagnostics::digraphs::node &input_node)
+{
+  dot::id dot_id (get_dot_id_for_node (input_node));
+
+  /* For now, we can only do either edges or children, not both
+     ...but see https://graphviz.org/docs/attrs/compound/  */
+
+  if (has_edges_p (input_node))
+    {
+      auto output_node
+	= std::make_unique<dot::node_stmt> (std::move (dot_id));
+      m_node_map[&input_node] = output_node.get ();
+      if (const char *label = input_node.get_label ())
+	output_node->set_label (dot::id (label));
+      return output_node;
+    }
+  else
+    {
+      auto output_node = std::make_unique<dot::subgraph> (std::move (dot_id));
+      m_node_map[&input_node] = output_node.get ();
+      if (const char *label = input_node.get_label ())
+	output_node->add_attr (dot::id ("label"), dot::id (label));
+      const int num_children = input_node.get_num_children ();
+      for (int i = 0; i < num_children; ++i)
+	{
+	  const digraph_node &input_child = input_node.get_child (i);
+	  auto dot_child_stmt = make_dot_node_from_digraph_node (input_child);
+	  output_node->m_stmt_list.add_stmt (std::move (dot_child_stmt));
+	}
+      return output_node;
+    }
+}
+
+std::unique_ptr<dot::edge_stmt>
+conversion_to_dot::
+make_dot_edge_from_digraph_edge (const digraph_edge &input_edge)
+{
+  const digraph_node &src_dnode = input_edge.get_src_node ();
+  const digraph_node &dst_dnode = input_edge.get_dst_node ();
+  auto output_edge
+    = std::make_unique<dot::edge_stmt>
+    (get_dot_id_for_node (src_dnode),
+     get_dot_id_for_node (dst_dnode));
+  if (const char *label = input_edge.get_label ())
+    output_edge->set_label (dot::id (label));
+  return output_edge;
+}
+
+dot::id
+conversion_to_dot::get_dot_id_for_node (const digraph_node &input_node)
+{
+  if (has_edges_p (input_node))
+    return input_node.get_id ();
+  else
+    return std::string ("cluster_") + input_node.get_id ();
+}
+
+bool
+conversion_to_dot::has_edges_p (const digraph_node &input_node)
+{
+  return m_nodes_with_edges.find (&input_node) != m_nodes_with_edges.end ();
+}
+
+// class object
+
+const char *
+diagnostics::digraphs::object::
+get_attr (const char *key_prefix, const char *key) const
+{
+  if (!m_property_bag)
+    return nullptr;
+  std::string prefixed_key = std::string (key_prefix) + key;
+  if (json::value *jv = m_property_bag->get (prefixed_key.c_str ()))
+    if (json::string *jstr = jv->dyn_cast_string ())
+      return jstr->get_string ();
+  return nullptr;
+}
+
+void
+diagnostics::digraphs::object::
+set_attr (const char *key_prefix, const char *key, const char *value)
+{
+  set_json_attr (key_prefix, key, std::make_unique<json::string> (value));
+}
+
+void
+diagnostics::digraphs::object::
+set_json_attr (const char *key_prefix, const char *key, std::unique_ptr<json::value> value)
+{
+  std::string prefixed_key = std::string (key_prefix) + key;
+  if (!m_property_bag)
+    m_property_bag = std::make_unique<json::object> ();
+  m_property_bag->set (prefixed_key.c_str (), std::move (value));
+}
+
+// class digraph
+
+DEBUG_FUNCTION void
+diagnostics::digraphs::digraph::dump () const
+{
+  make_json_sarif_graph ()->dump ();
+}
+
+std::unique_ptr<json::object>
+diagnostics::digraphs::digraph::make_json_sarif_graph () const
+{
+  return make_sarif_graph (*this, nullptr, nullptr);
+}
+
+std::unique_ptr<dot::graph>
+diagnostics::digraphs::digraph::make_dot_graph () const
+{
+  conversion_to_dot to_dot;
+  return to_dot.make_dot_graph_from_diagnostic_graph (*this);
+}
+
+std::unique_ptr<diagnostics::digraphs::digraph>
+diagnostics::digraphs::digraph::clone () const
+{
+  auto result = std::make_unique<diagnostics::digraphs::digraph> ();
+
+  if (get_property_bag ())
+    result->set_property_bag (get_property_bag ()->clone_as_object ());
+
+  std::map<diagnostics::digraphs::node *, diagnostics::digraphs::node *> node_mapping;
+
+  for (auto &iter : m_nodes)
+    result->add_node (iter->clone (*result, node_mapping));
+  for (auto &iter : m_edges)
+    result->add_edge (iter->clone (*result, node_mapping));
+
+  return result;
+}
+
+void
+diagnostics::digraphs::digraph::add_edge (const char *id,
+					  node &src_node,
+					  node &dst_node,
+					  const char *label)
+{
+  auto e = std::make_unique<digraph_edge> (*this,
+					   id,
+					   src_node,
+					   dst_node);
+  if (label)
+    e->set_label (label);
+  add_edge (std::move (e));
+}
+
+/* Utility function for edge ids: either use EDGE_ID, or
+   generate a unique one for when we don't care about the name.
+
+   Edges in SARIF "SHALL" have an id that's unique within the graph
+   (SARIF 2.1.0 §3.41.2).  This is so that graph traversals can refer
+   to edges by id (SARIF 2.1.0's §3.43.2 edgeId property).  */
+
+std::string
+diagnostics::digraphs::digraph::make_edge_id (const char *edge_id)
+{
+  /* If we have an id, use it.  */
+  if (edge_id)
+    return edge_id;
+
+  /* Otherwise, generate a unique one of the form "edgeN".  */
+  while (true)
+    {
+      auto candidate (std::string ("edge")
+		      + std::to_string (m_next_edge_id_index++));
+      auto iter = m_id_to_edge_map.find (candidate);
+      if (iter != m_id_to_edge_map.end ())
+	{
+	  // Try again with the next index...
+	  continue;
+	}
+      return candidate;
+    }
+}
+
+// class node
+
+DEBUG_FUNCTION void
+diagnostics::digraphs::node::dump () const
+{
+  to_json_sarif_node ()->dump ();
+}
+
+std::unique_ptr<json::object>
+diagnostics::digraphs::node::to_json_sarif_node () const
+{
+  return make_sarif_node (*this, nullptr, nullptr);
+}
+
+std::unique_ptr<diagnostics::digraphs::node>
+diagnostics::digraphs::node::clone (digraph &new_graph,
+				    std::map<node *, node *> &node_mapping) const
+{
+  auto result
+    = std::make_unique<diagnostics::digraphs::node> (new_graph,
+						     get_id ());
+  node_mapping.insert ({const_cast <node *> (this), result.get ()});
+
+  result->set_logical_loc (m_logical_loc);
+
+  if (get_property_bag ())
+    result->set_property_bag (get_property_bag ()->clone_as_object ());
+
+  for (auto &iter : m_children)
+    result->add_child (iter->clone (new_graph, node_mapping));
+
+  return result;
+}
+
+// class edge
+
+std::unique_ptr<digraph_edge>
+digraph_edge::clone (digraph &new_graph,
+		     const std::map<node *, node *> &node_mapping) const
+{
+  auto iter_new_src = node_mapping.find (&m_src_node);
+  gcc_assert (iter_new_src != node_mapping.end ());
+  auto iter_new_dst = node_mapping.find (&m_dst_node);
+  gcc_assert (iter_new_dst != node_mapping.end ());
+  auto result
+    = std::make_unique<digraph_edge> (new_graph,
+				      m_id.c_str (),
+				      *iter_new_src->second,
+				      *iter_new_dst->second);
+  if (get_property_bag ())
+    result->set_property_bag (get_property_bag ()->clone_as_object ());
+
+  return result;
+}
+
+DEBUG_FUNCTION void
+diagnostics::digraphs::edge::dump () const
+{
+  to_json_sarif_edge ()->dump ();
+}
+
+std::unique_ptr<json::object>
+diagnostics::digraphs::edge::to_json_sarif_edge () const
+{
+  return make_sarif_edge (*this, nullptr);
+}
+
+#if CHECKING_P
+
+namespace diagnostics {
+namespace selftest {
+
+static void
+test_empty_graph ()
+{
+  digraph g;
+
+  {
+    auto sarif = g.make_json_sarif_graph ();
+
+    pretty_printer pp;
+    sarif->print (&pp, true);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("{\"nodes\": [],\n"
+	" \"edges\": []}"));
+  }
+
+  {
+    auto dg = g.make_dot_graph ();
+
+    pretty_printer pp;
+    dot::writer w (pp);
+    dg->print (w);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("digraph {\n"
+	"}\n"));
+  }
+}
+
+static void
+test_simple_graph ()
+{
+#define KEY_PREFIX "/placeholder/"
+  auto g = std::make_unique<digraph> ();
+  g->set_description ("test graph");
+  g->set_attr (KEY_PREFIX, "date", "1066");
+
+  auto a = std::make_unique<digraph_node> (*g, "a");
+  auto b = std::make_unique<digraph_node> (*g, "b");
+  b->set_attr (KEY_PREFIX, "color", "red");
+  auto c = std::make_unique<digraph_node> (*g, "c");
+  c->set_label ("I am a node label");
+
+  auto e = std::make_unique<digraph_edge> (*g, nullptr, *a, *c);
+  e->set_attr (KEY_PREFIX, "status", "copacetic");
+  e->set_label ("I am an edge label");
+  g->add_edge (std::move (e));
+
+  g->add_node (std::move (a));
+
+  b->add_child (std::move (c));
+  g->add_node (std::move (b));
+#undef KEY_PREFIX
+
+  {
+    auto sarif = g->make_json_sarif_graph ();
+
+    pretty_printer pp;
+    sarif->print (&pp, true);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("{\"properties\": {\"/placeholder/date\": \"1066\"},\n"
+	" \"nodes\": [{\"id\": \"a\"},\n"
+	"           {\"id\": \"b\",\n"
+	"            \"properties\": {\"/placeholder/color\": \"red\"},\n"
+	"            \"children\": [{\"id\": \"c\"}]}],\n"
+	" \"edges\": [{\"id\": \"edge0\",\n"
+	"            \"properties\": {\"/placeholder/status\": \"copacetic\"},\n"
+	"            \"sourceNodeId\": \"a\",\n"
+	"            \"targetNodeId\": \"c\"}]}"));
+  }
+
+  {
+    auto dg = g->make_dot_graph ();
+
+    pretty_printer pp;
+    dot::writer w (pp);
+    dg->print (w);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("digraph {\n"
+	"    label=\"test graph\";\n"
+	"    a;\n"
+	"    \n"
+	"    subgraph cluster_b {\n"
+	"        c [label=\"I am a node label\"];\n"
+	"\n"
+	"    };\n"
+	"    a -> c [label=\"I am an edge label\"];\n"
+	"}\n"));
+  }
+}
+
+/* Run all of the selftests within this file.  */
+
+void
+digraphs_cc_tests ()
+{
+  test_empty_graph ();
+  test_simple_graph ();
+}
+
+} // namespace diagnostics::selftest
+} // namespace diagnostics
+
+#endif /* CHECKING_P */
diff --git a/gcc/diagnostics/digraphs.h b/gcc/diagnostics/digraphs.h
new file mode 100644
index 0000000..7193ee4
--- /dev/null
+++ b/gcc/diagnostics/digraphs.h
@@ -0,0 +1,379 @@
+/* Directed graphs associated with a diagnostic.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_DIGRAPHS_H
+#define GCC_DIAGNOSTICS_DIGRAPHS_H
+
+#include "json.h"
+#include "diagnostics/logical-locations.h"
+
+class graphviz_out;
+
+class sarif_graph;
+class sarif_node;
+class sarif_edge;
+
+namespace dot { class graph; }
+
+namespace diagnostics {
+namespace digraphs {
+
+/* A family of classes: digraph, node, and edge, closely related to
+   SARIF's graph, node, and edge types (SARIF v2.1.0 sections 3.39-3.41).
+
+   Nodes can have child nodes, allowing for arbitrarily deep nesting.
+   Edges can be between any pair of nodes (potentially at different
+   nesting levels).
+
+   Digraphs, nodes, and edges also optionally have a JSON property bag,
+   allowing round-tripping of arbitrary key/value pairs through SARIF.  */
+
+class digraph;
+class node;
+class edge;
+
+/* A base class for digraph, node, and edge to allow them to have
+   an optional JSON property bag.  */
+
+class object
+{
+public:
+  const char *
+  get_attr (const char *key_prefix,
+	    const char *key) const;
+
+  void
+  set_attr (const char *key_prefix,
+	    const char *key,
+	    const char *value);
+
+  void
+  set_json_attr (const char *key_prefix,
+		 const char *key,
+		 std::unique_ptr<json::value> value);
+
+  json::object *
+  get_property_bag () const { return m_property_bag.get (); }
+
+  void
+  set_property_bag (std::unique_ptr<json::object> property_bag)
+  {
+    m_property_bag = std::move (property_bag);
+  }
+
+private:
+  std::unique_ptr<json::object> m_property_bag;
+};
+
+// A directed graph, corresponding to SARIF v2.1.0 section 3.39.
+
+class digraph : public object
+{
+ public:
+  friend class node;
+  friend class edge;
+
+  digraph () : m_next_edge_id_index (0) {}
+  virtual ~digraph () {}
+
+  const char *
+  get_description () const
+  {
+    if (!m_description)
+      return nullptr;
+    return m_description->c_str ();
+  }
+
+  void
+  set_description (const char *desc)
+  {
+    if (desc)
+      m_description = std::make_unique<std::string> (desc);
+    else
+      m_description = nullptr;
+  }
+  void
+  set_description (std::string desc)
+  {
+    m_description = std::make_unique<std::string> (std::move (desc));
+  }
+
+  node *
+  get_node_by_id (const char *id) const
+  {
+    auto iter = m_id_to_node_map.find (id);
+    if (iter == m_id_to_node_map.end ())
+      return nullptr;
+    return iter->second;
+  }
+
+  edge *
+  get_edge_by_id (const char *id) const
+  {
+    auto iter = m_id_to_edge_map.find (id);
+    if (iter == m_id_to_edge_map.end ())
+      return nullptr;
+    return iter->second;
+  }
+
+  size_t
+  get_num_nodes () const
+  {
+    return m_nodes.size ();
+  }
+
+  node &
+  get_node (size_t idx) const
+  {
+    return *m_nodes[idx].get ();
+  }
+
+  size_t
+  get_num_edges () const
+  {
+    return m_edges.size ();
+  }
+
+  edge &
+  get_edge (size_t idx) const
+  {
+    return *m_edges[idx].get ();
+  }
+
+  void
+  dump () const;
+
+  std::unique_ptr<json::object>
+  make_json_sarif_graph () const;
+
+  std::unique_ptr<dot::graph>
+  make_dot_graph () const;
+
+  void
+  add_node (std::unique_ptr<node> n)
+  {
+    gcc_assert (n);
+    m_nodes.push_back (std::move (n));
+  }
+
+  void
+  add_edge (std::unique_ptr<edge> e)
+  {
+    gcc_assert (e);
+    m_edges.push_back (std::move (e));
+  }
+
+  void
+  add_edge (const char *id,
+	    node &src_node,
+	    node &dst_node,
+	    const char *label = nullptr);
+
+  std::unique_ptr<digraph> clone () const;
+
+ private:
+  void
+  add_node_id (std::string node_id, node &new_node)
+  {
+    m_id_to_node_map.insert ({std::move (node_id), &new_node});
+  }
+  void
+  add_edge_id (std::string edge_id, edge &new_edge)
+  {
+    m_id_to_edge_map.insert ({std::move (edge_id), &new_edge});
+  }
+
+  std::string
+  make_edge_id (const char *edge_id);
+
+  std::unique_ptr<std::string> m_description;
+  std::map<std::string, node *> m_id_to_node_map;
+  std::map<std::string, edge *> m_id_to_edge_map;
+  std::vector<std::unique_ptr<node>> m_nodes;
+  std::vector<std::unique_ptr<edge>> m_edges;
+  size_t m_next_edge_id_index;
+};
+
+// A node in a directed graph, corresponding to SARIF v2.1.0 section 3.40.
+
+class node : public object
+{
+ public:
+  virtual ~node () {}
+
+  node (digraph &g, std::string id)
+  : m_id (id),
+    m_physical_loc (UNKNOWN_LOCATION)
+  {
+    g.add_node_id (std::move (id), *this);
+  }
+  node (const node &) = delete;
+
+  std::string
+  get_id () const { return m_id; }
+
+  const char *
+  get_label () const
+  {
+    if (!m_label)
+      return nullptr;
+    return m_label->c_str ();
+  }
+
+  void
+  set_label (const char *label)
+  {
+    if (label)
+      m_label = std::make_unique<std::string> (label);
+    else
+      m_label = nullptr;
+  }
+  void
+  set_label (std::string label)
+  {
+    m_label = std::make_unique<std::string> (std::move (label));
+  }
+
+  size_t
+  get_num_children () const { return m_children.size (); }
+
+  node &
+  get_child (size_t idx) const { return *m_children[idx].get (); }
+
+  void
+  add_child (std::unique_ptr<node> child)
+  {
+    gcc_assert (child);
+    m_children.push_back (std::move (child));
+  }
+
+  location_t
+  get_physical_loc () const
+  {
+    return m_physical_loc;
+  }
+
+  void
+  set_physical_loc (location_t physical_loc)
+  {
+    m_physical_loc = physical_loc;
+  }
+
+  logical_locations::key
+  get_logical_loc () const
+  {
+    return m_logical_loc;
+  }
+
+  void
+  set_logical_loc (logical_locations::key logical_loc)
+  {
+    m_logical_loc = logical_loc;
+  }
+
+  void print (graphviz_out &gv) const;
+
+  void
+  dump () const;
+
+  std::unique_ptr<json::object>
+  to_json_sarif_node () const;
+
+  std::unique_ptr<node>
+  clone (digraph &new_graph,
+	 std::map<node *, node *> &node_mapping) const;
+
+ private:
+  std::string m_id;
+  std::unique_ptr<std::string> m_label;
+  std::vector<std::unique_ptr<node>> m_children;
+  location_t m_physical_loc;
+  logical_locations::key m_logical_loc;
+};
+
+// An edge in a directed graph, corresponding to SARIF v2.1.0 section 3.41.
+
+class edge : public object
+{
+ public:
+  virtual ~edge () {}
+
+  /* SARIF requires us to provide unique edge IDs within a graph,
+     but otherwise we don't need them.
+     Pass in nullptr for the id to get the graph to generate a unique
+     edge id for us.  */
+  edge (digraph &g,
+	const char *id,
+	node &src_node,
+	node &dst_node)
+  : m_id (g.make_edge_id (id)),
+    m_src_node (src_node),
+    m_dst_node (dst_node)
+  {
+    g.add_edge_id (m_id, *this);
+  }
+
+  std::string
+  get_id () const { return m_id; }
+
+  const char *
+  get_label () const
+  {
+    if (!m_label)
+      return nullptr;
+    return m_label->c_str ();
+  }
+
+  void
+  set_label (const char *label)
+  {
+    if (label)
+      m_label = std::make_unique<std::string> (label);
+    else
+      m_label = nullptr;
+  }
+
+  node &
+  get_src_node () const { return m_src_node; }
+
+  node &
+  get_dst_node () const { return m_dst_node; }
+
+  void
+  dump () const;
+
+  std::unique_ptr<json::object>
+  to_json_sarif_edge () const;
+
+  std::unique_ptr<edge>
+  clone (digraph &new_graph,
+	 const std::map<diagnostics::digraphs::node *, diagnostics::digraphs::node *> &node_mapping) const;
+
+private:
+  std::string m_id;
+  std::unique_ptr<std::string> m_label;
+  node &m_src_node;
+  node &m_dst_node;
+};
+
+} // namespace digraphs
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_DIGRAPHS_H */
diff --git a/gcc/diagnostic-event-id.h b/gcc/diagnostics/event-id.h
index 7a45a72..167599d 100644
--- a/gcc/diagnostic-event-id.h
+++ b/gcc/diagnostics/event-id.h
@@ -1,4 +1,4 @@
-/* A class for referring to events within a diagnostic_path.
+/* A class for referring to events within a diagnostics::paths::path.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>
 
@@ -18,10 +18,10 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_EVENT_ID_H
-#define GCC_DIAGNOSTIC_EVENT_ID_H
+#ifndef GCC_DIAGNOSTICS_EVENT_ID_H
+#define GCC_DIAGNOSTICS_EVENT_ID_H
 
-/* A class for referring to events within a diagnostic_path.
+/* A class for referring to events within a diagnostics::paths::path.
 
    They are stored as 0-based offsets into the events, but
    printed (e.g. via %@) as 1-based numbers.
@@ -30,8 +30,10 @@ along with GCC; see the file COPYING3.  If not see
    which would be shown to the user as "(1)", "(2)" and "(3)".
 
    This has its own header so that pretty-print.cc can use this
-   to implement "%@" without bringing in all of diagnostic_path
-   (which e.g. refers to "tree").  */
+   to implement "%@" without bringing in all of diagnostics::paths.
+
+   This has to be in the global namespace for compatibility with
+   c-format.cc in GCC 10 onwards.  */
 
 class diagnostic_event_id_t
 {
@@ -58,15 +60,24 @@ class diagnostic_event_id_t
   int m_index; // zero-based
 };
 
+namespace diagnostics {
+namespace paths {
+
+typedef diagnostic_event_id_t event_id_t;
+
+/* A type for compactly referring to a particular thread within a
+   diagnostics::paths::path.  Typically there is just one thread per path,
+   with id 0.  */
+typedef int thread_id_t;
+
+} // namespace paths
+} // namespace diagnostics
+
+
 /* A pointer to a diagnostic_event_id_t, for use with the "%@" format
    code, which will print a 1-based representation for it, with suitable
    colorization, e.g. "(1)".
    The %@ format code requires that known_p be true for the event ID. */
 typedef diagnostic_event_id_t *diagnostic_event_id_ptr;
 
-/* A type for compactly referring to a particular thread within a
-   diagnostic_path.  Typically there is just one thread per path,
-   with id 0.  */
-typedef int diagnostic_thread_id_t;
-
-#endif /* ! GCC_DIAGNOSTIC_EVENT_ID_H */
+#endif /* ! GCC_DIAGNOSTICS_EVENT_ID_H */
diff --git a/gcc/diagnostics/file-cache.cc b/gcc/diagnostics/file-cache.cc
new file mode 100644
index 0000000..febeb03
--- /dev/null
+++ b/gcc/diagnostics/file-cache.cc
@@ -0,0 +1,1083 @@
+/* Caching input files for use by diagnostics.
+   Copyright (C) 2004-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "cpplib.h"
+#include "diagnostics/file-cache.h"
+#include "selftest.h"
+
+#ifndef HAVE_ICONV
+#define HAVE_ICONV 0
+#endif
+
+namespace diagnostics {
+
+/* Input charset configuration.  */
+static const char *default_charset_callback (const char *)
+{
+  return nullptr;
+}
+
+void
+file_cache::initialize_input_context (diagnostic_input_charset_callback ccb,
+				      bool should_skip_bom)
+{
+  m_input_context.ccb = (ccb ? ccb : default_charset_callback);
+  m_input_context.should_skip_bom = should_skip_bom;
+}
+
+/* This is a cache used by get_next_line to store the content of a
+   file to be searched for file lines.  */
+class file_cache_slot
+{
+public:
+  file_cache_slot ();
+  ~file_cache_slot ();
+
+  void dump (FILE *out, int indent) const;
+  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
+
+  bool read_line_num (size_t line_num,
+		      char ** line, ssize_t *line_len);
+
+  /* Accessors.  */
+  const char *get_file_path () const { return m_file_path; }
+  unsigned get_use_count () const { return m_use_count; }
+  bool missing_trailing_newline_p () const
+  {
+    return m_missing_trailing_newline;
+  }
+  char_span get_full_file_content ();
+
+  void inc_use_count () { m_use_count++; }
+
+  bool create (const file_cache::input_context &in_context,
+	       const char *file_path, FILE *fp, unsigned highest_use_count);
+  void evict ();
+  void set_content (const char *buf, size_t sz);
+
+  static size_t tune (size_t line_record_size_)
+  {
+    size_t ret = line_record_size;
+    line_record_size = line_record_size_;
+    return ret;
+  }
+
+ private:
+  /* These are information used to store a line boundary.  */
+  class line_info
+  {
+  public:
+    /* The line number.  It starts from 1.  */
+    size_t line_num;
+
+    /* The position (byte count) of the beginning of the line,
+       relative to the file data pointer.  This starts at zero.  */
+    size_t start_pos;
+
+    /* The position (byte count) of the last byte of the line.  This
+       normally points to the '\n' character, or to one byte after the
+       last byte of the file, if the file doesn't contain a '\n'
+       character.  */
+    size_t end_pos;
+
+    line_info (size_t l, size_t s, size_t e)
+      : line_num (l), start_pos (s), end_pos (e)
+    {}
+
+    line_info ()
+      :line_num (0), start_pos (0), end_pos (0)
+    {}
+
+    static bool less_than(const line_info &a, const line_info &b)
+    {
+      return a.line_num < b.line_num;
+    }
+  };
+
+  bool needs_read_p () const;
+  bool needs_grow_p () const;
+  void maybe_grow ();
+  bool read_data ();
+  bool maybe_read_data ();
+  bool get_next_line (char **line, ssize_t *line_len);
+  bool read_next_line (char ** line, ssize_t *line_len);
+  bool goto_next_line ();
+
+  static const size_t buffer_size = 4 * 1024;
+  static size_t line_record_size;
+  static size_t recent_cached_lines_shift;
+
+  /* The number of time this file has been accessed.  This is used
+     to designate which file cache to evict from the cache
+     array.  */
+  unsigned m_use_count;
+
+  /* The file_path is the key for identifying a particular file in
+     the cache.  This copy is owned by the slot.  */
+  char *m_file_path;
+
+  FILE *m_fp;
+
+  /* True when an read error happened.  */
+  bool m_error;
+
+  /* This points to the content of the file that we've read so
+     far.  */
+  char *m_data;
+
+  /* The allocated buffer to be freed may start a little earlier than DATA,
+     e.g. if a UTF8 BOM was skipped at the beginning.  */
+  int m_alloc_offset;
+
+  /*  The size of the DATA array above.*/
+  size_t m_size;
+
+  /* The number of bytes read from the underlying file so far.  This
+     must be less (or equal) than SIZE above.  */
+  size_t m_nb_read;
+
+  /* The index of the beginning of the current line.  */
+  size_t m_line_start_idx;
+
+  /* The number of the previous line read.  This starts at 1.  Zero
+     means we've read no line so far.  */
+  size_t m_line_num;
+
+  /* Could this file be missing a trailing newline on its final line?
+     Initially true (to cope with empty files), set to true/false
+     as each line is read.  */
+  bool m_missing_trailing_newline;
+
+  /* This is a record of the beginning and end of the lines we've seen
+     while reading the file.  This is useful to avoid walking the data
+     from the beginning when we are asked to read a line that is
+     before LINE_START_IDX above.  When the lines exceed line_record_size
+     this is scaled down dynamically, with the line_info becoming anchors.  */
+  vec<line_info, va_heap> m_line_record;
+
+  /* A cache of the recently seen lines. This is maintained as a ring
+     buffer. */
+  vec<line_info, va_heap> m_line_recent;
+
+  /* First and last valid entry in m_line_recent.  */
+  size_t m_line_recent_last, m_line_recent_first;
+
+  void offset_buffer (int offset)
+  {
+    gcc_assert (offset < 0 ? m_alloc_offset + offset >= 0
+		: (size_t) offset <= m_size);
+    gcc_assert (m_data);
+    m_alloc_offset += offset;
+    m_data += offset;
+    m_size -= offset;
+  }
+
+};
+
+size_t file_cache_slot::line_record_size = 0;
+size_t file_cache_slot::recent_cached_lines_shift = 8;
+
+/* Tune file_cache.  */
+void
+file_cache::tune (size_t num_file_slots, size_t lines)
+{
+  if (file_cache_slot::tune (lines) != lines
+      || m_num_file_slots != num_file_slots)
+    {
+      delete[] m_file_slots;
+      m_file_slots = new file_cache_slot[num_file_slots];
+    }
+  m_num_file_slots = num_file_slots;
+}
+
+static const char *
+find_end_of_line (const char *s, size_t len);
+
+/* Lookup the cache used for the content of a given file accessed by
+   caret diagnostic.  Return the found cached file, or NULL if no
+   cached file was found.  */
+
+file_cache_slot *
+file_cache::lookup_file (const char *file_path)
+{
+  gcc_assert (file_path);
+
+  /* This will contain the found cached file.  */
+  file_cache_slot *r = NULL;
+  for (unsigned i = 0; i < m_num_file_slots; ++i)
+    {
+      file_cache_slot *c = &m_file_slots[i];
+      if (c->get_file_path () && !strcmp (c->get_file_path (), file_path))
+	{
+	  c->inc_use_count ();
+	  r = c;
+	}
+    }
+
+  if (r)
+    r->inc_use_count ();
+
+  return r;
+}
+
+/* Purge any mention of FILENAME from the cache of files used for
+   printing source code.  For use in selftests when working
+   with tempfiles.  */
+
+void
+file_cache::forcibly_evict_file (const char *file_path)
+{
+  gcc_assert (file_path);
+
+  file_cache_slot *r = lookup_file (file_path);
+  if (!r)
+    /* Not found.  */
+    return;
+
+  r->evict ();
+}
+
+/* Determine if FILE_PATH missing a trailing newline on its final line.
+   Only valid to call once all of the file has been loaded, by
+   requesting a line number beyond the end of the file.  */
+
+bool
+file_cache::missing_trailing_newline_p (const char *file_path)
+{
+  gcc_assert (file_path);
+
+  file_cache_slot *r = lookup_or_add_file (file_path);
+  return r->missing_trailing_newline_p ();
+}
+
+void
+file_cache::add_buffered_content (const char *file_path,
+				  const char *buffer,
+				  size_t sz)
+{
+  gcc_assert (file_path);
+
+  file_cache_slot *r = lookup_file (file_path);
+  if (!r)
+    {
+      unsigned highest_use_count = 0;
+      r = evicted_cache_tab_entry (&highest_use_count);
+      if (!r->create (m_input_context, file_path, nullptr, highest_use_count))
+	return;
+    }
+
+  r->set_content (buffer, sz);
+}
+
+void
+file_cache_slot::evict ()
+{
+  free (m_file_path);
+  m_file_path = NULL;
+  if (m_fp)
+    fclose (m_fp);
+  m_error = false;
+  m_fp = NULL;
+  m_nb_read = 0;
+  m_line_start_idx = 0;
+  m_line_num = 0;
+  m_line_record.truncate (0);
+  m_line_recent_first = 0;
+  m_line_recent_last = 0;
+  m_use_count = 0;
+  m_missing_trailing_newline = true;
+}
+
+/* Return the file cache that has been less used, recently, or the
+   first empty one.  If HIGHEST_USE_COUNT is non-null,
+   *HIGHEST_USE_COUNT is set to the highest use count of the entries
+   in the cache table.  */
+
+file_cache_slot*
+file_cache::evicted_cache_tab_entry (unsigned *highest_use_count)
+{
+  file_cache_slot *to_evict = &m_file_slots[0];
+  unsigned huc = to_evict->get_use_count ();
+  for (unsigned i = 1; i < m_num_file_slots; ++i)
+    {
+      file_cache_slot *c = &m_file_slots[i];
+      bool c_is_empty = (c->get_file_path () == NULL);
+
+      if (c->get_use_count () < to_evict->get_use_count ()
+	  || (to_evict->get_file_path () && c_is_empty))
+	/* We evict C because it's either an entry with a lower use
+	   count or one that is empty.  */
+	to_evict = c;
+
+      if (huc < c->get_use_count ())
+	huc = c->get_use_count ();
+
+      if (c_is_empty)
+	/* We've reached the end of the cache; subsequent elements are
+	   all empty.  */
+	break;
+    }
+
+  if (highest_use_count)
+    *highest_use_count = huc;
+
+  return to_evict;
+}
+
+/* Create the cache used for the content of a given file to be
+   accessed by caret diagnostic.  This cache is added to an array of
+   cache and can be retrieved by lookup_file_in_cache_tab.  This
+   function returns the created cache.  Note that only the last
+   m_num_file_slots files are cached.
+
+   This can return nullptr if the FILE_PATH can't be opened for
+   reading, or if the content can't be converted to the input_charset.  */
+
+file_cache_slot*
+file_cache::add_file (const char *file_path)
+{
+
+  FILE *fp = fopen (file_path, "r");
+  if (fp == NULL)
+    return NULL;
+
+  unsigned highest_use_count = 0;
+  file_cache_slot *r = evicted_cache_tab_entry (&highest_use_count);
+  if (!r->create (m_input_context, file_path, fp, highest_use_count))
+    return NULL;
+  return r;
+}
+
+/* Get a borrowed char_span to the full content of this file
+   as decoded according to the input charset, encoded as UTF-8.  */
+
+char_span
+file_cache_slot::get_full_file_content ()
+{
+  char *line;
+  ssize_t line_len;
+  while (get_next_line (&line, &line_len))
+    {
+    }
+  return char_span (m_data, m_nb_read);
+}
+
+/* Populate this slot for use on FILE_PATH and FP, dropping any
+   existing cached content within it.  */
+
+bool
+file_cache_slot::create (const file_cache::input_context &in_context,
+			 const char *file_path, FILE *fp,
+			 unsigned highest_use_count)
+{
+  m_file_path = file_path ? xstrdup (file_path) : nullptr;
+  if (m_fp)
+    fclose (m_fp);
+  m_error = false;
+  m_fp = fp;
+  if (m_alloc_offset)
+    offset_buffer (-m_alloc_offset);
+  m_nb_read = 0;
+  m_line_start_idx = 0;
+  m_line_num = 0;
+  m_line_recent_first = 0;
+  m_line_recent_last = 0;
+  m_line_record.truncate (0);
+  /* Ensure that this cache entry doesn't get evicted next time
+     add_file_to_cache_tab is called.  */
+  m_use_count = ++highest_use_count;
+  m_missing_trailing_newline = true;
+
+
+  /* Check the input configuration to determine if we need to do any
+     transformations, such as charset conversion or BOM skipping.  */
+  if (const char *input_charset = in_context.ccb (file_path))
+    {
+      /* Need a full-blown conversion of the input charset.  */
+      fclose (m_fp);
+      m_fp = NULL;
+      const cpp_converted_source cs
+	= cpp_get_converted_source (file_path, input_charset);
+      if (!cs.data)
+	return false;
+      if (m_data)
+	XDELETEVEC (m_data);
+      m_data = cs.data;
+      m_nb_read = m_size = cs.len;
+      m_alloc_offset = cs.data - cs.to_free;
+    }
+  else if (in_context.should_skip_bom)
+    {
+      if (read_data ())
+	{
+	  const int offset = cpp_check_utf8_bom (m_data, m_nb_read);
+	  offset_buffer (offset);
+	  m_nb_read -= offset;
+	}
+    }
+
+  return true;
+}
+
+void
+file_cache_slot::set_content (const char *buf, size_t sz)
+{
+  m_data = (char *)xmalloc (sz);
+  memcpy (m_data, buf, sz);
+  m_nb_read = m_size = sz;
+  m_alloc_offset = 0;
+
+  if (m_fp)
+    {
+      fclose (m_fp);
+      m_fp = nullptr;
+    }
+}
+
+/* file_cache's ctor.  */
+
+file_cache::file_cache ()
+: m_num_file_slots (16), m_file_slots (new file_cache_slot[m_num_file_slots])
+{
+  initialize_input_context (nullptr, false);
+}
+
+/* file_cache's dtor.  */
+
+file_cache::~file_cache ()
+{
+  delete[] m_file_slots;
+}
+
+void
+file_cache::dump (FILE *out, int indent) const
+{
+  for (size_t i = 0; i < m_num_file_slots; ++i)
+    {
+      fprintf (out, "%*sslot[%i]:\n", indent, "", (int)i);
+      m_file_slots[i].dump (out, indent + 2);
+    }
+}
+
+void
+file_cache::dump () const
+{
+  dump (stderr, 0);
+}
+
+/* Lookup the cache used for the content of a given file accessed by
+   caret diagnostic.  If no cached file was found, create a new cache
+   for this file, add it to the array of cached file and return
+   it.
+
+   This can return nullptr on a cache miss if FILE_PATH can't be opened for
+   reading, or if the content can't be converted to the input_charset.  */
+
+file_cache_slot*
+file_cache::lookup_or_add_file (const char *file_path)
+{
+  file_cache_slot *r = lookup_file (file_path);
+  if (r == NULL)
+    r = add_file (file_path);
+  return r;
+}
+
+/* Default constructor for a cache of file used by caret
+   diagnostic.  */
+
+file_cache_slot::file_cache_slot ()
+: m_use_count (0), m_file_path (NULL), m_fp (NULL), m_error (false), m_data (0),
+  m_alloc_offset (0), m_size (0), m_nb_read (0), m_line_start_idx (0),
+  m_line_num (0), m_missing_trailing_newline (true),
+  m_line_recent_last (0), m_line_recent_first (0)
+{
+  m_line_record.create (0);
+  m_line_recent.create (1U << recent_cached_lines_shift);
+  for (int i = 0; i < 1 << recent_cached_lines_shift; i++)
+    m_line_recent.quick_push (file_cache_slot::line_info (0, 0, 0));
+}
+
+/* Destructor for a cache of file used by caret diagnostic.  */
+
+file_cache_slot::~file_cache_slot ()
+{
+  free (m_file_path);
+  if (m_fp)
+    {
+      fclose (m_fp);
+      m_fp = NULL;
+    }
+  if (m_data)
+    {
+      offset_buffer (-m_alloc_offset);
+      XDELETEVEC (m_data);
+      m_data = 0;
+    }
+  m_line_record.release ();
+  m_line_recent.release ();
+}
+
+void
+file_cache_slot::dump (FILE *out, int indent) const
+{
+  if (!m_file_path)
+    {
+      fprintf (out, "%*s(unused)\n", indent, "");
+      return;
+    }
+  fprintf (out, "%*sfile_path: %s\n", indent, "", m_file_path);
+  fprintf (out, "%*sfp: %p\n", indent, "", (void *)m_fp);
+  fprintf (out, "%*sneeds_read_p: %i\n", indent, "", (int)needs_read_p ());
+  fprintf (out, "%*sneeds_grow_p: %i\n", indent, "", (int)needs_grow_p ());
+  fprintf (out, "%*suse_count: %i\n", indent, "", m_use_count);
+  fprintf (out, "%*ssize: %zi\n", indent, "", m_size);
+  fprintf (out, "%*snb_read: %zi\n", indent, "", m_nb_read);
+  fprintf (out, "%*sstart_line_idx: %zi\n", indent, "", m_line_start_idx);
+  fprintf (out, "%*sline_num: %zi\n", indent, "", m_line_num);
+  fprintf (out, "%*smissing_trailing_newline: %i\n",
+	   indent, "", (int)m_missing_trailing_newline);
+  fprintf (out, "%*sline records (%i):\n",
+	   indent, "", m_line_record.length ());
+  int idx = 0;
+  for (auto &line : m_line_record)
+    fprintf (out, "%*s[%i]: line %zi: byte offsets: %zi-%zi\n",
+	     indent + 2, "",
+	     idx++, line.line_num, line.start_pos, line.end_pos);
+}
+
+/* Returns TRUE iff the cache would need to be filled with data coming
+   from the file.  That is, either the cache is empty or full or the
+   current line is empty.  Note that if the cache is full, it would
+   need to be extended and filled again.  */
+
+bool
+file_cache_slot::needs_read_p () const
+{
+  return m_fp && (m_nb_read == 0
+	  || m_nb_read == m_size
+	  || (m_line_start_idx >= m_nb_read - 1));
+}
+
+/*  Return TRUE iff the cache is full and thus needs to be
+    extended.  */
+
+bool
+file_cache_slot::needs_grow_p () const
+{
+  return m_nb_read == m_size;
+}
+
+/* Grow the cache if it needs to be extended.  */
+
+void
+file_cache_slot::maybe_grow ()
+{
+  if (!needs_grow_p ())
+    return;
+
+  if (!m_data)
+    {
+      gcc_assert (m_size == 0 && m_alloc_offset == 0);
+      m_size = buffer_size;
+      m_data = XNEWVEC (char, m_size);
+    }
+  else
+    {
+      const int offset = m_alloc_offset;
+      offset_buffer (-offset);
+      m_size *= 2;
+      m_data = XRESIZEVEC (char, m_data, m_size);
+      offset_buffer (offset);
+    }
+}
+
+/*  Read more data into the cache.  Extends the cache if need be.
+    Returns TRUE iff new data could be read.  */
+
+bool
+file_cache_slot::read_data ()
+{
+  if (feof (m_fp) || ferror (m_fp))
+    return false;
+
+  maybe_grow ();
+
+  char * from = m_data + m_nb_read;
+  size_t to_read = m_size - m_nb_read;
+  size_t nb_read = fread (from, 1, to_read, m_fp);
+
+  if (ferror (m_fp))
+    {
+      m_error = true;
+      return false;
+    }
+
+  m_nb_read += nb_read;
+  return !!nb_read;
+}
+
+/* Read new data iff the cache needs to be filled with more data
+   coming from the file FP.  Return TRUE iff the cache was filled with
+   mode data.  */
+
+bool
+file_cache_slot::maybe_read_data ()
+{
+  if (!needs_read_p ())
+    return false;
+  return read_data ();
+}
+
+/* Helper function for file_cache_slot::get_next_line (), to find the end of
+   the next line.  Returns with the memchr convention, i.e. nullptr if a line
+   terminator was not found.  We need to determine line endings in the same
+   manner that libcpp does: any of \n, \r\n, or \r is a line ending.  */
+
+static const char *
+find_end_of_line (const char *s, size_t len)
+{
+  for (const auto end = s + len; s != end; ++s)
+    {
+      if (*s == '\n')
+	return s;
+      if (*s == '\r')
+	{
+	  const auto next = s + 1;
+	  if (next == end)
+	    {
+	      /* Don't find the line ending if \r is the very last character
+		 in the buffer; we do not know if it's the end of the file or
+		 just the end of what has been read so far, and we wouldn't
+		 want to break in the middle of what's actually a \r\n
+		 sequence.  Instead, we will handle the case of a file ending
+		 in a \r later.  */
+	      break;
+	    }
+	  return (*next == '\n' ? next : s);
+	}
+    }
+  return nullptr;
+}
+
+/* Read a new line from file FP, using C as a cache for the data
+   coming from the file.  Upon successful completion, *LINE is set to
+   the beginning of the line found.  *LINE points directly in the
+   line cache and is only valid until the next call of get_next_line.
+   *LINE_LEN is set to the length of the line.  Note that the line
+   does not contain any terminal delimiter.  This function returns
+   true if some data was read or process from the cache, false
+   otherwise.  Note that subsequent calls to get_next_line might
+   make the content of *LINE invalid.  */
+
+bool
+file_cache_slot::get_next_line (char **line, ssize_t *line_len)
+{
+  /* Fill the cache with data to process.  */
+  maybe_read_data ();
+
+  size_t remaining_size = m_nb_read - m_line_start_idx;
+  if (remaining_size == 0)
+    /* There is no more data to process.  */
+    return false;
+
+  const char *line_start = m_data + m_line_start_idx;
+
+  const char *next_line_start = NULL;
+  size_t len = 0;
+  const char *line_end = find_end_of_line (line_start, remaining_size);
+  if (line_end == NULL)
+    {
+      /* We haven't found an end-of-line delimiter in the cache.
+	 Fill the cache with more data from the file and look again.  */
+      while (maybe_read_data ())
+	{
+	  line_start = m_data + m_line_start_idx;
+	  remaining_size = m_nb_read - m_line_start_idx;
+	  line_end = find_end_of_line (line_start, remaining_size);
+	  if (line_end != NULL)
+	    {
+	      next_line_start = line_end + 1;
+	      break;
+	    }
+	}
+      if (line_end == NULL)
+	{
+	  /* We've loaded all the file into the cache and still no
+	     terminator.  Let's say the line ends up at one byte past the
+	     end of the file.  This is to stay consistent with the case
+	     of when the line ends up with a terminator and line_end points to
+	     that.  That consistency is useful below in the len calculation.
+
+	     If the file ends in a \r, we didn't identify it as a line
+	     terminator above, so do that now instead.  */
+	  line_end = m_data + m_nb_read;
+	  if (m_nb_read && line_end[-1] == '\r')
+	    {
+	      --line_end;
+	      m_missing_trailing_newline = false;
+	    }
+	  else
+	    m_missing_trailing_newline = true;
+	}
+      else
+	m_missing_trailing_newline = false;
+    }
+  else
+    {
+      next_line_start = line_end + 1;
+      m_missing_trailing_newline = false;
+    }
+
+  if (m_error)
+    return false;
+
+  /* At this point, we've found the end of the of line.  It either points to
+     the line terminator or to one byte after the last byte of the file.  */
+  gcc_assert (line_end != NULL);
+
+  len = line_end - line_start;
+
+  if (m_line_start_idx < m_nb_read)
+    *line = const_cast<char *> (line_start);
+
+  ++m_line_num;
+
+  /* Now update our line record so that re-reading lines from the
+     before m_line_start_idx is faster.  */
+  size_t rlen = m_line_record.length ();
+  /* Only update when beyond the previously cached region.  */
+  if (rlen == 0 || m_line_record[rlen - 1].line_num < m_line_num)
+    {
+      size_t spacing
+	= (rlen >= 2
+	   ? (m_line_record[rlen - 1].line_num
+	      - m_line_record[rlen - 2].line_num) : 1);
+      size_t delta
+	= rlen >= 1 ? m_line_num - m_line_record[rlen - 1].line_num : 1;
+
+      size_t max_size = line_record_size;
+      /* One anchor per hundred input lines.  */
+      if (max_size == 0)
+	max_size = m_line_num / 100;
+
+      /* If we're too far beyond drop half of the lines to rebalance.  */
+      if (rlen == max_size && delta >= spacing * 2)
+	{
+	  size_t j = 0;
+	  for (size_t i = 1; i < rlen; i += 2)
+	    m_line_record[j++] = m_line_record[i];
+	  m_line_record.truncate (j);
+	  rlen = j;
+	  spacing *= 2;
+	}
+
+      if (rlen < max_size && delta >= spacing)
+	{
+	  file_cache_slot::line_info li (m_line_num, m_line_start_idx,
+					 line_end - m_data);
+	  m_line_record.safe_push (li);
+	}
+    }
+
+  /* Cache recent tail lines separately for fast access. This assumes
+     most accesses do not skip backwards.  */
+  if (m_line_recent_last == m_line_recent_first
+      || m_line_recent[m_line_recent_last].line_num == m_line_num - 1)
+    {
+      size_t mask = ((size_t) 1 << recent_cached_lines_shift) - 1;
+      m_line_recent_last = (m_line_recent_last + 1) & mask;
+      if (m_line_recent_last == m_line_recent_first)
+	m_line_recent_first = (m_line_recent_first + 1) & mask;
+      m_line_recent[m_line_recent_last]
+	= file_cache_slot::line_info (m_line_num, m_line_start_idx,
+				      line_end - m_data);
+    }
+
+  /* Update m_line_start_idx so that it points to the next line to be
+     read.  */
+  if (next_line_start)
+    m_line_start_idx = next_line_start - m_data;
+  else
+    /* We didn't find any terminal '\n'.  Let's consider that the end
+       of line is the end of the data in the cache.  The next
+       invocation of get_next_line will either read more data from the
+       underlying file or return false early because we've reached the
+       end of the file.  */
+    m_line_start_idx = m_nb_read;
+
+  *line_len = len;
+
+  return true;
+}
+
+/* Consume the next bytes coming from the cache (or from its
+   underlying file if there are remaining unread bytes in the file)
+   until we reach the next end-of-line (or end-of-file).  There is no
+   copying from the cache involved.  Return TRUE upon successful
+   completion.  */
+
+bool
+file_cache_slot::goto_next_line ()
+{
+  char *l;
+  ssize_t len;
+
+  return get_next_line (&l, &len);
+}
+
+/* Read an arbitrary line number LINE_NUM from the file cached in C.
+   If the line was read successfully, *LINE points to the beginning
+   of the line in the file cache and *LINE_LEN is the length of the
+   line.  *LINE is not nul-terminated, but may contain zero bytes.
+   *LINE is only valid until the next call of read_line_num.
+   This function returns bool if a line was read.  */
+
+bool
+file_cache_slot::read_line_num (size_t line_num,
+				char ** line, ssize_t *line_len)
+{
+  gcc_assert (line_num > 0);
+
+  /* Is the line in the recent line cache?
+     This assumes the main file processing is only using
+     a single contiguous cursor with only temporary excursions.  */
+  if (m_line_recent_first != m_line_recent_last
+	&& m_line_recent[m_line_recent_first].line_num <= line_num
+	&& m_line_recent[m_line_recent_last].line_num >= line_num)
+    {
+      line_info &last = m_line_recent[m_line_recent_last];
+      size_t mask = (1U << recent_cached_lines_shift) - 1;
+      size_t idx = (m_line_recent_last - (last.line_num - line_num)) & mask;
+      line_info &recent = m_line_recent[idx];
+      gcc_assert (recent.line_num == line_num);
+      *line = m_data + recent.start_pos;
+      *line_len = recent.end_pos - recent.start_pos;
+      return true;
+    }
+
+  if (line_num <= m_line_num)
+    {
+      line_info l (line_num, 0, 0);
+      int i = m_line_record.lower_bound (l, line_info::less_than);
+      if (i == 0)
+	{
+	  m_line_start_idx = 0;
+	  m_line_num = 0;
+	}
+      else if (m_line_record[i - 1].line_num == line_num)
+	{
+	  /* We have the start/end of the line.  */
+	  *line = m_data + m_line_record[i - 1].start_pos;
+	  *line_len = m_line_record[i - 1].end_pos - m_line_record[i - 1].start_pos;
+	  return true;
+	}
+      else
+       {
+	 gcc_assert (m_line_record[i - 1].line_num < m_line_num);
+	 m_line_start_idx = m_line_record[i - 1].start_pos;
+	 m_line_num = m_line_record[i - 1].line_num - 1;
+       }
+    }
+
+  /*  Let's walk from line m_line_num up to line_num - 1, without
+      copying any line.  */
+  while (m_line_num < line_num - 1)
+    if (!goto_next_line ())
+      return false;
+
+  /* The line we want is the next one.  Let's read it.  */
+  return get_next_line (line, line_len);
+}
+
+/* Return the physical source line that corresponds to FILE_PATH/LINE.
+   The line is not nul-terminated.  The returned pointer is only
+   valid until the next call of location_get_source_line.
+   Note that the line can contain several null characters,
+   so the returned value's length has the actual length of the line.
+   If the function fails, a NULL char_span is returned.  */
+
+char_span
+file_cache::get_source_line (const char *file_path, int line)
+{
+  char *buffer = NULL;
+  ssize_t len;
+
+  if (line == 0)
+    return char_span (NULL, 0);
+
+  if (file_path == NULL)
+    return char_span (NULL, 0);
+
+  file_cache_slot *c = lookup_or_add_file (file_path);
+  if (c == NULL)
+    return char_span (NULL, 0);
+
+  bool read = c->read_line_num (line, &buffer, &len);
+  if (!read)
+    return char_span (NULL, 0);
+
+  return char_span (buffer, len);
+}
+
+char_span
+file_cache::get_source_file_content (const char *file_path)
+{
+  file_cache_slot *c = lookup_or_add_file (file_path);
+  if (c == nullptr)
+    return char_span (nullptr, 0);
+  return c->get_full_file_content ();
+}
+
+#if CHECKING_P
+
+namespace selftest {
+
+ using temp_source_file = ::selftest::temp_source_file;
+
+/* Verify reading of a specific line LINENUM in TMP, FC.  */
+
+static void
+check_line (temp_source_file &tmp, file_cache &fc, int linenum)
+{
+  char_span line = fc.get_source_line (tmp.get_filename (), linenum);
+  int n;
+  const char *b = line.get_buffer ();
+  size_t l = line.length ();
+  char buf[5];
+  ASSERT_LT (l, 5);
+  memcpy (buf, b, l);
+  buf[l] = '\0';
+  ASSERT_TRUE (sscanf (buf, "%d", &n) == 1);
+  ASSERT_EQ (n, linenum);
+}
+
+/* Test file cache replacement.  */
+
+static void
+test_replacement ()
+{
+  const int maxline = 1000;
+
+  char *vec = XNEWVEC (char, maxline * 5);
+  char *p = vec;
+  int i;
+  for (i = 1; i <= maxline; i++)
+    p += sprintf (p, "%d\n", i);
+
+  temp_source_file tmp (SELFTEST_LOCATION, ".txt", vec);
+  free (vec);
+  file_cache fc;
+
+  for (i = 2; i <= maxline; i++)
+    {
+      check_line (tmp, fc, i);
+      check_line (tmp, fc, i - 1);
+      if (i >= 10)
+	check_line (tmp, fc, i - 9);
+      if (i >= 350) /* Exceed the look behind cache.  */
+	check_line (tmp, fc, i - 300);
+    }
+  for (i = 5; i <= maxline; i += 100)
+    check_line (tmp, fc, i);
+  for (i = 1; i <= maxline; i++)
+    check_line (tmp, fc, i);
+}
+
+/* Verify reading of input files (e.g. for caret-based diagnostics).  */
+
+static void
+test_reading_source_line ()
+{
+  /* Create a tempfile and write some text to it.  */
+  temp_source_file tmp (SELFTEST_LOCATION, ".txt",
+			"01234567890123456789\n"
+			"This is the test text\n"
+			"This is the 3rd line");
+  file_cache fc;
+
+  /* Read back a specific line from the tempfile.  */
+  char_span source_line = fc.get_source_line (tmp.get_filename (), 3);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (20, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the 3rd line",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (tmp.get_filename (), 2);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (21, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the test text",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (tmp.get_filename (), 4);
+  ASSERT_FALSE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () == NULL);
+}
+
+/* Verify reading from buffers (e.g. for sarif-replay).  */
+
+static void
+test_reading_source_buffer ()
+{
+  const char *text = ("01234567890123456789\n"
+		      "This is the test text\n"
+		      "This is the 3rd line");
+  const char *filename = "foo.txt";
+  file_cache fc;
+  fc.add_buffered_content (filename, text, strlen (text));
+
+  /* Read back a specific line from the tempfile.  */
+  char_span source_line = fc.get_source_line (filename, 3);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (20, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the 3rd line",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (filename, 2);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (21, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the test text",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (filename, 4);
+  ASSERT_FALSE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () == NULL);
+}
+
+/* Run all of the selftests within this file.  */
+
+void
+file_cache_cc_tests ()
+{
+  test_reading_source_line ();
+  test_reading_source_buffer ();
+  test_replacement ();
+}
+
+} // namespace selftest
+
+#endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostics/file-cache.h b/gcc/diagnostics/file-cache.h
new file mode 100644
index 0000000..832a960
--- /dev/null
+++ b/gcc/diagnostics/file-cache.h
@@ -0,0 +1,125 @@
+/* Caching input files for use by diagnostics.
+   Copyright (C) 2004-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_FILE_CACHE_H
+#define GCC_DIAGNOSTICS_FILE_CACHE_H
+
+namespace diagnostics {
+
+/* A class capturing the bounds of a buffer, to allow for run-time
+   bounds-checking in a checked build.  */
+
+class char_span
+{
+ public:
+  char_span (const char *ptr, size_t n_elts) : m_ptr (ptr), m_n_elts (n_elts) {}
+
+  /* Test for a non-NULL pointer.  */
+  operator bool() const { return m_ptr; }
+
+  /* Get length, not including any 0-terminator (which may not be,
+     in fact, present).  */
+  size_t length () const { return m_n_elts; }
+
+  const char *get_buffer () const { return m_ptr; }
+
+  char operator[] (int idx) const
+  {
+    gcc_assert (idx >= 0);
+    gcc_assert ((size_t)idx < m_n_elts);
+    return m_ptr[idx];
+  }
+
+  char_span subspan (int offset, int n_elts) const
+  {
+    gcc_assert (offset >= 0);
+    gcc_assert (offset < (int)m_n_elts);
+    gcc_assert (n_elts >= 0);
+    gcc_assert (offset + n_elts <= (int)m_n_elts);
+    return char_span (m_ptr + offset, n_elts);
+  }
+
+  char *xstrdup () const
+  {
+    return ::xstrndup (m_ptr, m_n_elts);
+  }
+
+ private:
+  const char *m_ptr;
+  size_t m_n_elts;
+};
+
+/* Forward decl of slot within file_cache, so that the definition doesn't
+   need to be in this header.  */
+class file_cache_slot;
+
+/* A cache of source files for use when emitting diagnostics
+   (and in a few places in the C/C++ frontends).
+
+   Results are only valid until the next call to the cache, as
+   slots can be evicted.
+
+   Filenames are stored by pointer, and so must outlive the cache
+   instance.  */
+
+class file_cache
+{
+ public:
+  file_cache ();
+  ~file_cache ();
+
+  void dump (FILE *out, int indent) const;
+  void DEBUG_FUNCTION dump () const;
+
+  file_cache_slot *lookup_or_add_file (const char *file_path);
+  void forcibly_evict_file (const char *file_path);
+
+  /* See comments in diagnostic.h about the input conversion context.  */
+  struct input_context
+  {
+    diagnostic_input_charset_callback ccb;
+    bool should_skip_bom;
+  };
+  void initialize_input_context (diagnostic_input_charset_callback ccb,
+				 bool should_skip_bom);
+
+  char_span get_source_file_content (const char *file_path);
+  char_span get_source_line (const char *file_path, int line);
+  bool missing_trailing_newline_p (const char *file_path);
+
+  void add_buffered_content (const char *file_path,
+			     const char *buffer,
+			     size_t sz);
+
+  void tune (size_t num_file_slots, size_t lines);
+
+ private:
+  file_cache_slot *evicted_cache_tab_entry (unsigned *highest_use_count);
+  file_cache_slot *add_file (const char *file_path);
+  file_cache_slot *lookup_file (const char *file_path);
+
+ private:
+  size_t m_num_file_slots;
+  file_cache_slot *m_file_slots;
+  input_context m_input_context;
+};
+
+} // namespace diagnostics
+
+#endif // #ifndef GCC_DIAGNOSTICS_FILE_CACHE_H
diff --git a/gcc/diagnostic-format-html.cc b/gcc/diagnostics/html-sink.cc
index 473880f..934d8e2 100644
--- a/gcc/diagnostic-format-html.cc
+++ b/gcc/diagnostics/html-sink.cc
@@ -25,35 +25,39 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-format.h"
-#include "diagnostic-format-html.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-output-file.h"
-#include "diagnostic-buffer.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/html-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/output-file.h"
+#include "diagnostics/buffering.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 #include "pretty-print-format-impl.h"
 #include "pretty-print-urlifier.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
 #include "intl.h"
 #include "xml.h"
 #include "xml-printer.h"
-#include "diagnostic-state.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/state-graphs.h"
 #include "graphviz.h"
 #include "json.h"
 #include "selftest-xml.h"
 
+namespace diagnostics {
+
 // struct html_generation_options
 
 html_generation_options::html_generation_options ()
 : m_css (true),
   m_javascript (true),
   m_show_state_diagrams (false),
-  m_show_state_diagram_xml (false),
-  m_show_state_diagram_dot_src (false)
+  m_show_state_diagrams_sarif (false),
+  m_show_state_diagrams_dot_src (false)
 {
 }
 
@@ -61,19 +65,19 @@ class html_builder;
 
 /* Concrete buffering implementation subclass for HTML output.  */
 
-class diagnostic_html_format_buffer : public diagnostic_per_format_buffer
+class html_sink_buffer : public per_sink_buffer
 {
 public:
   friend class html_builder;
-  friend class html_output_format;
+  friend class html_sink;
 
-  diagnostic_html_format_buffer (html_builder &builder)
+  html_sink_buffer (html_builder &builder)
   : m_builder (builder)
   {}
 
   void dump (FILE *out, int indent) const final override;
   bool empty_p () const final override;
-  void move_to (diagnostic_per_format_buffer &dest) final override;
+  void move_to (per_sink_buffer &dest) final override;
   void clear () final override;
   void flush () final override;
 
@@ -105,9 +109,9 @@ private:
 class html_builder
 {
 public:
-  friend class diagnostic_html_format_buffer;
+  friend class html_sink_buffer;
 
-  html_builder (diagnostic_context &context,
+  html_builder (context &dc,
 		pretty_printer &pp,
 		const line_maps *line_maps,
 		const html_generation_options &html_gen_opts);
@@ -116,9 +120,11 @@ public:
   set_main_input_filename (const char *name);
 
   void on_report_diagnostic (const diagnostic_info &diagnostic,
-			     diagnostic_t orig_diag_kind,
-			     diagnostic_html_format_buffer *buffer);
-  void emit_diagram (const diagnostic_diagram &diagram);
+			     enum kind orig_diag_kind,
+			     html_sink_buffer *buffer);
+  void emit_diagram (const diagram &d);
+  void emit_global_graph (const lazily_created<digraphs::digraph> &);
+
   void end_group ();
 
   std::unique_ptr<xml::element> take_current_diagnostic ()
@@ -136,7 +142,7 @@ public:
   }
 
   std::unique_ptr<xml::element>
-  make_element_for_metadata (const diagnostic_metadata &metadata);
+  make_element_for_metadata (const metadata &);
 
   std::unique_ptr<xml::element>
   make_element_for_patch (const diagnostic_info &diagnostic);
@@ -147,7 +153,7 @@ public:
   }
 
   std::unique_ptr<xml::node>
-  maybe_make_state_diagram (const diagnostic_event &event);
+  maybe_make_state_diagram (const paths::event &event);
 
 private:
   void
@@ -155,7 +161,7 @@ private:
 
   std::unique_ptr<xml::element>
   make_element_for_diagnostic (const diagnostic_info &diagnostic,
-			       diagnostic_t orig_diag_kind,
+			       enum kind orig_diag_kind,
 			       bool alert);
 
   std::unique_ptr<xml::element>
@@ -172,21 +178,26 @@ private:
   void
   pop_nesting_level ();
 
-  diagnostic_context &m_context;
+  void
+  add_graph (const digraphs::digraph &dg,
+	     xml::element &parent_element);
+
+  context &m_context;
   pretty_printer *m_printer;
   const line_maps *m_line_maps;
   html_generation_options m_html_gen_opts;
-  const logical_location_manager *m_logical_loc_mgr;
+  const logical_locations::manager *m_logical_loc_mgr;
 
   std::unique_ptr<xml::document> m_document;
   xml::element *m_head_element;
   xml::element *m_title_element;
+  xml::element *m_body_element;
   xml::element *m_diagnostics_element;
   std::unique_ptr<xml::element> m_cur_diagnostic_element;
   std::vector<xml::element *> m_cur_nesting_levels;
   int m_next_diag_id; // for handing out unique IDs
   json::array m_ui_focus_ids;
-  logical_location m_last_logical_location;
+  logical_locations::key m_last_logical_location;
   location_t m_last_location;
   expanded_location m_last_expanded_location;
 };
@@ -207,12 +218,12 @@ make_span (std::string class_)
   return span;
 }
 
-/* class diagnostic_html_format_buffer : public diagnostic_per_format_buffer.  */
+/* class html_sink_buffer : public per_sink_buffer.  */
 
 void
-diagnostic_html_format_buffer::dump (FILE *out, int indent) const
+html_sink_buffer::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_html_format_buffer:\n", indent, "");
+  fprintf (out, "%*shtml_sink_buffer:\n", indent, "");
   int idx = 0;
   for (auto &result : m_results)
     {
@@ -224,29 +235,29 @@ diagnostic_html_format_buffer::dump (FILE *out, int indent) const
 }
 
 bool
-diagnostic_html_format_buffer::empty_p () const
+html_sink_buffer::empty_p () const
 {
   return m_results.empty ();
 }
 
 void
-diagnostic_html_format_buffer::move_to (diagnostic_per_format_buffer &base)
+html_sink_buffer::move_to (per_sink_buffer &base)
 {
-  diagnostic_html_format_buffer &dest
-    = static_cast<diagnostic_html_format_buffer &> (base);
+  html_sink_buffer &dest
+    = static_cast<html_sink_buffer &> (base);
   for (auto &&result : m_results)
     dest.m_results.push_back (std::move (result));
   m_results.clear ();
 }
 
 void
-diagnostic_html_format_buffer::clear ()
+html_sink_buffer::clear ()
 {
   m_results.clear ();
 }
 
 void
-diagnostic_html_format_buffer::flush ()
+html_sink_buffer::flush ()
 {
   for (auto &&result : m_results)
     m_builder.m_diagnostics_element->add_child (std::move (result));
@@ -390,17 +401,18 @@ struct html_doctypedecl : public xml::doctypedecl
 
 /* html_builder's ctor.  */
 
-html_builder::html_builder (diagnostic_context &context,
+html_builder::html_builder (context &dc,
 			    pretty_printer &pp,
 			    const line_maps *line_maps,
 			    const html_generation_options &html_gen_opts)
-: m_context (context),
+: m_context (dc),
   m_printer (&pp),
   m_line_maps (line_maps),
   m_html_gen_opts (html_gen_opts),
   m_logical_loc_mgr (nullptr),
   m_head_element (nullptr),
   m_title_element (nullptr),
+  m_body_element (nullptr),
   m_diagnostics_element (nullptr),
   m_next_diag_id (0),
   m_last_location (UNKNOWN_LOCATION),
@@ -408,7 +420,7 @@ html_builder::html_builder (diagnostic_context &context,
 {
   gcc_assert (m_line_maps);
 
-  if (auto client_data_hooks = context.get_client_data_hooks ())
+  if (auto client_data_hooks = dc.get_client_data_hooks ())
     m_logical_loc_mgr = client_data_hooks->get_logical_location_manager ();
 
   m_document = std::make_unique<xml::document> ();
@@ -447,6 +459,7 @@ html_builder::html_builder (diagnostic_context &context,
 
     {
       xml::auto_print_element body (xp, "body");
+      m_body_element = xp.get_insertion_point ();
       {
 	auto diagnostics_element = make_div ("gcc-diagnostic-list");
 	m_diagnostics_element = diagnostics_element.get ();
@@ -483,10 +496,10 @@ html_builder::add_stylesheet (std::string url)
 
 void
 html_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
-				    diagnostic_t orig_diag_kind,
-				    diagnostic_html_format_buffer *buffer)
+				    enum kind orig_diag_kind,
+				    html_sink_buffer *buffer)
 {
-  if (diagnostic.kind == DK_ICE || diagnostic.kind == DK_ICE_NOBT)
+  if (diagnostic.m_kind == kind::ice || diagnostic.m_kind == kind::ice_nobt)
     {
       /* Print a header for the remaining output to stderr, and
 	 return, attempting to print the usual ICE messages to
@@ -501,7 +514,7 @@ html_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
   if (m_cur_diagnostic_element && nesting_level > 0)
     alert = false;
   if (!m_cur_diagnostic_element)
-    m_last_logical_location = logical_location ();
+    m_last_logical_location = logical_locations::key ();
   auto diag_element
     = make_element_for_diagnostic (diagnostic, orig_diag_kind, alert);
   if (buffer)
@@ -594,44 +607,49 @@ print_pre_source (xml::printer &xp, const char *text)
 }
 
 std::unique_ptr<xml::node>
-html_builder::maybe_make_state_diagram (const diagnostic_event &event)
+html_builder::maybe_make_state_diagram (const paths::event &event)
 {
   if (!m_html_gen_opts.m_show_state_diagrams)
     return nullptr;
 
-  /* Get XML state document; if we're going to print it later, also request
+  if (!m_logical_loc_mgr)
+    return nullptr;
+
+  /* Get state graph; if we're going to print it later, also request
      the debug version.  */
-  auto xml_state
-    = event.maybe_make_xml_state (m_html_gen_opts.m_show_state_diagram_xml);
-  if (!xml_state)
+  auto state_graph
+    = event.maybe_make_diagnostic_state_graph
+	(m_html_gen_opts.m_show_state_diagrams_sarif);
+  if (!state_graph)
     return nullptr;
 
   // Convert it to .dot AST
-  auto graph = make_dot_graph_from_xml_state (*xml_state);
-  gcc_assert (graph);
+  auto dot_graph = state_graphs::make_dot_graph (*state_graph,
+						 *m_logical_loc_mgr);
+  gcc_assert (dot_graph);
 
   auto wrapper = std::make_unique<xml::element> ("div", false);
   xml::printer xp (*wrapper);
 
-  if (m_html_gen_opts.m_show_state_diagram_xml)
+  if (m_html_gen_opts.m_show_state_diagrams_sarif)
     {
-      // For debugging, show the XML src inline:
+      // For debugging, show the SARIF src inline:
       pretty_printer pp;
-      xml_state->write_as_xml (&pp, 0, true);
+      state_graph->make_json_sarif_graph ()->print (&pp, true);
       print_pre_source (xp, pp_formatted_text (&pp));
     }
 
-  if (m_html_gen_opts.m_show_state_diagram_dot_src)
+  if (m_html_gen_opts.m_show_state_diagrams_dot_src)
     {
       // For debugging, show the dot src inline:
       pretty_printer pp;
       dot::writer w (pp);
-      graph->print (w);
+      dot_graph->print (w);
       print_pre_source (xp, pp_formatted_text (&pp));
     }
 
   // Turn the .dot into SVG and splice into place
-  auto svg = dot::make_svg_from_graph (*graph);
+  auto svg = dot::make_svg_from_graph (*dot_graph);
   if (svg)
     xp.append (std::move (svg));
 
@@ -647,7 +665,7 @@ class html_path_label_writer : public html_label_writer
 public:
   html_path_label_writer (xml::printer &xp,
 			  html_builder &builder,
-			  const diagnostic_path &path,
+			  const paths::path &path,
 			  const std::string &event_id_prefix)
   : m_xp (xp),
     m_html_builder (builder),
@@ -668,7 +686,7 @@ public:
 
   void end_label () final override
   {
-    const diagnostic_event &event
+    const paths::event &event
       = m_path.get_event (m_curr_event_id.zero_based ());
     if (auto state_doc = m_html_builder.maybe_make_state_diagram (event))
     {
@@ -696,31 +714,31 @@ private:
 
   xml::printer &m_xp;
   html_builder &m_html_builder;
-  const diagnostic_path &m_path;
+  const paths::path &m_path;
   const std::string &m_event_id_prefix;
   int m_next_event_idx;
-  diagnostic_event_id_t m_curr_event_id;
+  paths::event_id_t m_curr_event_id;
 };
 
 /* See https://pf3.patternfly.org/v3/pattern-library/widgets/#alerts */
 static const char *
-get_pf_class_for_alert_div (diagnostic_t diag_kind)
+get_pf_class_for_alert_div (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_DEBUG:
-    case DK_NOTE:
+    case kind::debug:
+    case kind::note:
       return "alert alert-info";
 
-    case DK_ANACHRONISM:
-    case DK_WARNING:
+    case kind::anachronism:
+    case kind::warning:
       return "alert alert-warning";
 
-    case DK_ERROR:
-    case DK_SORRY:
-    case DK_ICE:
-    case DK_ICE_NOBT:
-    case DK_FATAL:
+    case kind::error:
+    case kind::sorry:
+    case kind::ice:
+    case kind::ice_nobt:
+    case kind::fatal:
       return "alert alert-danger";
 
     default:
@@ -729,23 +747,23 @@ get_pf_class_for_alert_div (diagnostic_t diag_kind)
 }
 
 static const char *
-get_pf_class_for_alert_icon (diagnostic_t diag_kind)
+get_pf_class_for_alert_icon (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_DEBUG:
-    case DK_NOTE:
+    case kind::debug:
+    case kind::note:
       return "pficon pficon-info";
 
-    case DK_ANACHRONISM:
-    case DK_WARNING:
+    case kind::anachronism:
+    case kind::warning:
       return "pficon pficon-warning-triangle-o";
 
-    case DK_ERROR:
-    case DK_SORRY:
-    case DK_ICE:
-    case DK_ICE_NOBT:
-    case DK_FATAL:
+    case kind::error:
+    case kind::sorry:
+    case kind::ice:
+    case kind::ice_nobt:
+    case kind::fatal:
       return "pficon pficon-error-circle-o";
 
     default:
@@ -754,57 +772,57 @@ get_pf_class_for_alert_icon (diagnostic_t diag_kind)
 }
 
 static const char *
-get_label_for_logical_location_kind (enum logical_location_kind kind)
+get_label_for_logical_location_kind (enum logical_locations::kind kind)
 {
   switch (kind)
     {
     default:
       gcc_unreachable ();
-    case logical_location_kind::unknown:
+    case logical_locations::kind::unknown:
       return nullptr;
 
     /* Kinds within executable code.  */
-    case logical_location_kind::function:
+    case logical_locations::kind::function:
       return "Function";
-    case logical_location_kind::member:
+    case logical_locations::kind::member:
       return "Member";
-    case logical_location_kind::module_:
+    case logical_locations::kind::module_:
       return "Module";
-    case logical_location_kind::namespace_:
+    case logical_locations::kind::namespace_:
       return "Namespace";
-    case logical_location_kind::type:
+    case logical_locations::kind::type:
       return "Type";
-    case logical_location_kind::return_type:
+    case logical_locations::kind::return_type:
       return "Return type";
-    case logical_location_kind::parameter:
+    case logical_locations::kind::parameter:
       return "Parameter";
-    case logical_location_kind::variable:
+    case logical_locations::kind::variable:
       return "Variable";
 
     /* Kinds within XML or HTML documents.  */
-    case logical_location_kind::element:
+    case logical_locations::kind::element:
       return "Element";
-    case logical_location_kind::attribute:
+    case logical_locations::kind::attribute:
       return "Attribute";
-    case logical_location_kind::text:
+    case logical_locations::kind::text:
       return "Text";
-    case logical_location_kind::comment:
+    case logical_locations::kind::comment:
       return "Comment";
-    case logical_location_kind::processing_instruction:
+    case logical_locations::kind::processing_instruction:
       return "Processing Instruction";
-    case logical_location_kind::dtd:
+    case logical_locations::kind::dtd:
       return "DTD";
-    case logical_location_kind::declaration:
+    case logical_locations::kind::declaration:
       return "Declaration";
 
   /* Kinds within JSON documents.  */
-    case logical_location_kind::object:
+    case logical_locations::kind::object:
       return "Object";
-    case logical_location_kind::array:
+    case logical_locations::kind::array:
       return "Array";
-    case logical_location_kind::property:
+    case logical_locations::kind::property:
       return "Property";
-    case logical_location_kind::value:
+    case logical_locations::kind::value:
       return "Value";
     }
 }
@@ -928,7 +946,7 @@ private:
 
 std::unique_ptr<xml::element>
 html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
-					   diagnostic_t orig_diag_kind,
+					   enum kind orig_diag_kind,
 					   bool alert)
 {
   const int diag_idx = m_next_diag_id++;
@@ -952,7 +970,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   diag_element->set_attr ("id", diag_id);
   if (alert)
     diag_element->set_attr ("class",
-			    get_pf_class_for_alert_div (diagnostic.kind));
+			    get_pf_class_for_alert_div (diagnostic.m_kind));
 
   xml::printer xp (*diag_element.get ());
   const size_t depth_within_alert_div = 1;
@@ -962,7 +980,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   if (alert)
     {
       xp.push_tag_with_class ("span",
-			      get_pf_class_for_alert_icon (diagnostic.kind),
+			      get_pf_class_for_alert_icon (diagnostic.m_kind),
 			      true);
       xp.add_text (" ");
       xp.pop_tag ("span");
@@ -986,7 +1004,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   if (show_severity)
   {
     xp.push_tag ("strong");
-    xp.add_text (_(get_diagnostic_kind_text (diagnostic.kind)));
+    xp.add_text (_(get_text_for_kind (diagnostic.m_kind)));
     xp.pop_tag ("strong");
     xp.add_text (" ");
   }
@@ -999,21 +1017,22 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   pp_clear_output_area (m_printer);
 
   // Add any metadata as a suffix to the message
-  if (diagnostic.metadata)
-    {
-      xp.add_text (" ");
-      xp.append (make_element_for_metadata (*diagnostic.metadata));
-    }
+  if (diagnostic.m_metadata)
+    if (auto e = make_element_for_metadata (*diagnostic.m_metadata))
+      {
+	xp.add_text (" ");
+	xp.append (std::move (e));
+      }
 
   // Add any option as a suffix to the message
 
   label_text option_text = label_text::take
-    (m_context.make_option_name (diagnostic.option_id,
-				 orig_diag_kind, diagnostic.kind));
+    (m_context.make_option_name (diagnostic.m_option_id,
+				 orig_diag_kind, diagnostic.m_kind));
   if (option_text.get ())
     {
       label_text option_url = label_text::take
-	(m_context.make_option_url (diagnostic.option_id));
+	(m_context.make_option_url (diagnostic.m_option_id));
 
       xp.add_text (" ");
       auto option_span = make_span ("gcc-option");
@@ -1045,7 +1064,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
       if (auto logical_loc = client_data_hooks->get_current_logical_location ())
 	if (logical_loc != m_last_logical_location)
 	  {
-	    enum logical_location_kind kind
+	    enum logical_locations::kind kind
 	      = m_logical_loc_mgr->get_kind (logical_loc);;
 	    if (const char *label = get_label_for_logical_location_kind (kind))
 	      if (const char *name_with_scope
@@ -1068,8 +1087,8 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
       if (s.line)
 	{
 	  add_labelled_value (xp, "line", "Line", std::to_string (s.line), false);
-	  diagnostic_column_policy column_policy (m_context);
-	  int converted_column = column_policy.converted_column (s);
+	  column_policy col_policy (m_context);
+	  int converted_column = col_policy.converted_column (s);
 	  if (converted_column >= 0)
 	    add_labelled_value (xp, "column", "Column",
 				std::to_string (converted_column),
@@ -1084,12 +1103,13 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
     // TODO: m_context.m_last_location should be moved into the sink
     location_t saved = m_context.m_last_location;
     m_context.m_last_location = m_last_location;
-    m_context.maybe_show_locus_as_html (*diagnostic.richloc,
-					m_context.m_source_printing,
-					diagnostic.kind,
-					xp,
-					nullptr,
-					nullptr);
+    m_context.maybe_show_locus_as_html
+      (*diagnostic.m_richloc,
+       m_context.get_source_printing_options (),
+       diagnostic.m_kind,
+       xp,
+       nullptr,
+       nullptr);
     m_context.m_last_location = saved;
     m_last_location = m_context.m_last_location;
   }
@@ -1097,7 +1117,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   gcc_assert (xp.get_num_open_tags () == depth_within_alert_div);
 
   /* Execution path.  */
-  if (auto path = diagnostic.richloc->get_path ())
+  if (auto path = diagnostic.m_richloc->get_path ())
     {
       xp.push_tag ("div");
       xp.set_attr ("id", "execution-path");
@@ -1116,7 +1136,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
       html_path_label_writer event_label_writer (xp, *this, *path,
 						 event_id_prefix);
 
-      diagnostic_source_print_policy dspp (m_context);
+      source_print_policy dspp (m_context);
       print_path_as_html (xp, *path, m_context, &event_label_writer,
 			  dspp);
 
@@ -1125,6 +1145,15 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
 
   gcc_assert (xp.get_num_open_tags () == depth_within_alert_div);
 
+  // Try to display any per-diagnostic graphs
+  if (diagnostic.m_metadata)
+    if (auto ldg = diagnostic.m_metadata->get_lazy_digraphs ())
+      {
+	auto &digraphs = ldg->get_or_create ();
+	for (auto &dg : digraphs)
+	  add_graph (*dg, *xp.get_insertion_point ());
+      }
+
   if (auto patch_element = make_element_for_patch (diagnostic))
     {
       xp.push_tag ("div");
@@ -1142,9 +1171,9 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
 std::unique_ptr<xml::element>
 html_builder::make_element_for_patch (const diagnostic_info &diagnostic)
 {
-  edit_context ec (m_context.get_file_cache ());
-  ec.add_fixits (diagnostic.richloc);
-  if (char *diff = ec.generate_diff (true))
+  changes::change_set edit (m_context.get_file_cache ());
+  edit.add_fixits (diagnostic.m_richloc);
+  if (char *diff = edit.generate_diff (true))
     {
       if (strlen (diff) > 0)
 	{
@@ -1182,11 +1211,11 @@ html_builder::make_metadata_element (label_text label,
 }
 
 std::unique_ptr<xml::element>
-html_builder::make_element_for_metadata (const diagnostic_metadata &metadata)
+html_builder::make_element_for_metadata (const metadata &m)
 {
   auto span_metadata = make_span ("gcc-metadata");
 
-  int cwe = metadata.get_cwe ();
+  int cwe = m.get_cwe ();
   if (cwe)
     {
       pretty_printer pp;
@@ -1197,23 +1226,26 @@ html_builder::make_element_for_metadata (const diagnostic_metadata &metadata)
 	(make_metadata_element (std::move (label), std::move (url)));
     }
 
-  for (unsigned idx = 0; idx < metadata.get_num_rules (); ++idx)
+  for (unsigned idx = 0; idx < m.get_num_rules (); ++idx)
     {
-      auto &rule = metadata.get_rule (idx);
+      auto &rule = m.get_rule (idx);
       label_text label = label_text::take (rule.make_description ());
       label_text url = label_text::take (rule.make_url ());
       span_metadata->add_child
 	(make_metadata_element (std::move (label), std::move (url)));
     }
 
+  if (span_metadata->m_children.empty ())
+    return nullptr;
+
   return span_metadata;
 }
 
-/* Implementation of diagnostic_context::m_diagrams.m_emission_cb
+/* Implementation of diagnostics::context::m_diagrams.m_emission_cb
    for HTML output.  */
 
 void
-html_builder::emit_diagram (const diagnostic_diagram &/*diagram*/)
+html_builder::emit_diagram (const diagram &)
 {
   /* We must be within the emission of a top-level diagnostic.  */
   gcc_assert (m_cur_diagnostic_element);
@@ -1221,6 +1253,35 @@ html_builder::emit_diagram (const diagnostic_diagram &/*diagram*/)
   // TODO: currently a no-op
 }
 
+void
+html_builder::add_graph (const digraphs::digraph &dg,
+			 xml::element &parent_element)
+{
+  if (auto dot_graph = dg.make_dot_graph ())
+    if (auto svg_element = dot::make_svg_from_graph (*dot_graph))
+      {
+	auto div = std::make_unique<xml::element> ("div", false);
+	div->set_attr ("class", "gcc-directed-graph");
+	xml::printer xp (*div);
+	if (const char *description = dg.get_description ())
+	  {
+	    xp.push_tag ("h2", true);
+	    xp.add_text (description);
+	    xp.pop_tag ("h2");
+	  }
+	xp.append (std::move (svg_element));
+	parent_element.add_child (std::move (div));
+      }
+}
+
+void
+html_builder::emit_global_graph (const lazily_created<digraphs::digraph> &ldg)
+{
+  auto &dg = ldg.get_or_create ();
+  gcc_assert (m_body_element);
+  add_graph (dg, *m_body_element);
+}
+
 /* Implementation of "end_group_cb" for HTML output.  */
 
 void
@@ -1257,10 +1318,10 @@ html_builder::flush_to_file (FILE *outf)
   fprintf (outf, "\n");
 }
 
-class html_output_format : public diagnostic_output_format
+class html_sink : public sink
 {
 public:
-  ~html_output_format ()
+  ~html_sink ()
   {
     /* Any diagnostics should have been handled by now.
        If not, then something's gone wrong with diagnostic
@@ -1272,8 +1333,8 @@ public:
 
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*shtml_output_format\n", indent, "");
-    diagnostic_output_format::dump (out, indent);
+    fprintf (out, "%*shtml_sink\n", indent, "");
+    sink::dump (out, indent);
   }
 
   void
@@ -1282,15 +1343,15 @@ public:
     m_builder.set_main_input_filename (name);
   }
 
-  std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () final override
+  std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () final override
   {
-    return std::make_unique<diagnostic_html_format_buffer> (m_builder);
+    return std::make_unique<html_sink_buffer> (m_builder);
   }
-  void set_buffer (diagnostic_per_format_buffer *base_buffer) final override
+  void set_buffer (per_sink_buffer *base_buffer) final override
   {
-    diagnostic_html_format_buffer *buffer
-      = static_cast<diagnostic_html_format_buffer *> (base_buffer);
+    html_sink_buffer *buffer
+      = static_cast<html_sink_buffer *> (base_buffer);
     m_buffer = buffer;
   }
 
@@ -1304,13 +1365,13 @@ public:
   }
   void
   on_report_diagnostic (const diagnostic_info &diagnostic,
-			diagnostic_t orig_diag_kind) final override
+			enum kind orig_diag_kind) final override
   {
     m_builder.on_report_diagnostic (diagnostic, orig_diag_kind, m_buffer);
   }
-  void on_diagram (const diagnostic_diagram &diagram) final override
+  void on_diagram (const diagram &d) final override
   {
-    m_builder.emit_diagram (diagram);
+    m_builder.emit_diagram (d);
   }
   void after_diagnostic (const diagnostic_info &) final override
   {
@@ -1334,6 +1395,13 @@ public:
     m_builder.set_printer (*get_printer ());
   }
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+    final override
+  {
+    m_builder.emit_global_graph (ldg);
+  }
+
   const xml::document &get_document () const
   {
     return m_builder.get_document ();
@@ -1342,41 +1410,41 @@ public:
   html_builder &get_builder () { return m_builder; }
 
 protected:
-  html_output_format (diagnostic_context &context,
-		      const line_maps *line_maps,
-		      const html_generation_options &html_gen_opts)
-  : diagnostic_output_format (context),
-    m_builder (context, *get_printer (), line_maps, html_gen_opts),
+  html_sink (context &dc,
+	     const line_maps *line_maps,
+	     const html_generation_options &html_gen_opts)
+  : sink (dc),
+    m_builder (dc, *get_printer (), line_maps, html_gen_opts),
     m_buffer (nullptr)
   {}
 
   html_builder m_builder;
-  diagnostic_html_format_buffer *m_buffer;
+  html_sink_buffer *m_buffer;
 };
 
-class html_file_output_format : public html_output_format
+class html_file_sink : public html_sink
 {
 public:
-  html_file_output_format (diagnostic_context &context,
-			   const line_maps *line_maps,
-			   const html_generation_options &html_gen_opts,
-			   diagnostic_output_file output_file)
-  : html_output_format (context, line_maps, html_gen_opts),
-    m_output_file (std::move (output_file))
+  html_file_sink (context &dc,
+		  const line_maps *line_maps,
+		  const html_generation_options &html_gen_opts,
+		  output_file output_file_)
+  : html_sink (dc, line_maps, html_gen_opts),
+    m_output_file (std::move (output_file_))
   {
     gcc_assert (m_output_file.get_open_file ());
     gcc_assert (m_output_file.get_filename ());
   }
-  ~html_file_output_format ()
+  ~html_file_sink ()
   {
     m_builder.flush_to_file (m_output_file.get_open_file ());
   }
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*shtml_file_output_format: %s\n",
+    fprintf (out, "%*shtml_file_sink: %s\n",
 	     indent, "",
 	     m_output_file.get_filename ());
-    diagnostic_output_format::dump (out, indent);
+    sink::dump (out, indent);
   }
   bool machine_readable_stderr_p () const final override
   {
@@ -1384,26 +1452,26 @@ public:
   }
 
 private:
-  diagnostic_output_file m_output_file;
+  output_file m_output_file;
 };
 
 /* Attempt to open BASE_FILE_NAME.html for writing.
-   Return a non-null diagnostic_output_file,
-   or return a null diagnostic_output_file and complain to CONTEXT
+   Return a non-null output_file,
+   or return a null output_file and complain to DC
    using LINE_MAPS.  */
 
-diagnostic_output_file
-diagnostic_output_format_open_html_file (diagnostic_context &context,
-					 line_maps *line_maps,
-					 const char *base_file_name)
+output_file
+open_html_output_file (context &dc,
+		       line_maps *line_maps,
+		       const char *base_file_name)
 {
   if (!base_file_name)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to determine filename for HTML output");
-      return diagnostic_output_file ();
+      return output_file ();
     }
 
   label_text filename = label_text::take (concat (base_file_name,
@@ -1413,26 +1481,26 @@ diagnostic_output_format_open_html_file (diagnostic_context &context,
   if (!outf)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to open %qs for HTML output: %m",
 	 filename.get ());
-      return diagnostic_output_file ();
+      return output_file ();
     }
-  return diagnostic_output_file (outf, true, std::move (filename));
+  return output_file (outf, true, std::move (filename));
 }
 
-std::unique_ptr<diagnostic_output_format>
-make_html_sink (diagnostic_context &context,
+std::unique_ptr<sink>
+make_html_sink (context &dc,
 		const line_maps &line_maps,
 		const html_generation_options &html_gen_opts,
-		diagnostic_output_file output_file)
+		output_file output_file_)
 {
   auto sink
-    = std::make_unique<html_file_output_format> (context,
-						 &line_maps,
-						 html_gen_opts,
-						 std::move (output_file));
+    = std::make_unique<html_file_sink> (dc,
+					&line_maps,
+					html_gen_opts,
+					std::move (output_file_));
   sink->update_printer ();
   return sink;
 }
@@ -1489,7 +1557,7 @@ test_token_printer ()
 
   {
     token_printer_test t;
-    diagnostic_event_id_t event_id (0);
+    paths::event_id_t event_id (0);
     pp_printf (&t.m_pp, "foo %@ bar", &event_id);
     ASSERT_XML_PRINT_EQ
       (t.m_top_element,
@@ -1497,26 +1565,26 @@ test_token_printer ()
   }
 }
 
-/* A subclass of html_output_format for writing selftests.
+/* A subclass of html_sink for writing selftests.
    The XML output is cached internally, rather than written
    out to a file.  */
 
-class test_html_diagnostic_context : public test_diagnostic_context
+class test_html_context : public test_context
 {
 public:
-  test_html_diagnostic_context ()
+  test_html_context ()
   {
     html_generation_options html_gen_opts;
     html_gen_opts.m_css = false;
     html_gen_opts.m_javascript = false;
-    auto sink = std::make_unique<html_buffered_output_format> (*this,
-							       line_table,
-							       html_gen_opts);
+    auto sink = std::make_unique<html_buffered_sink> (*this,
+						      line_table,
+						      html_gen_opts);
     sink->update_printer ();
     sink->set_main_input_filename ("(main input filename)");
     m_format = sink.get (); // borrowed
 
-    set_output_format (std::move (sink));
+    set_sink (std::move (sink));
   }
 
   const xml::document &get_document () const
@@ -1530,13 +1598,13 @@ public:
   }
 
 private:
-  class html_buffered_output_format : public html_output_format
+  class html_buffered_sink : public html_sink
   {
   public:
-    html_buffered_output_format (diagnostic_context &context,
-				 const line_maps *line_maps,
-				 const html_generation_options &html_gen_opts)
-    : html_output_format (context, line_maps, html_gen_opts)
+    html_buffered_sink (context &dc,
+			const line_maps *line_maps,
+			const html_generation_options &html_gen_opts)
+    : html_sink (dc, line_maps, html_gen_opts)
     {
     }
     bool machine_readable_stderr_p () const final override
@@ -1545,20 +1613,20 @@ private:
     }
   };
 
-  html_output_format *m_format; // borrowed
+  html_sink *m_format; // borrowed
 };
 
 /* Test of reporting a diagnostic at UNKNOWN_LOCATION to a
-   diagnostic_context and examining the generated XML document.
+   diagnostics::context and examining the generated XML document.
    Verify various basic properties. */
 
 static void
 test_simple_log ()
 {
-  test_html_diagnostic_context dc;
+  test_html_context dc;
 
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  dc.report (DK_ERROR, richloc, nullptr, 0, "this is a test: %qs", "foo");
+  dc.report (kind::error, richloc, nullptr, 0, "this is a test: %qs", "foo");
 
   const xml::document &doc  = dc.get_document ();
 
@@ -1586,13 +1654,13 @@ test_simple_log ()
 static void
 test_metadata ()
 {
-  test_html_diagnostic_context dc;
+  test_html_context dc;
   html_builder &b = dc.get_builder ();
 
   {
-    diagnostic_metadata metadata;
-    metadata.add_cwe (415);
-    auto element = b.make_element_for_metadata (metadata);
+    metadata m;
+    m.add_cwe (415);
+    auto element = b.make_element_for_metadata (m);
     ASSERT_XML_PRINT_EQ
       (*element,
        "<span class=\"gcc-metadata\">"
@@ -1607,11 +1675,11 @@ test_metadata ()
   }
 
   {
-    diagnostic_metadata metadata;
-    diagnostic_metadata::precanned_rule rule ("MISC-42",
-					      "http://example.com");
-    metadata.add_rule (rule);
-    auto element = b.make_element_for_metadata (metadata);
+    metadata m;
+    metadata::precanned_rule rule ("MISC-42",
+				   "http://example.com");
+    m.add_rule (rule);
+    auto element = b.make_element_for_metadata (m);
     ASSERT_XML_PRINT_EQ
       (*element,
        "<span class=\"gcc-metadata\">"
@@ -1629,9 +1697,9 @@ test_metadata ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_format_html_cc_tests ()
+html_sink_cc_tests ()
 {
-  auto_fix_quotes fix_quotes;
+  ::selftest::auto_fix_quotes fix_quotes;
   test_token_printer ();
   test_simple_log ();
   test_metadata ();
@@ -1640,3 +1708,5 @@ diagnostic_format_html_cc_tests ()
 } // namespace selftest
 
 #endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-format-html.h b/gcc/diagnostics/html-sink.h
index 09a97e0..d86bde8 100644
--- a/gcc/diagnostic-format-html.h
+++ b/gcc/diagnostics/html-sink.h
@@ -18,11 +18,13 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_HTML_H
-#define GCC_DIAGNOSTIC_FORMAT_HTML_H
+#ifndef GCC_DIAGNOSTICS_HTML_SINK_H
+#define GCC_DIAGNOSTICS_HTML_SINK_H
 
-#include "diagnostic-format.h"
-#include "diagnostic-output-file.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/output-file.h"
+
+namespace diagnostics {
 
 struct html_generation_options
 {
@@ -36,29 +38,31 @@ struct html_generation_options
   // If true, attempt to show state diagrams at events
   bool m_show_state_diagrams;
 
-  // If true, show the XML form of the state with such diagrams
-  bool m_show_state_diagram_xml;
+  // If true, show the SARIF form of the state with such diagrams
+  bool m_show_state_diagrams_sarif;
 
   // If true, show the .dot source used for the diagram
-  bool m_show_state_diagram_dot_src;
+  bool m_show_state_diagrams_dot_src;
 };
 
-extern diagnostic_output_file
-diagnostic_output_format_open_html_file (diagnostic_context &context,
-					 line_maps *line_maps,
-					 const char *base_file_name);
+extern diagnostics::output_file
+open_html_output_file (context &dc,
+		       line_maps *line_maps,
+		       const char *base_file_name);
 
-extern std::unique_ptr<diagnostic_output_format>
-make_html_sink (diagnostic_context &context,
+extern std::unique_ptr<sink>
+make_html_sink (context &dc,
 		const line_maps &line_maps,
 		const html_generation_options &html_gen_opts,
-		diagnostic_output_file output_file);
+		output_file output_file_);
 
 extern void
 print_path_as_html (xml::printer &xp,
-		    const diagnostic_path &path,
-		    diagnostic_context &dc,
+		    const paths::path &path,
+		    context &dc,
 		    html_label_writer *event_label_writer,
-		    const diagnostic_source_print_policy &dspp);
+		    const source_print_policy &dspp);
+
+} // namespace diagnostics
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_HTML_H */
+#endif /* ! GCC_DIAGNOSTICS_HTML_SINK_H */
diff --git a/gcc/diagnostic.def b/gcc/diagnostics/kinds.def
index ab2185c..2a0a0a6 100644
--- a/gcc/diagnostic.def
+++ b/gcc/diagnostics/kinds.def
@@ -16,40 +16,40 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-/* DK_UNSPECIFIED must be first so it has a value of zero.  We never
+/* kind::unspecified must be first so it has a value of zero.  We never
    assign this kind to an actual diagnostic, we only use this in
    variables that can hold a kind, to mean they have yet to have a
    kind specified.  I.e. they're uninitialized.  Within the diagnostic
    machinery, this kind also means "don't change the existing kind",
    meaning "no change is specified".  */
-DEFINE_DIAGNOSTIC_KIND (DK_UNSPECIFIED, "", NULL)
+DEFINE_DIAGNOSTIC_KIND (unspecified, "", NULL)
 
-/* If a diagnostic is set to DK_IGNORED, it won't get reported at all.
+/* If a diagnostic is set to kind::ignored, it won't get reported at all.
    This is used by the diagnostic machinery when it wants to disable a
    diagnostic without disabling the option which causes it.  */
-DEFINE_DIAGNOSTIC_KIND (DK_IGNORED, "", NULL)
+DEFINE_DIAGNOSTIC_KIND (ignored, "", NULL)
 
 /* The remainder are real diagnostic types.  */
-DEFINE_DIAGNOSTIC_KIND (DK_FATAL, "fatal error: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_ICE, "internal compiler error: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_ERROR, "error: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_SORRY, "sorry, unimplemented: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_WARNING, "warning: ", "warning")
-DEFINE_DIAGNOSTIC_KIND (DK_ANACHRONISM, "anachronism: ", "warning")
-DEFINE_DIAGNOSTIC_KIND (DK_NOTE, "note: ", "note")
-DEFINE_DIAGNOSTIC_KIND (DK_DEBUG, "debug: ", "note")
+DEFINE_DIAGNOSTIC_KIND (fatal, "fatal error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (ice, "internal compiler error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (error, "error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (sorry, "sorry, unimplemented: ", "error")
+DEFINE_DIAGNOSTIC_KIND (warning, "warning: ", "warning")
+DEFINE_DIAGNOSTIC_KIND (anachronism, "anachronism: ", "warning")
+DEFINE_DIAGNOSTIC_KIND (note, "note: ", "note")
+DEFINE_DIAGNOSTIC_KIND (debug, "debug: ", "note")
 
 /* For use when using the diagnostic_show_locus machinery to show
    a range of events within a path.  */
-DEFINE_DIAGNOSTIC_KIND (DK_DIAGNOSTIC_PATH, "path: ", "path")
+DEFINE_DIAGNOSTIC_KIND (path, "path: ", "path")
 
-/* These two would be re-classified as DK_WARNING or DK_ERROR, so the
+/* These two would be re-classified as kind::warning or kind::error, so the
 prefix does not matter.  */
-DEFINE_DIAGNOSTIC_KIND (DK_PEDWARN, "pedwarn: ", NULL)
-DEFINE_DIAGNOSTIC_KIND (DK_PERMERROR, "permerror: ", NULL)
-/* This one is just for counting DK_WARNING promoted to DK_ERROR
+DEFINE_DIAGNOSTIC_KIND (pedwarn, "pedwarn: ", NULL)
+DEFINE_DIAGNOSTIC_KIND (permerror, "permerror: ", NULL)
+/* This one is just for counting kind::warning promoted to kind::error
    due to -Werror and -Werror=warning.  */
-DEFINE_DIAGNOSTIC_KIND (DK_WERROR, "error: ", NULL)
-/* This is like DK_ICE, but backtrace is not printed.  Used in the driver
+DEFINE_DIAGNOSTIC_KIND (werror, "error: ", NULL)
+/* This is like kind::ICE, but backtrace is not printed.  Used in the driver
    when reporting fatal signal in the compiler.  */
-DEFINE_DIAGNOSTIC_KIND (DK_ICE_NOBT, "internal compiler error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (ice_nobt, "internal compiler error: ", "error")
diff --git a/gcc/diagnostics/kinds.h b/gcc/diagnostics/kinds.h
new file mode 100644
index 0000000..7b4a168
--- /dev/null
+++ b/gcc/diagnostics/kinds.h
@@ -0,0 +1,45 @@
+/* An enum used to discriminate severities of diagnostics.
+   Copyright (C) 1998-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_KINDS_H
+#define GCC_DIAGNOSTICS_KINDS_H
+
+namespace diagnostics {
+
+/* Constants used to discriminate diagnostics.  */
+enum class kind
+{
+#define DEFINE_DIAGNOSTIC_KIND(K, msgid, C) K,
+#include "diagnostics/kinds.def"
+#undef DEFINE_DIAGNOSTIC_KIND
+  last_diagnostic_kind,
+  /* This is used for tagging pragma pops in the diagnostic
+     classification history chain.  */
+  pop,
+  /* This is used internally to note that a diagnostic is enabled
+     without mandating any specific type.  */
+  any
+};
+
+extern const char *get_text_for_kind (enum diagnostics::kind);
+extern const char *get_color_for_kind (enum diagnostics::kind);
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_KINDS_H */
diff --git a/gcc/lazy-diagnostic-path.cc b/gcc/diagnostics/lazy-paths.cc
index 34743e3..f246eea 100644
--- a/gcc/lazy-diagnostic-path.cc
+++ b/gcc/diagnostics/lazy-paths.cc
@@ -23,60 +23,58 @@ along with GCC; see the file COPYING3.  If not see
 #define INCLUDE_VECTOR
 #include "system.h"
 #include "coretypes.h"
-#include "tree.h"
-#include "version.h"
-#include "intl.h"
 #include "diagnostic.h"
-#include "lazy-diagnostic-path.h"
+#include "diagnostics/lazy-paths.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "simple-diagnostic-path.h"
-#include "gcc-rich-location.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-paths.h"
+#include "diagnostics/text-sink.h"
 
-/* class lazy_diagnostic_path : public diagnostic_path.  */
+using namespace diagnostics::paths;
 
-/* Implementation of diagnostic_path vfuncs in terms of a lazily-generated
+/* class lazy_path : public path.  */
+
+/* Implementation of path vfuncs in terms of a lazily-generated
    path.  */
 
 unsigned
-lazy_diagnostic_path::num_events () const
+lazy_path::num_events () const
 {
   lazily_generate_path ();
   return m_inner_path->num_events ();
 }
 
-const diagnostic_event &
-lazy_diagnostic_path::get_event (int idx) const
+const event &
+lazy_path::get_event (int idx) const
 {
   lazily_generate_path ();
   return m_inner_path->get_event (idx);
 }
 
 unsigned
-lazy_diagnostic_path::num_threads () const
+lazy_path::num_threads () const
 {
   lazily_generate_path ();
   return m_inner_path->num_threads ();
 }
 
-const diagnostic_thread &
-lazy_diagnostic_path::get_thread (diagnostic_thread_id_t idx) const
+const thread &
+lazy_path::get_thread (thread_id_t idx) const
 {
   lazily_generate_path ();
   return m_inner_path->get_thread (idx);
 }
 
 bool
-lazy_diagnostic_path::same_function_p (int event_idx_a,
-					   int event_idx_b) const
+lazy_path::same_function_p (int event_idx_a,
+			    int event_idx_b) const
 {
   lazily_generate_path ();
   return m_inner_path->same_function_p (event_idx_a, event_idx_b);
 }
 
 void
-lazy_diagnostic_path::lazily_generate_path () const
+lazy_path::lazily_generate_path () const
 {
   if (!m_inner_path)
     m_inner_path = make_inner_path ();
@@ -85,30 +83,30 @@ lazy_diagnostic_path::lazily_generate_path () const
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
-class test_lazy_path : public lazy_diagnostic_path
+using auto_fix_quotes = ::selftest::auto_fix_quotes;
+
+class test_lazy_path : public lazy_path
 {
 public:
   test_lazy_path (pretty_printer &pp)
-  : lazy_diagnostic_path (m_logical_loc_mgr),
+  : lazy_path (m_logical_loc_mgr),
     m_pp (pp)
   {
   }
-  std::unique_ptr<diagnostic_path> make_inner_path () const final override
+  std::unique_ptr<path> make_inner_path () const final override
   {
-    tree fntype_void_void
-      = build_function_type_array (void_type_node, 0, nullptr);
-    tree fndecl_foo = build_fn_decl ("foo", fntype_void_void);
     auto path
-      = std::make_unique<simple_diagnostic_path> (m_logical_loc_mgr,
-						  &m_pp);
-    path->add_event (UNKNOWN_LOCATION, fndecl_foo, 0, "first %qs", "free");
-    path->add_event (UNKNOWN_LOCATION, fndecl_foo, 0, "double %qs", "free");
+      = std::make_unique<paths::selftest::test_path> (m_logical_loc_mgr,
+						      &m_pp);
+    path->add_event (UNKNOWN_LOCATION, "foo", 0, "first %qs", "free");
+    path->add_event (UNKNOWN_LOCATION, "foo", 0, "double %qs", "free");
     return path;
   }
 private:
-  const tree_logical_location_manager m_logical_loc_mgr;
+  mutable logical_locations::selftest::test_manager m_logical_loc_mgr;
   pretty_printer &m_pp;
 };
 
@@ -127,26 +125,26 @@ test_intraprocedural_path (pretty_printer *event_pp)
 		"double `free'");
 }
 
-/* Implementation of diagnostic_option_manager for which all
+/* Implementation of diagnostics::option_id_manager for which all
    options are disabled, for use in selftests.
-   Note that this is *not* called for diagnostic_option_id (0), which
+   Note that this is *not* called for option_id (0), which
    means "always warn"  */
 
-class all_warnings_disabled : public diagnostic_option_manager
+class all_warnings_disabled : public diagnostics::option_id_manager
 {
 public:
-  int option_enabled_p (diagnostic_option_id) const final override
+  int option_enabled_p (diagnostics::option_id) const final override
   {
     /* Treat all options as disabled.  */
     return 0;
   }
-  char *make_option_name (diagnostic_option_id,
-			  diagnostic_t,
-			  diagnostic_t) const final override
+  char *make_option_name (diagnostics::option_id,
+			  enum kind,
+			  enum kind) const final override
   {
     return nullptr;
   }
-  char *make_option_url (diagnostic_option_id) const final override
+  char *make_option_url (diagnostics::option_id) const final override
   {
     return nullptr;
   }
@@ -155,10 +153,10 @@ public:
 static void
 test_emission (pretty_printer *event_pp)
 {
-  struct test_rich_location : public gcc_rich_location
+  struct test_rich_location : public rich_location
   {
     test_rich_location (pretty_printer &event_pp)
-    : gcc_rich_location (UNKNOWN_LOCATION),
+    : rich_location (line_table, UNKNOWN_LOCATION),
       m_path (event_pp)
     {
       set_path (&m_path);
@@ -169,16 +167,16 @@ test_emission (pretty_printer *event_pp)
   /* Verify that we don't bother generating the inner path if the warning
      is skipped.  */
   {
-    test_diagnostic_context dc;
-    dc.set_option_manager (std::make_unique<all_warnings_disabled> (), 0);
+    test_context dc;
+    dc.set_option_id_manager (std::make_unique<all_warnings_disabled> (), 0);
 
     test_rich_location rich_loc (*event_pp);
     ASSERT_FALSE (rich_loc.m_path.generated_p ());
 
-    diagnostic_option_id option_id (42); // has to be non-zero
+    diagnostics::option_id opt_id (42); // has to be non-zero
     bool emitted
-      = dc.emit_diagnostic_with_group (DK_WARNING, rich_loc, nullptr,
-				       option_id,
+      = dc.emit_diagnostic_with_group (kind::warning, rich_loc, nullptr,
+				       opt_id,
 				       "this warning should be skipped");
     ASSERT_FALSE (emitted);
     ASSERT_FALSE (rich_loc.m_path.generated_p ());
@@ -187,23 +185,23 @@ test_emission (pretty_printer *event_pp)
   /* Verify that we *do* generate the inner path for a diagnostic that
      is emitted, such as an error.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
 
     test_rich_location rich_loc (*event_pp);
     ASSERT_FALSE (rich_loc.m_path.generated_p ());
 
     bool emitted
-      = dc.emit_diagnostic_with_group (DK_ERROR, rich_loc, nullptr, 0,
+      = dc.emit_diagnostic_with_group (kind::error, rich_loc, nullptr, 0,
 				       "this is a test");
     ASSERT_TRUE (emitted);
     ASSERT_TRUE (rich_loc.m_path.generated_p ());
 
     /* Verify that the path works as expected.  */
     dc.set_path_format (DPF_INLINE_EVENTS);
-    diagnostic_text_output_format sink (dc);
-    pp_buffer (sink.get_printer ())->m_flush_p = false;
-    sink.print_path (rich_loc.m_path);
-    ASSERT_STREQ (pp_formatted_text (sink.get_printer ()),
+    diagnostics::text_sink sink_ (dc);
+    pp_buffer (sink_.get_printer ())->m_flush_p = false;
+    sink_.print_path (rich_loc.m_path);
+    ASSERT_STREQ (pp_formatted_text (sink_.get_printer ()),
 		  "  `foo': event 1\n"
 		  " (1): first `free'\n"
 		  "  `foo': event 2\n"
@@ -214,7 +212,7 @@ test_emission (pretty_printer *event_pp)
 /* Run all of the selftests within this file.  */
 
 void
-lazy_diagnostic_path_cc_tests ()
+lazy_paths_cc_tests ()
 {
   /* In a few places we use the global dc's printer to determine
      colorization so ensure this off during the tests.  */
@@ -232,6 +230,7 @@ lazy_diagnostic_path_cc_tests ()
   pp_show_color (global_pp) = saved_show_color;
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/lazy-diagnostic-path.h b/gcc/diagnostics/lazy-paths.h
index 9609cab..01b921b 100644
--- a/gcc/lazy-diagnostic-path.h
+++ b/gcc/diagnostics/lazy-paths.h
@@ -18,13 +18,16 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_LAZY_DIAGNOSTIC_PATH_H
-#define GCC_LAZY_DIAGNOSTIC_PATH_H
+#ifndef GCC_DIAGNOSTICS_LAZY_PATHS_H
+#define GCC_DIAGNOSTICS_LAZY_PATHS_H
 
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 
-/* An implementation of diagnostic_path which has a trivial ctor
-   and lazily creates another diagnostic_path the first time the path
+namespace diagnostics {
+namespace paths {
+
+/* An implementation of diagnostics::paths::path which has a trivial ctor
+   and lazily creates another path the first time the path
    is queried, deferring to this inner path for all queries.
 
    Use this to avoid expensive path creation logic when creating
@@ -32,16 +35,16 @@ along with GCC; see the file COPYING3.  If not see
    is actually used by a diagnostic, and thus avoided for warnings that
    are disabled.  */
 
-class lazy_diagnostic_path : public diagnostic_path
+class lazy_path : public path
 {
  public:
-  virtual ~lazy_diagnostic_path () {}
+  virtual ~lazy_path () {}
 
   unsigned num_events () const final override;
-  const diagnostic_event & get_event (int idx) const final override;
+  const event & get_event (int idx) const final override;
   unsigned num_threads () const final override;
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override;
+  const thread &
+  get_thread (thread_id_t) const final override;
   bool
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override;
@@ -49,16 +52,19 @@ class lazy_diagnostic_path : public diagnostic_path
   bool generated_p () const { return m_inner_path != nullptr; }
 
 protected:
-  lazy_diagnostic_path (const logical_location_manager &logical_loc_mgr)
-  : diagnostic_path (logical_loc_mgr)
+  lazy_path (const logical_locations::manager &logical_loc_mgr)
+  : path (logical_loc_mgr)
   {
   }
 
  private:
   void lazily_generate_path () const;
-  virtual std::unique_ptr<diagnostic_path> make_inner_path () const = 0;
+  virtual std::unique_ptr<path> make_inner_path () const = 0;
 
-  mutable std::unique_ptr<diagnostic_path> m_inner_path;
+  mutable std::unique_ptr<path> m_inner_path;
 };
 
-#endif /* ! GCC_LAZY_DIAGNOSTIC_PATH_H */
+} // namespace paths
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_LAZY_PATHS_H */
diff --git a/gcc/logical-location.h b/gcc/diagnostics/logical-locations.h
index 3522934..b52a9b4 100644
--- a/gcc/logical-location.h
+++ b/gcc/diagnostics/logical-locations.h
@@ -18,18 +18,21 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_LOGICAL_LOCATION_H
-#define GCC_LOGICAL_LOCATION_H
+#ifndef GCC_DIAGNOSTICS_LOGICAL_LOCATIONS_H
+#define GCC_DIAGNOSTICS_LOGICAL_LOCATIONS_H
 
 #include "label-text.h"
 
+namespace diagnostics {
+namespace logical_locations {
+
 /* An enum for discriminating between different kinds of logical location
    for a diagnostic.
 
    Roughly corresponds to logicalLocation's "kind" property in SARIF v2.1.0
    (section 3.33.7).  */
 
-enum class logical_location_kind
+enum class kind
 {
   unknown,
 
@@ -68,78 +71,84 @@ enum class logical_location_kind
    libgdiagnostics internals), and without requiring heap allocation of an
    interface class when emitting a diagnostic.
 
-   To do this, we split the implementation into logical_location, which is
-   a wrapper around a (const void *), and logical_location_manager which
+   To do this, we split the implementation into logical_locations::key, which is
+   a wrapper around a (const void *), and logical_locations::manager which
    is provided by the client and has vfunc hooks for interpreting
-   logical_location instances.
+   key instances.
 
-   Every logical_location is associated with a logical_location_manager and
-   only has meaning in relation to that manager.
+   Every logical_locations::key is associated with a logical_locations::manager
+   and only has meaning in relation to that manager.
 
-   A "nullptr" within a logical_location means "no logical location".
+   A "nullptr" within a key means "no logical location".
 
    See tree-logical-location.h for concrete subclasses relating to trees,
    where the pointer is a const_tree.
 
-   See selftest-logical-location.h for a concrete subclass for selftests.  */
+   See diagnostics/selftest-logical-locations.h for a concrete subclass for
+   selftests.  */
 
-/* Abstract base class for giving meaning to logical_location values.
-   Typically there will just be one client-provided instance, of a
-   client-specific subclass.  */
+/* Extrinsic state for identifying a specific logical location.
+   This will be our logical location type.
+   This only makes sense with respect to a specific manager.
+   e.g. for a tree-based one it's a wrapper around "tree".
+
+   "nullptr" means "no logical location".
+
+   Note that there is no integration with GCC's garbage collector and thus
+   keys can't be long-lived.  */
 
-class logical_location_manager
+class key
 {
 public:
-  /* Extrinsic state for identifying a specific logical location.
-     This will be our logical_location type.
-     This only makes sense with respect to a specific manager.
-     e.g. for a tree-based one it's a wrapper around "tree".
-     "nullptr" means "no logical location".  */
-  class key
+  key () : m_ptr (nullptr) {}
+
+  static key from_ptr (const void *ptr)
   {
-  public:
-    key () : m_ptr (nullptr) {}
+    return key (ptr);
+  }
 
-    static key from_ptr (const void *ptr)
-    {
-      return key (ptr);
-    }
+  operator bool () const
+  {
+    return m_ptr != nullptr;
+  }
 
-    operator bool () const
-    {
-      return m_ptr != nullptr;
-    }
+  template <typename T>
+  T cast_to () const { return static_cast<T> (m_ptr); }
 
-    template <typename T>
-    T cast_to () const { return static_cast<T> (m_ptr); }
+  bool
+  operator== (const key &other) const
+  {
+    return m_ptr == other.m_ptr;
+  }
 
-    bool
-    operator== (const key &other) const
-    {
-      return m_ptr == other.m_ptr;
-    }
+  bool
+  operator!= (const key &other) const
+  {
+    return m_ptr != other.m_ptr;
+  }
 
-    bool
-    operator!= (const key &other) const
-    {
-      return m_ptr != other.m_ptr;
-    }
+  bool
+  operator< (const key &other) const
+  {
+    return m_ptr < other.m_ptr;
+  }
 
-    bool
-    operator< (const key &other) const
-    {
-      return m_ptr < other.m_ptr;
-    }
+private:
+  explicit key (const void *ptr) : m_ptr (ptr) {}
 
-  private:
-    explicit key (const void *ptr) : m_ptr (ptr) {}
+  const void *m_ptr;
+};
 
-    const void *m_ptr;
-  };
+/* Abstract base class for giving meaning to keys.
+   Typically there will just be one client-provided instance, of a
+   client-specific subclass.  */
 
-  virtual ~logical_location_manager () {}
+class manager
+{
+public:
+  virtual ~manager () {}
 
-  /* vfuncs for interpreting logical_location values.  */
+  /* vfuncs for interpreting keys.  */
 
   /* Get a string (or NULL) for K suitable for use by the SARIF logicalLocation
      "name" property (SARIF v2.1.0 section 3.33.4).  */
@@ -154,7 +163,7 @@ public:
   virtual const char *get_internal_name (key k) const = 0;
 
   /* Get what kind of SARIF logicalLocation K is (if any).  */
-  virtual enum logical_location_kind get_kind (key k) const = 0;
+  virtual enum kind get_kind (key k) const = 0;
 
   /* Get a string for location K in a form suitable for path output.  */
   virtual label_text get_name_for_path_output (key k) const = 0;
@@ -165,11 +174,7 @@ public:
   bool function_p (key k) const;
 };
 
-/* A logical location is a key for a given logical_location_manager.
-
-   Note that there is no integration with GCC's garbage collector and thus
-   logical_location instances can't be long-lived.  */
-
-typedef logical_location_manager::key logical_location;
+} // namespace diagnostics::logical_locations
+} // namespace diagnostics
 
-#endif /* GCC_LOGICAL_LOCATION_H.  */
+#endif /* GCC_DIAGNOSTICS_LOGICAL_LOCATIONS_H.  */
diff --git a/gcc/diagnostic-macro-unwinding.cc b/gcc/diagnostics/macro-unwinding.cc
index aa1b816..66bad1c 100644
--- a/gcc/diagnostic-macro-unwinding.cc
+++ b/gcc/diagnostics/macro-unwinding.cc
@@ -22,10 +22,12 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "tree.h"
 #include "diagnostic.h"
-#include "diagnostic-macro-unwinding.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/macro-unwinding.h"
+#include "diagnostics/text-sink.h"
 #include "intl.h"
 
+namespace diagnostics {
+
 /* This is a pair made of a location and the line map it originated
    from.  It's used in the maybe_unwind_expanded_macro_loc function
    below.  */
@@ -73,7 +75,7 @@ struct loc_map_pair
    function.  */
 
 void
-maybe_unwind_expanded_macro_loc (diagnostic_text_output_format &text_output,
+maybe_unwind_expanded_macro_loc (text_sink &text_output,
                                  location_t where)
 {
   const struct line_map *map;
@@ -215,8 +217,11 @@ maybe_unwind_expanded_macro_loc (diagnostic_text_output_format &text_output,
     that is similar to what is done for function call stacks, or
     template instantiation contexts.  */
 void
-virt_loc_aware_diagnostic_finalizer (diagnostic_text_output_format &text_output,
-				     const diagnostic_info *diagnostic)
+virt_loc_aware_text_finalizer (text_sink &text_output,
+			       const diagnostic_info *diagnostic)
 {
-  maybe_unwind_expanded_macro_loc (text_output, diagnostic_location (diagnostic));
+  maybe_unwind_expanded_macro_loc (text_output,
+				   diagnostic_location (diagnostic));
 }
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-macro-unwinding.h b/gcc/diagnostics/macro-unwinding.h
index ef1e55d..1f28d58 100644
--- a/gcc/diagnostic-macro-unwinding.h
+++ b/gcc/diagnostics/macro-unwinding.h
@@ -17,13 +17,17 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_MACRO_UNWINDING_H
-#define GCC_DIAGNOSTIC_MACRO_UNWINDING_H
+#ifndef GCC_DIAGNOSTICS_MACRO_UNWINDING_H
+#define GCC_DIAGNOSTICS_MACRO_UNWINDING_H
 
-void virt_loc_aware_diagnostic_finalizer (diagnostic_text_output_format &,
-					  const diagnostic_info *);
+namespace diagnostics {
 
-extern void maybe_unwind_expanded_macro_loc (diagnostic_text_output_format &,
+extern void virt_loc_aware_text_finalizer (text_sink &,
+					   const diagnostic_info *);
+
+extern void maybe_unwind_expanded_macro_loc (text_sink &,
 					     location_t where);
 
-#endif /* ! GCC_DIAGNOSTIC_MACRO_UNWINDING_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_MACRO_UNWINDING_H */
diff --git a/gcc/diagnostic-metadata.h b/gcc/diagnostics/metadata.h
index baeeb6e..c28f982 100644
--- a/gcc/diagnostic-metadata.h
+++ b/gcc/diagnostics/metadata.h
@@ -18,20 +18,31 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_METADATA_H
-#define GCC_DIAGNOSTIC_METADATA_H
+#ifndef GCC_DIAGNOSTICS_METADATA_H
+#define GCC_DIAGNOSTICS_METADATA_H
 
-class sarif_object;
+#include "lazily-created.h"
+
+namespace diagnostics {
+
+  class sarif_object;
+  namespace digraphs { class digraph; }
 
 /* A bundle of additional metadata that can be associated with a
    diagnostic.
 
    This supports an optional CWE identifier, and zero or more
-   "rules".  */
+   "rules".
+
+   Additionally, this provides a place to associate a diagnostic
+   with zero or more directed graphs.  */
 
-class diagnostic_metadata
+class metadata
 {
  public:
+  using lazy_digraphs
+  = lazily_created<std::vector<std::unique_ptr<digraphs::digraph>>>;
+
   /* Abstract base class for referencing a rule that has been violated,
      such as within a coding standard, or within a specification.  */
   class rule
@@ -64,8 +75,8 @@ class diagnostic_metadata
     const char *m_url;
   };
 
-  diagnostic_metadata () : m_cwe (0) {}
-  virtual ~diagnostic_metadata () {}
+  metadata () : m_cwe (0), m_lazy_digraphs (nullptr) {}
+  virtual ~metadata () {}
 
   /* Hook for SARIF output to allow for adding diagnostic-specific
      properties to  the result object's property bag.  */
@@ -87,9 +98,27 @@ class diagnostic_metadata
   unsigned get_num_rules () const { return m_rules.length (); }
   const rule &get_rule (unsigned idx) const { return *(m_rules[idx]); }
 
+  void
+  set_lazy_digraphs (const lazy_digraphs *lazy_digraphs_)
+  {
+    m_lazy_digraphs = lazy_digraphs_;
+  }
+
+  const lazy_digraphs *
+  get_lazy_digraphs () const
+  {
+    return m_lazy_digraphs;
+  }
+
  private:
   int m_cwe;
   auto_vec<const rule *> m_rules;
+
+  /* An optional way to create directed graphs associated with the
+     diagnostic, for the sinks that support this (e.g. SARIF).  */
+  const lazy_digraphs *m_lazy_digraphs;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_METADATA_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_METADATA_H */
diff --git a/gcc/diagnostics/option-classifier.cc b/gcc/diagnostics/option-classifier.cc
new file mode 100644
index 0000000..f98fd55
--- /dev/null
+++ b/gcc/diagnostics/option-classifier.cc
@@ -0,0 +1,222 @@
+/* Stacks of set of classifications of diagnostics.
+   Copyright (C) 1999-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "version.h"
+#include "diagnostic.h"
+
+namespace diagnostics {
+
+void
+option_classifier::init (int n_opts)
+{
+  m_n_opts = n_opts;
+  m_classify_diagnostic = XNEWVEC (enum kind, n_opts);
+  for (int i = 0; i < n_opts; i++)
+    m_classify_diagnostic[i] = kind::unspecified;
+  m_push_list = vNULL;
+  m_classification_history = vNULL;
+}
+
+void
+option_classifier::fini ()
+{
+  XDELETEVEC (m_classify_diagnostic);
+  m_classify_diagnostic = nullptr;
+  m_classification_history.release ();
+  m_push_list.release ();
+}
+
+/* Save the diagnostics::option_classifier state to F for PCH
+   output.  Returns 0 on success, -1 on error.  */
+
+int
+option_classifier::pch_save (FILE *f)
+{
+  unsigned int lengths[2] = { m_classification_history.length (),
+			      m_push_list.length () };
+  if (fwrite (lengths, sizeof (lengths), 1, f) != 1
+      || (lengths[0]
+	  && fwrite (m_classification_history.address (),
+		     sizeof (classification_change_t),
+		     lengths[0], f) != lengths[0])
+      || (lengths[1]
+	  && fwrite (m_push_list.address (), sizeof (int),
+		     lengths[1], f) != lengths[1]))
+    return -1;
+  return 0;
+}
+
+/* Read the diagnostics::option_classifier state from F for PCH
+   read.  Returns 0 on success, -1 on error.  */
+
+int
+option_classifier::pch_restore (FILE *f)
+{
+  unsigned int lengths[2];
+  if (fread (lengths, sizeof (lengths), 1, f) != 1)
+    return -1;
+  gcc_checking_assert (m_classification_history.is_empty ());
+  gcc_checking_assert (m_push_list.is_empty ());
+  m_classification_history.safe_grow (lengths[0]);
+  m_push_list.safe_grow (lengths[1]);
+  if ((lengths[0]
+       && fread (m_classification_history.address (),
+		 sizeof (classification_change_t),
+		 lengths[0], f) != lengths[0])
+      || (lengths[1]
+	  && fread (m_push_list.address (), sizeof (int),
+		    lengths[1], f) != lengths[1]))
+    return -1;
+  return 0;
+}
+
+/* Save all diagnostic classifications in a stack.  */
+
+void
+option_classifier::push ()
+{
+  m_push_list.safe_push (m_classification_history.length ());
+}
+
+/* Restore the topmost classification set off the stack.  If the stack
+   is empty, revert to the state based on command line parameters.  */
+
+void
+option_classifier::pop (location_t where)
+{
+  int jump_to;
+
+  if (!m_push_list.is_empty ())
+    jump_to = m_push_list.pop ();
+  else
+    jump_to = 0;
+
+  classification_change_t v = { where, jump_to, kind::pop };
+  m_classification_history.safe_push (v);
+}
+
+/* Interface to specify diagnostic kind overrides.  Returns the
+   previous setting, or kind::unspecified if the parameters are out of
+   range.  If OPTION_ID is zero, the new setting is for all the
+   diagnostics.  */
+
+enum kind
+option_classifier::classify_diagnostic (const context *dc,
+					option_id opt_id,
+					enum kind new_kind,
+					location_t where)
+{
+  enum kind old_kind;
+
+  if (opt_id.m_idx < 0
+      || opt_id.m_idx >= m_n_opts
+      || new_kind >= kind::last_diagnostic_kind)
+    return kind::unspecified;
+
+  old_kind = m_classify_diagnostic[opt_id.m_idx];
+
+  /* Handle pragmas separately, since we need to keep track of *where*
+     the pragmas were.  */
+  if (where != UNKNOWN_LOCATION)
+    {
+      unsigned i;
+
+      /* Record the command-line status, so we can reset it back on kind::pop. */
+      if (old_kind == kind::unspecified)
+	{
+	  old_kind = (!dc->option_enabled_p (opt_id)
+		      ? kind::ignored : kind::any);
+	  m_classify_diagnostic[opt_id.m_idx] = old_kind;
+	}
+
+      classification_change_t *p;
+      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
+	if (p->option == opt_id.m_idx)
+	  {
+	    old_kind = p->kind;
+	    break;
+	  }
+
+      classification_change_t v
+	= { where, opt_id.m_idx, new_kind };
+      m_classification_history.safe_push (v);
+    }
+  else
+    m_classify_diagnostic[opt_id.m_idx] = new_kind;
+
+  return old_kind;
+}
+
+/* Update the kind of DIAGNOSTIC based on its location(s), including
+   any of those in its inlining stack, relative to any
+     #pragma GCC diagnostic
+   directives recorded within this object.
+
+   Return the new kind of DIAGNOSTIC if it was updated, or kind::unspecified
+   otherwise.  */
+
+enum kind
+option_classifier::
+update_effective_level_from_pragmas (diagnostic_info *diagnostic) const
+{
+  if (m_classification_history.is_empty ())
+    return kind::unspecified;
+
+  /* Iterate over the locations, checking the diagnostic disposition
+     for the diagnostic at each.  If it's explicitly set as opposed
+     to unspecified, update the disposition for this instance of
+     the diagnostic and return it.  */
+  for (location_t loc: diagnostic->m_iinfo.m_ilocs)
+    {
+      /* FIXME: Stupid search.  Optimize later. */
+      unsigned int i;
+      classification_change_t *p;
+      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
+	{
+	  location_t pragloc = p->location;
+	  if (!linemap_location_before_p (line_table, pragloc, loc))
+	    continue;
+
+	  if (p->kind == kind::pop)
+	    {
+	      /* Move on to the next region.  */
+	      i = p->option;
+	      continue;
+	    }
+
+	  option_id opt_id = p->option;
+	  /* The option 0 is for all the diagnostics.  */
+	  if (opt_id == 0 || opt_id == diagnostic->m_option_id)
+	    {
+	      enum kind kind = p->kind;
+	      if (kind != diagnostics::kind::unspecified)
+		diagnostic->m_kind = kind;
+	      return kind;
+	    }
+	}
+    }
+
+  return kind::unspecified;
+}
+
+} // namespace diagnostics
diff --git a/gcc/diagnostics/option-classifier.h b/gcc/diagnostics/option-classifier.h
new file mode 100644
index 0000000..3b16c74
--- /dev/null
+++ b/gcc/diagnostics/option-classifier.h
@@ -0,0 +1,110 @@
+/* Stacks of set of classifications of diagnostics.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_OPTION_CLASSIFIER_H
+#define GCC_DIAGNOSTICS_OPTION_CLASSIFIER_H
+
+namespace diagnostics {
+
+/*  Forward declarations.  */
+class context;
+
+/* A stack of sets of classifications: each entry in the stack is
+   a mapping from option index to diagnostic severity that can be changed
+   via pragmas.  The stack can be pushed and popped.  */
+
+class option_classifier
+{
+public:
+  void init (int n_opts);
+  void fini ();
+
+  /* Save all diagnostic classifications in a stack.  */
+  void push ();
+
+  /* Restore the topmost classification set off the stack.  If the stack
+     is empty, revert to the state based on command line parameters.  */
+  void pop (location_t where);
+
+  bool option_unspecified_p (option_id opt_id) const
+  {
+    return get_current_override (opt_id) == kind::unspecified;
+  }
+
+  enum kind get_current_override (option_id opt_id) const
+  {
+    gcc_assert (opt_id.m_idx < m_n_opts);
+    return m_classify_diagnostic[opt_id.m_idx];
+  }
+
+  enum kind
+  classify_diagnostic (const context *context,
+		       option_id opt_id,
+		       enum kind new_kind,
+		       location_t where);
+
+  enum kind
+  update_effective_level_from_pragmas (diagnostic_info *diagnostic) const;
+
+  int pch_save (FILE *);
+  int pch_restore (FILE *);
+
+private:
+  /* Each time a diagnostic's classification is changed with a pragma,
+     we record the change and the location of the change in an array of
+     these structs.  */
+  struct classification_change_t
+  {
+    location_t location;
+
+    /* For kind::pop, this is the index of the corresponding push (as stored
+       in m_push_list).
+       Otherwise, this is an option index.  */
+    int option;
+
+    enum kind kind;
+  };
+
+  int m_n_opts;
+
+  /* For each option index that can be passed to warning() et al
+     (OPT_* from options.h when using this code with the core GCC
+     options), this array may contain a new kind that the diagnostic
+     should be changed to before reporting, or kind::unspecified to leave
+     it as the reported kind, or kind::ignored to not report it at
+     all.  */
+  enum kind *m_classify_diagnostic;
+
+  /* History of all changes to the classifications above.  This list
+     is stored in location-order, so we can search it, either
+     binary-wise or end-to-front, to find the most recent
+     classification for a given diagnostic, given the location of the
+     diagnostic.  */
+  vec<classification_change_t> m_classification_history;
+
+  /* For context::get_classification_history, declared later.  */
+  friend class context;
+
+  /* For pragma push/pop.  */
+  vec<int> m_push_list;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_OPTION_CLASSIFIER_H */
diff --git a/gcc/diagnostics/option-id-manager.h b/gcc/diagnostics/option-id-manager.h
new file mode 100644
index 0000000..08add5b
--- /dev/null
+++ b/gcc/diagnostics/option-id-manager.h
@@ -0,0 +1,56 @@
+/* Hooks for giving client-specific meaning to option ids.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_OPTION_ID_MANAGER_H
+#define GCC_DIAGNOSTICS_OPTION_ID_MANAGER_H
+
+namespace diagnostics {
+
+/* Abstract base class for the diagnostic subsystem to make queries
+   about command-line options.  */
+
+class option_id_manager
+{
+public:
+  virtual ~option_id_manager () {}
+
+  /* Return 1 if option OPT_ID is enabled, 0 if it is disabled,
+     or -1 if it isn't a simple on-off switch
+     (or if the value is unknown, typically set later in target).  */
+  virtual int option_enabled_p (option_id opt_id) const = 0;
+
+  /* Return malloced memory for the name of the option OPT_ID
+     which enabled a diagnostic, originally of type ORIG_DIAG_KIND but
+     possibly converted to DIAG_KIND by options such as -Werror.
+     May return NULL if no name is to be printed.
+     May be passed 0 as well as the index of a particular option.  */
+  virtual char *make_option_name (option_id opt_id,
+				  enum kind orig_diag_kind,
+				  enum kind diag_kind) const = 0;
+
+  /* Return malloced memory for a URL describing the option that controls
+     a diagnostic.
+     May return NULL if no URL is available.
+     May be passed 0 as well as the index of a particular option.  */
+  virtual char *make_option_url (option_id opt_id) const = 0;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_OPTION_ID_MANAGER_H */
diff --git a/gcc/diagnostics/option-id.h b/gcc/diagnostics/option-id.h
new file mode 100644
index 0000000..4132775
--- /dev/null
+++ b/gcc/diagnostics/option-id.h
@@ -0,0 +1,49 @@
+/* Declaration of struct diagnostics::option_id.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_OPTION_ID_H
+#define GCC_DIAGNOSTICS_OPTION_ID_H
+
+namespace diagnostics {
+
+/* A class to use for the ID of an option that controls
+   a particular diagnostic.
+   This is just a wrapper around "int", but better documents
+   the intent of the code.  */
+
+struct option_id
+{
+  option_id () : m_idx (0) {}
+
+  option_id (int idx) : m_idx (idx) {}
+  /* Ideally we'd take an enum opt_code here, but we don't
+     want to depend on its decl.  */
+
+  bool operator== (option_id other) const
+  {
+    return m_idx == other.m_idx;
+  }
+
+  int m_idx;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_OPTION_ID_H */
diff --git a/gcc/diagnostic-output-file.h b/gcc/diagnostics/output-file.h
index a0b2e1b..f936387 100644
--- a/gcc/diagnostic-output-file.h
+++ b/gcc/diagnostics/output-file.h
@@ -18,22 +18,24 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_OUTPUT_FILE_H
-#define GCC_DIAGNOSTIC_OUTPUT_FILE_H
+#ifndef GCC_DIAGNOSTICS_OUTPUT_FILE_H
+#define GCC_DIAGNOSTICS_OUTPUT_FILE_H
+
+namespace diagnostics {
 
 /* RAII class for wrapping a FILE * that could be borrowed or owned,
    along with the underlying filename.  */
 
-class diagnostic_output_file
+class output_file
 {
 public:
-  diagnostic_output_file ()
+  output_file ()
   : m_outf (nullptr),
     m_owned (false),
     m_filename ()
   {
   }
-  diagnostic_output_file (FILE *outf, bool owned, label_text filename)
+  output_file (FILE *outf, bool owned, label_text filename)
   : m_outf (outf),
     m_owned (owned),
     m_filename (std::move (filename))
@@ -42,7 +44,7 @@ public:
     if (m_owned)
       gcc_assert (m_outf);
   }
-  ~diagnostic_output_file ()
+  ~output_file ()
   {
     if (m_owned)
       {
@@ -50,8 +52,8 @@ public:
 	fclose (m_outf);
       }
   }
-  diagnostic_output_file (const diagnostic_output_file &other) = delete;
-  diagnostic_output_file (diagnostic_output_file &&other)
+  output_file (const output_file &other) = delete;
+  output_file (output_file &&other)
   : m_outf (other.m_outf),
     m_owned (other.m_owned),
     m_filename (std::move (other.m_filename))
@@ -63,10 +65,10 @@ public:
     if (m_owned)
       gcc_assert (m_outf);
   }
-  diagnostic_output_file &
-  operator= (const diagnostic_output_file &other) = delete;
-  diagnostic_output_file &
-  operator= (diagnostic_output_file &&other)
+  output_file &
+  operator= (const output_file &other) = delete;
+  output_file &
+  operator= (output_file &&other)
   {
     if (m_owned)
       {
@@ -91,8 +93,8 @@ public:
   FILE *get_open_file () const { return m_outf; }
   const char *get_filename () const { return m_filename.get (); }
 
-  static diagnostic_output_file
-  try_to_open (diagnostic_context &context,
+  static output_file
+  try_to_open (context &dc,
 	       line_maps *line_maps,
 	       const char *base_file_name,
 	       const char *extension,
@@ -104,4 +106,6 @@ private:
   label_text m_filename;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_OUTPUT_FILE_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_OUTPUT_FILE_H */
diff --git a/gcc/diagnostic-output-spec.cc b/gcc/diagnostics/output-spec.cc
index 25ef86f..83f128c 100644
--- a/gcc/diagnostic-output-spec.cc
+++ b/gcc/diagnostics/output-spec.cc
@@ -32,20 +32,21 @@ along with GCC; see the file COPYING3.  If not see
 #include "version.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-format.h"
-#include "diagnostic-format-html.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/color.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/html-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/sarif-sink.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 #include "pretty-print-markup.h"
-#include "diagnostic-output-spec.h"
+#include "diagnostics/output-spec.h"
 
 /* A namespace for handling the DSL of the arguments of
    -fdiagnostics-add-output= and -fdiagnostics-set-output=.  */
 
-namespace diagnostics_output_spec {
+namespace diagnostics {
+namespace output_spec {
 
 /* Decls.  */
 
@@ -56,7 +57,7 @@ struct scheme_name_and_params
 };
 
 /* Class for parsing the arguments of -fdiagnostics-add-output= and
-   -fdiagnostics-set-output=, and making diagnostic_output_format
+   -fdiagnostics-set-output=, and making sink
    instances (or issuing errors).  */
 
 class output_factory
@@ -72,9 +73,9 @@ public:
 
     const std::string &get_scheme_name () const { return m_scheme_name; }
 
-    virtual std::unique_ptr<diagnostic_output_format>
+    virtual std::unique_ptr<sink>
     make_sink (const context &ctxt,
-	       diagnostic_context &dc,
+	       diagnostics::context &dc,
 	       const char *unparsed_arg,
 	       const scheme_name_and_params &parsed_arg) const = 0;
 
@@ -145,9 +146,9 @@ public:
 
   output_factory ();
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg);
 
@@ -162,9 +163,9 @@ class text_scheme_handler : public output_factory::scheme_handler
 public:
   text_scheme_handler () : scheme_handler ("text") {}
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg) const final override;
 };
@@ -174,16 +175,16 @@ class sarif_scheme_handler : public output_factory::scheme_handler
 public:
   sarif_scheme_handler () : scheme_handler ("sarif") {}
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg) const final override;
 
 private:
   static sarif_generation_options
   make_sarif_gen_opts (enum sarif_version version,
-		       bool xml_state);
+		       bool state_graph);
 
   static std::unique_ptr<sarif_serialization_format>
   make_sarif_serialization_object (enum sarif_serialization_kind);
@@ -194,9 +195,9 @@ class html_scheme_handler : public output_factory::scheme_handler
 public:
   html_scheme_handler () : scheme_handler ("experimental-html") {}
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg) const final override;
 };
@@ -241,16 +242,16 @@ context::report_missing_key (const char *unparsed_arg,
      get_option_name (), scheme_name.c_str (), key.c_str (), metavar);
 }
 
-diagnostic_output_file
+output_file
 context::open_output_file (label_text &&filename) const
 {
   FILE *outf = fopen (filename.get (), "w");
   if (!outf)
     {
       report_error ("unable to open %qs: %m", filename.get ());
-      return diagnostic_output_file (nullptr, false, std::move (filename));
+      return output_file (nullptr, false, std::move (filename));
     }
-  return diagnostic_output_file (outf, true, std::move (filename));
+  return output_file (outf, true, std::move (filename));
 }
 
 static std::unique_ptr<scheme_name_and_params>
@@ -303,15 +304,15 @@ parse (const context &ctxt, const char *unparsed_arg)
   return std::make_unique<scheme_name_and_params> (std::move (result));
 }
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 context::parse_and_make_sink (const char *unparsed_arg,
-			      diagnostic_context &dc)
+			      diagnostics::context &dc)
 {
-  auto parsed_arg = diagnostics_output_spec::parse (*this, unparsed_arg);
+  auto parsed_arg = parse (*this, unparsed_arg);
   if (!parsed_arg)
     return nullptr;
 
-  diagnostics_output_spec::output_factory factory;
+  output_factory factory;
   return factory.make_sink (*this, dc, unparsed_arg, *parsed_arg);
 }
 
@@ -335,9 +336,9 @@ output_factory::get_scheme_handler (const std::string &scheme_name)
   return nullptr;
 }
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 output_factory::make_sink (const context &ctxt,
-			   diagnostic_context &dc,
+			   diagnostics::context &dc,
 			   const char *unparsed_arg,
 			   const scheme_name_and_params &parsed_arg)
 {
@@ -360,9 +361,9 @@ output_factory::make_sink (const context &ctxt,
 
 /* class text_scheme_handler : public output_factory::scheme_handler.  */
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 text_scheme_handler::make_sink (const context &ctxt,
-				diagnostic_context &dc,
+				diagnostics::context &dc,
 				const char *unparsed_arg,
 				const scheme_name_and_params &parsed_arg) const
 {
@@ -412,7 +413,7 @@ text_scheme_handler::make_sink (const context &ctxt,
       return nullptr;
     }
 
-  auto sink = std::make_unique<diagnostic_text_output_format> (dc);
+  auto sink = std::make_unique<diagnostics::text_sink> (dc);
   sink->set_show_nesting (show_nesting);
   sink->set_show_locations_in_nesting (show_locations_in_nesting);
   sink->set_show_nesting_levels (show_levels);
@@ -421,9 +422,9 @@ text_scheme_handler::make_sink (const context &ctxt,
 
 /* class sarif_scheme_handler : public output_factory::scheme_handler.  */
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 sarif_scheme_handler::make_sink (const context &ctxt,
-				 diagnostic_context &dc,
+				 diagnostics::context &dc,
 				 const char *unparsed_arg,
 				 const scheme_name_and_params &parsed_arg) const
 {
@@ -431,7 +432,7 @@ sarif_scheme_handler::make_sink (const context &ctxt,
   enum sarif_serialization_kind serialization_kind
     = sarif_serialization_kind::json;
   enum sarif_version version = sarif_version::v2_1_0;
-  bool xml_state = false;
+  bool state_graph = false;
   for (auto& iter : parsed_arg.m_kvs)
     {
       const std::string &key = iter.first;
@@ -469,10 +470,10 @@ sarif_scheme_handler::make_sink (const context &ctxt,
 	    return nullptr;
 	  continue;
 	}
-      if (key == "xml-state")
+      if (key == "state-graphs")
 	{
 	  if (!parse_bool_value (ctxt, unparsed_arg, key, value,
-				 xml_state))
+				 state_graph))
 	    return nullptr;
 	  continue;
 	}
@@ -481,16 +482,16 @@ sarif_scheme_handler::make_sink (const context &ctxt,
       auto_vec<const char *> known_keys;
       known_keys.safe_push ("file");
       known_keys.safe_push ("serialization");
+      known_keys.safe_push ("state-graphs");
       known_keys.safe_push ("version");
-      known_keys.safe_push ("xml-state");
       ctxt.report_unknown_key (unparsed_arg, key, get_scheme_name (),
 			       known_keys);
       return nullptr;
     }
 
-  diagnostic_output_file output_file;
+  output_file output_file_;
   if (filename.get ())
-    output_file = ctxt.open_output_file (std::move (filename));
+    output_file_ = ctxt.open_output_file (std::move (filename));
   else
     // Default filename
     {
@@ -503,17 +504,16 @@ sarif_scheme_handler::make_sink (const context &ctxt,
 				   "FILENAME");
 	  return nullptr;
 	}
-      output_file
-	= diagnostic_output_format_open_sarif_file
-	    (dc,
-	     ctxt.get_affected_location_mgr (),
-	     basename,
-	     serialization_kind);
+      output_file_
+	= open_sarif_output_file (dc,
+				  ctxt.get_affected_location_mgr (),
+				  basename,
+				  serialization_kind);
     }
-  if (!output_file)
+  if (!output_file_)
     return nullptr;
 
-  auto sarif_gen_opts = make_sarif_gen_opts (version, xml_state);
+  auto sarif_gen_opts = make_sarif_gen_opts (version, state_graph);
 
   auto serialization_obj = make_sarif_serialization_object (serialization_kind);
 
@@ -521,17 +521,17 @@ sarif_scheme_handler::make_sink (const context &ctxt,
 			       *ctxt.get_affected_location_mgr (),
 			       std::move (serialization_obj),
 			       sarif_gen_opts,
-			       std::move (output_file));
+			       std::move (output_file_));
   return sink;
 }
 
 sarif_generation_options
 sarif_scheme_handler::make_sarif_gen_opts (enum sarif_version version,
-					   bool xml_state)
+					   bool state_graph)
 {
   sarif_generation_options sarif_gen_opts;
   sarif_gen_opts.m_version = version;
-  sarif_gen_opts.m_xml_state = xml_state;
+  sarif_gen_opts.m_state_graph = state_graph;
   return sarif_gen_opts;
 }
 
@@ -551,9 +551,9 @@ make_sarif_serialization_object (enum sarif_serialization_kind kind)
 
 /* class html_scheme_handler : public output_factory::scheme_handler.  */
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 html_scheme_handler::make_sink (const context &ctxt,
-				diagnostic_context &dc,
+				diagnostics::context &dc,
 				const char *unparsed_arg,
 				const scheme_name_and_params &parsed_arg) const
 {
@@ -561,8 +561,8 @@ html_scheme_handler::make_sink (const context &ctxt,
   label_text filename;
   bool javascript = true;
   bool show_state_diagrams = false;
-  bool show_state_diagram_xml = false;
-  bool show_state_diagram_dot_src = false;
+  bool show_state_diagrams_sarif = false;
+  bool show_state_diagrams_dot_src = false;
   for (auto& iter : parsed_arg.m_kvs)
     {
       const std::string &key = iter.first;
@@ -593,17 +593,17 @@ html_scheme_handler::make_sink (const context &ctxt,
 	    return nullptr;
 	  continue;
 	}
-      if (key == "show-state-diagram-dot-src")
+      if (key == "show-state-diagrams-dot-src")
 	{
 	  if (!parse_bool_value (ctxt, unparsed_arg, key, value,
-				 show_state_diagram_dot_src))
+				 show_state_diagrams_dot_src))
 	    return nullptr;
 	  continue;
 	}
-      if (key == "show-state-diagram-xml")
+      if (key == "show-state-diagrams-sarif")
 	{
 	  if (!parse_bool_value (ctxt, unparsed_arg, key, value,
-				 show_state_diagram_xml))
+				 show_state_diagrams_sarif))
 	    return nullptr;
 	  continue;
 	}
@@ -615,15 +615,15 @@ html_scheme_handler::make_sink (const context &ctxt,
       known_keys.safe_push ("javascript");
       known_keys.safe_push ("show-state-diagrams");
       known_keys.safe_push ("show-state-diagram-dot-src");
-      known_keys.safe_push ("show-state-diagram-xml");
+      known_keys.safe_push ("show-state-diagram-sarif");
       ctxt.report_unknown_key (unparsed_arg, key, get_scheme_name (),
 			       known_keys);
       return nullptr;
     }
 
-  diagnostic_output_file output_file;
+  output_file output_file_;
   if (filename.get ())
-    output_file = ctxt.open_output_file (std::move (filename));
+    output_file_ = ctxt.open_output_file (std::move (filename));
   else
     // Default filename
     {
@@ -636,35 +636,37 @@ html_scheme_handler::make_sink (const context &ctxt,
 				   "FILENAME");
 	  return nullptr;
 	}
-      output_file
-	= diagnostic_output_format_open_html_file
+      output_file_
+	= open_html_output_file
 	    (dc,
 	     ctxt.get_affected_location_mgr (),
 	     basename);
     }
-  if (!output_file)
+  if (!output_file_)
     return nullptr;
 
   html_generation_options html_gen_opts;
   html_gen_opts.m_css = css;
   html_gen_opts.m_javascript = javascript;
   html_gen_opts.m_show_state_diagrams = show_state_diagrams;
-  html_gen_opts.m_show_state_diagram_xml = show_state_diagram_xml;
-  html_gen_opts.m_show_state_diagram_dot_src = show_state_diagram_dot_src;
+  html_gen_opts.m_show_state_diagrams_sarif = show_state_diagrams_sarif;
+  html_gen_opts.m_show_state_diagrams_dot_src = show_state_diagrams_dot_src;
 
   auto sink = make_html_sink (dc,
 			      *ctxt.get_affected_location_mgr (),
 			      html_gen_opts,
-			      std::move (output_file));
+			      std::move (output_file_));
   return sink;
 }
 
-} // namespace diagnostics_output_spec
+} // namespace output_spec
 
 #if CHECKING_P
 
 namespace selftest {
 
+using auto_fix_quotes = ::selftest::auto_fix_quotes;
+
 /* RAII class to temporarily override "progname" to the
    string "PROGNAME".  */
 
@@ -688,18 +690,18 @@ private:
 
 struct parser_test
 {
-  class test_spec_context : public diagnostics_output_spec::gcc_spec_context
+  class test_spec_context : public diagnostics::output_spec::dc_spec_context
   {
   public:
-    test_spec_context (diagnostic_context &dc,
+    test_spec_context (diagnostics::context &dc,
 		       line_maps *location_mgr,
 		       location_t loc,
 		       const char *option_name)
-    : gcc_spec_context (dc,
-			location_mgr,
-			location_mgr,
-			loc,
-			option_name)
+    : dc_spec_context (dc,
+		       location_mgr,
+		       location_mgr,
+		       loc,
+		       option_name)
     {
     }
 
@@ -713,15 +715,15 @@ struct parser_test
   parser_test ()
   : m_dc (),
     m_ctxt (m_dc, line_table, UNKNOWN_LOCATION, "-fOPTION="),
-    m_fmt (m_dc.get_output_format (0))
+    m_fmt (m_dc.get_sink (0))
   {
     pp_buffer (m_fmt.get_printer ())->m_flush_p = false;
   }
 
-  std::unique_ptr<diagnostics_output_spec::scheme_name_and_params>
+  std::unique_ptr<diagnostics::output_spec::scheme_name_and_params>
   parse (const char *unparsed_arg)
   {
-    return diagnostics_output_spec::parse (m_ctxt, unparsed_arg);
+    return diagnostics::output_spec::parse (m_ctxt, unparsed_arg);
   }
 
   bool execution_failed_p () const
@@ -736,9 +738,9 @@ struct parser_test
   }
 
 private:
-  test_diagnostic_context m_dc;
+  diagnostics::selftest::test_context m_dc;
   test_spec_context m_ctxt;
-  diagnostic_output_format &m_fmt;
+  diagnostics::sink &m_fmt;
 };
 
 /* Selftests.  */
@@ -838,12 +840,13 @@ test_output_arg_parsing ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_output_spec_cc_tests ()
+output_spec_cc_tests ()
 {
   test_output_arg_parsing ();
 }
 
-} // namespace selftest
-
+} // namespace diagnostics::selftest
 
 #endif /* #if CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-output-spec.h b/gcc/diagnostics/output-spec.h
index e02cdfe..c84d237 100644
--- a/gcc/diagnostic-output-spec.h
+++ b/gcc/diagnostics/output-spec.h
@@ -18,13 +18,14 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_OUTPUT_SPEC_H
-#define GCC_DIAGNOSTIC_OUTPUT_SPEC_H
+#ifndef GCC_DIAGNOSTICS_OUTPUT_SPEC_H
+#define GCC_DIAGNOSTICS_OUTPUT_SPEC_H
 
-#include "diagnostic-format.h"
-#include "diagnostic-output-file.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/output-file.h"
 
-namespace diagnostics_output_spec {
+namespace diagnostics {
+namespace output_spec {
 
 /* An abstract base class for handling the DSL of -fdiagnostics-add-output=
    and -fdiagnostics-set-output=.  */
@@ -32,9 +33,9 @@ namespace diagnostics_output_spec {
 class context
 {
  public:
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   parse_and_make_sink (const char *,
-		       diagnostic_context &dc);
+		       diagnostics::context &dc);
 
   void
   report_error (const char *gmsgid, ...) const
@@ -52,7 +53,7 @@ class context
 		      const std::string &scheme_name,
 		      const char *metavar) const;
 
-  diagnostic_output_file
+  output_file
   open_output_file (label_text &&filename) const;
 
   const char *
@@ -81,16 +82,16 @@ protected:
   line_maps *m_affected_location_mgr;
 };
 
-/* A subclass that implements reporting errors via a diagnostic_context.  */
+/* A subclass that implements reporting errors via a diagnostics::context.  */
 
-struct gcc_spec_context : public diagnostics_output_spec::context
+struct dc_spec_context : public output_spec::context
 {
 public:
-  gcc_spec_context (diagnostic_context &dc,
-		    line_maps *affected_location_mgr,
-		    line_maps *control_location_mgr,
-		    location_t loc,
-		    const char *option_name)
+  dc_spec_context (diagnostics::context &dc,
+		   line_maps *affected_location_mgr,
+		   line_maps *control_location_mgr,
+		   location_t loc,
+		   const char *option_name)
   : context (option_name, affected_location_mgr),
     m_dc (dc),
     m_control_location_mgr (control_location_mgr),
@@ -102,15 +103,16 @@ public:
   {
     m_dc.begin_group ();
     rich_location richloc (m_control_location_mgr, m_loc);
-    m_dc.diagnostic_impl (&richloc, nullptr, -1, gmsgid, ap, DK_ERROR);
+    m_dc.diagnostic_impl (&richloc, nullptr, -1, gmsgid, ap, kind::error);
     m_dc.end_group ();
   }
 
-  diagnostic_context &m_dc;
+  diagnostics::context &m_dc;
   line_maps *m_control_location_mgr;
   location_t m_loc;
 };
 
-} // namespace diagnostics_output_spec
+} // namespace output_spec
+} // namespace diagnostics
 
-#endif
+#endif // #ifndef GCC_DIAGNOSTICS_OUTPUT_SPEC_H
diff --git a/gcc/diagnostic-path-output.cc b/gcc/diagnostics/paths-output.cc
index 439fd33..a3ac9a0 100644
--- a/gcc/diagnostic-path-output.cc
+++ b/gcc/diagnostics/paths-output.cc
@@ -26,20 +26,20 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-macro-unwinding.h"
+#include "diagnostics/macro-unwinding.h"
 #include "intl.h"
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 #include "gcc-rich-location.h"
-#include "diagnostic-color.h"
-#include "diagnostic-event-id.h"
-#include "diagnostic-label-effects.h"
+#include "diagnostics/color.h"
+#include "diagnostics/event-id.h"
+#include "diagnostics/source-printing-effects.h"
 #include "pretty-print-markup.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "selftest-diagnostic-path.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-paths.h"
 #include "text-art/theme.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-format-html.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/html-sink.h"
 #include "xml.h"
 #include "xml-printer.h"
 
@@ -54,17 +54,20 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace {
 
+using namespace diagnostics;
+using namespace diagnostics::paths;
+
 /* A bundle of state for printing a path.  */
 
 class path_print_policy
 {
 public:
-  path_print_policy (const diagnostic_text_output_format &text_output)
+  path_print_policy (const diagnostics::text_sink &text_output)
   : m_source_policy (text_output.get_context ())
   {
   }
 
-  path_print_policy (const diagnostic_context &dc)
+  path_print_policy (const diagnostics::context &dc)
   : m_source_policy (dc)
   {
   }
@@ -75,26 +78,26 @@ public:
     return m_source_policy.get_diagram_theme ();
   }
 
-  const diagnostic_source_print_policy &
+  const diagnostics::source_print_policy &
   get_source_policy () const { return m_source_policy; }
 
 private:
-  diagnostic_source_print_policy m_source_policy;
+  diagnostics::source_print_policy m_source_policy;
 };
 
 /* Subclass of range_label for showing a particular event
-   when showing a consecutive run of events within a diagnostic_path as
+   when showing a consecutive run of events within a diagnostic path as
    labelled ranges within one gcc_rich_location.  */
 
 class path_label : public range_label
 {
  public:
-  path_label (const diagnostic_path &path,
+  path_label (const path &path_,
 	      const pretty_printer &ref_pp,
 	      unsigned start_idx,
 	      bool colorize,
 	      bool allow_emojis)
-  : m_path (path),
+  : m_path (path_),
     m_ref_pp (ref_pp),
     m_start_idx (start_idx), m_effects (*this),
     m_colorize (colorize), m_allow_emojis (allow_emojis)
@@ -103,18 +106,17 @@ class path_label : public range_label
   label_text get_text (unsigned range_idx) const final override
   {
     unsigned event_idx = m_start_idx + range_idx;
-    const diagnostic_event &event = m_path.get_event (event_idx);
+    const event &ev = m_path.get_event (event_idx);
 
-    const diagnostic_event::meaning meaning (event.get_meaning ());
+    const event::meaning meaning (ev.get_meaning ());
 
     auto pp = m_ref_pp.clone ();
     pp_show_color (pp.get ()) = m_colorize;
-    diagnostic_event_id_t event_id (event_idx);
-
+    event_id_t event_id (event_idx);
     pp_printf (pp.get (), "%@", &event_id);
     pp_space (pp.get ());
 
-    if (meaning.m_verb == diagnostic_event::verb::danger
+    if (meaning.m_verb == event::verb::danger
 	&& m_allow_emojis)
       {
 	pp_unicode_character (pp.get (), 0x26A0); /* U+26A0 WARNING SIGN.  */
@@ -128,14 +130,15 @@ class path_label : public range_label
 	pp_string (pp.get (), "  ");
       }
 
-    event.print_desc (*pp.get ());
+    ev.print_desc (*pp.get ());
 
     label_text result
       = label_text::take (xstrdup (pp_formatted_text (pp.get ())));
     return result;
   }
 
-  const label_effects *get_effects (unsigned /*range_idx*/) const final override
+  const diagnostics::label_effects *
+  get_effects (unsigned /*range_idx*/) const final override
   {
     return &m_effects;
   }
@@ -150,28 +153,28 @@ class path_label : public range_label
     }
     bool has_in_edge (unsigned range_idx) const final override
     {
-      if (const diagnostic_event *prev_event
+      if (const event *prev_event
 	    = m_path_label.get_prev_event (range_idx))
 	return prev_event->connect_to_next_event_p ();
       return false;
     }
     bool has_out_edge (unsigned range_idx) const final override
     {
-      const diagnostic_event &event = m_path_label.get_event (range_idx);
-      return event.connect_to_next_event_p ();
+      const event &ev = m_path_label.get_event (range_idx);
+      return ev.connect_to_next_event_p ();
     }
 
   private:
     const path_label &m_path_label;
   };
 
-  const diagnostic_event &get_event (unsigned range_idx) const
+  const event &get_event (unsigned range_idx) const
   {
     unsigned event_idx = m_start_idx + range_idx;
     return m_path.get_event (event_idx);
   }
 
-  const diagnostic_event *get_prev_event (unsigned range_idx) const
+  const event *get_prev_event (unsigned range_idx) const
   {
     if (m_start_idx + range_idx == 0)
       return nullptr;
@@ -179,7 +182,7 @@ class path_label : public range_label
     return &m_path.get_event (event_idx);
   }
 
-  const diagnostic_path &m_path;
+  const path &m_path;
   const pretty_printer &m_ref_pp;
   unsigned m_start_idx;
   path_label_effects m_effects;
@@ -188,20 +191,20 @@ class path_label : public range_label
 };
 
 /* Return true if E1 and E2 can be consolidated into the same run of events
-   when printing a diagnostic_path.  */
+   when printing a diagnostic path.  */
 
 static bool
-can_consolidate_events (const diagnostic_path &path,
-			const diagnostic_event &e1,
+can_consolidate_events (const path &p,
+			const event &e1,
 			unsigned ev1_idx,
-			const diagnostic_event &e2,
+			const event &e2,
 			unsigned ev2_idx,
 			bool check_locations)
 {
   if (e1.get_thread_id () != e2.get_thread_id ())
     return false;
 
-  if (!path.same_function_p (ev1_idx, ev2_idx))
+  if (!p.same_function_p (ev1_idx, ev2_idx))
     return false;
 
   if (e1.get_stack_depth () != e2.get_stack_depth ())
@@ -231,16 +234,16 @@ struct event_range;
 struct path_summary;
 class thread_event_printer;
 
-/* A bundle of information about all of the events in a diagnostic_path
+/* A bundle of information about all of the events in a diagnostic path
    relating to a specific path, for use by path_summary.  */
 
 class per_thread_summary
 {
 public:
-  per_thread_summary (const diagnostic_path &path,
-		      const logical_location_manager &logical_loc_mgr,
+  per_thread_summary (const path &path_,
+		      const logical_locations::manager &logical_loc_mgr,
 		      label_text name, unsigned swimlane_idx)
-  : m_path (path),
+  : m_path (path_),
     m_logical_loc_mgr (logical_loc_mgr),
     m_name (std::move (name)),
     m_swimlane_idx (swimlane_idx),
@@ -267,8 +270,8 @@ private:
   friend class thread_event_printer;
   friend struct event_range;
 
-  const diagnostic_path &m_path;
-  const logical_location_manager &m_logical_loc_mgr;
+  const path &m_path;
+  const logical_locations::manager &m_logical_loc_mgr;
 
   const label_text m_name;
 
@@ -279,7 +282,7 @@ private:
   // The event ranges specific to this thread:
   auto_vec<event_range *> m_event_ranges;
 
-  const diagnostic_event *m_last_event;
+  const event *m_last_event;
 
   int m_min_depth;
   int m_max_depth;
@@ -291,7 +294,7 @@ private:
 struct stack_frame
 {
   stack_frame (std::unique_ptr<stack_frame> parent,
-	       logical_location logical_loc,
+	       logical_locations::key logical_loc,
 	       int stack_depth)
   : m_parent (std::move (parent)),
     m_logical_loc (logical_loc),
@@ -299,7 +302,7 @@ struct stack_frame
   {}
 
   std::unique_ptr<stack_frame> m_parent;
-  logical_location m_logical_loc;
+  logical_locations::key m_logical_loc;
   const int m_stack_depth;
 };
 
@@ -309,9 +312,9 @@ struct stack_frame
 static std::unique_ptr<stack_frame>
 begin_html_stack_frame (xml::printer &xp,
 			std::unique_ptr<stack_frame> parent,
-			logical_location logical_loc,
+			logical_locations::key logical_loc,
 			int stack_depth,
-			const logical_location_manager *logical_loc_mgr)
+			const logical_locations::manager *logical_loc_mgr)
 {
   if (logical_loc)
     {
@@ -398,7 +401,7 @@ emit_svg_arrow (xml::printer &xp, int old_depth, int new_depth)
   xp.add_raw (pp_formatted_text (pp));
 }
 
-/* A range of consecutive events within a diagnostic_path, all within the
+/* A range of consecutive events within a diagnostic path, all within the
    same thread, and with the same fndecl and stack_depth, and which are suitable
    to print with a single call to diagnostic_show_locus.  */
 struct event_range
@@ -423,7 +426,7 @@ struct event_range
        the events already on this source line, *and* a new event at COLUMN.  */
     bool
     can_add_label_for_event_p (bool has_in_edge,
-			       const diagnostic_event *prev_event,
+			       const event *prev_event,
 			       bool has_out_edge,
 			       int column) const
     {
@@ -488,20 +491,20 @@ struct event_range
     int m_max_label_source_column;
   };
 
-  event_range (const diagnostic_path &path,
+  event_range (const path &path_,
 	       const pretty_printer &ref_pp,
 	       unsigned start_idx,
-	       const diagnostic_event &initial_event,
+	       const event &initial_event,
 	       per_thread_summary &t,
 	       bool show_event_links,
 	       bool colorize_labels,
 	       bool allow_emojis)
-  : m_path (path),
+  : m_path (path_),
     m_initial_event (initial_event),
     m_logical_loc (initial_event.get_logical_location ()),
     m_stack_depth (initial_event.get_stack_depth ()),
     m_start_idx (start_idx), m_end_idx (start_idx),
-    m_path_label (path, ref_pp,
+    m_path_label (path_, ref_pp,
 		  start_idx, colorize_labels, allow_emojis),
     m_richloc (initial_event.get_location (), &m_path_label, nullptr),
     m_thread_id (initial_event.get_thread_id ()),
@@ -516,7 +519,7 @@ struct event_range
 	per_source_line_info &source_line_info
 	  = get_per_source_line_info (exploc.line);
 
-	const diagnostic_event *prev_thread_event = t.m_last_event;
+	const event *prev_thread_event = t.m_last_event;
 	const bool has_in_edge
 	  = (prev_thread_event
 	     ? prev_thread_event->connect_to_next_event_p ()
@@ -540,7 +543,7 @@ struct event_range
   }
 
   bool maybe_add_event (const path_print_policy &policy,
-			const diagnostic_event &new_ev,
+			const event &new_ev,
 			unsigned new_ev_idx,
 			bool check_rich_locations)
   {
@@ -557,7 +560,7 @@ struct event_range
       (line_table, new_ev.get_location (), LOCATION_ASPECT_CARET);
     per_source_line_info &source_line_info
       = get_per_source_line_info (exploc.line);
-    const diagnostic_event *prev_event = nullptr;
+    const event *prev_event = nullptr;
     if (new_ev_idx > 0)
       prev_event = &m_path.get_event (new_ev_idx - 1);
     const bool has_in_edge = (prev_event
@@ -591,24 +594,24 @@ struct event_range
      call to diagnostic_show_locus.  */
 
   void print_as_text (pretty_printer &pp,
-		      diagnostic_text_output_format &text_output,
-		      diagnostic_source_effect_info *effect_info)
+		      diagnostics::text_sink &text_output,
+		      diagnostics::source_effect_info *effect_info)
   {
     location_t initial_loc = m_initial_event.get_location ();
 
-    diagnostic_context &dc = text_output.get_context ();
+    diagnostics::context &dc = text_output.get_context ();
 
     /* Emit a span indicating the filename (and line/column) if the
        line has changed relative to the last call to
        diagnostic_show_locus.  */
-    if (dc.m_source_printing.enabled)
+    if (dc.get_source_printing_options ().enabled)
       {
 	expanded_location exploc
 	  = linemap_client_expand_location_to_spelling_point
 	  (line_table, initial_loc, LOCATION_ASPECT_CARET);
 	if (exploc.file != LOCATION_FILE (dc.m_last_location))
 	  {
-	    diagnostic_location_print_policy loc_policy (text_output);
+	    diagnostics::location_print_policy loc_policy (text_output);
 	    loc_policy.print_text_span_start (dc, pp, exploc);
 	  }
       }
@@ -624,7 +627,7 @@ struct event_range
       {
 	for (unsigned i = m_start_idx; i <= m_end_idx; i++)
 	  {
-	    const diagnostic_event &iter_event = m_path.get_event (i);
+	    const event &iter_event = m_path.get_event (i);
 	    diagnostic_event_id_t event_id (i);
 	    pp_printf (&pp, " %@: ", &event_id);
 	    iter_event.print_desc (pp);
@@ -635,7 +638,7 @@ struct event_range
 
     /* Call diagnostic_show_locus to show the events using labels.  */
     diagnostic_show_locus (&dc, text_output.get_source_printing_options (),
-			   &m_richloc, DK_DIAGNOSTIC_PATH, &pp,
+			   &m_richloc, diagnostics::kind::path, &pp,
 			   effect_info);
 
     /* If we have a macro expansion, show the expansion to the user.  */
@@ -650,8 +653,8 @@ struct event_range
      call to diagnostic_show_locus_as_html.  */
 
   void print_as_html (xml::printer &xp,
-		      diagnostic_context &dc,
-		      diagnostic_source_effect_info *effect_info,
+		      diagnostics::context &dc,
+		      diagnostics::source_effect_info *effect_info,
 		      html_label_writer *event_label_writer)
   {
     location_t initial_loc = m_initial_event.get_location ();
@@ -659,14 +662,14 @@ struct event_range
     /* Emit a span indicating the filename (and line/column) if the
        line has changed relative to the last call to
        diagnostic_show_locus.  */
-    if (dc.m_source_printing.enabled)
+    if (dc.get_source_printing_options ().enabled)
       {
 	expanded_location exploc
 	  = linemap_client_expand_location_to_spelling_point
 	  (line_table, initial_loc, LOCATION_ASPECT_CARET);
 	if (exploc.file != LOCATION_FILE (dc.m_last_location))
 	  {
-	    diagnostic_location_print_policy loc_policy (dc);
+	    diagnostics::location_print_policy loc_policy (dc);
 	    loc_policy.print_html_span_start (dc, xp, exploc);
 	  }
       }
@@ -682,7 +685,7 @@ struct event_range
       {
 	for (unsigned i = m_start_idx; i <= m_end_idx; i++)
 	  {
-	    const diagnostic_event &iter_event = m_path.get_event (i);
+	    const event &iter_event = m_path.get_event (i);
 	    diagnostic_event_id_t event_id (i);
 	    pretty_printer pp;
 	    pp_printf (&pp, " %@: ", &event_id);
@@ -698,29 +701,29 @@ struct event_range
 
     /* Call diagnostic_show_locus_as_html to show the source,
        showing events using labels.  */
-    diagnostic_show_locus_as_html (&dc, dc.m_source_printing,
-				   &m_richloc, DK_DIAGNOSTIC_PATH, xp,
+    diagnostic_show_locus_as_html (&dc, dc.get_source_printing_options (),
+				   &m_richloc, diagnostics::kind::path, xp,
 				   effect_info, event_label_writer);
 
     // TODO: show macro expansions
   }
 
-  const diagnostic_path &m_path;
-  const diagnostic_event &m_initial_event;
-  logical_location m_logical_loc;
+  const path &m_path;
+  const event &m_initial_event;
+  logical_locations::key m_logical_loc;
   int m_stack_depth;
   unsigned m_start_idx;
   unsigned m_end_idx;
   path_label m_path_label;
   gcc_rich_location m_richloc;
-  diagnostic_thread_id_t m_thread_id;
+  thread_id_t m_thread_id;
   per_thread_summary &m_per_thread_summary;
   hash_map<int_hash<int, -1, -2>,
 	   per_source_line_info> m_source_line_info_map;
   bool m_show_event_links;
 };
 
-/* A struct for grouping together the events in a diagnostic_path into
+/* A struct for grouping together the events in a path into
    ranges of events, partitioned by thread and by stack frame (i.e. by fndecl
    and stack depth).  */
 
@@ -728,19 +731,19 @@ struct path_summary
 {
   path_summary (const path_print_policy &policy,
 		const pretty_printer &ref_pp,
-		const diagnostic_path &path,
+		const path &path_,
 		bool check_rich_locations,
 		bool colorize = false,
 		bool show_event_links = true);
 
-  const logical_location_manager &get_logical_location_manager () const
+  const logical_locations::manager &get_logical_location_manager () const
   {
     return m_logical_loc_mgr;
   }
   unsigned get_num_ranges () const { return m_ranges.length (); }
   bool multithreaded_p () const { return m_per_thread_summary.length () > 1; }
 
-  const per_thread_summary &get_events_for_thread_id (diagnostic_thread_id_t tid)
+  const per_thread_summary &get_events_for_thread_id (thread_id_t tid)
   {
     per_thread_summary **slot = m_thread_id_to_events.get (tid);
     gcc_assert (slot);
@@ -748,23 +751,23 @@ struct path_summary
     return **slot;
   }
 
-  const logical_location_manager &m_logical_loc_mgr;
+  const logical_locations::manager &m_logical_loc_mgr;
   auto_delete_vec <event_range> m_ranges;
   auto_delete_vec <per_thread_summary> m_per_thread_summary;
-  hash_map<int_hash<diagnostic_thread_id_t, -1, -2>,
+  hash_map<int_hash<thread_id_t, -1, -2>,
 	   per_thread_summary *> m_thread_id_to_events;
 
 private:
   per_thread_summary &
-  get_or_create_events_for_thread_id (const diagnostic_path &path,
-				      diagnostic_thread_id_t tid)
+  get_or_create_events_for_thread_id (const path &path_,
+				      thread_id_t tid)
   {
     if (per_thread_summary **slot = m_thread_id_to_events.get (tid))
       return **slot;
 
-    const diagnostic_thread &thread = path.get_thread (tid);
+    const thread &thread = path_.get_thread (tid);
     per_thread_summary *pts
-      = new per_thread_summary (path,
+      = new per_thread_summary (path_,
 				m_logical_loc_mgr,
 				thread.get_name (false),
 				m_per_thread_summary.length ());
@@ -798,40 +801,40 @@ per_thread_summary::interprocedural_p () const
 
 path_summary::path_summary (const path_print_policy &policy,
 			    const pretty_printer &ref_pp,
-			    const diagnostic_path &path,
+			    const path &path_,
 			    bool check_rich_locations,
 			    bool colorize,
 			    bool show_event_links)
-: m_logical_loc_mgr (path.get_logical_location_manager ())
+: m_logical_loc_mgr (path_.get_logical_location_manager ())
 {
-  const unsigned num_events = path.num_events ();
+  const unsigned num_events = path_.num_events ();
 
   event_range *cur_event_range = nullptr;
   for (unsigned idx = 0; idx < num_events; idx++)
     {
-      const diagnostic_event &event = path.get_event (idx);
-      const diagnostic_thread_id_t thread_id = event.get_thread_id ();
+      const event &ev = path_.get_event (idx);
+      const thread_id_t thread_id = ev.get_thread_id ();
       per_thread_summary &pts
-	= get_or_create_events_for_thread_id (path, thread_id);
+	= get_or_create_events_for_thread_id (path_, thread_id);
 
-      pts.update_depth_limits (event.get_stack_depth ());
+      pts.update_depth_limits (ev.get_stack_depth ());
 
       if (cur_event_range)
 	if (cur_event_range->maybe_add_event (policy,
-					      event,
+					      ev,
 					      idx, check_rich_locations))
 	  continue;
 
       auto theme = policy.get_diagram_theme ();
       const bool allow_emojis = theme ? theme->emojis_p () : false;
-      cur_event_range = new event_range (path, ref_pp,
-					 idx, event, pts,
+      cur_event_range = new event_range (path_, ref_pp,
+					 idx, ev, pts,
 					 show_event_links,
 					 colorize,
 					 allow_emojis);
       m_ranges.safe_push (cur_event_range);
       pts.m_event_ranges.safe_push (cur_event_range);
-      pts.m_last_event = &event;
+      pts.m_last_event = &ev;
     }
 }
 
@@ -881,11 +884,11 @@ public:
   }
 
   void
-  print_swimlane_for_event_range_as_text (diagnostic_text_output_format &text_output,
+  print_swimlane_for_event_range_as_text (diagnostics::text_sink &text_output,
 					  pretty_printer *pp,
-					  const logical_location_manager &logical_loc_mgr,
+					  const logical_locations::manager &logical_loc_mgr,
 					  event_range *range,
-					  diagnostic_source_effect_info *effect_info)
+					  diagnostics::source_effect_info *effect_info)
   {
     gcc_assert (pp);
     const char *const line_color = "path";
@@ -1041,11 +1044,11 @@ public:
   }
 
   void
-  print_swimlane_for_event_range_as_html (diagnostic_context &dc,
+  print_swimlane_for_event_range_as_html (diagnostics::context &dc,
 					  xml::printer &xp,
 					  html_label_writer *event_label_writer,
 					  event_range *range,
-					  diagnostic_source_effect_info *effect_info)
+					  diagnostics::source_effect_info *effect_info)
   {
     range->print_as_html (xp, dc, effect_info, event_label_writer);
     m_num_printed++;
@@ -1107,7 +1110,7 @@ private:
 
 static void
 print_path_summary_as_text (const path_summary &ps,
-			    diagnostic_text_output_format &text_output,
+			    diagnostics::text_sink &text_output,
 			    bool show_depths)
 {
   pretty_printer *const pp = text_output.get_printer ();
@@ -1135,7 +1138,7 @@ print_path_summary_as_text (const path_summary &ps,
       thread_event_printer &tep = thread_event_printers[swimlane_idx];
       /* Wire up any trailing out-edge from previous range to leading in-edge
 	 of this range.  */
-      diagnostic_source_effect_info effect_info;
+      diagnostics::source_effect_info effect_info;
       effect_info.m_leading_in_edge_column = last_out_edge_column;
       tep.print_swimlane_for_event_range_as_text
 	(text_output, pp,
@@ -1149,7 +1152,7 @@ print_path_summary_as_text (const path_summary &ps,
 
 static void
 print_path_summary_as_html (const path_summary &ps,
-			    diagnostic_context &dc,
+			    diagnostics::context &dc,
 			    xml::printer &xp,
 			    html_label_writer *event_label_writer,
 			    bool show_depths)
@@ -1158,7 +1161,7 @@ print_path_summary_as_html (const path_summary &ps,
   for (auto t : ps.m_per_thread_summary)
     thread_event_printers.push_back (thread_event_printer (*t, show_depths));
 
-  const logical_location_manager *logical_loc_mgr
+  const logical_locations::manager *logical_loc_mgr
     = dc.get_logical_location_manager ();
 
   xp.push_tag_with_class ("div", "event-ranges", false);
@@ -1173,7 +1176,7 @@ print_path_summary_as_html (const path_summary &ps,
       const int swimlane_idx
 	= range->m_per_thread_summary.get_swimlane_index ();
 
-      const logical_location this_logical_loc = range->m_logical_loc;
+      const logical_locations::key this_logical_loc = range->m_logical_loc;
       const int this_depth = range->m_stack_depth;
       if (curr_frame)
 	{
@@ -1261,7 +1264,7 @@ print_path_summary_as_html (const path_summary &ps,
       thread_event_printer &tep = thread_event_printers[swimlane_idx];
       /* Wire up any trailing out-edge from previous range to leading in-edge
 	 of this range.  */
-      diagnostic_source_effect_info effect_info;
+      diagnostics::source_effect_info effect_info;
       effect_info.m_leading_in_edge_column = last_out_edge_column;
       tep.print_swimlane_for_event_range_as_html (dc, xp, event_label_writer,
 						  range, &effect_info);
@@ -1284,8 +1287,8 @@ print_path_summary_as_html (const path_summary &ps,
 class element_event_desc : public pp_element
 {
 public:
-  element_event_desc (const diagnostic_event &event)
-  : m_event (event)
+  element_event_desc (const event &event_)
+  : m_event (event_)
   {
   }
 
@@ -1297,15 +1300,15 @@ public:
   }
 
 private:
-  const diagnostic_event &m_event;
+  const event &m_event;
 };
 
 /* Print PATH according to the context's path_format.  */
 
 void
-diagnostic_text_output_format::print_path (const diagnostic_path &path)
+diagnostics::text_sink::print_path (const path &path_)
 {
-  const unsigned num_events = path.num_events ();
+  const unsigned num_events = path_.num_events ();
 
   switch (get_context ().get_path_format ())
     {
@@ -1316,36 +1319,36 @@ diagnostic_text_output_format::print_path (const diagnostic_path &path)
     case DPF_SEPARATE_EVENTS:
       {
 	/* A note per event.  */
-	auto &logical_loc_mgr = path.get_logical_location_manager ();
+	auto &logical_loc_mgr = path_.get_logical_location_manager ();
 	for (unsigned i = 0; i < num_events; i++)
 	  {
-	    const diagnostic_event &event = path.get_event (i);
-	    element_event_desc e_event_desc (event);
+	    const event &ev = path_.get_event (i);
+	    element_event_desc e_event_desc (ev);
 	    diagnostic_event_id_t event_id (i);
 	    if (get_context ().show_path_depths_p ())
 	      {
-		int stack_depth = event.get_stack_depth ();
+		int stack_depth = ev.get_stack_depth ();
 		/* -fdiagnostics-path-format=separate-events doesn't print
 		   fndecl information, so with -fdiagnostics-show-path-depths
 		   print the fndecls too, if any.  */
-		if (logical_location logical_loc
-		      = event.get_logical_location ())
+		if (logical_locations::key logical_loc
+		      = ev.get_logical_location ())
 		  {
 		    label_text name
 		      (logical_loc_mgr.get_name_for_path_output (logical_loc));
-		    inform (event.get_location (),
+		    inform (ev.get_location (),
 			    "%@ %e (fndecl %qs, depth %i)",
 			    &event_id, &e_event_desc,
 			    name.get (), stack_depth);
 		  }
 		else
-		  inform (event.get_location (),
+		  inform (ev.get_location (),
 			  "%@ %e (depth %i)",
 			  &event_id, &e_event_desc,
 			  stack_depth);
 	      }
 	    else
-	      inform (event.get_location (),
+	      inform (ev.get_location (),
 		      "%@ %e", &event_id, &e_event_desc);
 	  }
       }
@@ -1361,7 +1364,7 @@ diagnostic_text_output_format::print_path (const diagnostic_path &path)
 	const bool show_event_links = m_source_printing.show_event_links_p;
 	path_summary summary (policy,
 			      *pp,
-			      path,
+			      path_,
 			      check_rich_locations,
 			      colorize,
 			      show_event_links);
@@ -1376,25 +1379,25 @@ diagnostic_text_output_format::print_path (const diagnostic_path &path)
     }
 }
 
-/* Print PATH as HTML to XP, using DC and DSPP for settings.
+/* Print PATH_ as HTML to XP, using DC and DSPP for settings.
    If non-null, use EVENT_LABEL_WRITER when writing events.  */
 
 void
-print_path_as_html (xml::printer &xp,
-		    const diagnostic_path &path,
-		    diagnostic_context &dc,
-		    html_label_writer *event_label_writer,
-		    const diagnostic_source_print_policy &dspp)
+diagnostics::print_path_as_html (xml::printer &xp,
+				 const path &path_,
+				 context &dc,
+				 html_label_writer *event_label_writer,
+				 const source_print_policy &dspp)
 {
   path_print_policy policy (dc);
   const bool check_rich_locations = true;
   const bool colorize = false;
-  const diagnostic_source_printing_options &source_printing_opts
+  const source_printing_options &source_printing_opts
     = dspp.get_options ();
   const bool show_event_links = source_printing_opts.show_event_links_p;
   path_summary summary (policy,
 			*dc.get_reference_printer (),
-			path,
+			path_,
 			check_rich_locations,
 			colorize,
 			show_event_links);
@@ -1404,18 +1407,27 @@ print_path_as_html (xml::printer &xp,
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace paths {
 namespace selftest {
 
-/* Return true iff all events in PATH have locations for which column data
+using location = ::selftest::location;
+using line_table_case = ::selftest::line_table_case;
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+
+using test_context = diagnostics::selftest::test_context;
+
+/* Return true iff all events in PATH_ have locations for which column data
    is available, so that selftests that require precise string output can
    bail out for awkward line_table cases.  */
 
 static bool
-path_events_have_column_data_p (const diagnostic_path &path)
+path_events_have_column_data_p (const path &path_)
 {
-  for (unsigned idx = 0; idx < path.num_events (); idx++)
+  for (unsigned idx = 0; idx < path_.num_events (); idx++)
     {
-      location_t event_loc = path.get_event (idx).get_location ();
+      location_t event_loc = path_.get_event (idx).get_location ();
       if (line_table->get_pure_location (event_loc)
 	  > LINE_MAP_MAX_LOCATION_WITH_COLS)
 	return false;
@@ -1432,11 +1444,12 @@ path_events_have_column_data_p (const diagnostic_path &path)
 static void
 test_empty_path (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   ASSERT_FALSE (path.interprocedural_p ());
 
-  test_diagnostic_context dc;
-  diagnostic_text_output_format text_output (dc);
+  test_context dc;
+  text_sink text_output (dc);
   path_print_policy policy (text_output);
   path_summary summary (policy, *event_pp, path, false);
   ASSERT_EQ (summary.get_num_ranges (), 0);
@@ -1451,15 +1464,16 @@ test_empty_path (pretty_printer *event_pp)
 static void
 test_intraprocedural_path (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   const char *const funcname = "foo";
   path.add_event (UNKNOWN_LOCATION, funcname, 0, "first %qs", "free");
   path.add_event (UNKNOWN_LOCATION, funcname, 0, "double %qs", "free");
 
   ASSERT_FALSE (path.interprocedural_p ());
 
-  test_diagnostic_context dc;
-  diagnostic_text_output_format text_output (dc);
+  selftest::test_context dc;
+  text_sink text_output (dc);
   path_print_policy policy (text_output);
   path_summary summary (policy, *event_pp, path, false, false, false);
   ASSERT_EQ (summary.get_num_ranges (), 1);
@@ -1476,7 +1490,8 @@ test_intraprocedural_path (pretty_printer *event_pp)
 static void
 test_interprocedural_path_1 (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_entry ("test", 0);
   path.add_call ("test", 0, "make_boxed_int");
   path.add_call ("make_boxed_int", 1, "wrapped_malloc");
@@ -1495,8 +1510,8 @@ test_interprocedural_path_1 (pretty_printer *event_pp)
   ASSERT_TRUE (path.interprocedural_p ());
 
   {
-    test_diagnostic_context dc;
-    diagnostic_text_output_format text_output (dc, nullptr, false);
+    selftest::test_context dc;
+    text_sink text_output (dc, nullptr, false);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     ASSERT_EQ (summary.get_num_ranges (), 9);
@@ -1556,9 +1571,9 @@ test_interprocedural_path_1 (pretty_printer *event_pp)
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     print_path_summary_as_text (summary, text_output, true);
@@ -1623,7 +1638,8 @@ test_interprocedural_path_1 (pretty_printer *event_pp)
 static void
 test_interprocedural_path_2 (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_entry ("foo", 0);
   path.add_call ("foo", 0, "bar");
   path.add_call ("bar", 1, "baz");
@@ -1634,8 +1650,8 @@ test_interprocedural_path_2 (pretty_printer *event_pp)
   ASSERT_TRUE (path.interprocedural_p ());
 
   {
-    test_diagnostic_context dc;
-    diagnostic_text_output_format text_output (dc);
+    selftest::test_context dc;
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     ASSERT_EQ (summary.get_num_ranges (), 5);
@@ -1670,9 +1686,9 @@ test_interprocedural_path_2 (pretty_printer *event_pp)
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     print_path_summary_as_text (summary, text_output, true);
@@ -1707,12 +1723,13 @@ test_interprocedural_path_2 (pretty_printer *event_pp)
 }
 
 /* Verify that print_path_summary is sane in the face of a recursive
-   diagnostic_path.  */
+   diagnostic path.  */
 
 static void
 test_recursion (pretty_printer *event_pp)
 {
- test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_entry ("factorial", 0);
   for (int depth = 0; depth < 3; depth++)
     path.add_call ("factorial", depth, "factorial");
@@ -1721,10 +1738,10 @@ test_recursion (pretty_printer *event_pp)
   ASSERT_TRUE (path.interprocedural_p ());
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
 
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     ASSERT_EQ (summary.get_num_ranges (), 4);
@@ -1753,10 +1770,10 @@ test_recursion (pretty_printer *event_pp)
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
 
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     print_path_summary_as_text (summary, text_output, true);
@@ -1789,7 +1806,7 @@ test_recursion (pretty_printer *event_pp)
 class control_flow_test
 {
 public:
-  control_flow_test (const location &loc,
+  control_flow_test (const selftest::location &loc,
 		     const line_table_case &case_,
 		     const char *content)
   : m_tmp_file (loc, ".c", content,
@@ -1858,7 +1875,8 @@ test_control_flow_1 (const line_table_case &case_,
   const location_t conditional = t.get_line_and_column (3, 7);
   const location_t cfg_dest = t.get_line_and_column (5, 10);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (conditional, nullptr, 0,
 		  "following %qs branch (when %qs is NULL)...",
 		  "false", "p");
@@ -1875,10 +1893,10 @@ test_control_flow_1 (const line_table_case &case_,
 
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1901,10 +1919,10 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = false;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (false);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1924,11 +1942,11 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_event_links_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_line_numbers (true);
+    dc.show_event_links (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1951,11 +1969,11 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_event_links_p = false;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_line_numbers (true);
+    dc.show_event_links (false);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1975,10 +1993,10 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    dc.m_source_printing.show_event_links_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2001,11 +2019,11 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2055,7 +2073,8 @@ test_control_flow_2 (const line_table_case &case_,
   const location_t loop_body_start = t.get_line_and_columns (5, 12, 17);
   const location_t loop_body_end = t.get_line_and_columns (5, 5, 9, 17);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (iter_test, nullptr, 0, "infinite loop here");
 
   path.add_event (iter_test, nullptr, 0, "looping from here...");
@@ -2072,11 +2091,11 @@ test_control_flow_2 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2142,7 +2161,8 @@ test_control_flow_3 (const line_table_case &case_,
   const location_t iter_test = t.get_line_and_column (3, 19);
   const location_t iter_next = t.get_line_and_columns (3, 22, 24);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (iter_test, nullptr, 0, "infinite loop here");
 
   path.add_event (iter_test, nullptr, 0, "looping from here...");
@@ -2159,11 +2179,11 @@ test_control_flow_3 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2208,7 +2228,8 @@ assert_cfg_edge_path_streq (const location &loc,
 			    const location_t dst_loc,
 			    const char *expected_str)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (src_loc, nullptr, 0, "from here...");
   path.connect_to_next_event ();
 
@@ -2217,11 +2238,11 @@ assert_cfg_edge_path_streq (const location &loc,
   if (!path_events_have_column_data_p (path))
     return;
 
-  test_diagnostic_context dc;
+  selftest::test_context dc;
   dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-  dc.m_source_printing.show_event_links_p = true;
-  dc.m_source_printing.show_line_numbers_p = true;
-  diagnostic_text_output_format text_output (dc);
+  dc.show_event_links (true);
+  dc.show_line_numbers (true);
+  text_sink text_output (dc);
   path_print_policy policy (text_output);
   path_summary summary (policy, *event_pp, path, true);
   print_path_summary_as_text (summary, text_output, false);
@@ -2512,7 +2533,8 @@ test_control_flow_5 (const line_table_case &case_,
 
   control_flow_test t (SELFTEST_LOCATION, case_, content);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   /* (1) */
   path.add_event (t.get_line_and_column (1, 6), nullptr, 0,
 		  "following %qs branch (when %qs is non-NULL)...",
@@ -2541,11 +2563,11 @@ test_control_flow_5 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2601,7 +2623,8 @@ test_control_flow_6 (const line_table_case &case_,
 
   control_flow_test t (SELFTEST_LOCATION, case_, content);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   /* (1) */
   path.add_event (t.get_line_and_columns (6, 25, 35), nullptr, 0,
 		  "allocated here");
@@ -2630,11 +2653,11 @@ test_control_flow_6 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2687,24 +2710,30 @@ control_flow_tests (const line_table_case &case_)
   test_control_flow_6 (case_, &pp);
 }
 
+} // namespace diagnostics::paths::selftest
+} // namespace diagnostics::paths
+
+namespace selftest { // diagnostics::selftest
+
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_path_output_cc_tests ()
+paths_output_cc_tests ()
 {
   pretty_printer pp;
   pp_show_color (&pp) = false;
 
-  auto_fix_quotes fix_quotes;
-  test_empty_path (&pp);
-  test_intraprocedural_path (&pp);
-  test_interprocedural_path_1 (&pp);
-  test_interprocedural_path_2 (&pp);
-  test_recursion (&pp);
-  for_each_line_table_case (control_flow_tests);
+  ::selftest::auto_fix_quotes fix_quotes;
+  diagnostics::paths::selftest::test_empty_path (&pp);
+  diagnostics::paths::selftest::test_intraprocedural_path (&pp);
+  diagnostics::paths::selftest::test_interprocedural_path_1 (&pp);
+  diagnostics::paths::selftest::test_interprocedural_path_2 (&pp);
+  diagnostics::paths::selftest::test_recursion (&pp);
+  for_each_line_table_case (diagnostics::paths::selftest::control_flow_tests);
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #if __GNUC__ >= 10
 #  pragma GCC diagnostic pop
diff --git a/gcc/diagnostic-path.cc b/gcc/diagnostics/paths.cc
index 3e169da..bc769c4 100644
--- a/gcc/diagnostic-path.cc
+++ b/gcc/diagnostics/paths.cc
@@ -26,8 +26,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-path.h"
-#include "xml.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/state-graphs.h"
 
 /* Disable warnings about missing quoting in GCC diagnostics for the print
    calls below.  */
@@ -36,12 +36,15 @@ along with GCC; see the file COPYING3.  If not see
 #  pragma GCC diagnostic ignored "-Wformat-diag"
 #endif
 
-/* class diagnostic_event.  */
+using namespace diagnostics;
+using namespace diagnostics::paths;
 
-/* struct diagnostic_event::meaning.  */
+/* class diagnostics::paths::event.  */
+
+/* struct event::meaning.  */
 
 void
-diagnostic_event::meaning::dump_to_pp (pretty_printer *pp) const
+event::meaning::dump_to_pp (pretty_printer *pp) const
 {
   bool need_comma = false;
   pp_character (pp, '{');
@@ -71,7 +74,7 @@ diagnostic_event::meaning::dump_to_pp (pretty_printer *pp) const
    threadFlowLocation "kinds" property (SARIF v2.1.0 section 3.38.8).  */
 
 const char *
-diagnostic_event::meaning::maybe_get_verb_str (enum verb v)
+event::meaning::maybe_get_verb_str (enum verb v)
 {
   switch (v)
     {
@@ -102,7 +105,7 @@ diagnostic_event::meaning::maybe_get_verb_str (enum verb v)
    threadFlowLocation "kinds" property (SARIF v2.1.0 section 3.38.8).  */
 
 const char *
-diagnostic_event::meaning::maybe_get_noun_str (enum noun n)
+event::meaning::maybe_get_noun_str (enum noun n)
 {
   switch (n)
     {
@@ -129,7 +132,7 @@ diagnostic_event::meaning::maybe_get_noun_str (enum noun n)
    threadFlowLocation "kinds" property (SARIF v2.1.0 section 3.38.8).  */
 
 const char *
-diagnostic_event::meaning::maybe_get_property_str (enum property p)
+event::meaning::maybe_get_property_str (enum property p)
 {
   switch (p)
     {
@@ -148,7 +151,7 @@ diagnostic_event::meaning::maybe_get_property_str (enum property p)
    (for debugging/logging purposes).  */
 
 label_text
-diagnostic_event::get_desc (pretty_printer &ref_pp) const
+event::get_desc (pretty_printer &ref_pp) const
 {
   auto pp = ref_pp.clone ();
   pp_show_color (pp.get ()) = false;
@@ -156,31 +159,31 @@ diagnostic_event::get_desc (pretty_printer &ref_pp) const
   return label_text::take (xstrdup (pp_formatted_text (pp.get ())));
 }
 
-// Base implementation of diagnostic_event::maybe_make_xml_state
+// Base implementation of event::maybe_make_diagnostic_state_graph
 
-std::unique_ptr<xml::document>
-diagnostic_event::maybe_make_xml_state (bool) const
+std::unique_ptr<digraphs::digraph>
+event::maybe_make_diagnostic_state_graph (bool) const
 {
-  // Don't attempt to make a state document:
+  // Don't attempt to make a state graph:
   return nullptr;
 }
 
-/* class diagnostic_path.  */
+/* class diagnostics::paths::path.  */
 
-/* Subroutine of diagnostic_path::interprocedural_p.
+/* Subroutine of path::interprocedural_p.
    Look for the first event in this path that is within a function
    i.e. has a non-null logical location for which function_p is true.
    If found, write its index to *OUT_IDX and return true.
    Otherwise return false.  */
 
 bool
-diagnostic_path::get_first_event_in_a_function (unsigned *out_idx) const
+path::get_first_event_in_a_function (unsigned *out_idx) const
 {
   const unsigned num = num_events ();
   for (unsigned i = 0; i < num; i++)
     {
-      const diagnostic_event &event = get_event (i);
-      if (logical_location logical_loc = event.get_logical_location ())
+      const event &event = get_event (i);
+      if (logical_locations::key logical_loc = event.get_logical_location ())
 	if (m_logical_loc_mgr.function_p (logical_loc))
 	  {
 	    *out_idx = i;
@@ -194,14 +197,14 @@ diagnostic_path::get_first_event_in_a_function (unsigned *out_idx) const
    function, or false if it is purely intraprocedural.  */
 
 bool
-diagnostic_path::interprocedural_p () const
+path::interprocedural_p () const
 {
   /* Ignore leading events that are outside of any function.  */
   unsigned first_fn_event_idx;
   if (!get_first_event_in_a_function (&first_fn_event_idx))
     return false;
 
-  const diagnostic_event &first_fn_event = get_event (first_fn_event_idx);
+  const event &first_fn_event = get_event (first_fn_event_idx);
   int first_fn_stack_depth = first_fn_event.get_stack_depth ();
 
   const unsigned num = num_events ();
@@ -218,10 +221,10 @@ diagnostic_path::interprocedural_p () const
 /* Print PATH by emitting a dummy "note" associated with it.  */
 
 DEBUG_FUNCTION
-void debug (diagnostic_path *path)
+void debug (path *p)
 {
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  richloc.set_path (path);
+  richloc.set_path (p);
   inform (&richloc, "debug path");
 }
 
diff --git a/gcc/diagnostic-path.h b/gcc/diagnostics/paths.h
index e68f768..d30c420 100644
--- a/gcc/diagnostic-path.h
+++ b/gcc/diagnostics/paths.h
@@ -18,19 +18,28 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_PATH_H
-#define GCC_DIAGNOSTIC_PATH_H
+#ifndef GCC_DIAGNOSTICS_PATHS_H
+#define GCC_DIAGNOSTICS_PATHS_H
 
 #include "diagnostic.h" /* for ATTRIBUTE_GCC_DIAG.  */
-#include "diagnostic-event-id.h"
-#include "logical-location.h"
-
-namespace xml { class document; }
-
-class sarif_builder;
-class sarif_object;
-
-/* A diagnostic_path is an optional additional piece of metadata associated
+#include "diagnostics/event-id.h"
+#include "diagnostics/logical-locations.h"
+
+namespace diagnostics {
+  namespace digraphs {
+    class digraph;
+  } // namespace digraphs
+  namespace logical_locations {
+    class manager;
+  } // logical_locations
+  class sarif_builder;
+  class sarif_object;
+} //namespace diagnostics
+
+namespace diagnostics {
+namespace paths {
+
+/* A diagnostics::paths::path is an optional additional piece of metadata associated
    with a diagnostic (via its rich_location).
 
    It describes a sequence of events predicted by the compiler that
@@ -68,9 +77,9 @@ class sarif_object;
 /* Abstract base classes, describing events within a path, and the paths
    themselves.  */
 
-/* One event within a diagnostic_path.  */
+/* One event within a path.  */
 
-class diagnostic_event
+class event
 {
  public:
   /* Enums for giving a sense of what this event means.
@@ -142,7 +151,7 @@ class diagnostic_event
     enum property m_property;
   };
 
-  virtual ~diagnostic_event () {}
+  virtual ~event () {}
 
   virtual location_t get_location () const = 0;
 
@@ -154,7 +163,7 @@ class diagnostic_event
   virtual void print_desc (pretty_printer &pp) const = 0;
 
   /* Get a logical location for this event, or null if there is none.  */
-  virtual logical_location get_logical_location () const = 0;
+  virtual logical_locations::key get_logical_location () const = 0;
 
   virtual meaning get_meaning () const = 0;
 
@@ -162,7 +171,7 @@ class diagnostic_event
      next event (e.g. to highlight control flow).  */
   virtual bool connect_to_next_event_p () const = 0;
 
-  virtual diagnostic_thread_id_t get_thread_id () const = 0;
+  virtual thread_id_t get_thread_id () const = 0;
 
   /* Hook for SARIF output to allow for adding diagnostic-specific
      properties to the threadFlowLocation object's property bag.  */
@@ -173,36 +182,36 @@ class diagnostic_event
   }
 
   /* Hook for capturing state at this event, potentially for visualizing
-     in HTML output.  */
-  virtual std::unique_ptr<xml::document>
-  maybe_make_xml_state (bool debug) const;
+     in HTML output, or for adding to SARIF.  */
+  virtual std::unique_ptr<digraphs::digraph>
+  maybe_make_diagnostic_state_graph (bool debug) const;
 
   label_text get_desc (pretty_printer &ref_pp) const;
 };
 
 /* Abstract base class representing a thread of execution within
-   a diagnostic_path.
-   Each diagnostic_event is associated with one thread.
-   Typically there is just one thread per diagnostic_path. */
+   a diagnostics::paths::path.
+   Each event is associated with one thread.
+   Typically there is just one thread per diagnostics::paths::path. */
 
-class diagnostic_thread
+class thread
 {
 public:
-  virtual ~diagnostic_thread () {}
+  virtual ~thread () {}
   virtual label_text get_name (bool can_colorize) const = 0;
 };
 
 /* Abstract base class for getting at a sequence of events.  */
 
-class diagnostic_path
+class path
 {
  public:
-  virtual ~diagnostic_path () {}
+  virtual ~path () {}
   virtual unsigned num_events () const = 0;
-  virtual const diagnostic_event & get_event (int idx) const = 0;
+  virtual const event & get_event (int idx) const = 0;
   virtual unsigned num_threads () const = 0;
-  virtual const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const = 0;
+  virtual const thread &
+  get_thread (thread_id_t) const = 0;
 
   /* Return true iff the two events are both within the same function,
      or both outside of any function.  */
@@ -213,13 +222,13 @@ class diagnostic_path
   bool interprocedural_p () const;
   bool multithreaded_p () const;
 
-  const logical_location_manager &get_logical_location_manager () const
+  const logical_locations::manager &get_logical_location_manager () const
   {
     return m_logical_loc_mgr;
   }
 
 protected:
-  diagnostic_path (const logical_location_manager &logical_loc_mgr)
+  path (const logical_locations::manager &logical_loc_mgr)
   : m_logical_loc_mgr (logical_loc_mgr)
   {
   }
@@ -227,12 +236,15 @@ protected:
 private:
   bool get_first_event_in_a_function (unsigned *out_idx) const;
 
-  const logical_location_manager &m_logical_loc_mgr;
+  const logical_locations::manager &m_logical_loc_mgr;
 };
 
+} // namespace paths
+} // namespace diagnostics
+
 /* Concrete subclasses of the above can be found in
    simple-diagnostic-path.h.  */
 
-extern void debug (diagnostic_path *path);
+extern void debug (diagnostics::paths::path *path);
 
-#endif /* ! GCC_DIAGNOSTIC_PATH_H */
+#endif /* ! GCC_DIAGNOSTICS_PATHS_H */
diff --git a/gcc/diagnostic-format-sarif.cc b/gcc/diagnostics/sarif-sink.cc
index 14cdbc2..4738ae9 100644
--- a/gcc/diagnostic-format-sarif.cc
+++ b/gcc/diagnostics/sarif-sink.cc
@@ -27,23 +27,25 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-format.h"
-#include "diagnostic-buffer.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/state-graphs.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/buffering.h"
 #include "json.h"
 #include "cpplib.h"
-#include "logical-location.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-diagram.h"
+#include "diagnostics/logical-locations.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/diagram.h"
 #include "text-art/canvas.h"
-#include "diagnostic-format-sarif.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/text-sink.h"
 #include "ordered-hash-map.h"
 #include "sbitmap.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "selftest-diagnostic-show-locus.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-source-printing.h"
 #include "selftest-json.h"
 #include "text-range-label.h"
 #include "pretty-print-format-impl.h"
@@ -52,6 +54,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "backtrace.h"
 #include "xml.h"
 
+namespace diagnostics {
+
 /* A json::array where the values are "unique" as per
    SARIF v2.1.0 section 3.7.3 ("Array properties with unique values").  */
 
@@ -448,9 +452,9 @@ public:
 
   void
   on_nested_diagnostic (const diagnostic_info &diagnostic,
-			diagnostic_t orig_diag_kind,
+			enum kind orig_diag_kind,
 			sarif_builder &builder);
-  void on_diagram (const diagnostic_diagram &diagram,
+  void on_diagram (const diagram &d,
 		   sarif_builder &builder);
 
 private:
@@ -530,22 +534,22 @@ public:
   unsigned get_index_within_parent () const { return m_idx_within_parent; }
 
   sarif_thread_flow &
-  get_or_append_thread_flow (const diagnostic_thread &thread,
-			     diagnostic_thread_id_t thread_id);
+  get_or_append_thread_flow (const paths::thread &thread,
+			     paths::thread_id_t thread_id);
 
   sarif_thread_flow &
-  get_thread_flow (diagnostic_thread_id_t thread_id);
+  get_thread_flow (paths::thread_id_t thread_id);
 
   void add_location (sarif_thread_flow_location &);
 
   sarif_thread_flow_location &
-  get_thread_flow_loc_obj (diagnostic_event_id_t event_id) const;
+  get_thread_flow_loc_obj (paths::event_id_t event_id) const;
 
 private:
   sarif_result &m_parent;
   const unsigned m_idx_within_parent;
 
-  hash_map<int_hash<diagnostic_thread_id_t, -1, -2>,
+  hash_map<int_hash<paths::thread_id_t, -1, -2>,
 	   sarif_thread_flow *> m_thread_id_map; // borrowed ptr
   json::array *m_thread_flows_arr; // borrowed
 
@@ -561,7 +565,7 @@ class sarif_thread_flow : public sarif_object
 {
 public:
   sarif_thread_flow (sarif_code_flow &parent,
-		     const diagnostic_thread &thread,
+		     const paths::thread &thread,
 		     unsigned idx_within_parent);
 
   sarif_code_flow &get_parent () const { return m_parent; }
@@ -658,20 +662,20 @@ public:
   render (const sarif_builder &builder) const = 0;
 };
 
-/* Concrete buffering implementation subclass for JSON output.  */
+/* Concrete buffering implementation subclass for SARIF output.  */
 
-class diagnostic_sarif_format_buffer : public diagnostic_per_format_buffer
+class sarif_sink_buffer : public per_sink_buffer
 {
 public:
-  friend class sarif_output_format;
+  friend class sarif_sink;
 
-  diagnostic_sarif_format_buffer (sarif_builder &builder)
+  sarif_sink_buffer (sarif_builder &builder)
   : m_builder (builder)
   {}
 
   void dump (FILE *out, int indent) const final override;
   bool empty_p () const final override;
-  void move_to (diagnostic_per_format_buffer &dest) final override;
+  void move_to (per_sink_buffer &dest) final override;
   void clear () final override;
   void flush () final override;
 
@@ -741,15 +745,15 @@ sarif_serialization_format_json::write_to_file (FILE *outf,
    - doesn't capture -Werror cleanly
    - doesn't capture inlining information (can SARIF handle this?)
    - doesn't capture macro expansion information (can SARIF handle this?).
-   - doesn't capture any diagnostic_metadata::rules associated with
+   - doesn't capture any diagnostics::metadata::rules associated with
      a diagnostic.  */
 
 class sarif_builder
 {
 public:
-  friend class diagnostic_sarif_format_buffer;
+  friend class sarif_sink_buffer;
 
-  sarif_builder (diagnostic_context &context,
+  sarif_builder (diagnostics::context &dc,
 		 pretty_printer &printer,
 		 const line_maps *line_maps,
 		 std::unique_ptr<sarif_serialization_format> serialization_format,
@@ -761,7 +765,7 @@ public:
     m_printer = &printer;
   }
 
-  const logical_location_manager *
+  const logical_locations::manager *
   get_logical_location_manager () const
   {
     return m_logical_loc_mgr;
@@ -771,11 +775,14 @@ public:
   set_main_input_filename (const char *name);
 
   void on_report_diagnostic (const diagnostic_info &diagnostic,
-			     diagnostic_t orig_diag_kind,
-			     diagnostic_sarif_format_buffer *buffer);
-  void emit_diagram (const diagnostic_diagram &diagram);
+			     enum kind orig_diag_kind,
+			     sarif_sink_buffer *buffer);
+  void emit_diagram (const diagram &d);
   void end_group ();
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &);
+
   std::unique_ptr<sarif_result> take_current_result ()
   {
     return std::move (m_cur_group_result);
@@ -789,9 +796,9 @@ public:
 		      const diagnostic_info &diagnostic,
 		      enum diagnostic_artifact_role role);
   std::unique_ptr<sarif_location>
-  make_location_object (sarif_location_manager &loc_mgr,
+  make_location_object (sarif_location_manager *loc_mgr,
 			const rich_location &rich_loc,
-			logical_location logical_loc,
+			logical_locations::key logical_loc,
 			enum diagnostic_artifact_role role);
   std::unique_ptr<sarif_location>
   make_location_object (sarif_location_manager &loc_mgr,
@@ -800,7 +807,7 @@ public:
   std::unique_ptr<sarif_message>
   make_message_object (const char *msg) const;
   std::unique_ptr<sarif_message>
-  make_message_object_for_diagram (const diagnostic_diagram &diagram);
+  make_message_object_for_diagram (const diagram &d);
   std::unique_ptr<sarif_artifact_content>
   maybe_make_artifact_content_object (const char *filename) const;
 
@@ -813,7 +820,7 @@ public:
     return m_current_code_flow;
   }
 
-  diagnostic_context &get_context () const { return m_context; }
+  diagnostics::context &get_context () const { return m_context; }
   pretty_printer *get_printer () const { return m_printer; }
   token_printer &get_token_printer () { return m_token_printer; }
   enum sarif_version get_version () const { return m_sarif_gen_opts.m_version; }
@@ -829,7 +836,7 @@ public:
   const sarif_generation_options &get_opts () const { return m_sarif_gen_opts; }
 
   std::unique_ptr<sarif_logical_location>
-  make_minimal_sarif_logical_location (logical_location);
+  make_minimal_sarif_logical_location (logical_locations::key);
 
 private:
   class sarif_token_printer : public token_printer
@@ -847,7 +854,7 @@ private:
 
   std::unique_ptr<sarif_result>
   make_result_object (const diagnostic_info &diagnostic,
-		      diagnostic_t orig_diag_kind,
+		      enum kind orig_diag_kind,
 		      unsigned idx_within_parent);
   void
   add_any_include_chain (sarif_location_manager &loc_mgr,
@@ -855,22 +862,22 @@ private:
 			 location_t where);
   void
   set_any_logical_locs_arr (sarif_location &location_obj,
-			    logical_location logical_loc);
+			    logical_locations::key logical_loc);
   std::unique_ptr<sarif_location>
   make_location_object (sarif_location_manager &loc_mgr,
-			const diagnostic_event &event,
+			const paths::event &event,
 			enum diagnostic_artifact_role role);
   std::unique_ptr<sarif_code_flow>
   make_code_flow_object (sarif_result &result,
 			 unsigned idx_within_parent,
-			 const diagnostic_path &path);
+			 const paths::path &path);
   void
   populate_thread_flow_location_object (sarif_result &result,
 					sarif_thread_flow_location &thread_flow_loc_obj,
-					const diagnostic_event &event,
+					const paths::event &event,
 					int event_execution_idx);
   std::unique_ptr<json::array>
-  maybe_make_kinds_array (diagnostic_event::meaning m) const;
+  maybe_make_kinds_array (paths::event::meaning m) const;
   std::unique_ptr<sarif_physical_location>
   maybe_make_physical_location_object (location_t loc,
 				       enum diagnostic_artifact_role role,
@@ -890,7 +897,7 @@ private:
   make_region_object_for_hint (const fixit_hint &hint) const;
 
   int
-  ensure_sarif_logical_location_for (logical_location k);
+  ensure_sarif_logical_location_for (logical_locations::key k);
 
   std::unique_ptr<sarif_multiformat_message_string>
   make_multiformat_message_string (const char *msg) const;
@@ -911,7 +918,7 @@ private:
   make_tool_component_reference_object_for_cwe () const;
   std::unique_ptr<sarif_reporting_descriptor>
   make_reporting_descriptor_object_for_warning (const diagnostic_info &diagnostic,
-						diagnostic_t orig_diag_kind,
+						enum kind orig_diag_kind,
 						const char *option_text);
   std::unique_ptr<sarif_reporting_descriptor>
   make_reporting_descriptor_object_for_cwe_id (int cwe_id) const;
@@ -943,12 +950,12 @@ private:
   std::unique_ptr<json::object>
   make_stack_from_backtrace ();
 
-  diagnostic_context &m_context;
+  diagnostics::context &m_context;
   pretty_printer *m_printer;
   const line_maps *m_line_maps;
   sarif_token_printer m_token_printer;
 
-  const logical_location_manager *m_logical_loc_mgr;
+  const logical_locations::manager *m_logical_loc_mgr;
 
   /* The JSON object for the invocation object.  */
   std::unique_ptr<sarif_invocation> m_invocation_obj;
@@ -974,6 +981,8 @@ private:
 
   std::unique_ptr<sarif_array_of_unique<sarif_logical_location>> m_cached_logical_locs;
 
+  std::unique_ptr<sarif_array_of_unique<sarif_graph>> m_run_graphs;
+
   int m_tabstop;
 
   std::unique_ptr<sarif_serialization_format> m_serialization_format;
@@ -1053,10 +1062,10 @@ sarif_invocation::add_notification_for_ice (const diagnostic_info &diagnostic,
 void
 sarif_invocation::prepare_to_flush (sarif_builder &builder)
 {
-  const diagnostic_context &context = builder.get_context ();
+  const context &dc = builder.get_context ();
 
   /* "executionSuccessful" property (SARIF v2.1.0 section 3.20.14).  */
-  if (context.execution_failed_p ())
+  if (dc.execution_failed_p ())
     m_success = false;
   set_bool ("executionSuccessful", m_success);
 
@@ -1065,7 +1074,7 @@ sarif_invocation::prepare_to_flush (sarif_builder &builder)
 
   /* Call client hook, allowing it to create a custom property bag for
      this object (SARIF v2.1.0 section 3.8) e.g. for recording time vars.  */
-  if (auto client_data_hooks = context.get_client_data_hooks ())
+  if (auto client_data_hooks = dc.get_client_data_hooks ())
     client_data_hooks->add_sarif_invocation_properties (*this);
 
   // "endTimeUtc" property (SARIF v2.1.0 section 3.20.8);
@@ -1298,15 +1307,15 @@ sarif_location_manager::process_worklist_item (sarif_builder &builder,
 
 void
 sarif_result::on_nested_diagnostic (const diagnostic_info &diagnostic,
-				    diagnostic_t /*orig_diag_kind*/,
+				    enum kind /*orig_diag_kind*/,
 				    sarif_builder &builder)
 {
   /* We don't yet generate meaningful logical locations for notes;
      sometimes these will related to current_function_decl, but
      often they won't.  */
   auto location_obj
-    = builder.make_location_object (*this, *diagnostic.richloc,
-				    logical_location (),
+    = builder.make_location_object (this, *diagnostic.m_richloc,
+				    logical_locations::key (),
 				    diagnostic_artifact_role::result_file);
   auto message_obj
     = builder.make_message_object (pp_formatted_text (builder.get_printer ()));
@@ -1329,11 +1338,11 @@ sarif_result::on_nested_diagnostic (const diagnostic_info &diagnostic,
    (SARIF v2.1.0 section 3.28.5).  */
 
 void
-sarif_result::on_diagram (const diagnostic_diagram &diagram,
+sarif_result::on_diagram (const diagram &d,
 			  sarif_builder &builder)
 {
   auto location_obj = std::make_unique<sarif_location> ();
-  auto message_obj = builder.make_message_object_for_diagram (diagram);
+  auto message_obj = builder.make_message_object_for_diagram (d);
   location_obj->set<sarif_message> ("message", std::move (message_obj));
 
   add_related_location (std::move (location_obj), builder);
@@ -1565,8 +1574,8 @@ sarif_code_flow::sarif_code_flow (sarif_result &parent,
 }
 
 sarif_thread_flow &
-sarif_code_flow::get_or_append_thread_flow (const diagnostic_thread &thread,
-					    diagnostic_thread_id_t thread_id)
+sarif_code_flow::get_or_append_thread_flow (const paths::thread &thread,
+					    paths::thread_id_t thread_id)
 {
   sarif_thread_flow **slot = m_thread_id_map.get (thread_id);
   if (slot)
@@ -1582,7 +1591,7 @@ sarif_code_flow::get_or_append_thread_flow (const diagnostic_thread &thread,
 }
 
 sarif_thread_flow &
-sarif_code_flow::get_thread_flow (diagnostic_thread_id_t thread_id)
+sarif_code_flow::get_thread_flow (paths::thread_id_t thread_id)
 {
   sarif_thread_flow **slot = m_thread_id_map.get (thread_id);
   gcc_assert (slot); // it must already have one
@@ -1596,7 +1605,7 @@ sarif_code_flow::add_location (sarif_thread_flow_location &tfl_obj)
 }
 
 sarif_thread_flow_location &
-sarif_code_flow::get_thread_flow_loc_obj (diagnostic_event_id_t event_id) const
+sarif_code_flow::get_thread_flow_loc_obj (paths::event_id_t event_id) const
 {
   gcc_assert (event_id.known_p ());
   gcc_assert ((size_t)event_id.zero_based () < m_all_tfl_objs.size ());
@@ -1608,7 +1617,7 @@ sarif_code_flow::get_thread_flow_loc_obj (diagnostic_event_id_t event_id) const
 /* class sarif_thread_flow : public sarif_object.  */
 
 sarif_thread_flow::sarif_thread_flow (sarif_code_flow &parent,
-				      const diagnostic_thread &thread,
+				      const paths::thread &thread,
 				      unsigned idx_within_parent)
 : m_parent (parent),
   m_idx_within_parent (idx_within_parent)
@@ -1643,19 +1652,19 @@ sarif_thread_flow::add_location ()
 
 /* sarif_builder's ctor.  */
 
-sarif_builder::sarif_builder (diagnostic_context &context,
+sarif_builder::sarif_builder (diagnostics::context &dc,
 			      pretty_printer &printer,
 			      const line_maps *line_maps,
 			      std::unique_ptr<sarif_serialization_format> serialization_format,
 			      const sarif_generation_options &sarif_gen_opts)
-: m_context (context),
+: m_context (dc),
   m_printer (&printer),
   m_line_maps (line_maps),
   m_token_printer (*this),
   m_logical_loc_mgr (nullptr),
   m_invocation_obj
     (std::make_unique<sarif_invocation> (*this,
-					 context.get_original_argv ())),
+					 dc.get_original_argv ())),
   m_results_array (new json::array ()),
   m_cur_group_result (nullptr),
   m_seen_any_relative_paths (false),
@@ -1663,7 +1672,9 @@ sarif_builder::sarif_builder (diagnostic_context &context,
   m_rules_arr (new json::array ()),
   m_cached_logical_locs
     (std::make_unique<sarif_array_of_unique<sarif_logical_location>> ()),
-  m_tabstop (context.m_tabstop),
+  m_run_graphs
+    (std::make_unique<sarif_array_of_unique<sarif_graph>> ()),
+  m_tabstop (dc.m_tabstop),
   m_serialization_format (std::move (serialization_format)),
   m_sarif_gen_opts (sarif_gen_opts),
   m_next_result_idx (0),
@@ -1672,7 +1683,7 @@ sarif_builder::sarif_builder (diagnostic_context &context,
   gcc_assert (m_line_maps);
   gcc_assert (m_serialization_format);
 
-  if (auto client_data_hooks = context.get_client_data_hooks ())
+  if (auto client_data_hooks = dc.get_client_data_hooks ())
     m_logical_loc_mgr = client_data_hooks->get_logical_location_manager ();
 }
 
@@ -1725,10 +1736,10 @@ bt_callback (void *data, uintptr_t pc, const char *filename, int lineno,
   if (filename == nullptr && function == nullptr)
     return 0;
 
-  /* Skip functions in diagnostic.cc or diagnostic-global-context.cc.  */
+  /* Skip functions in context.cc or diagnostic-global-context.cc.  */
   if (closure->m_frames_arr->size () == 0
       && filename != nullptr
-      && (strcmp (lbasename (filename), "diagnostic.cc") == 0
+      && (strcmp (lbasename (filename), "context.cc") == 0
 	  || strcmp (lbasename (filename),
 		     "diagnostic-global-context.cc") == 0))
     return 0;
@@ -1829,12 +1840,12 @@ sarif_builder::set_main_input_filename (const char *name)
 
 void
 sarif_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
-				     diagnostic_t orig_diag_kind,
-				     diagnostic_sarif_format_buffer *buffer)
+				     enum kind orig_diag_kind,
+				     sarif_sink_buffer *buffer)
 {
   pp_output_formatted_text (m_printer, m_context.get_urlifier ());
 
-  if (diagnostic.kind == DK_ICE || diagnostic.kind == DK_ICE_NOBT)
+  if (diagnostic.m_kind == kind::ice || diagnostic.m_kind == kind::ice_nobt)
     {
       std::unique_ptr<json::object> stack = make_stack_from_backtrace ();
       m_invocation_obj->add_notification_for_ice (diagnostic, *this,
@@ -1872,15 +1883,15 @@ sarif_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
     }
 }
 
-/* Implementation of diagnostic_context::m_diagrams.m_emission_cb
+/* Implementation of diagnostics::context::m_diagrams.m_emission_cb
    for SARIF output.  */
 
 void
-sarif_builder::emit_diagram (const diagnostic_diagram &diagram)
+sarif_builder::emit_diagram (const diagram &d)
 {
   /* We must be within the emission of a top-level diagnostic.  */
   gcc_assert (m_cur_group_result);
-  m_cur_group_result->on_diagram (diagram, *this);
+  m_cur_group_result->on_diagram (d, *this);
 }
 
 /* Implementation of "end_group_cb" for SARIF output.  */
@@ -1895,6 +1906,17 @@ sarif_builder::end_group ()
     }
 }
 
+void
+sarif_builder::
+report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+{
+  auto &dg = ldg.get_or_create ();
+
+  /* Presumably the location manager must be nullptr; see
+     https://github.com/oasis-tcs/sarif-spec/issues/712  */
+  m_run_graphs->append (make_sarif_graph (dg, this, nullptr));
+}
+
 /* Create a top-level object, and add it to all the results
    (and other entities) we've seen so far, moving ownership
    to the object.  */
@@ -1927,16 +1949,16 @@ sarif_builder::flush_to_file (FILE *outf)
    Return nullptr if there isn't one.  */
 
 static const char *
-maybe_get_sarif_level (diagnostic_t diag_kind)
+maybe_get_sarif_level (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_WARNING:
+    case kind::warning:
       return "warning";
-    case DK_ERROR:
+    case kind::error:
       return "error";
-    case DK_NOTE:
-    case DK_ANACHRONISM:
+    case kind::note:
+    case kind::anachronism:
       return "note";
     default:
       return nullptr;
@@ -1948,10 +1970,10 @@ maybe_get_sarif_level (diagnostic_t diag_kind)
    have anything better to use.  */
 
 static char *
-make_rule_id_for_diagnostic_kind (diagnostic_t diag_kind)
+make_rule_id_for_diagnostic_kind (enum kind diag_kind)
 {
   /* Lose the trailing ": ".  */
-  const char *kind_text = get_diagnostic_kind_text (diag_kind);
+  const char *kind_text = get_text_for_kind (diag_kind);
   size_t len = strlen (kind_text);
   gcc_assert (len > 2);
   gcc_assert (kind_text[len - 2] == ':');
@@ -1965,7 +1987,7 @@ make_rule_id_for_diagnostic_kind (diagnostic_t diag_kind)
 
 std::unique_ptr<sarif_result>
 sarif_builder::make_result_object (const diagnostic_info &diagnostic,
-				   diagnostic_t orig_diag_kind,
+				   enum kind orig_diag_kind,
 				   unsigned idx_within_parent)
 {
   auto result_obj = std::make_unique<sarif_result> (idx_within_parent);
@@ -1973,8 +1995,8 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
   /* "ruleId" property (SARIF v2.1.0 section 3.27.5).  */
   /* Ideally we'd have an option_name for these.  */
   if (char *option_text
-	= m_context.make_option_name (diagnostic.option_id,
-				      orig_diag_kind, diagnostic.kind))
+	= m_context.make_option_name (diagnostic.m_option_id,
+				      orig_diag_kind, diagnostic.m_kind))
     {
       /* Lazily create reportingDescriptor objects for and add to m_rules_arr.
 	 Set ruleId referencing them.  */
@@ -2004,10 +2026,10 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
       free (rule_id);
     }
 
-  if (diagnostic.metadata)
+  if (diagnostic.m_metadata)
     {
       /* "taxa" property (SARIF v2.1.0 section 3.27.8).  */
-      if (int cwe_id = diagnostic.metadata->get_cwe ())
+      if (int cwe_id = diagnostic.m_metadata->get_cwe ())
 	{
 	  auto taxa_arr = std::make_unique<json::array> ();
 	  taxa_arr->append<sarif_reporting_descriptor_reference>
@@ -2015,13 +2037,13 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
 	  result_obj->set<json::array> ("taxa", std::move (taxa_arr));
 	}
 
-      diagnostic.metadata->maybe_add_sarif_properties (*result_obj);
+      diagnostic.m_metadata->maybe_add_sarif_properties (*result_obj);
 
-      /* We don't yet support diagnostic_metadata::rule.  */
+      /* We don't yet support diagnostics::metadata::rule.  */
     }
 
   /* "level" property (SARIF v2.1.0 section 3.27.10).  */
-  if (const char *sarif_level = maybe_get_sarif_level (diagnostic.kind))
+  if (const char *sarif_level = maybe_get_sarif_level (diagnostic.m_kind))
     result_obj->set_string ("level", sarif_level);
 
   /* "message" property (SARIF v2.1.0 section 3.27.11).  */
@@ -2038,7 +2060,7 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
 			 diagnostic_artifact_role::result_file));
 
   /* "codeFlows" property (SARIF v2.1.0 section 3.27.18).  */
-  if (const diagnostic_path *path = diagnostic.richloc->get_path ())
+  if (const paths::path *path = diagnostic.m_richloc->get_path ())
     {
       auto code_flows_arr = std::make_unique<json::array> ();
       const unsigned code_flow_index = 0;
@@ -2049,12 +2071,25 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
       result_obj->set<json::array> ("codeFlows", std::move (code_flows_arr));
     }
 
+  // "graphs" property (SARIF v2.1.0 section 3.27.19).  */
+  if (diagnostic.m_metadata)
+    if (auto ldg = diagnostic.m_metadata->get_lazy_digraphs ())
+      {
+	auto &digraphs = ldg->get_or_create ();
+	auto graphs_arr = std::make_unique<json::array> ();
+	for (auto &iter : digraphs)
+	  graphs_arr->append (make_sarif_graph (*iter, this,
+						result_obj.get ()));
+	if (graphs_arr->size () > 0)
+	  result_obj->set<json::array> ("graphs", std::move (graphs_arr));
+      }
+
   /* The "relatedLocations" property (SARIF v2.1.0 section 3.27.22) is
      set up later, if any nested diagnostics occur within this diagnostic
      group.  */
 
   /* "fixes" property (SARIF v2.1.0 section 3.27.30).  */
-  const rich_location *richloc = diagnostic.richloc;
+  const rich_location *richloc = diagnostic.m_richloc;
   if (richloc->get_num_fixit_hints ())
     {
       auto fix_arr = std::make_unique<json::array> ();
@@ -2071,7 +2106,7 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
 std::unique_ptr<sarif_reporting_descriptor>
 sarif_builder::
 make_reporting_descriptor_object_for_warning (const diagnostic_info &diagnostic,
-					      diagnostic_t /*orig_diag_kind*/,
+					      enum kind /*orig_diag_kind*/,
 					      const char *option_text)
 {
   auto reporting_desc = std::make_unique<sarif_reporting_descriptor> ();
@@ -2083,7 +2118,7 @@ make_reporting_descriptor_object_for_warning (const diagnostic_info &diagnostic,
      it seems redundant compared to "id".  */
 
   /* "helpUri" property (SARIF v2.1.0 section 3.49.12).  */
-  if (char *option_url = m_context.make_option_url (diagnostic.option_id))
+  if (char *option_url = m_context.make_option_url (diagnostic.m_option_id))
     {
       reporting_desc->set_string ("helpUri", option_url);
       free (option_url);
@@ -2171,12 +2206,12 @@ sarif_builder::make_locations_arr (sarif_location_manager &loc_mgr,
 				   enum diagnostic_artifact_role role)
 {
   auto locations_arr = std::make_unique<json::array> ();
-  logical_location logical_loc;
+  logical_locations::key logical_loc;
   if (auto client_data_hooks = m_context.get_client_data_hooks ())
     logical_loc = client_data_hooks->get_current_logical_location ();
 
   auto location_obj
-    = make_location_object (loc_mgr, *diagnostic.richloc, logical_loc, role);
+    = make_location_object (&loc_mgr, *diagnostic.m_richloc, logical_loc, role);
   /* Don't add entirely empty location objects to the array.  */
   if (!location_obj->is_empty ())
     locations_arr->append<sarif_location> (std::move (location_obj));
@@ -2191,7 +2226,7 @@ sarif_builder::make_locations_arr (sarif_location_manager &loc_mgr,
 void
 sarif_builder::
 set_any_logical_locs_arr (sarif_location &location_obj,
-			  logical_location logical_loc)
+			  logical_locations::key logical_loc)
 {
   if (!logical_loc)
     return;
@@ -2210,12 +2245,14 @@ set_any_logical_locs_arr (sarif_location &location_obj,
 /* Make a "location" object (SARIF v2.1.0 section 3.28) for RICH_LOC
    and LOGICAL_LOC.
    Use LOC_MGR for any locations that need "id" values, and for
-   any worklist items.  */
+   any worklist items.
+   Note that we might not always have a LOC_MGR; see
+   https://github.com/oasis-tcs/sarif-spec/issues/712  */
 
 std::unique_ptr<sarif_location>
-sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
+sarif_builder::make_location_object (sarif_location_manager *loc_mgr,
 				     const rich_location &rich_loc,
-				     logical_location logical_loc,
+				     logical_locations::key logical_loc,
 				     enum diagnostic_artifact_role role)
 {
   class escape_nonascii_renderer : public content_renderer
@@ -2230,21 +2267,22 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
     std::unique_ptr<sarif_multiformat_message_string>
     render (const sarif_builder &builder) const final override
     {
-      diagnostic_context dc;
+      diagnostics::context dc;
       diagnostic_initialize (&dc, 0);
-      dc.m_source_printing.enabled = true;
-      dc.m_source_printing.colorize_source_p = false;
-      dc.m_source_printing.show_labels_p = true;
-      dc.m_source_printing.show_line_numbers_p = true;
+      auto &source_printing_opts = dc.get_source_printing_options ();
+      source_printing_opts.enabled = true;
+      source_printing_opts.colorize_source_p = false;
+      source_printing_opts.show_labels_p = true;
+      source_printing_opts.show_line_numbers_p = true;
 
       rich_location my_rich_loc (m_richloc);
       my_rich_loc.set_escape_on_output (true);
 
-      diagnostic_source_print_policy source_policy (dc);
+      source_print_policy source_policy (dc);
       dc.set_escape_format (m_escape_format);
-      diagnostic_text_output_format text_output (dc);
+      text_sink text_output (dc);
       source_policy.print (*text_output.get_printer (),
-			   my_rich_loc, DK_ERROR, nullptr);
+			   my_rich_loc, kind::error, nullptr);
 
       const char *buf = pp_formatted_text (text_output.get_printer ());
       std::unique_ptr<sarif_multiformat_message_string> result
@@ -2309,8 +2347,8 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 
 	/* Add related locations for any secondary locations in RICH_LOC
 	   that don't have labels (and thus aren't added to "annotations"). */
-	if (i > 0 && !handled)
-	  loc_mgr.add_relationship_to_worklist
+	if (loc_mgr && i > 0 && !handled)
+	  loc_mgr->add_relationship_to_worklist
 	    (*location_obj.get (),
 	     sarif_location_manager::worklist_item::kind::unlabelled_secondary_location,
 	     range->m_loc);
@@ -2321,7 +2359,8 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 				      std::move (annotations_arr));
   }
 
-  add_any_include_chain (loc_mgr, *location_obj.get (), loc);
+  if (loc_mgr)
+    add_any_include_chain (*loc_mgr, *location_obj.get (), loc);
 
   /* A flag for hinting that the diagnostic involves issues at the
      level of character encodings (such as homoglyphs, or misleading
@@ -2340,7 +2379,7 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 /* If WHERE was #included from somewhere, add a worklist item
    to LOC_MGR to lazily add a location for the #include location,
    and relationships between it and the LOCATION_OBJ.
-   Compare with diagnostic_context::report_current_module, but rather
+   Compare with diagnostics::context::report_current_module, but rather
    than iterating the current chain, we add the next edge and iterate
    in the worklist, so that edges are only added once.  */
 
@@ -2392,11 +2431,11 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 }
 
 /* Make a "location" object (SARIF v2.1.0 section 3.28) for EVENT
-   within a diagnostic_path.  */
+   within a paths::path.  */
 
 std::unique_ptr<sarif_location>
 sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
-				     const diagnostic_event &event,
+				     const paths::event &event,
 				     enum diagnostic_artifact_role role)
 {
   auto location_obj = std::make_unique<sarif_location> ();
@@ -2409,7 +2448,7 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 						std::move (phs_loc_obj));
 
   /* "logicalLocations" property (SARIF v2.1.0 section 3.28.4).  */
-  logical_location logical_loc = event.get_logical_location ();
+  logical_locations::key logical_loc = event.get_logical_location ();
   set_any_logical_locs_arr (*location_obj, logical_loc);
 
   /* "message" property (SARIF v2.1.0 section 3.28.5).  */
@@ -2730,57 +2769,59 @@ sarif_builder::make_region_object_for_hint (const fixit_hint &hint) const
    Return nullptr if unknown.  */
 
 static const char *
-maybe_get_sarif_kind (enum logical_location_kind kind)
+maybe_get_sarif_kind (enum logical_locations::kind kind)
 {
+  using namespace logical_locations;
+
   switch (kind)
     {
     default:
       gcc_unreachable ();
-    case logical_location_kind::unknown:
+    case logical_locations::kind::unknown:
       return nullptr;
 
     /* Kinds within executable code.  */
-    case logical_location_kind::function:
+    case logical_locations::kind::function:
       return "function";
-    case logical_location_kind::member:
+    case logical_locations::kind::member:
       return "member";
-    case logical_location_kind::module_:
+    case logical_locations::kind::module_:
       return "module";
-    case logical_location_kind::namespace_:
+    case logical_locations::kind::namespace_:
       return "namespace";
-    case logical_location_kind::type:
+    case logical_locations::kind::type:
       return "type";
-    case logical_location_kind::return_type:
+    case logical_locations::kind::return_type:
       return "returnType";
-    case logical_location_kind::parameter:
+    case logical_locations::kind::parameter:
       return "parameter";
-    case logical_location_kind::variable:
+    case logical_locations::kind::variable:
       return "variable";
 
     /* Kinds within XML or HTML documents.  */
-    case logical_location_kind::element:
+    case logical_locations::kind::element:
       return "element";
-    case logical_location_kind::attribute:
+    case logical_locations::kind::attribute:
       return "attribute";
-    case logical_location_kind::text:
+    case logical_locations::kind::text:
       return "text";
-    case logical_location_kind::comment:
+    case logical_locations::kind::comment:
       return "comment";
-    case logical_location_kind::processing_instruction:
+    case logical_locations::kind::processing_instruction:
       return "processingInstruction";
-    case logical_location_kind::dtd:
+    case logical_locations::kind::dtd:
       return "dtd";
-    case logical_location_kind::declaration:
+    case logical_locations::kind::declaration:
       return "declaration";
 
     /* Kinds within JSON documents.  */
-    case logical_location_kind::object:
+    case logical_locations::kind::object:
       return "object";
-    case logical_location_kind::array:
+    case logical_locations::kind::array:
       return "array";
-    case logical_location_kind::property:
+    case logical_locations::kind::property:
       return "property";
-    case logical_location_kind::value:
+    case logical_locations::kind::value:
       return "value";
     }
 }
@@ -2792,20 +2833,156 @@ maybe_get_sarif_kind (enum logical_location_kind kind)
 void
 sarif_property_bag::set_logical_location (const char *property_name,
 					  sarif_builder &builder,
-					  logical_location logical_loc)
+					  logical_locations::key logical_loc)
 {
   set<sarif_logical_location>
     (property_name,
      builder.make_minimal_sarif_logical_location (logical_loc));
 }
 
+static void
+copy_any_property_bag (const digraphs::object &input_obj,
+		       sarif_object &output_obj)
+{
+  if (input_obj.get_property_bag ())
+    {
+      const json::object &old_bag = *input_obj.get_property_bag ();
+      sarif_property_bag &new_bag = output_obj.get_or_create_properties ();
+      for (size_t i = 0; i < old_bag.get_num_keys (); ++i)
+	{
+	  const char *key = old_bag.get_key (i);
+	  json::value *val = old_bag.get (key);
+	  new_bag.set (key, val->clone ());
+	}
+    }
+}
+
+std::unique_ptr<sarif_graph>
+make_sarif_graph (const digraphs::digraph &g,
+		  sarif_builder *builder,
+		  sarif_location_manager *sarif_location_mgr)
+{
+  auto result = std::make_unique<sarif_graph> ();
+
+  // 3.39.2 description property
+  if (const char *desc = g.get_description ())
+    if (builder)
+      result->set<sarif_message> ("description",
+				  builder->make_message_object (desc));
+
+  copy_any_property_bag (g, *result);
+
+  // 3.39.3 nodes property
+  auto nodes_arr = std::make_unique<json::array> ();
+  const int num_nodes = g.get_num_nodes ();
+  for (int i = 0; i < num_nodes; ++i)
+    nodes_arr->append (make_sarif_node (g.get_node (i),
+					builder,
+					sarif_location_mgr));
+  result->set ("nodes", std::move (nodes_arr));
+
+  // 3.39.4 edges property
+  auto edges_arr = std::make_unique<json::array> ();
+  const int num_edges = g.get_num_edges ();
+  for (int i = 0; i < num_edges; ++i)
+    edges_arr->append (make_sarif_edge (g.get_edge (i), builder));
+  result->set ("edges", std::move (edges_arr));
+
+  return result;
+}
+
+std::unique_ptr<sarif_node>
+make_sarif_node (const digraphs::node &n,
+		 sarif_builder *builder,
+		 sarif_location_manager *sarif_location_mgr)
+{
+  auto result = std::make_unique<sarif_node> ();
+
+  // 3.40.2 id property
+  result->set_string ("id", n.get_id ().c_str ());
+
+  copy_any_property_bag (n, *result);
+
+  // 3.40.3 label property
+  if (const char *label = n.get_label ())
+    if (builder)
+      result->set<sarif_message> ("label",
+				  builder->make_message_object (label));
+
+  // 3.40.4 location property
+  if (n.get_logical_loc ()
+      || n.get_physical_loc () != UNKNOWN_LOCATION)
+    if (builder)
+      {
+	rich_location rich_loc
+	  (line_table, n.get_physical_loc ());
+	auto loc_obj
+	  = builder->make_location_object
+	      (sarif_location_mgr,
+	       rich_loc,
+	       n.get_logical_loc (),
+	       diagnostic_artifact_role::scanned_file);
+	result->set<sarif_location> ("location",
+				     std::move (loc_obj));
+    }
+
+  // 3.40.5 children property
+  if (const int num_children = n.get_num_children ())
+    {
+      auto children_arr = std::make_unique<json::array> ();
+      for (int i = 0; i < num_children; ++i)
+	children_arr->append (make_sarif_node (n.get_child (i),
+					       builder,
+					       sarif_location_mgr));
+      result->set ("children", std::move (children_arr));
+    }
+
+  return result;
+}
+
+std::unique_ptr<sarif_edge>
+make_sarif_edge (const digraphs::edge &e,
+		 sarif_builder *builder)
+{
+  auto result = std::make_unique<sarif_edge> ();
+
+  // 3.41.2 id property
+  result->set_string ("id", e.get_id ().c_str ());
+
+  copy_any_property_bag (e, *result);
+
+  // 3.41.3 label property
+  if (const char *label = e.get_label ())
+    if (builder)
+      result->set<sarif_message> ("label",
+				  builder->make_message_object (label));
+
+  // 3.41.4 sourceNodeId property
+  result->set_string ("sourceNodeId", e.get_src_node ().get_id ().c_str ());
+
+  // 3.41.5 targetNodeId property
+  result->set_string ("targetNodeId", e.get_dst_node ().get_id ().c_str ());
+
+  return result;
+}
+
+void
+sarif_property_bag::set_graph (const char *property_name,
+			       sarif_builder &builder,
+			       sarif_location_manager *sarif_location_mgr,
+			       const digraphs::digraph &g)
+{
+  set<sarif_graph> (property_name,
+		    make_sarif_graph (g, &builder, sarif_location_mgr));
+}
+
 /* Ensure that m_cached_logical_locs has a "logicalLocation" object
    (SARIF v2.1.0 section 3.33) for K, and return its index within the
    array.  */
 
 int
 sarif_builder::
-ensure_sarif_logical_location_for (logical_location k)
+ensure_sarif_logical_location_for (logical_locations::key k)
 {
   gcc_assert (m_logical_loc_mgr);
 
@@ -2823,7 +3000,7 @@ ensure_sarif_logical_location_for (logical_location k)
     sarif_logical_loc->set_string ("decoratedName", internal_name);
 
   /* "kind" property (SARIF v2.1.0 section 3.33.7).  */
-  enum logical_location_kind kind = m_logical_loc_mgr->get_kind (k);
+  enum logical_locations::kind kind = m_logical_loc_mgr->get_kind (k);
   if (const char *sarif_kind_str = maybe_get_sarif_kind (kind))
     sarif_logical_loc->set_string ("kind", sarif_kind_str);
 
@@ -2849,7 +3026,7 @@ ensure_sarif_logical_location_for (logical_location k)
 
 std::unique_ptr<sarif_logical_location>
 sarif_builder::
-make_minimal_sarif_logical_location (logical_location logical_loc)
+make_minimal_sarif_logical_location (logical_locations::key logical_loc)
 {
   gcc_assert (m_logical_loc_mgr);
 
@@ -2874,7 +3051,7 @@ make_minimal_sarif_logical_location (logical_location logical_loc)
 
 label_text
 make_sarif_url_for_event (const sarif_code_flow *code_flow,
-			  diagnostic_event_id_t event_id)
+			  paths::event_id_t event_id)
 {
   gcc_assert (event_id.known_p ());
 
@@ -2908,7 +3085,7 @@ make_sarif_url_for_event (const sarif_code_flow *code_flow,
 std::unique_ptr<sarif_code_flow>
 sarif_builder::make_code_flow_object (sarif_result &result,
 				      unsigned idx_within_parent,
-				      const diagnostic_path &path)
+				      const paths::path &path)
 {
   auto code_flow_obj
     = std::make_unique <sarif_code_flow> (result, idx_within_parent);
@@ -2920,8 +3097,8 @@ sarif_builder::make_code_flow_object (sarif_result &result,
      SARIF file.  */
   for (unsigned i = 0; i < path.num_events (); i++)
     {
-      const diagnostic_event &event = path.get_event (i);
-      const diagnostic_thread_id_t thread_id = event.get_thread_id ();
+      const paths::event &event = path.get_event (i);
+      const paths::thread_id_t thread_id = event.get_thread_id ();
 
       sarif_thread_flow &thread_flow_obj
 	= code_flow_obj->get_or_append_thread_flow (path.get_thread (thread_id),
@@ -2933,7 +3110,7 @@ sarif_builder::make_code_flow_object (sarif_result &result,
   m_current_code_flow = code_flow_obj.get ();
   for (unsigned i = 0; i < path.num_events (); i++)
     {
-      const diagnostic_event &event = path.get_event (i);
+      const paths::event &event = path.get_event (i);
       sarif_thread_flow_location &thread_flow_loc_obj
 	= code_flow_obj->get_thread_flow_loc_obj (i);
       populate_thread_flow_location_object (result,
@@ -2953,24 +3130,28 @@ void
 sarif_builder::
 populate_thread_flow_location_object (sarif_result &result,
 				      sarif_thread_flow_location &tfl_obj,
-				      const diagnostic_event &ev,
+				      const paths::event &ev,
 				      int event_execution_idx)
 {
-  /* Give diagnostic_event subclasses a chance to add custom properties
+  /* Give paths::event subclasses a chance to add custom properties
      via a property bag.  */
   ev.maybe_add_sarif_properties (*this, tfl_obj);
 
-  if (get_opts ().m_xml_state)
-    if (auto xml_state = ev.maybe_make_xml_state (true))
+  if (get_opts ().m_state_graph)
+    if (auto state_graph = ev.maybe_make_diagnostic_state_graph (true))
       {
 	sarif_property_bag &props = tfl_obj.get_or_create_properties ();
 
-	pretty_printer pp;
-	xml_state->write_as_xml (&pp, 0, true);
-
-#define PROPERTY_PREFIX "gcc/diagnostic_event/"
-	props.set_string (PROPERTY_PREFIX "xml_state",
-			  pp_formatted_text (&pp));
+#define PROPERTY_PREFIX "gcc/diagnostics/paths/event/"
+	props.set_graph (PROPERTY_PREFIX "state_graph",
+			 *this,
+			 /* Use RESULT for any related locations in the graph's
+			    nodes.
+			    It's not clear if this is correct; see:
+			    https://github.com/oasis-tcs/sarif-spec/issues/712
+			 */
+			 &result,
+			 *state_graph);
 #undef PROPERTY_PREFIX
       }
 
@@ -2980,7 +3161,7 @@ populate_thread_flow_location_object (sarif_result &result,
      make_location_object (result, ev, diagnostic_artifact_role::traced_file));
 
   /* "kinds" property (SARIF v2.1.0 section 3.38.8).  */
-  diagnostic_event::meaning m = ev.get_meaning ();
+  paths::event::meaning m = ev.get_meaning ();
   if (auto kinds_arr = maybe_make_kinds_array (m))
     tfl_obj.set<json::array> ("kinds", std::move (kinds_arr));
 
@@ -3003,22 +3184,25 @@ populate_thread_flow_location_object (sarif_result &result,
    Otherwise, return nullptr.  */
 
 std::unique_ptr<json::array>
-sarif_builder::maybe_make_kinds_array (diagnostic_event::meaning m) const
+sarif_builder::
+maybe_make_kinds_array (paths::event::meaning m) const
 {
-  if (m.m_verb == diagnostic_event::verb::unknown
-      && m.m_noun == diagnostic_event::noun::unknown
-      && m.m_property == diagnostic_event::property::unknown)
+  using namespace paths;
+
+  if (m.m_verb == event::verb::unknown
+      && m.m_noun == event::noun::unknown
+      && m.m_property == event::property::unknown)
     return nullptr;
 
   auto kinds_arr = std::make_unique<json::array> ();
   if (const char *verb_str
-	= diagnostic_event::meaning::maybe_get_verb_str (m.m_verb))
+	= event::meaning::maybe_get_verb_str (m.m_verb))
     kinds_arr->append_string (verb_str);
   if (const char *noun_str
-	= diagnostic_event::meaning::maybe_get_noun_str (m.m_noun))
+	= event::meaning::maybe_get_noun_str (m.m_noun))
     kinds_arr->append_string (noun_str);
   if (const char *property_str
-	= diagnostic_event::meaning::maybe_get_property_str (m.m_property))
+	= event::meaning::maybe_get_property_str (m.m_property))
     kinds_arr->append_string (property_str);
   return kinds_arr;
 }
@@ -3069,18 +3253,18 @@ sarif_builder::make_message_object (const char *msg) const
   return message_obj;
 }
 
-/* Make a "message" object (SARIF v2.1.0 section 3.11) for DIAGRAM.
+/* Make a "message" object (SARIF v2.1.0 section 3.11) for D.
    We emit the diagram as a code block within the Markdown part
    of the message.  */
 
 std::unique_ptr<sarif_message>
-sarif_builder::make_message_object_for_diagram (const diagnostic_diagram &diagram)
+sarif_builder::make_message_object_for_diagram (const diagram &d)
 {
   auto message_obj = std::make_unique<sarif_message> ();
 
   /* "text" property (SARIF v2.1.0 section 3.11.8).  */
   set_string_property_escaping_braces (*message_obj,
-				       "text", diagram.get_alt_text ());
+				       "text", d.get_alt_text ());
 
   pretty_printer *const pp = m_printer;
   char *saved_prefix = pp_take_prefix (pp);
@@ -3089,7 +3273,7 @@ sarif_builder::make_message_object_for_diagram (const diagnostic_diagram &diagra
   /* "To produce a code block in Markdown, simply indent every line of
      the block by at least 4 spaces or 1 tab."
      Here we use 4 spaces.  */
-  diagram.get_canvas ().print_to_pp (pp, "    ");
+  d.get_canvas ().print_to_pp (pp, "    ");
   pp_set_prefix (pp, saved_prefix);
 
   /* "markdown" property (SARIF v2.1.0 section 3.11.9).  */
@@ -3234,6 +3418,11 @@ make_run_object (std::unique_ptr<sarif_invocation> invocation_obj,
 				 std::move (m_cached_logical_locs));
     }
 
+  // "graphs" property (SARIF v2.1.0 3.14.20)
+  if (m_run_graphs->size () > 0)
+    run_obj->set<json::array> ("graphs",
+			       std::move (m_run_graphs));
+
   return run_obj;
 }
 
@@ -3257,7 +3446,8 @@ sarif_builder::make_tool_object ()
 	class my_plugin_visitor : public client_version_info :: plugin_visitor
 	{
 	public:
-	  void on_plugin (const diagnostic_client_plugin_info &p) final override
+	  void
+	  on_plugin (const client_plugin_info &p) final override
 	  {
 	    /* Create a "toolComponent" object (SARIF v2.1.0 section 3.19)
 	       for the plugin.  */
@@ -3608,12 +3798,12 @@ sarif_builder::make_artifact_content_object (const char *text) const
   return content_obj;
 }
 
-/* class diagnostic_sarif_format_buffer : public diagnostic_per_format_buffer.  */
+/* class sarif_sink_buffer : public per_sink_buffer.  */
 
 void
-diagnostic_sarif_format_buffer::dump (FILE *out, int indent) const
+sarif_sink_buffer::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_sarif_format_buffer:\n", indent, "");
+  fprintf (out, "%*ssarif_sink_buffer:\n", indent, "");
   int idx = 0;
   for (auto &result : m_results)
     {
@@ -3625,29 +3815,29 @@ diagnostic_sarif_format_buffer::dump (FILE *out, int indent) const
 }
 
 bool
-diagnostic_sarif_format_buffer::empty_p () const
+sarif_sink_buffer::empty_p () const
 {
   return m_results.empty ();
 }
 
 void
-diagnostic_sarif_format_buffer::move_to (diagnostic_per_format_buffer &base)
+sarif_sink_buffer::move_to (per_sink_buffer &base)
 {
-  diagnostic_sarif_format_buffer &dest
-    = static_cast<diagnostic_sarif_format_buffer &> (base);
+  sarif_sink_buffer &dest
+    = static_cast<sarif_sink_buffer &> (base);
   for (auto &&result : m_results)
     dest.m_results.push_back (std::move (result));
   m_results.clear ();
 }
 
 void
-diagnostic_sarif_format_buffer::clear ()
+sarif_sink_buffer::clear ()
 {
   m_results.clear ();
 }
 
 void
-diagnostic_sarif_format_buffer::flush ()
+sarif_sink_buffer::flush ()
 {
   for (auto &&result : m_results)
     {
@@ -3657,10 +3847,10 @@ diagnostic_sarif_format_buffer::flush ()
   m_results.clear ();
 }
 
-class sarif_output_format : public diagnostic_output_format
+class sarif_sink : public sink
 {
 public:
-  ~sarif_output_format ()
+  ~sarif_sink ()
   {
     /* Any sarifResult objects should have been handled by now.
        If not, then something's gone wrong with diagnostic
@@ -3672,8 +3862,8 @@ public:
 
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*ssarif_output_format\n", indent, "");
-    diagnostic_output_format::dump (out, indent);
+    fprintf (out, "%*ssarif_sink\n", indent, "");
+    sink::dump (out, indent);
   }
 
   void
@@ -3682,15 +3872,15 @@ public:
     m_builder.set_main_input_filename (name);
   }
 
-  std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () final override
+  std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () final override
   {
-    return std::make_unique<diagnostic_sarif_format_buffer> (m_builder);
+    return std::make_unique<sarif_sink_buffer> (m_builder);
   }
-  void set_buffer (diagnostic_per_format_buffer *base_buffer) final override
+  void set_buffer (per_sink_buffer *base_buffer) final override
   {
-    diagnostic_sarif_format_buffer *buffer
-      = static_cast<diagnostic_sarif_format_buffer *> (base_buffer);
+    sarif_sink_buffer *buffer
+      = static_cast<sarif_sink_buffer *> (base_buffer);
     m_buffer = buffer;
   }
 
@@ -3726,53 +3916,60 @@ public:
   }
   void
   on_report_diagnostic (const diagnostic_info &diagnostic,
-			diagnostic_t orig_diag_kind) final override
+			enum kind orig_diag_kind) final override
   {
     m_builder.on_report_diagnostic (diagnostic, orig_diag_kind, m_buffer);
   }
-  void on_diagram (const diagnostic_diagram &diagram) final override
+  void on_diagram (const diagram &d) final override
   {
-    m_builder.emit_diagram (diagram);
+    m_builder.emit_diagram (d);
   }
   void after_diagnostic (const diagnostic_info &) final override
   {
     /* No-op.  */
   }
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+    final override
+  {
+    m_builder.report_global_digraph (ldg);
+  }
+
   sarif_builder &get_builder () { return m_builder; }
 
   size_t num_results () const { return m_builder.num_results (); }
   sarif_result &get_result (size_t idx) { return m_builder.get_result (idx); }
 
 protected:
-  sarif_output_format (diagnostic_context &context,
-		       const line_maps *line_maps,
-		       std::unique_ptr<sarif_serialization_format> serialization_format,
-		       const sarif_generation_options &sarif_gen_opts)
-  : diagnostic_output_format (context),
-    m_builder (context, *get_printer (), line_maps,
+  sarif_sink (context &dc,
+	      const line_maps *line_maps,
+	      std::unique_ptr<sarif_serialization_format> serialization_format,
+	      const sarif_generation_options &sarif_gen_opts)
+  : sink (dc),
+    m_builder (dc, *get_printer (), line_maps,
 	       std::move (serialization_format), sarif_gen_opts),
     m_buffer (nullptr)
   {}
 
   sarif_builder m_builder;
-  diagnostic_sarif_format_buffer *m_buffer;
+  sarif_sink_buffer *m_buffer;
 };
 
-class sarif_stream_output_format : public sarif_output_format
+class sarif_stream_sink : public sarif_sink
 {
 public:
-  sarif_stream_output_format (diagnostic_context &context,
-			      const line_maps *line_maps,
-			      std::unique_ptr<sarif_serialization_format> serialization_format,
-			      const sarif_generation_options &sarif_gen_opts,
-			      FILE *stream)
-  : sarif_output_format (context, line_maps,
-			 std::move (serialization_format), sarif_gen_opts),
+  sarif_stream_sink (context &dc,
+		     const line_maps *line_maps,
+		     std::unique_ptr<sarif_serialization_format> serialization_format,
+		     const sarif_generation_options &sarif_gen_opts,
+		     FILE *stream)
+  : sarif_sink (dc, line_maps,
+		std::move (serialization_format), sarif_gen_opts),
     m_stream (stream)
   {
   }
-  ~sarif_stream_output_format ()
+  ~sarif_stream_sink ()
   {
     m_builder.flush_to_file (m_stream);
   }
@@ -3784,31 +3981,32 @@ private:
   FILE *m_stream;
 };
 
-class sarif_file_output_format : public sarif_output_format
+class sarif_file_sink : public sarif_sink
 {
 public:
-  sarif_file_output_format (diagnostic_context &context,
-			    const line_maps *line_maps,
-			    std::unique_ptr<sarif_serialization_format> serialization_format,
-			    const sarif_generation_options &sarif_gen_opts,
-			    diagnostic_output_file output_file)
-  : sarif_output_format (context, line_maps,
-			 std::move (serialization_format), sarif_gen_opts),
-    m_output_file (std::move (output_file))
+  sarif_file_sink (context &dc,
+		   const line_maps *line_maps,
+		   std::unique_ptr<sarif_serialization_format> serialization_format,
+		   const sarif_generation_options &sarif_gen_opts,
+		   output_file output_file_)
+  : sarif_sink (dc, line_maps,
+		std::move (serialization_format),
+		sarif_gen_opts),
+    m_output_file (std::move (output_file_))
   {
     gcc_assert (m_output_file.get_open_file ());
     gcc_assert (m_output_file.get_filename ());
   }
-  ~sarif_file_output_format ()
+  ~sarif_file_sink ()
   {
     m_builder.flush_to_file (m_output_file.get_open_file ());
   }
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*ssarif_file_output_format: %s\n",
+    fprintf (out, "%*ssarif_file_sink: %s\n",
 	     indent, "",
 	     m_output_file.get_filename ());
-    diagnostic_output_format::dump (out, indent);
+    sink::dump (out, indent);
   }
   bool machine_readable_stderr_p () const final override
   {
@@ -3816,7 +4014,7 @@ public:
   }
 
 private:
-  diagnostic_output_file m_output_file;
+  output_file m_output_file;
 };
 
 /* Print the start of an embedded link to PP, as per 3.11.6.  */
@@ -3939,52 +4137,52 @@ sarif_builder::sarif_token_printer::print_tokens (pretty_printer *pp,
    to a file).
    Return a reference to *FMT.  */
 
-static diagnostic_output_format &
-diagnostic_output_format_init_sarif (diagnostic_context &context,
-				     std::unique_ptr<sarif_output_format> fmt)
+static sink &
+init_sarif_sink (context &dc,
+		 std::unique_ptr<sarif_sink> fmt)
 {
   gcc_assert (fmt);
-  diagnostic_output_format &out = *fmt;
+  sink &out = *fmt;
 
   fmt->update_printer ();
 
-  context.set_output_format (std::move (fmt));
+  dc.set_sink (std::move (fmt));
 
   return out;
 }
 
-/* Populate CONTEXT in preparation for SARIF output to stderr.
+/* Populate DC in preparation for SARIF output to stderr.
    Return a reference to the new sink.  */
 
-diagnostic_output_format &
-diagnostic_output_format_init_sarif_stderr (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted)
+sink &
+init_sarif_stderr (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted)
 {
   gcc_assert (line_maps);
   const sarif_generation_options sarif_gen_opts;
   auto serialization
     = std::make_unique<sarif_serialization_format_json> (formatted);
-  return diagnostic_output_format_init_sarif
-    (context,
-     std::make_unique<sarif_stream_output_format> (context,
-						   line_maps,
-						   std::move (serialization),
-						   sarif_gen_opts,
-						   stderr));
+  return init_sarif_sink
+    (dc,
+     std::make_unique<sarif_stream_sink> (dc,
+					  line_maps,
+					  std::move (serialization),
+					  sarif_gen_opts,
+					  stderr));
 }
 
 /* Attempt to open "BASE_FILE_NAME""EXTENSION" for writing.
-   Return a non-null diagnostic_output_file,
-   or return a null diagnostic_output_file and complain to CONTEXT
+   Return a non-null output_file,
+   or return a null output_file and complain to DC
    using LINE_MAPS.  */
 
-diagnostic_output_file
-diagnostic_output_file::try_to_open (diagnostic_context &context,
-				     line_maps *line_maps,
-				     const char *base_file_name,
-				     const char *extension,
-				     bool is_binary)
+output_file
+output_file::try_to_open (context &dc,
+			  line_maps *line_maps,
+			  const char *base_file_name,
+			  const char *extension,
+			  bool is_binary)
 {
   gcc_assert (extension);
   gcc_assert (extension[0] == '.');
@@ -3992,10 +4190,10 @@ diagnostic_output_file::try_to_open (diagnostic_context &context,
   if (!base_file_name)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to determine filename for SARIF output");
-      return diagnostic_output_file ();
+      return output_file ();
     }
 
   label_text filename = label_text::take (concat (base_file_name,
@@ -4005,25 +4203,25 @@ diagnostic_output_file::try_to_open (diagnostic_context &context,
   if (!outf)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to open %qs for diagnostic output: %m",
 	 filename.get ());
-      return diagnostic_output_file ();
+      return output_file ();
     }
-  return diagnostic_output_file (outf, true, std::move (filename));
+  return output_file (outf, true, std::move (filename));
 }
 
 /* Attempt to open BASE_FILE_NAME.sarif for writing JSON.
-   Return a non-null diagnostic_output_file,
-   or return a null diagnostic_output_file and complain to CONTEXT
+   Return a non-null output_file,
+   or return a null output_file and complain to DC
    using LINE_MAPS.  */
 
-diagnostic_output_file
-diagnostic_output_format_open_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  const char *base_file_name,
-					  enum sarif_serialization_kind serialization_kind)
+output_file
+open_sarif_output_file (context &dc,
+			line_maps *line_maps,
+			const char *base_file_name,
+			enum sarif_serialization_kind serialization_kind)
 {
   const char *suffix;
   bool is_binary;
@@ -4037,78 +4235,78 @@ diagnostic_output_format_open_sarif_file (diagnostic_context &context,
       break;
     }
 
-  return diagnostic_output_file::try_to_open (context,
-					      line_maps,
-					      base_file_name,
-					      suffix,
-					      is_binary);
+  return output_file::try_to_open (dc,
+				   line_maps,
+				   base_file_name,
+				   suffix,
+				   is_binary);
 }
 
-/* Populate CONTEXT in preparation for SARIF output to a file named
+/* Populate DC in preparation for SARIF output to a file named
    BASE_FILE_NAME.sarif.
    Return a reference to the new sink.  */
 
-diagnostic_output_format &
-diagnostic_output_format_init_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  bool formatted,
-					  const char *base_file_name)
+sink &
+init_sarif_file (context &dc,
+		 line_maps *line_maps,
+		 bool formatted,
+		 const char *base_file_name)
 {
   gcc_assert (line_maps);
 
-  diagnostic_output_file output_file
-    = diagnostic_output_format_open_sarif_file (context,
-						line_maps,
-						base_file_name,
-						sarif_serialization_kind::json);
+  output_file output_file_
+    = open_sarif_output_file (dc,
+			      line_maps,
+			      base_file_name,
+			      sarif_serialization_kind::json);
   auto serialization
     = std::make_unique<sarif_serialization_format_json> (formatted);
 
   const sarif_generation_options sarif_gen_opts;
-  return diagnostic_output_format_init_sarif
-    (context,
-     std::make_unique<sarif_file_output_format> (context,
-						 line_maps,
-						 std::move (serialization),
-						 sarif_gen_opts,
-						 std::move (output_file)));
+  return init_sarif_sink
+    (dc,
+     std::make_unique<sarif_file_sink> (dc,
+					line_maps,
+					std::move (serialization),
+					sarif_gen_opts,
+					std::move (output_file_)));
 }
 
-/* Populate CONTEXT in preparation for SARIF output to STREAM.
+/* Populate DC in preparation for SARIF output to STREAM.
    Return a reference to the new sink.  */
 
-diagnostic_output_format &
-diagnostic_output_format_init_sarif_stream (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted,
-					    FILE *stream)
+sink &
+init_sarif_stream (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted,
+		   FILE *stream)
 {
   gcc_assert (line_maps);
   const sarif_generation_options sarif_gen_opts;
   auto serialization
     = std::make_unique<sarif_serialization_format_json> (formatted);
-  return diagnostic_output_format_init_sarif
-    (context,
-     std::make_unique<sarif_stream_output_format> (context,
-						   line_maps,
-						   std::move (serialization),
-						   sarif_gen_opts,
-						   stream));
+  return init_sarif_sink
+    (dc,
+     std::make_unique<sarif_stream_sink> (dc,
+					  line_maps,
+					  std::move (serialization),
+					  sarif_gen_opts,
+					  stream));
 }
 
-std::unique_ptr<diagnostic_output_format>
-make_sarif_sink (diagnostic_context &context,
+std::unique_ptr<sink>
+make_sarif_sink (context &dc,
 		 const line_maps &line_maps,
 		 std::unique_ptr<sarif_serialization_format> serialization,
 		 const sarif_generation_options &sarif_gen_opts,
-		 diagnostic_output_file output_file)
+		 output_file output_file_)
 {
   auto sink
-    = std::make_unique<sarif_file_output_format> (context,
-						  &line_maps,
-						  std::move (serialization),
-						  sarif_gen_opts,
-						  std::move (output_file));
+    = std::make_unique<sarif_file_sink> (dc,
+					 &line_maps,
+					 std::move (serialization),
+					 sarif_gen_opts,
+					 std::move (output_file_));
   sink->update_printer ();
   return sink;
 }
@@ -4117,7 +4315,7 @@ make_sarif_sink (diagnostic_context &context,
 
 sarif_generation_options::sarif_generation_options ()
 : m_version (sarif_version::v2_1_0),
-  m_xml_state (false)
+  m_state_graph (false)
 {
 }
 
@@ -4125,6 +4323,9 @@ sarif_generation_options::sarif_generation_options ()
 
 namespace selftest {
 
+using auto_fix_quotes = ::selftest::auto_fix_quotes;
+using line_table_case = ::selftest::line_table_case;
+
 static void
 test_sarif_array_of_unique_1 ()
 {
@@ -4195,45 +4396,44 @@ test_sarif_array_of_unique_2 ()
   ASSERT_JSON_INT_PROPERTY_EQ (arr[1], "index", 1);
 }
 
-/* A subclass of sarif_output_format for writing selftests.
+/* A subclass of sarif_sink for writing selftests.
    The JSON output is cached internally, rather than written
    out to a file.  */
 
-class test_sarif_diagnostic_context : public test_diagnostic_context
+class test_sarif_diagnostic_context : public test_context
 {
 public:
   test_sarif_diagnostic_context (const char *main_input_filename,
 				 const sarif_generation_options &sarif_gen_opts)
   {
-    auto format = std::make_unique<buffered_output_format> (*this,
-							    line_table,
-							    true,
-							    sarif_gen_opts);
-    m_format = format.get (); // borrowed
-    diagnostic_output_format_init_sarif (*this, std::move (format));
-    m_format->set_main_input_filename (main_input_filename);
+    auto sink_ = std::make_unique<buffered_sink> (*this,
+						  line_table,
+						  true,
+						  sarif_gen_opts);
+    m_sink = sink_.get (); // borrowed
+    init_sarif_sink (*this, std::move (sink_));
+    m_sink->set_main_input_filename (main_input_filename);
   }
 
   std::unique_ptr<sarif_log> flush_to_object ()
   {
-    return m_format->flush_to_object ();
+    return m_sink->flush_to_object ();
   }
 
-  size_t num_results () const { return m_format->num_results (); }
-  sarif_result &get_result (size_t idx) { return m_format->get_result (idx); }
+  size_t num_results () const { return m_sink->num_results (); }
+  sarif_result &get_result (size_t idx) { return m_sink->get_result (idx); }
 
 private:
-  class buffered_output_format : public sarif_output_format
+  class buffered_sink : public sarif_sink
   {
   public:
-    buffered_output_format (diagnostic_context &context,
-			    const line_maps *line_maps,
-			    bool formatted,
-			    const sarif_generation_options &sarif_gen_opts)
-    : sarif_output_format (context, line_maps,
-			   std::make_unique<sarif_serialization_format_json>
-			     (formatted),
-			   sarif_gen_opts)
+    buffered_sink (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted,
+		   const sarif_generation_options &sarif_gen_opts)
+    : sarif_sink (dc, line_maps,
+		  std::make_unique<sarif_serialization_format_json> (formatted),
+		  sarif_gen_opts)
     {
     }
     bool machine_readable_stderr_p () const final override
@@ -4246,7 +4446,7 @@ private:
     }
   };
 
-  buffered_output_format *m_format; // borrowed
+  buffered_sink *m_sink; // borrowed
 };
 
 /* Test making a sarif_location for a complex rich_location
@@ -4254,16 +4454,16 @@ private:
 
 static void
 test_make_location_object (const sarif_generation_options &sarif_gen_opts,
-			   const line_table_case &case_)
+			   const ::selftest::line_table_case &case_)
 {
-  diagnostic_show_locus_fixture_one_liner_utf8 f (case_);
+  source_printing_fixture_one_liner_utf8 f (case_);
   location_t line_end = linemap_position_for_column (line_table, 31);
 
   /* Don't attempt to run the tests if column data might be unavailable.  */
   if (line_end > LINE_MAP_MAX_LOCATION_WITH_COLS)
     return;
 
-  test_diagnostic_context dc;
+  test_context dc;
   pretty_printer pp;
   sarif_builder builder
     (dc, pp, line_table,
@@ -4299,7 +4499,7 @@ test_make_location_object (const sarif_generation_options &sarif_gen_opts,
 
   std::unique_ptr<sarif_location> location_obj
     = builder.make_location_object
-	(result, richloc, logical_location (),
+	(&result, richloc, logical_locations::key (),
 	 diagnostic_artifact_role::analysis_target);
   ASSERT_NE (location_obj, nullptr);
 
@@ -4376,7 +4576,7 @@ test_make_location_object (const sarif_generation_options &sarif_gen_opts,
 }
 
 /* Test of reporting a diagnostic at UNKNOWN_LOCATION to a
-   diagnostic_context and examining the generated sarif_log.
+   diagnostics::context and examining the generated sarif_log.
    Verify various basic properties. */
 
 static void
@@ -4385,7 +4585,7 @@ test_simple_log (const sarif_generation_options &sarif_gen_opts)
   test_sarif_diagnostic_context dc ("MAIN_INPUT_FILENAME", sarif_gen_opts);
 
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  dc.report (DK_ERROR, richloc, nullptr, 0, "this is a test: %i", 42);
+  dc.report (kind::error, richloc, nullptr, 0, "this is a test: %i", 42);
 
   auto log_ptr = dc.flush_to_object ();
 
@@ -4507,7 +4707,7 @@ test_simple_log_2 (const sarif_generation_options &sarif_gen_opts,
     /* 000000000111111
        123456789012345.  */
     = "unsinged int i;\n";
-  diagnostic_show_locus_fixture f (case_, content);
+  source_printing_fixture f (case_, content);
   location_t line_end = linemap_position_for_column (line_table, 31);
 
   /* Don't attempt to run the tests if column data might be unavailable.  */
@@ -4522,7 +4722,7 @@ test_simple_log_2 (const sarif_generation_options &sarif_gen_opts,
 		     linemap_position_for_column (line_table, 8));
 
   rich_location richloc (line_table, typo_loc);
-  dc.report (DK_ERROR, richloc, nullptr, 0,
+  dc.report (kind::error, richloc, nullptr, 0,
 	     "did you misspell %qs again?",
 	     "unsigned");
 
@@ -4650,7 +4850,7 @@ test_message_with_embedded_link (const sarif_generation_options &sarif_gen_opts)
   {
     test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
     rich_location richloc (line_table, UNKNOWN_LOCATION);
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "before %{text%} after",
 	       "http://example.com");
     std::unique_ptr<sarif_log> log = dc.flush_to_object ();
@@ -4673,7 +4873,7 @@ test_message_with_embedded_link (const sarif_generation_options &sarif_gen_opts)
 #  pragma GCC diagnostic push
 #  pragma GCC diagnostic ignored "-Wformat-diag"
 #endif
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "Prohibited term used in %{para[0]\\spans[2]%}.",
 	       "1");
 #if __GNUC__ >= 10
@@ -4707,7 +4907,7 @@ test_message_with_embedded_link (const sarif_generation_options &sarif_gen_opts)
     test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
     dc.push_owned_urlifier (std::make_unique<test_urlifier> ());
     rich_location richloc (line_table, UNKNOWN_LOCATION);
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "foo %<-foption%> %<unrecognized%> bar");
     std::unique_ptr<sarif_log> log = dc.flush_to_object ();
 
@@ -4728,7 +4928,7 @@ test_message_with_braces (const sarif_generation_options &sarif_gen_opts)
   {
     test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
     rich_location richloc (line_table, UNKNOWN_LOCATION);
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "open brace: %qs close brace: %qs",
 	       "{", "}");
     std::unique_ptr<sarif_log> log = dc.flush_to_object ();
@@ -4745,26 +4945,26 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
 {
   test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
 
-  diagnostic_buffer buf_a (dc);
-  diagnostic_buffer buf_b (dc);
+  diagnostics::buffer buf_a (dc);
+  diagnostics::buffer buf_b (dc);
 
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
-  ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 0);
-  ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 0);
-  ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+  ASSERT_EQ (dc.diagnostic_count (kind::error), 0);
+  ASSERT_EQ (buf_a.diagnostic_count (kind::error), 0);
+  ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
   ASSERT_EQ (dc.num_results (), 0);
   ASSERT_TRUE (buf_a.empty_p ());
   ASSERT_TRUE (buf_b.empty_p ());
 
   /* Unbuffered diagnostic.  */
   {
-    dc.report (DK_ERROR, rich_loc, nullptr, 0,
+    dc.report (kind::error, rich_loc, nullptr, 0,
 	       "message 1");
 
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 0);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 0);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 1);
     sarif_result &result_obj = dc.get_result (0);
     auto message_obj = get_message_from_result (result_obj);
@@ -4777,11 +4977,11 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Buffer diagnostic into buffer A.  */
   {
     dc.set_diagnostic_buffer (&buf_a);
-    dc.report (DK_ERROR, rich_loc, nullptr, 0,
+    dc.report (kind::error, rich_loc, nullptr, 0,
 	       "message in buffer a");
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 1);
     ASSERT_FALSE (buf_a.empty_p ());
     ASSERT_TRUE (buf_b.empty_p ());
@@ -4790,11 +4990,11 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Buffer diagnostic into buffer B.  */
   {
     dc.set_diagnostic_buffer (&buf_b);
-    dc.report (DK_ERROR, rich_loc, nullptr, 0,
+    dc.report (kind::error, rich_loc, nullptr, 0,
 	       "message in buffer b");
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 1);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 1);
     ASSERT_EQ (dc.num_results (), 1);
     ASSERT_FALSE (buf_a.empty_p ());
     ASSERT_FALSE (buf_b.empty_p ());
@@ -4803,9 +5003,9 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Flush buffer B to dc.  */
   {
     dc.flush_diagnostic_buffer (buf_b);
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 2);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 2);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 2);
     sarif_result &result_1_obj = dc.get_result (1);
     auto message_1_obj = get_message_from_result (result_1_obj);
@@ -4818,9 +5018,9 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Clear buffer A.  */
   {
     dc.clear_diagnostic_buffer (buf_a);
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 2);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 0);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 2);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 0);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 2);
     ASSERT_TRUE (buf_a.empty_p ());
     ASSERT_TRUE (buf_b.empty_p ());
@@ -4857,7 +5057,7 @@ run_line_table_case_tests_per_version (const line_table_case &case_)
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_format_sarif_cc_tests ()
+sarif_sink_cc_tests ()
 {
   test_sarif_array_of_unique_1 ();
   test_sarif_array_of_unique_2 ();
@@ -4871,6 +5071,8 @@ diagnostic_format_sarif_cc_tests ()
   for_each_line_table_case (run_line_table_case_tests_per_version);
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
 
 #endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-format-sarif.h b/gcc/diagnostics/sarif-sink.h
index c3ae330..9f8a73f 100644
--- a/gcc/diagnostic-format-sarif.h
+++ b/gcc/diagnostics/sarif-sink.h
@@ -18,13 +18,21 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_SARIF_H
-#define GCC_DIAGNOSTIC_FORMAT_SARIF_H
+#ifndef GCC_DIAGNOSTICS_SARIF_SINK_H
+#define GCC_DIAGNOSTICS_SARIF_SINK_H
 
 #include "json.h"
-#include "diagnostic-format.h"
-#include "diagnostic-output-file.h"
-#include "logical-location.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/output-file.h"
+#include "diagnostics/logical-locations.h"
+
+namespace diagnostics {
+
+namespace digraphs {
+  class digraph;
+  class node;
+  class edge;
+}
 
 /* Enum for choosing what format to serializing the generated SARIF into.  */
 
@@ -35,26 +43,26 @@ enum class sarif_serialization_kind
    num_values
 };
 
-extern diagnostic_output_file
-diagnostic_output_format_open_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  const char *base_file_name,
-					  enum sarif_serialization_kind serialization_kind);
-
-extern diagnostic_output_format &
-diagnostic_output_format_init_sarif_stderr (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted);
-extern diagnostic_output_format &
-diagnostic_output_format_init_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  bool formatted,
-					  const char *base_file_name);
-extern diagnostic_output_format &
-diagnostic_output_format_init_sarif_stream (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted,
-					    FILE *stream);
+extern output_file
+open_sarif_output_file (context &dc,
+			line_maps *line_maps,
+			const char *base_file_name,
+			enum sarif_serialization_kind serialization_kind);
+
+extern sink &
+init_sarif_stderr (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted);
+extern sink &
+init_sarif_file (context &dc,
+		 line_maps *line_maps,
+		 bool formatted,
+		 const char *base_file_name);
+extern sink &
+init_sarif_stream (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted,
+		   FILE *stream);
 
 /* Abstract base class for handling JSON output vs other kinds of
    serialization of the json tree.  */
@@ -101,17 +109,18 @@ struct sarif_generation_options
   sarif_generation_options ();
 
   enum sarif_version m_version;
-  bool m_xml_state;
+  bool m_state_graph;
 };
 
-extern std::unique_ptr<diagnostic_output_format>
-make_sarif_sink (diagnostic_context &context,
+extern std::unique_ptr<sink>
+make_sarif_sink (context &dc,
 		 const line_maps &line_maps,
 		 std::unique_ptr<sarif_serialization_format> serialization_format,
 		 const sarif_generation_options &sarif_gen_opts,
-		 diagnostic_output_file output_file);
+		 output_file output_file_);
 
 class sarif_builder;
+class sarif_location_manager;
 
 /* Concrete subclass of json::object for SARIF property bags
    (SARIF v2.1.0 section 3.8).  */
@@ -121,7 +130,11 @@ class sarif_property_bag : public json::object
 public:
   void set_logical_location (const char *property_name,
 			     sarif_builder &,
-			     logical_location logical_loc);
+			     logical_locations::key logical_loc);
+  void set_graph (const char *property_name,
+		  sarif_builder &,
+		  sarif_location_manager *sarif_location_mgr,
+		  const digraphs::digraph &g);
 };
 
 /* Concrete subclass of json::object for SARIF objects that can
@@ -136,4 +149,41 @@ public:
   sarif_property_bag &get_or_create_properties ();
 };
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_SARIF_H */
+/* Subclass of sarif_object for SARIF "graph" objects
+   (SARIF v2.1.0 section 3.39).  */
+
+class sarif_graph : public sarif_object
+{
+};
+
+/* Subclass of sarif_object for SARIF "node" objects
+   (SARIF v2.1.0 section 3.40).  */
+
+class sarif_node : public sarif_object
+{
+};
+
+/* Subclass of sarif_object for SARIF "edge" objects
+   (SARIF v2.1.0 section 3.41).  */
+
+class sarif_edge : public sarif_object
+{
+};
+
+extern std::unique_ptr<sarif_graph>
+make_sarif_graph (const digraphs::digraph &g,
+		  sarif_builder *builder,
+		  sarif_location_manager *sarif_location_mgr);
+
+extern std::unique_ptr<sarif_node>
+make_sarif_node (const digraphs::node &n,
+		 sarif_builder *builder,
+		 sarif_location_manager *sarif_location_mgr);
+
+extern std::unique_ptr<sarif_edge>
+make_sarif_edge (const digraphs::edge &e,
+		 sarif_builder *builder);
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_SARIF_SINK_H */
diff --git a/gcc/selftest-diagnostic.cc b/gcc/diagnostics/selftest-context.cc
index eeee2eb..2b6dd0b 100644
--- a/gcc/selftest-diagnostic.cc
+++ b/gcc/diagnostics/selftest-context.cc
@@ -21,61 +21,65 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
-/* Implementation of class selftest::test_diagnostic_context.  */
+/* Implementation of class diagnostics::selftest::test_context.  */
 
-test_diagnostic_context::test_diagnostic_context ()
+test_context::test_context ()
 {
   diagnostic_initialize (this, 0);
   pp_show_color (get_reference_printer ()) = false;
-  m_source_printing.enabled = true;
-  m_source_printing.show_labels_p = true;
+
+  auto &source_printing_opts = get_source_printing_options ();
+  source_printing_opts.enabled = true;
+  source_printing_opts.show_labels_p = true;
   m_show_column = true;
-  diagnostic_start_span (this) = start_span_cb;
-  m_source_printing.min_margin_width = 6;
-  m_source_printing.max_width = 80;
-  pp_buffer (get_output_format (0).get_printer ())->m_flush_p = false;
+  start_span (this) = start_span_cb;
+  source_printing_opts.min_margin_width = 6;
+  source_printing_opts.max_width = 80;
+  pp_buffer (get_sink (0).get_printer ())->m_flush_p = false;
 }
 
-test_diagnostic_context::~test_diagnostic_context ()
+test_context::~test_context ()
 {
   diagnostic_finish (this);
 }
 
-/* Implementation of diagnostic_start_span_fn, hiding the
+/* Implementation of diagnostics::start_span_fn, hiding the
    real filename (to avoid printing the names of tempfiles).  */
 
 void
-test_diagnostic_context::
-start_span_cb (const diagnostic_location_print_policy &loc_policy,
-	       to_text &sink,
+test_context::
+start_span_cb (const location_print_policy &loc_policy,
+	       to_text &html_or_text,
 	       expanded_location exploc)
 {
   exploc.file = "FILENAME";
-  default_diagnostic_start_span_fn<to_text> (loc_policy, sink, exploc);
+  default_start_span_fn<to_text>
+    (loc_policy, html_or_text, exploc);
 }
 
 bool
-test_diagnostic_context::report (diagnostic_t kind,
-				 rich_location &richloc,
-				 const diagnostic_metadata *metadata,
-				 diagnostic_option_id option,
-				 const char * fmt, ...)
+test_context::report (enum kind kind,
+		      rich_location &richloc,
+		      const metadata *metadata_,
+		      option_id opt_id,
+		      const char * fmt, ...)
 {
   va_list ap;
   va_start (ap, fmt);
   begin_group ();
-  bool result = diagnostic_impl (&richloc, metadata, option, fmt, &ap, kind);
+  bool result = diagnostic_impl (&richloc, metadata_, opt_id, fmt, &ap, kind);
   end_group ();
   va_end (ap);
   return result;
@@ -85,15 +89,16 @@ test_diagnostic_context::report (diagnostic_t kind,
    Return the text buffer from the printer.  */
 
 const char *
-test_diagnostic_context::test_show_locus (rich_location &richloc)
+test_context::test_show_locus (rich_location &richloc)
 {
   pretty_printer *pp = get_reference_printer ();
   gcc_assert (pp);
-  diagnostic_source_print_policy source_policy (*this);
-  source_policy.print (*pp, richloc, DK_ERROR, nullptr);
+  source_print_policy source_policy (*this);
+  source_policy.print (*pp, richloc, kind::error, nullptr);
   return pp_formatted_text (pp);
 }
 
 } // namespace selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/selftest-diagnostic.h b/gcc/diagnostics/selftest-context.h
index 4a43310..cfba997 100644
--- a/gcc/selftest-diagnostic.h
+++ b/gcc/diagnostics/selftest-context.h
@@ -17,29 +17,30 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_DIAGNOSTIC_H
-#define GCC_SELFTEST_DIAGNOSTIC_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_CONTEXT_H
+#define GCC_DIAGNOSTICS_SELFTEST_CONTEXT_H
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
-/* Convenience subclass of diagnostic_context for testing
+/* Convenience subclass of diagnostics::context for testing
    the diagnostic subsystem.  */
 
-class test_diagnostic_context : public diagnostic_context
+class test_context : public context
 {
  public:
-  test_diagnostic_context ();
-  ~test_diagnostic_context ();
+  test_context ();
+  ~test_context ();
 
-  /* Implementation of diagnostic_start_span_fn, hiding the
+  /* Implementation of diagnostics::start_span_fn, hiding the
      real filename (to avoid printing the names of tempfiles).  */
   static void
-  start_span_cb (const diagnostic_location_print_policy &,
+  start_span_cb (const location_print_policy &,
 		 to_text &sink,
 		 expanded_location exploc);
 
@@ -47,17 +48,46 @@ class test_diagnostic_context : public diagnostic_context
      should only be called on a context that uses a non-standard formatter
      that e.g. gathers the results in memory, rather than emits to stderr.  */
   bool
-  report (diagnostic_t kind,
+  report (enum kind kind,
 	  rich_location &richloc,
-	  const diagnostic_metadata *metadata,
-	  diagnostic_option_id option,
+	  const metadata *,
+	  option_id opt_id,
 	  const char * fmt, ...) ATTRIBUTE_GCC_DIAG(6,7);
 
   const char *test_show_locus (rich_location &richloc);
+
+  /* Setters for the context's source_printing_options
+     for use in selftests.  */
+  void colorize_source (bool val)
+  {
+    get_source_printing_options ().colorize_source_p = val;
+  }
+  void show_labels (bool val)
+  {
+    get_source_printing_options ().show_labels_p = val;
+  }
+  void show_line_numbers (bool val)
+  {
+    get_source_printing_options ().show_line_numbers_p = val;
+  }
+  void show_ruler (bool val)
+  {
+    get_source_printing_options ().show_ruler_p = val;
+  }
+  void show_event_links (bool val)
+  {
+    get_source_printing_options ().show_event_links_p = val;
+  }
+  void set_caret_char (unsigned idx, char ch)
+  {
+    gcc_assert (idx < rich_location::STATICALLY_ALLOCATED_RANGES);
+    get_source_printing_options ().caret_chars[idx] = ch;
+  }
 };
 
 } // namespace selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-#endif /* GCC_SELFTEST_DIAGNOSTIC_H */
+#endif /* GCC_DIAGNOSTICS_SELFTEST_CONTEXT_H */
diff --git a/gcc/selftest-logical-location.cc b/gcc/diagnostics/selftest-logical-locations.cc
index 63b9fb4..8ba4233 100644
--- a/gcc/selftest-logical-location.cc
+++ b/gcc/diagnostics/selftest-logical-locations.cc
@@ -1,4 +1,4 @@
-/* Concrete subclass of logical_location for use in selftests.
+/* Concrete subclass of logical_locations::manager for use in selftests.
    Copyright (C) 2024-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -22,22 +22,24 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "selftest.h"
-#include "selftest-logical-location.h"
+#include "diagnostics/selftest-logical-locations.h"
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace logical_locations {
 namespace selftest {
 
-/* class test_logical_location_manager : public logical_location_manager.  */
+/* class test_manager : public manager.  */
 
-test_logical_location_manager::~test_logical_location_manager ()
+test_manager::~test_manager ()
 {
   for (auto iter : m_name_to_item_map)
     delete iter.second;
 }
 
 const char *
-test_logical_location_manager::get_short_name (key k) const
+test_manager::get_short_name (key k) const
 {
   auto item = item_from_key (k);
   if (!item)
@@ -46,43 +48,43 @@ test_logical_location_manager::get_short_name (key k) const
 }
 
 const char *
-test_logical_location_manager::get_name_with_scope (key k) const
+test_manager::get_name_with_scope (key k) const
 {
   auto item = item_from_key (k);
   return item->m_name;
 }
 
 const char *
-test_logical_location_manager::get_internal_name (key k) const
+test_manager::get_internal_name (key k) const
 {
   auto item = item_from_key (k);
   return item->m_name;
 }
 
-enum logical_location_kind
-test_logical_location_manager::get_kind (key k) const
+enum diagnostics::logical_locations::kind
+test_manager::get_kind (key k) const
 {
   auto item = item_from_key (k);
   return item->m_kind;
 }
 
 label_text
-test_logical_location_manager::get_name_for_path_output (key k) const
+test_manager::get_name_for_path_output (key k) const
 {
   auto item = item_from_key (k);
   return label_text::borrow (item->m_name);
 }
 
-logical_location
-test_logical_location_manager::
+diagnostics::logical_locations::key
+test_manager::
 logical_location_from_funcname (const char *funcname)
 {
   const item *i = item_from_funcname (funcname);
   return key::from_ptr (i);
 }
 
-const test_logical_location_manager::item *
-test_logical_location_manager::item_from_funcname (const char *funcname)
+const test_manager::item *
+test_manager::item_from_funcname (const char *funcname)
 {
   if (!funcname)
     return nullptr;
@@ -90,7 +92,7 @@ test_logical_location_manager::item_from_funcname (const char *funcname)
   if (item **slot = m_name_to_item_map.get (funcname))
     return *slot;
 
-  item *i = new item (logical_location_kind::function, funcname);
+  item *i = new item (kind::function, funcname);
   m_name_to_item_map.put (funcname, i);
   return i;
 }
@@ -98,14 +100,14 @@ test_logical_location_manager::item_from_funcname (const char *funcname)
 /* Run all of the selftests within this file.  */
 
 void
-selftest_logical_location_cc_tests ()
+selftest_logical_locations_cc_tests ()
 {
-  test_logical_location_manager mgr;
+  test_manager mgr;
 
   ASSERT_FALSE (mgr.logical_location_from_funcname (nullptr));
 
-  logical_location loc_foo = mgr.logical_location_from_funcname ("foo");
-  logical_location loc_bar = mgr.logical_location_from_funcname ("bar");
+  key loc_foo = mgr.logical_location_from_funcname ("foo");
+  key loc_bar = mgr.logical_location_from_funcname ("bar");
 
   ASSERT_NE (loc_foo, loc_bar);
 
@@ -113,6 +115,8 @@ selftest_logical_location_cc_tests ()
   ASSERT_STREQ (mgr.get_short_name (loc_bar), "bar");
 }
 
-} // namespace selftest
+} // namespace diagnostics::logical_locations::selftest
+} // namespace diagnostics::logical_locations
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/selftest-logical-location.h b/gcc/diagnostics/selftest-logical-locations.h
index d9bf38f..c14a282 100644
--- a/gcc/selftest-logical-location.h
+++ b/gcc/diagnostics/selftest-logical-locations.h
@@ -1,4 +1,4 @@
-/* Concrete subclass of logical_location for use in selftests.
+/* Concrete subclass of logical_locations::manager for use in selftests.
    Copyright (C) 2024-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -18,56 +18,58 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_LOGICAL_LOCATION_H
-#define GCC_SELFTEST_LOGICAL_LOCATION_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_LOGICAL_LOCATIONS_H
+#define GCC_DIAGNOSTICS_SELFTEST_LOGICAL_LOCATIONS_H
 
-#include "logical-location.h"
+#include "diagnostics/logical-locations.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace logical_locations {
 namespace selftest {
 
-/* Concrete subclass of logical_location_manager for use in selftests.  */
+/* Concrete subclass of logical_locations::manager for use in selftests.  */
 
-class test_logical_location_manager : public logical_location_manager
+class test_manager : public manager
 {
 public:
-  ~test_logical_location_manager ();
+  ~test_manager ();
 
   const char *get_short_name (key) const final override;
   const char *get_name_with_scope (key) const final override;
   const char *get_internal_name (key) const final override;
-  enum logical_location_kind get_kind (key) const final override;
+  kind get_kind (key) const final override;
   label_text get_name_for_path_output (key) const final override;
   key get_parent (key) const final override
   {
     return key ();
   }
 
-  logical_location
+  key
   logical_location_from_funcname (const char *funcname);
 
 private:
   struct item
   {
-    item (enum logical_location_kind kind,
+    item (kind kind_,
 	  const char *name)
-    : m_kind (kind),
+    : m_kind (kind_),
       m_name (name)
     {
     }
 
-    enum logical_location_kind m_kind;
+    kind m_kind;
     const char *m_name;
   };
 
   const item *
   item_from_funcname (const char *funcname);
 
-  static const item *item_from_key (logical_location k)
+  static const item *item_from_key (key k)
   {
     return k.cast_to<const item *> ();
   }
@@ -75,9 +77,10 @@ private:
   hash_map<nofree_string_hash, item *> m_name_to_item_map;
 };
 
-} // namespace selftest
+} // namespace diagnostics::logical_locations::selftest
+} // namespace diagnostics::logical_locations::
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-
-#endif /* GCC_SELFTEST_LOGICAL_LOCATION_H.  */
+#endif /* GCC_DIAGNOSTICS_SELFTEST_LOGICAL_LOCATIONS_H.  */
diff --git a/gcc/selftest-diagnostic-path.cc b/gcc/diagnostics/selftest-paths.cc
index 04372e7..56ce7ff 100644
--- a/gcc/selftest-diagnostic-path.cc
+++ b/gcc/diagnostics/selftest-paths.cc
@@ -24,66 +24,69 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "version.h"
-#include "demangle.h"
-#include "backtrace.h"
 #include "diagnostic.h"
-#include "selftest-diagnostic-path.h"
+#include "diagnostics/selftest-paths.h"
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace paths {
 namespace selftest {
 
-/* class test_diagnostic_path : public diagnostic_path.  */
+/* class test_path : public diagnostics::paths::path.  */
 
-test_diagnostic_path::test_diagnostic_path (pretty_printer *event_pp)
-: diagnostic_path (m_test_logical_loc_mgr),
+test_path::
+test_path (logical_locations::selftest::test_manager &logical_loc_mgr,
+	   pretty_printer *event_pp)
+: path (logical_loc_mgr),
+  m_test_logical_loc_mgr (logical_loc_mgr),
   m_event_pp (event_pp)
 {
   add_thread ("main");
 }
 
-/* Implementation of diagnostic_path::num_events vfunc for
-   test_diagnostic_path: simply get the number of events in the vec.  */
+/* Implementation of path::num_events vfunc for
+   test_path: simply get the number of events in the vec.  */
 
 unsigned
-test_diagnostic_path::num_events () const
+test_path::num_events () const
 {
   return m_events.length ();
 }
 
-/* Implementation of diagnostic_path::get_event vfunc for
-   test_diagnostic_path: simply return the event in the vec.  */
+/* Implementation of path::get_event vfunc for
+   test_path: simply return the event in the vec.  */
 
-const diagnostic_event &
-test_diagnostic_path::get_event (int idx) const
+const event &
+test_path::get_event (int idx) const
 {
   return *m_events[idx];
 }
 
 unsigned
-test_diagnostic_path::num_threads () const
+test_path::num_threads () const
 {
   return m_threads.length ();
 }
 
-const diagnostic_thread &
-test_diagnostic_path::get_thread (diagnostic_thread_id_t idx) const
+const thread &
+test_path::get_thread (thread_id_t idx) const
 {
   return *m_threads[idx];
 }
 
 bool
-test_diagnostic_path::same_function_p (int event_idx_a,
-				       int event_idx_b) const
+test_path::same_function_p (int event_idx_a,
+			    int event_idx_b) const
 {
   return (m_events[event_idx_a]->get_logical_location ()
 	  == m_events[event_idx_b]->get_logical_location ());
 }
 
-diagnostic_thread_id_t
-test_diagnostic_path::add_thread (const char *name)
+thread_id_t
+test_path::add_thread (const char *name)
 {
-  m_threads.safe_push (new test_diagnostic_thread (name));
+  m_threads.safe_push (new test_thread (name));
   return m_threads.length () - 1;
 }
 
@@ -95,11 +98,11 @@ test_diagnostic_path::add_thread (const char *name)
 
    Return the id of the new event.  */
 
-diagnostic_event_id_t
-test_diagnostic_path::add_event (location_t loc,
-				 const char *funcname,
-				 int depth,
-				 const char *fmt, ...)
+event_id_t
+test_path::add_event (location_t loc,
+		      const char *funcname,
+		      int depth,
+		      const char *fmt, ...)
 {
   pretty_printer *pp = m_event_pp;
   pp_clear_output_area (pp);
@@ -117,24 +120,24 @@ test_diagnostic_path::add_event (location_t loc,
 
   va_end (ap);
 
-  test_diagnostic_event *new_event
-    = new test_diagnostic_event (loc,
-				 logical_location_from_funcname (funcname),
-				 depth,
-				 pp_formatted_text (pp));
+  test_event *new_event
+    = new test_event (loc,
+		      logical_location_from_funcname (funcname),
+		      depth,
+		      pp_formatted_text (pp));
   m_events.safe_push (new_event);
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
-diagnostic_event_id_t
-test_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
-					location_t loc,
-					const char *funcname,
-					int depth,
-					const char *fmt, ...)
+event_id_t
+test_path::add_thread_event (thread_id_t thread_id,
+			     location_t loc,
+			     const char *funcname,
+			     int depth,
+			     const char *fmt, ...)
 {
   pretty_printer *pp = m_event_pp;
   pp_clear_output_area (pp);
@@ -153,52 +156,52 @@ test_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
 
   va_end (ap);
 
-  test_diagnostic_event *new_event
-    = new test_diagnostic_event (loc,
-				 logical_location_from_funcname (funcname),
-				 depth,
-				 pp_formatted_text (pp),
-				 thread_id);
+  test_event *new_event
+    = new test_event (loc,
+		      logical_location_from_funcname (funcname),
+		      depth,
+		      pp_formatted_text (pp),
+		      thread_id);
   m_events.safe_push (new_event);
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
 /* Mark the most recent event on this path (which must exist) as being
    connected to the next one to be added.  */
 
 void
-test_diagnostic_path::connect_to_next_event ()
+test_path::connect_to_next_event ()
 {
   gcc_assert (m_events.length () > 0);
   m_events[m_events.length () - 1]->connect_to_next_event ();
 }
 
 void
-test_diagnostic_path::add_entry (const char *callee_name,
-				 int stack_depth,
-				 diagnostic_thread_id_t thread_id)
+test_path::add_entry (const char *callee_name,
+		      int stack_depth,
+		      thread_id_t thread_id)
 {
   add_thread_event (thread_id, UNKNOWN_LOCATION, callee_name, stack_depth,
 		    "entering %qs", callee_name);
 }
 
 void
-test_diagnostic_path::add_return (const char *caller_name,
-				  int stack_depth,
-				  diagnostic_thread_id_t thread_id)
+test_path::add_return (const char *caller_name,
+		       int stack_depth,
+		       thread_id_t thread_id)
 {
   add_thread_event (thread_id, UNKNOWN_LOCATION, caller_name, stack_depth,
 		    "returning to %qs", caller_name);
 }
 
 void
-test_diagnostic_path::add_call (const char *caller_name,
-				int caller_stack_depth,
-				const char *callee_name,
-				diagnostic_thread_id_t thread_id)
+test_path::add_call (const char *caller_name,
+		     int caller_stack_depth,
+		     const char *callee_name,
+		     thread_id_t thread_id)
 {
   add_thread_event (thread_id, UNKNOWN_LOCATION,
 		    caller_name, caller_stack_depth,
@@ -206,22 +209,22 @@ test_diagnostic_path::add_call (const char *caller_name,
   add_entry (callee_name, caller_stack_depth + 1, thread_id);
 }
 
-logical_location
-test_diagnostic_path::logical_location_from_funcname (const char *funcname)
+logical_locations::key
+test_path::logical_location_from_funcname (const char *funcname)
 {
   return m_test_logical_loc_mgr.logical_location_from_funcname (funcname);
 }
 
-/* struct test_diagnostic_event.  */
+/* struct test_event.  */
 
-/* test_diagnostic_event's ctor.  */
+/* test_event's ctor.  */
 
-test_diagnostic_event::
-test_diagnostic_event (location_t loc,
-		       logical_location logical_loc,
-		       int depth,
-		       const char *desc,
-		       diagnostic_thread_id_t thread_id)
+test_event::
+test_event (location_t loc,
+	    logical_location logical_loc,
+	    int depth,
+	    const char *desc,
+	    thread_id_t thread_id)
 : m_loc (loc),
   m_logical_loc (logical_loc),
   m_depth (depth), m_desc (xstrdup (desc)),
@@ -230,13 +233,15 @@ test_diagnostic_event (location_t loc,
 {
 }
 
-/* test_diagnostic_event's dtor.  */
+/* test_event's dtor.  */
 
-test_diagnostic_event::~test_diagnostic_event ()
+test_event::~test_event ()
 {
   free (m_desc);
 }
 
-} // namespace selftest
+} // namespace diagnostics::paths::selftest
+} // namespace diagnostics::paths
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/selftest-diagnostic-path.h b/gcc/diagnostics/selftest-paths.h
index 8829943..fe628f6 100644
--- a/gcc/selftest-diagnostic-path.h
+++ b/gcc/diagnostics/selftest-paths.h
@@ -18,17 +18,19 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_DIAGNOSTIC_PATH_H
-#define GCC_SELFTEST_DIAGNOSTIC_PATH_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_PATHS_H
+#define GCC_DIAGNOSTICS_SELFTEST_PATHS_H
 
-#include "diagnostic-path.h"
-#include "selftest-logical-location.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/selftest-logical-locations.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace paths {
 namespace selftest {
 
 /* Concrete subclasses of the abstract base classes
@@ -36,17 +38,20 @@ namespace selftest {
 
    This code should have no dependency on "tree".  */
 
-/* An implementation of diagnostic_event.  */
+/* An implementation of diagnostics::paths::event.  */
 
-class test_diagnostic_event : public diagnostic_event
+class test_event : public event
 {
  public:
-  test_diagnostic_event (location_t loc,
-			 logical_location logical_loc,
-			 int depth,
-			 const char *desc,
-			 diagnostic_thread_id_t thread_id = 0);
-  ~test_diagnostic_event ();
+  using logical_location = logical_locations::key;
+  using thread_id_t = paths::thread_id_t;
+
+  test_event (location_t loc,
+	      logical_location logical_loc,
+	      int depth,
+	      const char *desc,
+	      thread_id_t thread_id = 0);
+  ~test_event ();
 
   location_t get_location () const final override { return m_loc; }
   int get_stack_depth () const final override { return m_depth; }
@@ -66,7 +71,7 @@ class test_diagnostic_event : public diagnostic_event
   {
     return m_connected_to_next_event;
   }
-  diagnostic_thread_id_t get_thread_id () const final override
+  thread_id_t get_thread_id () const final override
   {
     return m_thread_id;
   }
@@ -82,15 +87,15 @@ class test_diagnostic_event : public diagnostic_event
   int m_depth;
   char *m_desc; // has been formatted; doesn't get i18n-ed
   bool m_connected_to_next_event;
-  diagnostic_thread_id_t m_thread_id;
+  thread_id_t m_thread_id;
 };
 
-/* A simple implementation of diagnostic_thread.  */
+/* A simple implementation of diagnostics::paths::thread.  */
 
-class test_diagnostic_thread : public diagnostic_thread
+class test_thread : public thread
 {
 public:
-  test_diagnostic_thread (const char *name) : m_name (name) {}
+  test_thread (const char *name) : m_name (name) {}
   label_text get_name (bool) const final override
   {
     return label_text::borrow (m_name);
@@ -100,33 +105,34 @@ private:
   const char *m_name; // has been i18n-ed and formatted
 };
 
-/* A concrete subclass of diagnostic_path for implementing selftests
-   - a vector of test_diagnostic_event instances
+/* A concrete subclass of diagnostics::paths::path for implementing selftests
+   - a vector of test_event instances
    - adds member functions for adding test event
    - does no translation of its events
    - has no dependency on "tree".  */
 
-class test_diagnostic_path : public diagnostic_path
+class test_path : public path
 {
  public:
-  test_diagnostic_path (pretty_printer *event_pp);
+  test_path (logical_locations::selftest::test_manager &logical_loc_mgr,
+	     pretty_printer *event_pp);
 
   unsigned num_events () const final override;
-  const diagnostic_event & get_event (int idx) const final override;
+  const event & get_event (int idx) const final override;
   unsigned num_threads () const final override;
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override;
+  const thread &
+  get_thread (thread_id_t) const final override;
   bool
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override;
 
-  diagnostic_thread_id_t add_thread (const char *name);
+  thread_id_t add_thread (const char *name);
 
-  diagnostic_event_id_t add_event (location_t loc, const char *funcname, int depth,
-				   const char *fmt, ...)
+  event_id_t add_event (location_t loc, const char *funcname, int depth,
+			const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(5,6);
-  diagnostic_event_id_t
-  add_thread_event (diagnostic_thread_id_t thread_id,
+  event_id_t
+  add_thread_event (thread_id_t thread_id,
 		    location_t loc, const char *funcname, int depth,
 		    const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(6,7);
@@ -134,28 +140,30 @@ class test_diagnostic_path : public diagnostic_path
   void connect_to_next_event ();
 
   void add_entry (const char *callee_name, int stack_depth,
-		  diagnostic_thread_id_t thread_id = 0);
+		  thread_id_t thread_id = 0);
   void add_return (const char *caller_name, int stack_depth,
-		   diagnostic_thread_id_t thread_id = 0);
+		   thread_id_t thread_id = 0);
   void add_call (const char *caller_name,
 		 int caller_stack_depth,
 		 const char *callee_name,
-		 diagnostic_thread_id_t thread_id = 0);
+		 thread_id_t thread_id = 0);
 
  private:
-  logical_location
+  logical_locations::key
   logical_location_from_funcname (const char *funcname);
 
-  test_logical_location_manager m_test_logical_loc_mgr;
-  auto_delete_vec<test_diagnostic_thread> m_threads;
-  auto_delete_vec<test_diagnostic_event> m_events;
+  logical_locations::selftest::test_manager &m_test_logical_loc_mgr;
+  auto_delete_vec<test_thread> m_threads;
+  auto_delete_vec<test_event> m_events;
 
   /* (for use by add_event).  */
   pretty_printer *m_event_pp;
 };
 
-} // namespace selftest
+} // namespace diagnostics::paths::selftest
+} // namespace diagnostics::paths
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-#endif /* ! GCC_SELFTEST_DIAGNOSTIC_PATH_H */
+#endif /* ! GCC_DIAGNOSTICS_SELFTEST_PATHS_H */
diff --git a/gcc/selftest-diagnostic-show-locus.h b/gcc/diagnostics/selftest-source-printing.h
index 37a18b2..451c120 100644
--- a/gcc/selftest-diagnostic-show-locus.h
+++ b/gcc/diagnostics/selftest-source-printing.h
@@ -17,16 +17,18 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_DIAGNOSTIC_SHOW_LOCUS_H
-#define GCC_SELFTEST_DIAGNOSTIC_SHOW_LOCUS_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_SOURCE_PRINTING_H
+#define GCC_DIAGNOSTICS_SELFTEST_SOURCE_PRINTING_H
 
 #include "selftest.h"
+#include "diagnostics/file-cache.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
 /* RAII class for use in selftests involving diagnostic_show_locus.
@@ -37,10 +39,10 @@ namespace selftest {
      push a line_map starting at the first line of the temporary file
    - provide a file_cache.  */
 
-struct diagnostic_show_locus_fixture
+struct source_printing_fixture
 {
-  diagnostic_show_locus_fixture (const line_table_case &case_,
-				 const char *content);
+  source_printing_fixture (const ::selftest::line_table_case &case_,
+			   const char *content);
 
   const char *get_filename () const
   {
@@ -48,8 +50,8 @@ struct diagnostic_show_locus_fixture
   }
 
   const char *m_content;
-  temp_source_file m_tmp_source_file;
-  line_table_test m_ltt;
+  ::selftest::temp_source_file m_tmp_source_file;
+  ::selftest::line_table_test m_ltt;
   file_cache m_fc;
 };
 
@@ -69,14 +71,15 @@ struct diagnostic_show_locus_fixture
    Here SS represents the two display columns for the U+1F602 emoji and
    P represents the one display column for the U+03C0 pi symbol.  */
 
-struct diagnostic_show_locus_fixture_one_liner_utf8
-  : public diagnostic_show_locus_fixture
+struct source_printing_fixture_one_liner_utf8
+  : public source_printing_fixture
 {
-  diagnostic_show_locus_fixture_one_liner_utf8 (const line_table_case &case_);
+  source_printing_fixture_one_liner_utf8 (const ::selftest::line_table_case &case_);
 };
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-#endif /* GCC_SELFTEST_DIAGNOSTIC_SHOW_LOCUS_H */
+#endif /* GCC_DIAGNOSTICS_SELFTEST_SOURCE_PRINTING_H */
diff --git a/gcc/diagnostic-format.h b/gcc/diagnostics/sink.h
index 07db2cd..ac4e0fb64 100644
--- a/gcc/diagnostic-format.h
+++ b/gcc/diagnostics/sink.h
@@ -18,21 +18,23 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_H
-#define GCC_DIAGNOSTIC_FORMAT_H
+#ifndef GCC_DIAGNOSTICS_SINK_H
+#define GCC_DIAGNOSTICS_SINK_H
 
 #include "diagnostic.h"
 
-class diagnostic_per_format_buffer;
+namespace diagnostics {
+
+class per_sink_buffer;
 
 /* Abstract base class for a particular output format for diagnostics;
    each value of -fdiagnostics-output-format= will have its own
    implementation.  */
 
-class diagnostic_output_format
+class sink
 {
 public:
-  virtual ~diagnostic_output_format () {}
+  virtual ~sink () {}
 
   virtual void dump (FILE *out, int indent) const;
 
@@ -40,16 +42,16 @@ public:
      e.g. for titles of HTML, for SARIF's artifact metadata.  */
   virtual void set_main_input_filename (const char *) {}
 
-  /* Vfunc for making an appropriate diagnostic_per_format_buffer
+  /* Vfunc for making an appropriate per_sink_buffer
      subclass for this format.  */
-  virtual std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () = 0;
+  virtual std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () = 0;
 
-  /* Vfunc to be called when call a diagnostic_buffer is set on
-     a diagnostic_context, to update this format.  The per_format_buffer
-     will be one created by make_per_format_buffer above and thus be
+  /* Vfunc to be called when call a diagnostics::buffer is set on
+     a diagnostics::context, to update this format.  The per_sink_buffer
+     will be one created by make_per_sink_buffer above and thus be
      of the correct subclass.  */
-  virtual void set_buffer (diagnostic_per_format_buffer *) = 0;
+  virtual void set_buffer (per_sink_buffer *) = 0;
 
   virtual void on_begin_group () = 0;
   virtual void on_end_group () = 0;
@@ -57,22 +59,25 @@ public:
   /* Vfunc with responsibility for phase 3 of formatting the message
      and "printing" the result.  */
   virtual void on_report_diagnostic (const diagnostic_info &,
-				     diagnostic_t orig_diag_kind) = 0;
+				     enum kind orig_diag_kind) = 0;
 
   virtual void on_report_verbatim (text_info &);
 
-  virtual void on_diagram (const diagnostic_diagram &diagram) = 0;
+  virtual void on_diagram (const diagram &diag) = 0;
   virtual void after_diagnostic (const diagnostic_info &) = 0;
   virtual bool machine_readable_stderr_p () const = 0;
   virtual bool follows_reference_printer_p () const = 0;
 
-  /* Vfunc called when the diagnostic_context changes its
+  /* Vfunc called when the diagnostics::context changes its
      reference printer (either to a new subclass of pretty_printer
      or when color/url options change).
      Subclasses should update their m_printer accordingly.  */
   virtual void update_printer () = 0;
 
-  diagnostic_context &get_context () const { return m_context; }
+  virtual void
+  report_global_digraph (const lazily_created<digraphs::digraph> &) = 0;
+
+  context &get_context () const { return m_context; }
   pretty_printer *get_printer () const { return m_printer.get (); }
 
   text_art::theme *get_diagram_theme () const
@@ -83,21 +88,23 @@ public:
   void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
 
 protected:
-  diagnostic_output_format (diagnostic_context &context)
-  : m_context (context),
-    m_printer (context.clone_printer ())
+  sink (context &dc)
+  : m_context (dc),
+    m_printer (dc.clone_printer ())
   {}
 
 protected:
-  diagnostic_context &m_context;
+  context &m_context;
   std::unique_ptr<pretty_printer> m_printer;
 };
 
 extern void
-diagnostic_output_format_init (diagnostic_context &,
-			       const char *main_input_filename_,
-			       const char *base_file_name,
-			       enum diagnostics_output_format,
-			       bool json_formatting);
+output_format_init (context &,
+		    const char *main_input_filename_,
+		    const char *base_file_name,
+		    enum diagnostics_output_format,
+		    bool json_formatting);
+
+} // namespace diagnostics
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_H */
+#endif /* ! GCC_DIAGNOSTICS_SINK_H */
diff --git a/gcc/diagnostic-label-effects.h b/gcc/diagnostics/source-printing-effects.h
index 1d877b0d..3c4b1bd 100644
--- a/gcc/diagnostic-label-effects.h
+++ b/gcc/diagnostics/source-printing-effects.h
@@ -18,8 +18,10 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_LABEL_EFFECTS_H
-#define GCC_DIAGNOSTIC_LABEL_EFFECTS_H
+#ifndef GCC_DIAGNOSTICS_SOURCE_PRINTING_EFFECTS_H
+#define GCC_DIAGNOSTICS_SOURCE_PRINTING_EFFECTS_H
+
+namespace diagnostics {
 
 /* Abstract base class for describing special effects when printing
    a label when quoting source code.  */
@@ -37,10 +39,10 @@ public:
 
 /* A class to hold state when quoting a run of lines of source code.  */
 
-class diagnostic_source_effect_info
+class source_effect_info
 {
 public:
-  diagnostic_source_effect_info ()
+  source_effect_info ()
   : m_leading_in_edge_column (-1),
     m_trailing_out_edge_column (-1)
   {
@@ -55,4 +57,6 @@ public:
   int m_trailing_out_edge_column;
 };
 
-#endif /* GCC_DIAGNOSTIC_LABEL_EFFECTS_H */
+} // namespace diagnostics
+
+#endif /* GCC_DIAGNOSTICS_SOURCE_PRINTING_EFFECTS_H */
diff --git a/gcc/diagnostics/source-printing-options.h b/gcc/diagnostics/source-printing-options.h
new file mode 100644
index 0000000..362b691
--- /dev/null
+++ b/gcc/diagnostics/source-printing-options.h
@@ -0,0 +1,76 @@
+/* Options relating to printing the user's source code.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_SOURCE_PRINTING_OPTIONS_H
+#define GCC_DIAGNOSTICS_SOURCE_PRINTING_OPTIONS_H
+
+namespace diagnostics {
+
+/* A bundle of options relating to printing the user's source code
+   (potentially with a margin, underlining, labels, etc).  */
+
+struct source_printing_options
+{
+  /* True if we should print the source line with a caret indicating
+     the location.
+     Corresponds to -fdiagnostics-show-caret.  */
+  bool enabled;
+
+  /* Maximum width of the source line printed.  */
+  int max_width;
+
+  /* Character used at the caret when printing source locations.  */
+  char caret_chars[rich_location::STATICALLY_ALLOCATED_RANGES];
+
+  /* When printing source code, should the characters at carets and ranges
+     be colorized? (assuming colorization is on at all).
+     This should be true for frontends that generate range information
+     (so that the ranges of code are colorized),
+     and false for frontends that merely specify points within the
+     source code (to avoid e.g. colorizing just the first character in
+     a token, which would look strange).  */
+  bool colorize_source_p;
+
+  /* When printing source code, should labelled ranges be printed?
+     Corresponds to -fdiagnostics-show-labels.  */
+  bool show_labels_p;
+
+  /* When printing source code, should there be a left-hand margin
+     showing line numbers?
+     Corresponds to -fdiagnostics-show-line-numbers.  */
+  bool show_line_numbers_p;
+
+  /* If printing source code, what should the minimum width of the margin
+     be?  Line numbers will be right-aligned, and padded to this width.
+     Corresponds to -fdiagnostics-minimum-margin-width=VALUE.  */
+  int min_margin_width;
+
+  /* Usable by plugins; if true, print a debugging ruler above the
+     source output.  */
+  bool show_ruler_p;
+
+  /* When printing events in an inline path, should we print lines
+     visualizing links between related events (e.g. for CFG paths)?
+     Corresponds to -fdiagnostics-show-event-links.  */
+  bool show_event_links_p;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_SOURCE_PRINTING_OPTIONS_H */
diff --git a/gcc/diagnostic-show-locus.cc b/gcc/diagnostics/source-printing.cc
index da7637c..94b1c2d 100644
--- a/gcc/diagnostic-show-locus.cc
+++ b/gcc/diagnostics/source-printing.cc
@@ -25,20 +25,19 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "version.h"
-#include "demangle.h"
 #include "intl.h"
-#include "backtrace.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 #include "gcc-rich-location.h"
 #include "text-range-label.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "selftest-diagnostic-show-locus.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-source-printing.h"
 #include "cpplib.h"
 #include "text-art/types.h"
 #include "text-art/theme.h"
-#include "diagnostic-label-effects.h"
+#include "diagnostics/source-printing-effects.h"
+#include "diagnostics/file-cache.h"
 #include "xml.h"
 #include "xml-printer.h"
 
@@ -96,7 +95,7 @@ class colorizer
  public:
   colorizer (pretty_printer &pp,
 	     const rich_location &richloc,
-	     diagnostic_t diagnostic_kind);
+	     enum diagnostics::kind diagnostic_kind);
   ~colorizer ();
 
   void set_range (int range_idx)
@@ -116,7 +115,7 @@ class colorizer
        two colors for the secondary locations.
        But if we're printing a run of events in a diagnostic path, that
        makes no sense, so print all of them with the same colorization.  */
-    if (m_diagnostic_kind == DK_DIAGNOSTIC_PATH)
+    if (m_diagnostic_kind == diagnostics::kind::path)
       set_state (0);
     else
       set_state (range_idx);
@@ -141,7 +140,7 @@ class colorizer
 
   pretty_printer &m_pp;
   const rich_location &m_richloc;
-  diagnostic_t m_diagnostic_kind;
+  enum diagnostics::kind m_diagnostic_kind;
   int m_current_state;
   const char *m_range1;
   const char *m_range2;
@@ -203,7 +202,7 @@ enum column_unit {
 class exploc_with_display_col : public expanded_location
 {
  public:
-  exploc_with_display_col (file_cache &fc,
+  exploc_with_display_col (diagnostics::file_cache &fc,
 			   const expanded_location &exploc,
 			   const cpp_char_column_policy &policy,
 			   enum location_aspect aspect)
@@ -375,9 +374,9 @@ struct char_display_policy : public cpp_char_column_policy
  public:
   char_display_policy (int tabstop,
 		       int (*width_cb) (cppchar_t c),
-		       void (*print_text_cb) (to_text &sink,
+		       void (*print_text_cb) (diagnostics::to_text &text_out,
 					      const cpp_decoded_char &cp),
-		       void (*print_html_cb) (to_html &sink,
+		       void (*print_html_cb) (diagnostics::to_html &html_out,
 					      const cpp_decoded_char &cp))
   : cpp_char_column_policy (tabstop, width_cb),
     m_print_text_cb (print_text_cb),
@@ -385,21 +384,23 @@ struct char_display_policy : public cpp_char_column_policy
   {
   }
 
-  void (*m_print_text_cb) (to_text &sink,
+  void (*m_print_text_cb) (diagnostics::to_text &text_out,
 			   const cpp_decoded_char &cp);
-  void (*m_print_html_cb) (to_html &sink,
+  void (*m_print_html_cb) (diagnostics::to_html &html_out,
 			   const cpp_decoded_char &cp);
 };
 
-template <typename Sink> class layout_printer;
+template <typename TextOrHtml> class layout_printer;
 
 } // anonymous namespace
 
+namespace diagnostics {
+
 /* This code is written generically to write either:
    - text, to a pretty_printer, potentially with colorization codes, or
    - html, to an xml::printer, with nested HTML tags.
 
-   This is handled via a "Sink" template, which is either to_text
+   This is handled via a "TextOrHtml" template, which is either to_text
    or to_html.  */
 
 /* Writing text output.  */
@@ -519,8 +520,8 @@ struct to_text
   }
 
   void
-  invoke_start_span_fn (const diagnostic_source_print_policy &source_policy,
-			const diagnostic_location_print_policy &loc_policy,
+  invoke_start_span_fn (const source_print_policy &source_policy,
+			const location_print_policy &loc_policy,
 			const expanded_location &exploc)
   {
     source_policy.get_text_start_span_fn () (loc_policy, *this, exploc);
@@ -548,8 +549,8 @@ struct to_html
   struct auto_check_tag_nesting : public xml::auto_check_tag_nesting
   {
   public:
-    auto_check_tag_nesting (to_html &sink)
-    : xml::auto_check_tag_nesting (sink.m_xp)
+    auto_check_tag_nesting (to_html &out)
+    : xml::auto_check_tag_nesting (out.m_xp)
     {
     }
   };
@@ -654,8 +655,8 @@ struct to_html
   }
 
   void
-  invoke_start_span_fn (const diagnostic_source_print_policy &source_policy,
-			const diagnostic_location_print_policy &loc_policy,
+  invoke_start_span_fn (const source_print_policy &source_policy,
+			const location_print_policy &loc_policy,
 			const expanded_location &exploc)
   {
     source_policy.get_html_start_span_fn () (loc_policy, *this, exploc);
@@ -687,66 +688,67 @@ private:
 };
 
 void
-diagnostic_location_print_policy::
-print_text_span_start (const diagnostic_context &dc,
+location_print_policy::
+print_text_span_start (const diagnostics::context &dc,
 		       pretty_printer &pp,
 		       const expanded_location &exploc)
 {
-  to_text sink (pp, nullptr);
-  diagnostic_source_print_policy source_policy (dc);
-  source_policy.get_text_start_span_fn () (*this, sink, exploc);
+  to_text out (pp, nullptr);
+  source_print_policy source_policy (dc);
+  source_policy.get_text_start_span_fn () (*this, out, exploc);
 }
 
 void
-diagnostic_location_print_policy::
-print_html_span_start (const diagnostic_context &dc,
+location_print_policy::
+print_html_span_start (const diagnostics::context &dc,
 		       xml::printer &xp,
 		       const expanded_location &exploc)
 {
-  to_html sink (xp, nullptr, nullptr);
-  diagnostic_source_print_policy source_policy (dc);
-  source_policy.get_html_start_span_fn () (*this, sink, exploc);
+  to_html out (xp, nullptr, nullptr);
+  source_print_policy source_policy (dc);
+  source_policy.get_html_start_span_fn () (*this, out, exploc);
 }
 
 pretty_printer *
-get_printer (to_text &sink)
+get_printer (to_text &out)
 {
-  return &sink.m_pp;
+  return &out.m_pp;
 }
 
 template<>
 void
-default_diagnostic_start_span_fn<to_text> (const diagnostic_location_print_policy &loc_policy,
-					   to_text &sink,
-					   expanded_location exploc)
+default_start_span_fn<to_text> (const location_print_policy &loc_policy,
+				to_text &out,
+				expanded_location exploc)
 {
-  const diagnostic_column_policy &column_policy
-    = loc_policy.get_column_policy ();
+  const column_policy &column_policy_ = loc_policy.get_column_policy ();
   label_text text
-    = column_policy.get_location_text (exploc,
-				       loc_policy.show_column_p (),
-				       pp_show_color (&sink.m_pp));
-  pp_string (&sink.m_pp, text.get ());
-  pp_newline (&sink.m_pp);
+    = column_policy_.get_location_text (exploc,
+					loc_policy.show_column_p (),
+					pp_show_color (&out.m_pp));
+  pp_string (&out.m_pp, text.get ());
+  pp_newline (&out.m_pp);
 }
 
 template<>
 void
-default_diagnostic_start_span_fn<to_html> (const diagnostic_location_print_policy &loc_policy,
-					   to_html &sink,
-					   expanded_location exploc)
+default_start_span_fn<to_html> (const location_print_policy &loc_policy,
+				to_html &out,
+				expanded_location exploc)
 {
-  const diagnostic_column_policy &column_policy
+  const column_policy &column_policy_
     = loc_policy.get_column_policy ();
   label_text text
-    = column_policy.get_location_text (exploc,
-				       loc_policy.show_column_p (),
-				       false);
-  sink.m_xp.push_tag_with_class ("span", "location", true);
-  sink.m_xp.add_text (text.get ());
-  sink.m_xp.pop_tag ("span");
+    = column_policy_.get_location_text (exploc,
+					loc_policy.show_column_p (),
+					false);
+  out.m_xp.push_tag_with_class ("span", "location", true);
+  out.m_xp.add_text (text.get ());
+  out.m_xp.pop_tag ("span");
 }
 
+} // namespace diagnostics
+
 namespace {
 
 /* A class to control the overall layout when printing a diagnostic.
@@ -761,12 +763,12 @@ namespace {
 class layout
 {
  public:
-  friend class layout_printer<to_text>;
-  friend class layout_printer<to_html>;
+  friend class layout_printer<diagnostics::to_text>;
+  friend class layout_printer<diagnostics::to_html>;
 
-  layout (const diagnostic_source_print_policy &source_policy,
+  layout (const diagnostics::source_print_policy &source_policy,
 	  const rich_location &richloc,
-	  diagnostic_source_effect_info *effect_info = nullptr);
+	  diagnostics::source_effect_info *effect_info = nullptr);
 
   bool maybe_add_location_range (const location_range *loc_range,
 				 unsigned original_idx,
@@ -811,12 +813,12 @@ class layout
  private:
   bool compatible_locations_p (location_t loc_a, location_t loc_b) const;
 
-  const diagnostic_source_printing_options &m_options;
+  const diagnostics::source_printing_options &m_options;
   const line_maps *m_line_table;
-  file_cache &m_file_cache;
+  diagnostics::file_cache &m_file_cache;
   const text_art::ascii_theme m_fallback_theme;
   const text_art::theme &m_theme;
-  diagnostic_source_effect_info *m_effect_info;
+  diagnostics::source_effect_info *m_effect_info;
   char_display_policy m_char_policy;
   location_t m_primary_loc;
   exploc_with_display_col m_exploc;
@@ -838,19 +840,19 @@ enum class margin_kind
 };
 
 /* A bundle of state for printing a particular layout
-   to a particular Sink (either to_text or to_html).  */
-template <typename Sink>
+   to a particular TextOrHtml (either to_text or to_html).  */
+template <typename TextOrHtml>
 class layout_printer
 {
 public:
-  layout_printer (Sink &sink,
+  layout_printer (TextOrHtml &text_or_html,
 		  const layout &layout,
 		  bool is_diagnostic_path);
 
-  void print (const diagnostic_source_print_policy &source_policy);
+  void print (const diagnostics::source_print_policy &source_policy);
 
 private:
-  const diagnostic_source_printing_options &
+  const diagnostics::source_printing_options &
   get_options () const { return m_layout.m_options; }
 
   const text_art::theme &
@@ -880,7 +882,7 @@ private:
   void set_outside_range ();
 
 private:
-  Sink &m_sink;
+  TextOrHtml &m_text_or_html;
   const layout &m_layout;
   bool m_is_diagnostic_path;
 
@@ -944,7 +946,7 @@ private:
 
 colorizer::colorizer (pretty_printer &pp,
 		      const rich_location &richloc,
-		      diagnostic_t diagnostic_kind) :
+		      enum diagnostics::kind diagnostic_kind) :
   m_pp (pp),
   m_richloc (richloc),
   m_diagnostic_kind (diagnostic_kind),
@@ -1022,7 +1024,7 @@ colorizer::begin_state (int state)
       pp_string
 	(&m_pp,
 	 colorize_start (pp_show_color (&m_pp),
-			 diagnostic_get_color_for_kind (m_diagnostic_kind)));
+			 get_color_for_kind (m_diagnostic_kind)));
       break;
 
     case 1:
@@ -1206,7 +1208,8 @@ layout_range::has_in_edge () const
 {
   if (!m_label)
     return false;
-  const label_effects *effects = m_label->get_effects (m_original_idx);
+  const diagnostics::label_effects *effects
+    = m_label->get_effects (m_original_idx);
   if (!effects)
     return false;
 
@@ -1220,7 +1223,8 @@ layout_range::has_out_edge () const
 {
   if (!m_label)
     return false;
-  const label_effects *effects = m_label->get_effects (m_original_idx);
+  const diagnostics::label_effects *effects
+    = m_label->get_effects (m_original_idx);
   if (!effects)
     return false;
 
@@ -1247,7 +1251,7 @@ static cpp_char_column_policy def_policy ()
    e.g. in test_diagnostic_show_locus_one_liner_utf8().  */
 
 static layout_range
-make_range (file_cache &fc,
+make_range (diagnostics::file_cache &fc,
 	    int start_line, int start_col, int end_line, int end_col)
 {
   const expanded_location start_exploc
@@ -1276,7 +1280,7 @@ make_range (file_cache &fc,
 static void
 test_layout_range_for_single_point ()
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
   layout_range point = make_range (fc, 7, 10, 7, 10);
 
   /* Tests for layout_range::contains_point.  */
@@ -1313,7 +1317,7 @@ test_layout_range_for_single_point ()
 static void
 test_layout_range_for_single_line ()
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
   layout_range example_a = make_range (fc, 2, 22, 2, 38);
 
   /* Tests for layout_range::contains_point.  */
@@ -1356,7 +1360,7 @@ test_layout_range_for_single_line ()
 static void
 test_layout_range_for_multiple_lines ()
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
   layout_range example_b = make_range (fc, 3, 14, 5, 8);
 
   /* Tests for layout_range::contains_point.  */
@@ -1567,9 +1571,9 @@ fixit_cmp (const void *p_a, const void *p_b)
 /* Callback for char_display_policy::m_print_cb for printing source chars
    when not escaping the source.  */
 
-template <class Sink>
+template <class TextOrHtml>
 static void
-default_print_decoded_ch (Sink &sink,
+default_print_decoded_ch (TextOrHtml &text_or_html,
 			  const cpp_decoded_char &decoded_ch)
 {
   for (const char *ptr = decoded_ch.m_start_byte;
@@ -1577,11 +1581,11 @@ default_print_decoded_ch (Sink &sink,
     {
       if (*ptr == '\0' || *ptr == '\r')
 	{
-	  sink.add_space ();
+	  text_or_html.add_space ();
 	  continue;
 	}
 
-      sink.add_utf8_byte (*ptr);
+      text_or_html.add_utf8_byte (*ptr);
     }
 }
 
@@ -1609,9 +1613,9 @@ escape_as_bytes_width (cppchar_t ch)
 /* Callback for char_display_policy::m_print_cb for printing source chars
    when escaping with DIAGNOSTICS_ESCAPE_FORMAT_BYTES.  */
 
-template <typename Sink>
+template <typename TextOrHtml>
 static void
-escape_as_bytes_print (Sink &sink,
+escape_as_bytes_print (TextOrHtml &text_or_html,
 		       const cpp_decoded_char &decoded_ch)
 {
   if (!decoded_ch.m_valid_ch)
@@ -1621,14 +1625,14 @@ escape_as_bytes_print (Sink &sink,
 	{
 	  char buf[16];
 	  sprintf (buf, "<%02x>", (unsigned char)*iter);
-	  sink.add_text (buf);
+	  text_or_html.add_text (buf);
 	}
       return;
     }
 
   cppchar_t ch = decoded_ch.m_ch;
   if (ch < 0x80 && ISPRINT (ch))
-    sink.add_character (ch);
+    text_or_html.add_character (ch);
   else
     {
       for (const char *iter = decoded_ch.m_start_byte;
@@ -1636,7 +1640,7 @@ escape_as_bytes_print (Sink &sink,
 	{
 	  char buf[16];
 	  sprintf (buf, "<%02x>", (unsigned char)*iter);
-	  sink.add_text (buf);
+	  text_or_html.add_text (buf);
 	}
     }
 }
@@ -1666,40 +1670,40 @@ escape_as_unicode_width (cppchar_t ch)
 /* Callback for char_display_policy::m_print_cb for printing source chars
    when escaping with DIAGNOSTICS_ESCAPE_FORMAT_UNICODE.  */
 
-template <typename Sink>
+template <typename TextOrHtml>
 static void
-escape_as_unicode_print (Sink &sink,
+escape_as_unicode_print (TextOrHtml &text_or_html,
 			 const cpp_decoded_char &decoded_ch)
 {
   if (!decoded_ch.m_valid_ch)
     {
-      escape_as_bytes_print<Sink> (sink, decoded_ch);
+      escape_as_bytes_print<TextOrHtml> (text_or_html, decoded_ch);
       return;
     }
 
   cppchar_t ch = decoded_ch.m_ch;
   if (ch < 0x80 && ISPRINT (ch))
-    sink.add_character (ch);
+    text_or_html.add_character (ch);
   else
     {
       char buf[16];
       sprintf (buf, "<U+%04X>", ch);
-      sink.add_text (buf);
+      text_or_html.add_text (buf);
     }
 }
 
 /* Populate a char_display_policy based on SOURCE_POLICY and RICHLOC.  */
 
 static char_display_policy
-make_char_policy (const diagnostic_source_print_policy &source_policy,
+make_char_policy (const diagnostics::source_print_policy &source_policy,
 		  const rich_location &richloc)
 {
   /* The default is to not escape non-ASCII bytes.  */
   char_display_policy result
     (source_policy.get_column_policy ().get_tabstop (),
      cpp_wcwidth,
-     default_print_decoded_ch<to_text>,
-     default_print_decoded_ch<to_html>);
+     default_print_decoded_ch<diagnostics::to_text>,
+     default_print_decoded_ch<diagnostics::to_html>);
 
   /* If the diagnostic suggests escaping non-ASCII bytes, then
      use policy from user-supplied options.  */
@@ -1712,13 +1716,15 @@ make_char_policy (const diagnostic_source_print_policy &source_policy,
 	  gcc_unreachable ();
 	case DIAGNOSTICS_ESCAPE_FORMAT_UNICODE:
 	  result.m_width_cb = escape_as_unicode_width;
-	  result.m_print_text_cb = escape_as_unicode_print<to_text>;
-	  result.m_print_html_cb = escape_as_unicode_print<to_html>;
+	  result.m_print_text_cb
+	    = escape_as_unicode_print<diagnostics::to_text>;
+	  result.m_print_html_cb
+	    = escape_as_unicode_print<diagnostics::to_html>;
 	  break;
 	case DIAGNOSTICS_ESCAPE_FORMAT_BYTES:
 	  result.m_width_cb = escape_as_bytes_width;
-	  result.m_print_text_cb = escape_as_bytes_print<to_text>;
-	  result.m_print_html_cb = escape_as_bytes_print<to_html>;
+	  result.m_print_text_cb = escape_as_bytes_print<diagnostics::to_text>;
+	  result.m_print_html_cb = escape_as_bytes_print<diagnostics::to_html>;
 	  break;
 	}
     }
@@ -1735,11 +1741,11 @@ make_char_policy (const diagnostic_source_print_policy &source_policy,
    Determine the range of lines that we will print, splitting them
    up into an ordered list of disjoint spans of contiguous line numbers.
    Determine m_x_offset_display, to ensure that the primary caret
-   will fit within the max_width provided by the diagnostic_context.  */
+   will fit within the max_width provided by the diagnostics::context.  */
 
-layout::layout (const diagnostic_source_print_policy &source_policy,
+layout::layout (const diagnostics::source_print_policy &source_policy,
 		const rich_location &richloc,
-		diagnostic_source_effect_info *effect_info)
+		diagnostics::source_effect_info *effect_info)
 : m_options (source_policy.get_options ()),
   m_line_table (richloc.get_line_table ()),
   m_file_cache (source_policy.get_file_cache ()),
@@ -1927,25 +1933,25 @@ layout::will_show_line_p (linenum_type row) const
 
 template<>
 void
-layout_printer<to_text>::print_gap_in_line_numbering ()
+layout_printer<diagnostics::to_text>::print_gap_in_line_numbering ()
 {
   gcc_assert (m_layout.m_options.show_line_numbers_p);
 
-  m_sink.emit_text_prefix ();
+  m_text_or_html.emit_text_prefix ();
 
   for (int i = 0; i < m_layout.get_linenum_width () + 1; i++)
-    m_sink.add_character ('.');
+    m_text_or_html.add_character ('.');
 
-  m_sink.add_newline ();
+  m_text_or_html.add_newline ();
 }
 
 template<>
 void
-layout_printer<to_html>::print_gap_in_line_numbering ()
+layout_printer<diagnostics::to_html>::print_gap_in_line_numbering ()
 {
   gcc_assert (m_layout.m_options.show_line_numbers_p);
 
-  m_sink.add_raw_html
+  m_text_or_html.add_raw_html
     ("<tbody class=\"line-span-jump\">\n"
      "<tr class=\"line-span-jump-row\">"
      "<td class=\"linenum-gap\">[...]</td>"
@@ -2169,7 +2175,7 @@ layout::calculate_linenum_width ()
   int highest_line = last_span->m_last_line;
   if (highest_line < 0)
     highest_line = 0;
-  m_linenum_width = num_digits (highest_line);
+  m_linenum_width = diagnostics::num_digits (highest_line);
   /* If we're showing jumps in the line-numbering, allow at least 3 chars.  */
   if (m_line_spans.length () > 1)
     m_linenum_width = MAX (m_linenum_width, 3);
@@ -2196,8 +2202,9 @@ layout::calculate_x_offset_display ()
       return;
     }
 
-  const char_span line = m_file_cache.get_source_line (m_exploc.file,
-						       m_exploc.line);
+  const diagnostics::char_span line
+    = m_file_cache.get_source_line (m_exploc.file,
+				    m_exploc.line);
   if (!line)
     {
       /* Nothing to do, we couldn't find the source line.  */
@@ -2264,32 +2271,32 @@ layout::calculate_x_offset_display ()
    colorization and tab expansion, this function tracks the line position in
    both byte and display column units.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 line_bounds
-layout_printer<Sink>::print_source_line (linenum_type row,
+layout_printer<TextOrHtml>::print_source_line (linenum_type row,
 					 const char *line,
 					 int line_bytes)
 {
-  m_sink.colorize_text_ensure_normal ();
-  m_sink.push_html_tag ("tr", true);
-  m_sink.emit_text_prefix ();
+  m_text_or_html.colorize_text_ensure_normal ();
+  m_text_or_html.push_html_tag ("tr", true);
+  m_text_or_html.emit_text_prefix ();
   if (m_layout.m_options.show_line_numbers_p)
     {
-      m_sink.push_html_tag_with_class ("td", "linenum", true);
-      int width = num_digits (row);
+      m_text_or_html.push_html_tag_with_class ("td", "linenum", true);
+      int width = diagnostics::num_digits (row);
       for (int i = 0; i < m_layout.get_linenum_width () - width; i++)
-	m_sink.add_space ();
+	m_text_or_html.add_space ();
       char buf[20];
       sprintf (buf, "%i", row);
-      m_sink.add_text (buf);
-      if (Sink::is_text ())
-	m_sink.add_text (" |");
-      m_sink.pop_html_tag ("td");
+      m_text_or_html.add_text (buf);
+      if (TextOrHtml::is_text ())
+	m_text_or_html.add_text (" |");
+      m_text_or_html.pop_html_tag ("td");
     }
 
-  m_sink.push_html_tag_with_class ("td", "left-margin", true);
+  m_text_or_html.push_html_tag_with_class ("td", "left-margin", true);
   print_leftmost_column ();
-  m_sink.pop_html_tag ("td");
+  m_text_or_html.pop_html_tag ("td");
 
   /* We will stop printing the source line at any trailing whitespace.  */
   line_bytes = get_line_bytes_without_trailing_whitespace (line,
@@ -2300,7 +2307,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
      tab expansion, and for implementing m_x_offset_display.  */
   cpp_display_width_computation dw (line, line_bytes, m_layout.m_char_policy);
 
-  m_sink.push_html_tag_with_class ("td", "source", true);
+  m_text_or_html.push_html_tag_with_class ("td", "source", true);
 
   /* Skip the first m_x_offset_display display columns.  In case the leading
      portion that will be skipped ends with a character with wcwidth > 1, then
@@ -2312,7 +2319,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
   for (int skipped_display_cols
 	 = dw.advance_display_cols (m_layout.m_x_offset_display);
        skipped_display_cols > m_layout.m_x_offset_display; --skipped_display_cols)
-    m_sink.add_space ();
+    m_text_or_html.add_space ();
 
   /* Print the line and compute the line_bounds.  */
   line_bounds lbounds;
@@ -2356,7 +2363,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
 	  /* The returned display width is the number of spaces into which the
 	     tab should be expanded.  */
 	  for (int i = 0; i != this_display_width; ++i)
-	    m_sink.add_space ();
+	    m_text_or_html.add_space ();
 	  continue;
 	}
 
@@ -2370,7 +2377,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
 	}
 
       /* Output the character.  */
-      m_sink.print_decoded_char (m_layout.m_char_policy, cp);
+      m_text_or_html.print_decoded_char (m_layout.m_char_policy, cp);
       c = dw.next_byte ();
     }
   set_outside_range ();
@@ -2399,9 +2406,9 @@ layout::should_print_annotation_line_p (linenum_type row) const
 /* Print the leftmost column after the margin, which is used for showing
    links between labels (e.g. for CFG edges in execution paths).  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_leftmost_column ()
+layout_printer<TextOrHtml>::print_leftmost_column ()
 {
   if (!get_options ().show_event_links_p)
     gcc_assert (m_link_lhs_state == link_lhs_state::none);
@@ -2411,33 +2418,33 @@ layout_printer<Sink>::print_leftmost_column ()
     default:
       gcc_unreachable ();
     case link_lhs_state::none:
-      m_sink.add_space ();
+      m_text_or_html.add_space ();
       break;
     case link_lhs_state::rewinding_to_lhs:
       {
-	m_sink.colorize_text_for_cfg_edge ();
+	m_text_or_html.colorize_text_for_cfg_edge ();
 	const cppchar_t ch = get_theme ().get_cppchar
 	  (text_art::theme::cell_kind::CFG_FROM_LEFT_TO_DOWN);
-	m_sink.add_character (ch);
-	m_sink.colorize_text_ensure_normal ();
+	m_text_or_html.add_character (ch);
+	m_text_or_html.colorize_text_ensure_normal ();
       }
       break;
     case link_lhs_state::at_lhs:
       {
-	m_sink.colorize_text_for_cfg_edge ();
+	m_text_or_html.colorize_text_for_cfg_edge ();
 	const cppchar_t ch = get_theme ().get_cppchar
 	  (text_art::theme::cell_kind::CFG_DOWN);
-	m_sink.add_character (ch);
-	m_sink.colorize_text_ensure_normal ();
+	m_text_or_html.add_character (ch);
+	m_text_or_html.colorize_text_ensure_normal ();
       }
       break;
     case link_lhs_state::indenting_to_dest:
       {
-	m_sink.colorize_text_for_cfg_edge ();
+	m_text_or_html.colorize_text_for_cfg_edge ();
 	const cppchar_t ch = get_theme ().get_cppchar
 	  (text_art::theme::cell_kind::CFG_FROM_DOWN_TO_RIGHT);
-	m_sink.add_character (ch);
-	m_sink.colorize_text_ensure_normal ();
+	m_text_or_html.add_character (ch);
+	m_text_or_html.colorize_text_ensure_normal ();
       }
       break;
     }
@@ -2450,16 +2457,16 @@ layout_printer<Sink>::print_leftmost_column ()
    After any left margin, print a leftmost column, which is used for
    showing links between labels (e.g. for CFG edges in execution paths).
 
-   For text sinks, this also first prints the text prefix.
-   For html sinks, this also pushes <tr> and <td> open tags, where the
+   For text output, this also first prints the text prefix.
+   For html output, this also pushes <tr> and <td> open tags, where the
    <td> is for the coming annotations.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::start_annotation_line (enum margin_kind margin)
+layout_printer<TextOrHtml>::start_annotation_line (enum margin_kind margin)
 {
-  m_sink.emit_text_prefix ();
-  m_sink.push_html_tag ("tr", true);
+  m_text_or_html.emit_text_prefix ();
+  m_text_or_html.push_html_tag ("tr", true);
 
   char margin_char = (margin == margin_kind::insertion
 		      ? '+'
@@ -2469,29 +2476,29 @@ layout_printer<Sink>::start_annotation_line (enum margin_kind margin)
     {
       /* Print the margin.  If MARGIN_CHAR != ' ', then print up to 3
 	 of it, right-aligned, padded with spaces.  */
-      m_sink.push_html_tag_with_class ("td", "linenum", true);
+      m_text_or_html.push_html_tag_with_class ("td", "linenum", true);
       int i;
       for (i = 0; i < m_layout.m_linenum_width - 3; i++)
-	m_sink.add_space ();
+	m_text_or_html.add_space ();
       for (; i < m_layout.m_linenum_width; i++)
-	m_sink.add_character (margin_char);
-      if (Sink::is_text ())
-	m_sink.add_text (" |");
-      m_sink.pop_html_tag ("td");
+	m_text_or_html.add_character (margin_char);
+      if (TextOrHtml::is_text ())
+	m_text_or_html.add_text (" |");
+      m_text_or_html.pop_html_tag ("td");
     }
 
-  m_sink.push_html_tag_with_class ("td", "left-margin", true);
+  m_text_or_html.push_html_tag_with_class ("td", "left-margin", true);
   if (margin == margin_kind::insertion)
-    m_sink.add_character (margin_char);
+    m_text_or_html.add_character (margin_char);
   else
     print_leftmost_column ();
-  m_sink.pop_html_tag ("td");
+  m_text_or_html.pop_html_tag ("td");
 
-  m_sink.push_html_tag_with_class ("td",
-				   (margin == margin_kind::ruler
-				    ? "ruler"
-				    : "annotation"),
-				   true);
+  m_text_or_html.push_html_tag_with_class ("td",
+					   (margin == margin_kind::ruler
+					    ? "ruler"
+					    : "annotation"),
+					   true);
 }
 
 /* End a source or annotation line: text implementation.
@@ -2499,10 +2506,10 @@ layout_printer<Sink>::start_annotation_line (enum margin_kind margin)
 
 template<>
 void
-layout_printer<to_text>::end_line ()
+layout_printer<diagnostics::to_text>::end_line ()
 {
-  m_sink.colorize_text_ensure_normal ();
-  m_sink.add_newline ();
+  m_text_or_html.colorize_text_ensure_normal ();
+  m_text_or_html.add_newline ();
 }
 
 /* End a source or annotation line: HTML implementation.
@@ -2510,18 +2517,18 @@ layout_printer<to_text>::end_line ()
 
 template<>
 void
-layout_printer<to_html>::end_line ()
+layout_printer<diagnostics::to_html>::end_line ()
 {
-  m_sink.pop_html_tag ("td");
-  m_sink.pop_html_tag ("tr");
+  m_text_or_html.pop_html_tag ("td");
+  m_text_or_html.pop_html_tag ("tr");
 }
 
 /* Handle the various transitions between being-in-range and
    not-being-in-a-range, and between ranges.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::set_in_range (int range_idx)
+layout_printer<TextOrHtml>::set_in_range (int range_idx)
 {
   if (m_was_in_range_p)
     {
@@ -2541,9 +2548,9 @@ layout_printer<Sink>::set_in_range (int range_idx)
   m_last_range_idx = range_idx;
 }
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::set_outside_range ()
+layout_printer<TextOrHtml>::set_outside_range ()
 {
   if (m_was_in_range_p)
     /* transition from "in a range" to "not in a range".  */
@@ -2554,9 +2561,9 @@ layout_printer<Sink>::set_outside_range ()
 /* Print a line consisting of the caret/underlines for the given
    source line.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_annotation_line (linenum_type row,
+layout_printer<TextOrHtml>::print_annotation_line (linenum_type row,
 					     const line_bounds lbounds)
 {
   int x_bound = m_layout.get_x_bound_for_row (row,
@@ -2590,15 +2597,15 @@ layout_printer<Sink>::print_annotation_line (linenum_type row,
 		caret_char = get_options ().caret_chars[state.range_idx];
 	      else
 		caret_char = '^';
-	      m_sink.add_character (caret_char);
+	      m_text_or_html.add_character (caret_char);
 	    }
 	  else
-	    m_sink.add_character ('~');
+	    m_text_or_html.add_character ('~');
 	}
       else
 	{
 	  /* Not in a range.  */
-	  m_sink.add_character (' ');
+	  m_text_or_html.add_character (' ');
 	}
     }
 
@@ -2697,54 +2704,54 @@ public:
 
 template<>
 void
-layout_printer<to_text>::begin_label (int range_idx,
-				      bool is_label_text)
+layout_printer<diagnostics::to_text>::begin_label (int range_idx,
+						   bool is_label_text)
 {
   /* Colorize the text, unless it's for labels for events in a
-     diagnostic_path.  */
+     diagnostic path.  */
   if (is_label_text && m_is_diagnostic_path)
     return;
 
-  gcc_assert (m_sink.m_colorizer);
-  m_sink.m_colorizer->set_range (range_idx);
+  gcc_assert (m_text_or_html.m_colorizer);
+  m_text_or_html.m_colorizer->set_range (range_idx);
 }
 
 template<>
 void
-layout_printer<to_html>::begin_label (int range_idx,
-				      bool is_label_text)
+layout_printer<diagnostics::to_html>::begin_label (int range_idx,
+						   bool is_label_text)
 {
-  if (is_label_text && m_sink.m_html_label_writer)
-    m_sink.m_html_label_writer->begin_label ();
+  if (is_label_text && m_text_or_html.m_html_label_writer)
+    m_text_or_html.m_html_label_writer->begin_label ();
 
   if (const char *highlight_color
-	= m_sink.get_highlight_color_for_range_idx (range_idx))
-    m_sink.m_xp.push_tag_with_class ("span", highlight_color);
+	= m_text_or_html.get_highlight_color_for_range_idx (range_idx))
+    m_text_or_html.m_xp.push_tag_with_class ("span", highlight_color);
 }
 
 template<>
 void
-layout_printer<to_text>::end_label (int, bool)
+layout_printer<diagnostics::to_text>::end_label (int, bool)
 {
-  m_sink.colorize_text_ensure_normal ();
+  m_text_or_html.colorize_text_ensure_normal ();
 }
 
 template<>
 void
-layout_printer<to_html>::end_label (int range_idx,
-				    bool is_label_text)
+layout_printer<diagnostics::to_html>::end_label (int range_idx,
+						 bool is_label_text)
 {
-  if (m_sink.get_highlight_color_for_range_idx (range_idx))
-    m_sink.m_xp.pop_tag ("span");
+  if (m_text_or_html.get_highlight_color_for_range_idx (range_idx))
+    m_text_or_html.m_xp.pop_tag ("span");
 
-  if (is_label_text && m_sink.m_html_label_writer)
-    m_sink.m_html_label_writer->end_label ();
+  if (is_label_text && m_text_or_html.m_html_label_writer)
+    m_text_or_html.m_html_label_writer->end_label ();
 }
 
 /* Print any labels in this row.  */
-template <typename Sink>
+template <typename TextOrHtml>
 void
-layout_printer<Sink>::print_any_labels (linenum_type row)
+layout_printer<TextOrHtml>::print_any_labels (linenum_type row)
 {
   int i;
   auto_vec<line_label> labels;
@@ -2885,20 +2892,20 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		       . ^~~~~~~~~~~~~
 		       . this text.  */
 		    gcc_assert (get_options ().show_event_links_p);
-		    m_sink.colorize_text_for_cfg_edge ();
+		    m_text_or_html.colorize_text_for_cfg_edge ();
 		    const cppchar_t right= get_theme ().get_cppchar
 		      (text_art::theme::cell_kind::CFG_RIGHT);
 		    while (column < label->m_column - 1)
 		      {
-			m_sink.add_character (right);
+			m_text_or_html.add_character (right);
 			column++;
 		      }
 		    if (column == label->m_column - 1)
 		      {
-			m_sink.add_character ('>');
+			m_text_or_html.add_character ('>');
 			column++;
 		      }
-		    m_sink.colorize_text_ensure_normal ();
+		    m_text_or_html.colorize_text_ensure_normal ();
 		    m_link_lhs_state = link_lhs_state::none;
 		    label_line_with_in_edge = -1;
 		  }
@@ -2907,7 +2914,7 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		gcc_assert (column == label->m_column);
 
 		begin_label (label->m_state_idx, true);
-		m_sink.add_text (label->m_text.m_buffer);
+		m_text_or_html.add_text (label->m_text.m_buffer);
 		end_label (label->m_state_idx, true);
 
 		column += label->m_display_width;
@@ -2924,13 +2931,13 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		      (text_art::theme::cell_kind::CFG_RIGHT);
 		    const cppchar_t from_right_to_down= get_theme ().get_cppchar
 		      (text_art::theme::cell_kind::CFG_FROM_RIGHT_TO_DOWN);
-		    m_sink.colorize_text_for_cfg_edge ();
-		    m_sink.add_space ();
-		    m_sink.add_character (right);
-		    m_sink.add_character ('>');
-		    m_sink.add_character (right);
-		    m_sink.add_character (from_right_to_down);
-		    m_sink.colorize_text_ensure_normal ();
+		    m_text_or_html.colorize_text_for_cfg_edge ();
+		    m_text_or_html.add_space ();
+		    m_text_or_html.add_character (right);
+		    m_text_or_html.add_character ('>');
+		    m_text_or_html.add_character (right);
+		    m_text_or_html.add_character (from_right_to_down);
+		    m_text_or_html.colorize_text_ensure_normal ();
 		    column += 5;
 		    m_link_rhs_column = column - 1;
 		  }
@@ -2940,7 +2947,7 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		gcc_assert (column <= label->m_column);
 		move_to_column (&column, label->m_column, true);
 		begin_label (label->m_state_idx, false);
-		m_sink.add_character ('|');
+		m_text_or_html.add_character ('|');
 		end_label (label->m_state_idx, false);
 		column++;
 	      }
@@ -2951,11 +2958,11 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 	if (m_link_rhs_column != -1 && column < m_link_rhs_column)
 	  {
 	    move_to_column (&column, m_link_rhs_column, true);
-	    m_sink.colorize_text_for_cfg_edge ();
+	    m_text_or_html.colorize_text_for_cfg_edge ();
 	    const cppchar_t down= get_theme ().get_cppchar
 	      (text_art::theme::cell_kind::CFG_DOWN);
-	    m_sink.add_character (down);
-	    m_sink.colorize_text_ensure_normal ();
+	    m_text_or_html.add_character (down);
+	    m_text_or_html.colorize_text_ensure_normal ();
 	  }
 
 	end_line ();
@@ -2969,10 +2976,10 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
       int column = 1 + m_layout.m_x_offset_display;
       start_annotation_line (margin_kind::normal);
       move_to_column (&column, m_link_rhs_column, true);
-      m_sink.colorize_text_for_cfg_edge ();
+      m_text_or_html.colorize_text_for_cfg_edge ();
       const cppchar_t down= get_theme ().get_cppchar
 	(text_art::theme::cell_kind::CFG_DOWN);
-      m_sink.add_character (down);
+      m_text_or_html.add_character (down);
       end_line ();
     }
 
@@ -2990,9 +2997,9 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
    They are printed on lines of their own, before the source line
    itself, with a leading '+'.  */
 
-template <typename Sink>
+template <typename TextOrHtml>
 void
-layout_printer<Sink>::print_leading_fixits (linenum_type row)
+layout_printer<TextOrHtml>::print_leading_fixits (linenum_type row)
 {
   for (unsigned int i = 0; i < m_layout.m_fixit_hints.length (); i++)
     {
@@ -3012,14 +3019,14 @@ layout_printer<Sink>::print_leading_fixits (linenum_type row)
 	     and the inserted line with "insert" colorization
 	     helps them stand out from each other, and from
 	     the surrounding text.  */
-	  m_sink.colorize_text_ensure_normal ();
+	  m_text_or_html.colorize_text_ensure_normal ();
 	  start_annotation_line (margin_kind::insertion);
-	  m_sink.colorize_text_for_fixit_insert ();
+	  m_text_or_html.colorize_text_for_fixit_insert ();
 	  /* Print all but the trailing newline of the fix-it hint.
 	     We have to print the newline separately to avoid
 	     getting additional pp prefixes printed.  */
 	  for (size_t i = 0; i < hint->get_length () - 1; i++)
-	    m_sink.add_character (hint->get_string ()[i]);
+	    m_text_or_html.add_character (hint->get_string ()[i]);
 	  end_line ();
 	}
     }
@@ -3154,7 +3161,7 @@ public:
 
 /* Get the range of bytes or display columns that HINT would affect.  */
 static column_range
-get_affected_range (file_cache &fc,
+get_affected_range (diagnostics::file_cache &fc,
 		    const cpp_char_column_policy &policy,
 		    const fixit_hint *hint, enum column_unit col_unit)
 {
@@ -3184,7 +3191,7 @@ get_affected_range (file_cache &fc,
 /* Get the range of display columns that would be printed for HINT.  */
 
 static column_range
-get_printed_columns (file_cache &fc,
+get_printed_columns (diagnostics::file_cache &fc,
 		     const cpp_char_column_policy &policy,
 		     const fixit_hint *hint)
 {
@@ -3246,7 +3253,7 @@ public:
     m_display_cols = cpp_display_width (m_text, m_byte_length, m_policy);
   }
 
-  void overwrite (int dst_offset, const char_span &src_span)
+  void overwrite (int dst_offset, const diagnostics::char_span &src_span)
   {
     gcc_assert (dst_offset >= 0);
     gcc_assert (dst_offset + src_span.length () < m_alloc_sz);
@@ -3307,7 +3314,7 @@ correction::ensure_terminated ()
 class line_corrections
 {
 public:
-  line_corrections (file_cache &fc,
+  line_corrections (diagnostics::file_cache &fc,
 		    const char_display_policy &policy,
 		    const char *filename,
 		    linenum_type row)
@@ -3318,7 +3325,7 @@ public:
 
   void add_hint (const fixit_hint *hint);
 
-  file_cache &m_file_cache;
+  diagnostics::file_cache &m_file_cache;
   const char_display_policy &m_policy;
   const char *m_filename;
   linenum_type m_row;
@@ -3341,9 +3348,12 @@ line_corrections::~line_corrections ()
 class source_line
 {
 public:
-  source_line (file_cache &fc, const char *filename, int line);
+  source_line (diagnostics::file_cache &fc, const char *filename, int line);
 
-  char_span as_span () { return char_span (chars, width); }
+  diagnostics::char_span as_span ()
+  {
+    return diagnostics::char_span (chars, width);
+  }
 
   const char *chars;
   int width;
@@ -3351,9 +3361,11 @@ public:
 
 /* source_line's ctor.  */
 
-source_line::source_line (file_cache &fc, const char *filename, int line)
+source_line::source_line (diagnostics::file_cache &fc,
+			  const char *filename,
+			  int line)
 {
-  char_span span = fc.get_source_line (filename, line);
+  diagnostics::char_span span = fc.get_source_line (filename, line);
   chars = span.get_buffer ();
   width = span.length ();
 }
@@ -3417,9 +3429,10 @@ line_corrections::add_hint (const fixit_hint *hint)
 		(old_byte_len,
 		 line.as_span ().subspan (between.start - 1,
 					  between.finish + 1 - between.start));
-	      last_correction->overwrite (old_byte_len + between_byte_len,
-					  char_span (hint->get_string (),
-						     hint->get_length ()));
+	      last_correction->overwrite
+		(old_byte_len + between_byte_len,
+		 diagnostics::char_span (hint->get_string (),
+					 hint->get_length ()));
 	      last_correction->m_byte_length = new_byte_len;
 	      last_correction->ensure_terminated ();
 	      last_correction->m_affected_bytes.finish
@@ -3450,11 +3463,11 @@ line_corrections::add_hint (const fixit_hint *hint)
    Fix-it hints that insert new lines are handled separately,
    in layout::print_leading_fixits.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_trailing_fixits (linenum_type row)
+layout_printer<TextOrHtml>::print_trailing_fixits (linenum_type row)
 {
-  typename Sink::auto_check_tag_nesting sentinel (m_sink);
+  typename TextOrHtml::auto_check_tag_nesting sentinel (m_text_or_html);
 
   /* Build a list of correction instances for the line,
      potentially consolidating hints (for the sake of readability).  */
@@ -3490,9 +3503,9 @@ layout_printer<Sink>::print_trailing_fixits (linenum_type row)
 	  /* This assumes the insertion just affects one line.  */
 	  int start_column = c->m_printed_columns.start;
 	  move_to_column (&column, start_column, true);
-	  m_sink.colorize_text_for_fixit_insert ();
-	  m_sink.add_text (c->m_text);
-	  m_sink.colorize_text_ensure_normal ();
+	  m_text_or_html.colorize_text_for_fixit_insert ();
+	  m_text_or_html.add_text (c->m_text);
+	  m_text_or_html.colorize_text_ensure_normal ();
 	  column += c->m_display_cols;
 	}
       else
@@ -3508,10 +3521,10 @@ layout_printer<Sink>::print_trailing_fixits (linenum_type row)
 	      || c->m_byte_length == 0)
 	    {
 	      move_to_column (&column, start_column, true);
-	      m_sink.colorize_text_for_fixit_delete ();
+	      m_text_or_html.colorize_text_for_fixit_delete ();
 	      for (; column <= finish_column; column++)
-		m_sink.add_character ('-');
-	      m_sink.colorize_text_ensure_normal ();
+		m_text_or_html.add_character ('-');
+	      m_text_or_html.colorize_text_ensure_normal ();
 	    }
 	  /* Print the replacement text.  REPLACE also covers
 	     removals, so only do this extra work (potentially starting
@@ -3519,9 +3532,9 @@ layout_printer<Sink>::print_trailing_fixits (linenum_type row)
 	  if (c->m_byte_length > 0)
 	    {
 	      move_to_column (&column, start_column, true);
-	      m_sink.colorize_text_for_fixit_insert ();
-	      m_sink.add_text (c->m_text);
-	      m_sink.colorize_text_ensure_normal ();
+	      m_text_or_html.colorize_text_for_fixit_insert ();
+	      m_text_or_html.add_text (c->m_text);
+	      m_text_or_html.colorize_text_ensure_normal ();
 	      column += c->m_display_cols;
 	    }
 	}
@@ -3627,9 +3640,9 @@ layout::get_x_bound_for_row (linenum_type row, int caret_column,
    and updating *COLUMN.  If ADD_LEFT_MARGIN, then print the (empty)
    left margin after any newline.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::move_to_column (int *column,
+layout_printer<TextOrHtml>::move_to_column (int *column,
 				      int dest_column,
 				      bool add_left_margin)
 {
@@ -3646,9 +3659,9 @@ layout_printer<Sink>::move_to_column (int *column,
     {
       /* For debugging column issues, it can be helpful to replace this
 	 add_space call with
-	   m_sink.add_character ('0' + (*column % 10));
+	   m_text_or_html.add_character ('0' + (*column % 10));
 	 to visualize the changing value of "*column".  */
-      m_sink.add_space ();
+      m_text_or_html.add_space ();
       (*column)++;
     }
 }
@@ -3656,11 +3669,11 @@ layout_printer<Sink>::move_to_column (int *column,
 /* For debugging layout issues, render a ruler giving column numbers
    (after the 1-column indent).  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::show_ruler (int max_column)
+layout_printer<TextOrHtml>::show_ruler (int max_column)
 {
-  m_sink.push_html_tag_with_class("thead", "ruler", false);
+  m_text_or_html.push_html_tag_with_class("thead", "ruler", false);
 
   /* Hundreds.  */
   if (max_column > 99)
@@ -3670,9 +3683,9 @@ layout_printer<Sink>::show_ruler (int max_column)
 	   column <= max_column;
 	   ++column)
 	if (column % 10 == 0)
-	  m_sink.add_character ('0' + (column / 100) % 10);
+	  m_text_or_html.add_character ('0' + (column / 100) % 10);
 	else
-	  m_sink.add_space ();
+	  m_text_or_html.add_space ();
       end_line ();
     }
 
@@ -3682,9 +3695,9 @@ layout_printer<Sink>::show_ruler (int max_column)
        column <= max_column;
        ++column)
     if (column % 10 == 0)
-      m_sink.add_character ('0' + (column / 10) % 10);
+      m_text_or_html.add_character ('0' + (column / 10) % 10);
     else
-      m_sink.add_space ();
+      m_text_or_html.add_space ();
   end_line ();
 
   /* Units.  */
@@ -3692,23 +3705,23 @@ layout_printer<Sink>::show_ruler (int max_column)
   for (int column = 1 + m_layout.m_x_offset_display;
        column <= max_column;
        ++column)
-    m_sink.add_character ('0' + (column % 10));
+    m_text_or_html.add_character ('0' + (column % 10));
   end_line ();
 
-  m_sink.pop_html_tag("thead"); // thead
+  m_text_or_html.pop_html_tag("thead"); // thead
 }
 
 /* Print leading fix-its (for new lines inserted before the source line)
    then the source line, followed by an annotation line
    consisting of any caret/underlines, then any fixits.
    If the source line can't be read, print nothing.  */
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_line (linenum_type row)
+layout_printer<TextOrHtml>::print_line (linenum_type row)
 {
-  typename Sink::auto_check_tag_nesting sentinel (m_sink);
+  typename TextOrHtml::auto_check_tag_nesting sentinel (m_text_or_html);
 
-  char_span line
+  diagnostics::char_span line
     = m_layout.m_file_cache.get_source_line (m_layout.m_exploc.file, row);
   if (!line)
     return;
@@ -3730,9 +3743,9 @@ layout_printer<Sink>::print_line (linenum_type row)
    showing the link entering at the top right and emerging
    at the bottom left.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_any_right_to_left_edge_lines ()
+layout_printer<TextOrHtml>::print_any_right_to_left_edge_lines ()
 {
   if (m_link_rhs_column == -1)
     /* Can also happen if the out-edge had UNKNOWN_LOCATION.  */
@@ -3745,25 +3758,25 @@ layout_printer<Sink>::print_any_right_to_left_edge_lines ()
 
   int column = 1 + m_layout.m_x_offset_display;
   move_to_column (&column, m_link_rhs_column, true);
-  m_sink.colorize_text_for_cfg_edge ();
+  m_text_or_html.colorize_text_for_cfg_edge ();
   const cppchar_t down= get_theme ().get_cppchar
     (text_art::theme::cell_kind::CFG_DOWN);
-  m_sink.add_character (down);
+  m_text_or_html.add_character (down);
   end_line ();
 
   /* Print the line with "┌──────────────────────────────────────────┘".  */
   m_link_lhs_state = link_lhs_state::rewinding_to_lhs;
   start_annotation_line (margin_kind::normal);
-  m_sink.colorize_text_for_cfg_edge ();
+  m_text_or_html.colorize_text_for_cfg_edge ();
   const cppchar_t left= get_theme ().get_cppchar
     (text_art::theme::cell_kind::CFG_LEFT);
   for (int column = 1 + m_layout.m_x_offset_display;
        column < m_link_rhs_column;
        ++column)
-    m_sink.add_character (left);
+    m_text_or_html.add_character (left);
   const cppchar_t from_down_to_left = get_theme ().get_cppchar
     (text_art::theme::cell_kind::CFG_FROM_DOWN_TO_LEFT);
-  m_sink.add_character (from_down_to_left);
+  m_text_or_html.add_character (from_down_to_left);
   end_line ();
 
   /* We now have a link line on the LHS,
@@ -3772,11 +3785,11 @@ layout_printer<Sink>::print_any_right_to_left_edge_lines ()
   m_link_rhs_column = -1;
 }
 
-template<typename Sink>
-layout_printer<Sink>::layout_printer (Sink &sink,
+template<typename TextOrHtml>
+layout_printer<TextOrHtml>::layout_printer (TextOrHtml &text_or_html,
 				      const layout &layout,
 				      bool is_diagnostic_path)
-: m_sink (sink),
+: m_text_or_html (text_or_html),
   m_layout (layout),
   m_is_diagnostic_path (is_diagnostic_path),
   m_was_in_range_p (false),
@@ -3801,7 +3814,7 @@ layout_printer<Sink>::layout_printer (Sink &sink,
 
 bool
 gcc_rich_location::
-add_location_if_nearby (const diagnostic_source_print_policy &policy,
+add_location_if_nearby (const diagnostics::source_print_policy &policy,
 			location_t loc,
 			bool restrict_to_current_line_spans,
 			const range_label *label)
@@ -3825,27 +3838,29 @@ add_location_if_nearby (const diagnostic_source_print_policy &policy,
 
 bool
 gcc_rich_location::
-add_location_if_nearby (const diagnostic_context &dc,
+add_location_if_nearby (const diagnostics::context &dc,
 			location_t loc,
 			bool restrict_to_current_line_spans,
 			const range_label *label)
 {
-  diagnostic_source_print_policy source_policy (dc,
-						dc.m_source_printing);
+  diagnostics::source_print_policy
+    source_policy (dc,
+		   dc.get_source_printing_options ());
   return add_location_if_nearby (source_policy, loc,
 				 restrict_to_current_line_spans, label);
 }
 
+namespace diagnostics {
 
-/* As per diagnostic_source_print_policy::print, but don't print anything
+/* As per diagnostics::source_print_policy::print, but don't print anything
    if source printing is disabled, or if the location hasn't changed.  */
 
 void
-diagnostic_context::maybe_show_locus (const rich_location &richloc,
-				      const diagnostic_source_printing_options &opts,
-				      diagnostic_t diagnostic_kind,
-				      pretty_printer &pp,
-				      diagnostic_source_effect_info *effects)
+context::maybe_show_locus (const rich_location &richloc,
+			   const source_printing_options &opts,
+			   enum kind diagnostic_kind,
+			   pretty_printer &pp,
+			   source_effect_info *effects)
 {
   const location_t loc = richloc.get_loc ();
   /* Do nothing if source-printing has been disabled.  */
@@ -3866,7 +3881,7 @@ diagnostic_context::maybe_show_locus (const rich_location &richloc,
 
   m_last_location = loc;
 
-  diagnostic_source_print_policy source_policy (*this, opts);
+  source_print_policy source_policy (*this, opts);
   source_policy.print (pp, richloc, diagnostic_kind, effects);
 }
 
@@ -3874,12 +3889,12 @@ diagnostic_context::maybe_show_locus (const rich_location &richloc,
    If non-null, use LABEL_WRITER when writing labelled ranges.  */
 
 void
-diagnostic_context::maybe_show_locus_as_html (const rich_location &richloc,
-					      const diagnostic_source_printing_options &opts,
-					      diagnostic_t diagnostic_kind,
-					      xml::printer &xp,
-					      diagnostic_source_effect_info *effects,
-					      html_label_writer *label_writer)
+context::maybe_show_locus_as_html (const rich_location &richloc,
+				   const source_printing_options &opts,
+				   enum kind diagnostic_kind,
+				   xml::printer &xp,
+				   source_effect_info *effects,
+				   html_label_writer *label_writer)
 {
   const location_t loc = richloc.get_loc ();
   /* Do nothing if source-printing has been disabled.  */
@@ -3900,13 +3915,13 @@ diagnostic_context::maybe_show_locus_as_html (const rich_location &richloc,
 
   m_last_location = loc;
 
-  diagnostic_source_print_policy source_policy (*this, opts);
+  source_print_policy source_policy (*this, opts);
   source_policy.print_as_html (xp, richloc, diagnostic_kind, effects,
 			       label_writer);
 }
 
-diagnostic_source_print_policy::
-diagnostic_source_print_policy (const diagnostic_context &dc)
+source_print_policy::
+source_print_policy (const context &dc)
 : m_options (dc.m_source_printing),
   m_location_policy (dc),
   m_text_start_span_cb (dc.m_text_callbacks.m_text_start_span),
@@ -3917,9 +3932,8 @@ diagnostic_source_print_policy (const diagnostic_context &dc)
 {
 }
 
-diagnostic_source_print_policy::
-diagnostic_source_print_policy (const diagnostic_context &dc,
-				const diagnostic_source_printing_options &opts)
+source_print_policy::source_print_policy (const context &dc,
+					  const source_printing_options &opts)
 : m_options (opts),
   m_location_policy (dc),
   m_text_start_span_cb (dc.m_text_callbacks.m_text_start_span),
@@ -3936,17 +3950,16 @@ diagnostic_source_print_policy (const diagnostic_context &dc,
    If EFFECTS is non-null, then use and update it.  */
 
 void
-diagnostic_source_print_policy::print (pretty_printer &pp,
-				       const rich_location &richloc,
-				       diagnostic_t diagnostic_kind,
-				       diagnostic_source_effect_info *effects)
-  const
+source_print_policy::print (pretty_printer &pp,
+			    const rich_location &richloc,
+			    enum kind diagnostic_kind,
+			    source_effect_info *effects) const
 {
   layout layout (*this, richloc, effects);
   colorizer col (pp, richloc, diagnostic_kind);
-  to_text sink (pp, col);
-  layout_printer<to_text> lp (sink, layout,
-			      diagnostic_kind == DK_DIAGNOSTIC_PATH);
+  to_text text_or_html (pp, col);
+  layout_printer<to_text> lp (text_or_html, layout,
+			      diagnostic_kind == diagnostics::kind::path);
   lp.print (*this);
 }
 
@@ -3954,28 +3967,30 @@ diagnostic_source_print_policy::print (pretty_printer &pp,
    If non-null, use LABEL_WRITER when writing labelled ranges.  */
 
 void
-diagnostic_source_print_policy::print_as_html (xml::printer &xp,
-					       const rich_location &richloc,
-					       diagnostic_t diagnostic_kind,
-					       diagnostic_source_effect_info *effects,
-					       html_label_writer *label_writer)
-  const
+source_print_policy::print_as_html (xml::printer &xp,
+				    const rich_location &richloc,
+				    enum kind diagnostic_kind,
+				    source_effect_info *effects,
+				    html_label_writer *label_writer) const
 {
   layout layout (*this, richloc, effects);
-  to_html sink (xp, &richloc, label_writer);
-  layout_printer<to_html> lp (sink, layout,
-			      diagnostic_kind == DK_DIAGNOSTIC_PATH);
+  to_html text_or_html (xp, &richloc, label_writer);
+  layout_printer<to_html> lp (text_or_html, layout,
+			      diagnostic_kind == diagnostics::kind::path);
   xml::auto_check_tag_nesting sentinel (xp);
   lp.print (*this);
 }
 
-template <typename Sink>
+} // namespace diagnostics
+
+template <typename TextOrHtml>
 void
-layout_printer<Sink>::print (const diagnostic_source_print_policy &source_policy)
+layout_printer<TextOrHtml>::
+print (const diagnostics::source_print_policy &source_policy)
 {
-  typename Sink::auto_check_tag_nesting sentinel (m_sink);
+  typename TextOrHtml::auto_check_tag_nesting sentinel (m_text_or_html);
 
-  m_sink.push_html_tag_with_class ("table", "locus", false);
+  m_text_or_html.push_html_tag_with_class ("table", "locus", false);
 
   if (get_options ().show_ruler_p)
     show_ruler (m_layout.m_x_offset_display + get_options ().max_width);
@@ -3998,13 +4013,13 @@ layout_printer<Sink>::print (const diagnostic_source_print_policy &source_policy
 	    {
 	      expanded_location exploc
 		= m_layout.get_expanded_location (line_span);
-	      const diagnostic_location_print_policy &
+	      const diagnostics::location_print_policy &
 		loc_policy = source_policy.get_location_policy ();
-	      m_sink.invoke_start_span_fn (source_policy, loc_policy, exploc);
+	      m_text_or_html.invoke_start_span_fn (source_policy, loc_policy, exploc);
 	    }
 	}
 
-      m_sink.push_html_tag_with_class ("tbody", "line-span", false);
+      m_text_or_html.push_html_tag_with_class ("tbody", "line-span", false);
 
       /* Iterate over the lines within this span (using linenum_arith_t to
 	 avoid overflow with 0xffffffff causing an infinite loop).  */
@@ -4013,30 +4028,37 @@ layout_printer<Sink>::print (const diagnostic_source_print_policy &source_policy
 	   row <= last_line; row++)
 	print_line (row);
 
-      m_sink.pop_html_tag ("tbody");
+      m_text_or_html.pop_html_tag ("tbody");
     }
 
   if (auto effect_info = m_layout.m_effect_info)
     effect_info->m_trailing_out_edge_column = m_link_rhs_column;
 
-  m_sink.pop_html_tag ("table");
+  m_text_or_html.pop_html_tag ("table");
 }
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
+using test_context = diagnostics::selftest::test_context;
+
+using line_table_case = ::selftest::line_table_case;
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+
 static std::unique_ptr<xml::node>
 make_element_for_locus (const rich_location &rich_loc,
-			diagnostic_t kind,
-			diagnostic_context &dc)
+			enum kind kind,
+			diagnostics::context &dc)
 {
   dc.m_last_location = UNKNOWN_LOCATION;
 
   xml::element wrapper ("wrapper", false);
   xml::printer xp (wrapper);
   dc.maybe_show_locus_as_html (rich_loc,
-			       dc.m_source_printing,
+			       dc.get_source_printing_options (),
 			       kind,
 			       xp,
 			       nullptr,
@@ -4049,8 +4071,8 @@ make_element_for_locus (const rich_location &rich_loc,
 
 static label_text
 make_raw_html_for_locus (const rich_location &rich_loc,
-			 diagnostic_t kind,
-			 diagnostic_context &dc)
+			 enum kind kind,
+			 diagnostics::context &dc)
 {
   auto node = make_element_for_locus (rich_loc, kind, dc);
   pretty_printer pp;
@@ -4061,9 +4083,9 @@ make_raw_html_for_locus (const rich_location &rich_loc,
 
 /* Selftests for diagnostic_show_locus.  */
 
-diagnostic_show_locus_fixture::
-diagnostic_show_locus_fixture (const line_table_case &case_,
-			       const char *content)
+source_printing_fixture::
+source_printing_fixture (const line_table_case &case_,
+			 const char *content)
 : m_content (content),
   m_tmp_source_file (SELFTEST_LOCATION, ".c", content),
   m_ltt (case_),
@@ -4076,10 +4098,10 @@ diagnostic_show_locus_fixture (const line_table_case &case_,
 /* Populate a char_display_policy based on DC and RICHLOC.  */
 
 static char_display_policy
-make_char_policy (const diagnostic_context &dc,
+make_char_policy (const diagnostics::context &dc,
 		  const rich_location &richloc)
 {
-  diagnostic_source_print_policy source_policy (dc);
+  diagnostics::source_print_policy source_policy (dc);
   return ::make_char_policy (source_policy, richloc);
 }
 
@@ -4101,7 +4123,7 @@ test_display_widths ()
 
   /* No escaping.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     char_display_policy policy (make_char_policy (dc, richloc));
     ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 1);
     ASSERT_EQ (cpp_display_width (emoji, strlen (emoji), policy), 2);
@@ -4112,7 +4134,7 @@ test_display_widths ()
   richloc.set_escape_on_output (true);
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
     char_display_policy policy (make_char_policy (dc, richloc));
     ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 8);
@@ -4124,7 +4146,7 @@ test_display_widths ()
   }
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
     char_display_policy policy (make_char_policy (dc, richloc));
     ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 8);
@@ -4149,13 +4171,14 @@ test_offset_impl (int caret_byte_col, int max_width,
 		  int expected_x_offset_display,
 		  int left_margin = test_left_margin)
 {
-  test_diagnostic_context dc;
-  dc.m_source_printing.max_width = max_width;
-  /* diagnostic_context::min_margin_width sets the minimum space reserved for
+  test_context dc;
+  auto &source_printing_opts = dc.get_source_printing_options ();
+  source_printing_opts.max_width = max_width;
+  /* min_margin_width sets the minimum space reserved for
      the line number plus one space after.  */
-  dc.m_source_printing.min_margin_width = left_margin - test_linenum_sep + 1;
-  dc.m_source_printing.show_line_numbers_p = true;
-  diagnostic_source_print_policy source_policy (dc);
+  source_printing_opts.min_margin_width = left_margin - test_linenum_sep + 1;
+  dc.show_line_numbers (true);
+  diagnostics::source_print_policy source_policy (dc);
   rich_location richloc (line_table,
 			 linemap_position_for_column (line_table,
 						      caret_byte_col));
@@ -4188,7 +4211,7 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
      no multibyte characters earlier on the line.  */
   const int emoji_col = 102;
 
-  diagnostic_show_locus_fixture f (case_, content);
+  source_printing_fixture f (case_, content);
 
   linemap_add (line_table, LC_ENTER, false, f.get_filename (), 1);
 
@@ -4202,7 +4225,7 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
   ASSERT_EQ (1, LOCATION_LINE (line_end));
   ASSERT_EQ (line_bytes, LOCATION_COLUMN (line_end));
 
-  char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
+  diagnostics::char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
   ASSERT_EQ (line_display_cols,
 	     cpp_display_width (lspan.get_buffer (), lspan.length (),
 				def_policy ()));
@@ -4265,21 +4288,22 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
 
   /* Test that the source line is offset as expected when printed.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.max_width = small_width - 6;
-    dc.m_source_printing.min_margin_width
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    source_printing_opts.max_width = small_width - 6;
+    source_printing_opts.min_margin_width
       = test_left_margin - test_linenum_sep + 1;
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_ruler_p = true;
-    diagnostic_source_print_policy policy (dc);
+    dc.show_line_numbers (true);
+    dc.show_ruler (true);
+    diagnostics::source_print_policy policy (dc);
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
 							emoji_col));
     layout test_layout (policy, richloc, nullptr);
     colorizer col (*dc.get_reference_printer (),
-		   richloc, DK_ERROR);
-    to_text sink (*dc.get_reference_printer (), col);
-    layout_printer<to_text> lp (sink, test_layout, false);
+		   richloc, diagnostics::kind::error);
+    diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+    layout_printer<diagnostics::to_text> lp (text_or_html, test_layout, false);
     lp.print (policy);
     ASSERT_STREQ ("     |         1         \n"
 		  "     |         1         \n"
@@ -4295,21 +4319,22 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
      the first emoji in the middle of the UTF-8 sequence.  Check that we replace
      it with a padding space in this case.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.max_width = small_width - 5;
-    dc.m_source_printing.min_margin_width
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    source_printing_opts.max_width = small_width - 5;
+    source_printing_opts.min_margin_width
       = test_left_margin - test_linenum_sep + 1;
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_ruler_p = true;
-    diagnostic_source_print_policy policy (dc);
+    dc.show_line_numbers (true);
+    dc.show_ruler (true);
+    diagnostics::source_print_policy policy (dc);
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
 							emoji_col + 2));
     layout test_layout (dc, richloc, nullptr);
     colorizer col (*dc.get_reference_printer (),
-		   richloc, DK_ERROR);
-    to_text sink (*dc.get_reference_printer (), col);
-    layout_printer<to_text> lp (sink, test_layout, false);
+		   richloc, diagnostics::kind::error);
+    diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+    layout_printer<diagnostics::to_text> lp (text_or_html, test_layout, false);
     lp.print (policy);
     ASSERT_STREQ ("     |        1         1 \n"
 		  "     |        1         2 \n"
@@ -4354,7 +4379,7 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
   ASSERT_EQ (7, extra_width[10]);
 
   temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
-  file_cache fc;
+  diagnostics::file_cache fc;
   line_table_test ltt (case_);
 
   linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 1);
@@ -4366,7 +4391,7 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
     return;
 
   /* Check that cpp_display_width handles the tabs as expected.  */
-  char_span lspan = fc.get_source_line (tmp.get_filename (), 1);
+  diagnostics::char_span lspan = fc.get_source_line (tmp.get_filename (), 1);
   ASSERT_EQ ('\t', *(lspan.get_buffer () + (tab_col - 1)));
   for (int tabstop = 1; tabstop != num_tabstops; ++tabstop)
     {
@@ -4386,14 +4411,15 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
 						      tab_col + 1));
   for (int tabstop = 1; tabstop != num_tabstops; ++tabstop)
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.m_tabstop = tabstop;
-      diagnostic_source_print_policy policy (dc);
+      diagnostics::source_print_policy policy (dc);
       layout test_layout (policy, richloc, nullptr);
       colorizer col (*dc.get_reference_printer (),
-		     richloc, DK_ERROR);
-      to_text sink (*dc.get_reference_printer (), col);
-      layout_printer<to_text> lp (sink, test_layout, false);
+		     richloc, diagnostics::kind::error);
+      diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+      layout_printer<diagnostics::to_text> lp
+	(text_or_html, test_layout, false);
       lp.print (policy);
       const char *out = pp_formatted_text (dc.get_reference_printer ());
       ASSERT_EQ (nullptr, strchr (out, '\t'));
@@ -4409,19 +4435,21 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
      over.  */
   for (int tabstop = 1; tabstop != num_tabstops; ++tabstop)
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.m_tabstop = tabstop;
       static const int small_width = 24;
-      dc.m_source_printing.max_width = small_width - 4;
-      dc.m_source_printing.min_margin_width
+      auto &source_printing_opts = dc.get_source_printing_options ();
+      source_printing_opts.max_width = small_width - 4;
+      source_printing_opts.min_margin_width
 	= test_left_margin - test_linenum_sep + 1;
-      dc.m_source_printing.show_line_numbers_p = true;
-      diagnostic_source_print_policy policy (dc);
+      dc.show_line_numbers (true);
+      diagnostics::source_print_policy policy (dc);
       layout test_layout (policy, richloc, nullptr);
       colorizer col (*dc.get_reference_printer (),
-		     richloc, DK_ERROR);
-      to_text sink (*dc.get_reference_printer (), col);
-      layout_printer<to_text> lp (sink, test_layout, false);
+		     richloc, diagnostics::kind::error);
+      diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+      layout_printer<diagnostics::to_text> lp
+	(text_or_html, test_layout, false);
       lp.print (policy);
 
       /* We have arranged things so that two columns will be printed before
@@ -4447,7 +4475,7 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
 static void
 test_diagnostic_show_locus_unknown_location ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   rich_location richloc (line_table, UNKNOWN_LOCATION);
   ASSERT_STREQ ("", dc.test_show_locus (richloc));
 }
@@ -4467,7 +4495,7 @@ test_diagnostic_show_locus_unknown_location ()
 static void
 test_one_liner_simple_caret ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 10);
   rich_location richloc (line_table, caret);
   ASSERT_STREQ (" foo = bar.field;\n"
@@ -4481,7 +4509,7 @@ test_one_liner_simple_caret ()
 static void
 test_one_liner_no_column ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 0);
   rich_location richloc (line_table, caret);
   ASSERT_STREQ (" foo = bar.field;\n",
@@ -4493,7 +4521,7 @@ test_one_liner_no_column ()
 static void
 test_one_liner_caret_and_range ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 10);
   location_t start = linemap_position_for_column (line_table, 7);
   location_t finish = linemap_position_for_column (line_table, 15);
@@ -4504,8 +4532,8 @@ test_one_liner_caret_and_range ()
 		dc.test_show_locus (richloc));
 
   {
-    test_diagnostic_context dc;
-    auto out = make_raw_html_for_locus (richloc, DK_ERROR, dc);
+    test_context dc;
+    auto out = make_raw_html_for_locus (richloc, diagnostics::kind::error, dc);
      ASSERT_STREQ
        ("<table class=\"locus\">\n"
 	"  <tbody class=\"line-span\">\n"
@@ -4516,9 +4544,9 @@ test_one_liner_caret_and_range ()
 	out.get ());
   }
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_line_numbers_p = true;
-    auto out = make_raw_html_for_locus (richloc, DK_ERROR, dc);
+    test_context dc;
+    dc.show_line_numbers (true);
+    auto out = make_raw_html_for_locus (richloc, diagnostics::kind::error, dc);
      ASSERT_STREQ
        ("<table class=\"locus\">\n"
 	"  <tbody class=\"line-span\">\n"
@@ -4535,24 +4563,24 @@ test_one_liner_caret_and_range ()
 static void
 test_one_liner_multiple_carets_and_ranges ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t foo
     = make_location (linemap_position_for_column (line_table, 2),
 		     linemap_position_for_column (line_table, 1),
 		     linemap_position_for_column (line_table, 3));
-  dc.m_source_printing.caret_chars[0] = 'A';
+  dc.set_caret_char (0, 'A');
 
   location_t bar
     = make_location (linemap_position_for_column (line_table, 8),
 		     linemap_position_for_column (line_table, 7),
 		     linemap_position_for_column (line_table, 9));
-  dc.m_source_printing.caret_chars[1] = 'B';
+  dc.set_caret_char (1, 'B');
 
   location_t field
     = make_location (linemap_position_for_column (line_table, 13),
 		     linemap_position_for_column (line_table, 11),
 		     linemap_position_for_column (line_table, 15));
-  dc.m_source_printing.caret_chars[2] = 'C';
+  dc.set_caret_char (2, 'C');
 
   rich_location richloc (line_table, foo);
   richloc.add_range (bar, SHOW_RANGE_WITH_CARET);
@@ -4567,7 +4595,7 @@ test_one_liner_multiple_carets_and_ranges ()
 static void
 test_one_liner_fixit_insert_before ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 7);
   rich_location richloc (line_table, caret);
   richloc.add_fixit_insert_before ("&");
@@ -4582,7 +4610,7 @@ test_one_liner_fixit_insert_before ()
 static void
 test_one_liner_fixit_insert_after ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 1);
   location_t finish = linemap_position_for_column (line_table, 3);
   location_t foo = make_location (start, start, finish);
@@ -4609,7 +4637,7 @@ test_one_liner_fixit_remove ()
 
   /* Normal.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "          ^~~~~~\n"
 		  "          ------\n",
@@ -4618,7 +4646,7 @@ test_one_liner_fixit_remove ()
 
   /* Test of adding a prefix.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     pp_prefixing_rule (dc.get_reference_printer ()) = DIAGNOSTICS_SHOW_PREFIX_EVERY_LINE;
     pp_set_prefix (dc.get_reference_printer (), xstrdup ("TEST PREFIX:"));
     ASSERT_STREQ ("TEST PREFIX: foo = bar.field;\n"
@@ -4629,9 +4657,10 @@ test_one_liner_fixit_remove ()
 
   /* Normal, with ruler.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_ruler_p = true;
-    dc.m_source_printing.max_width = 104;
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    dc.show_ruler (true);
+    source_printing_opts.max_width = 104;
     ASSERT_STREQ ("          0         0         0         0         0         0         0         0         0         1    \n"
 		  "          1         2         3         4         5         6         7         8         9         0    \n"
 		  " 12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234\n"
@@ -4643,9 +4672,10 @@ test_one_liner_fixit_remove ()
 
   /* Test of adding a prefix, with ruler.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_ruler_p = true;
-    dc.m_source_printing.max_width = 50;
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    dc.show_ruler (true);
+    source_printing_opts.max_width = 50;
     pp_prefixing_rule (dc.get_reference_printer ()) = DIAGNOSTICS_SHOW_PREFIX_EVERY_LINE;
     pp_set_prefix (dc.get_reference_printer (), xstrdup ("TEST PREFIX:"));
     ASSERT_STREQ ("TEST PREFIX:          1         2         3         4         5\n"
@@ -4658,10 +4688,11 @@ test_one_liner_fixit_remove ()
 
   /* Test of adding a prefix, with ruler and line numbers.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_ruler_p = true;
-    dc.m_source_printing.max_width = 50;
-    dc.m_source_printing.show_line_numbers_p = true;
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    dc.show_ruler (true);
+    source_printing_opts.max_width = 50;
+    dc.show_line_numbers (true);
     pp_prefixing_rule (dc.get_reference_printer ()) = DIAGNOSTICS_SHOW_PREFIX_EVERY_LINE;
     pp_set_prefix (dc.get_reference_printer (), xstrdup ("TEST PREFIX:"));
     ASSERT_STREQ ("TEST PREFIX:      |          1         2         3         4         5\n"
@@ -4678,7 +4709,7 @@ test_one_liner_fixit_remove ()
 static void
 test_one_liner_fixit_replace ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 11);
   location_t finish = linemap_position_for_column (line_table, 15);
   location_t field = make_location (start, start, finish);
@@ -4696,7 +4727,7 @@ test_one_liner_fixit_replace ()
 static void
 test_one_liner_fixit_replace_non_equal_range ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   location_t start = linemap_position_for_column (line_table, 11);
   location_t finish = linemap_position_for_column (line_table, 15);
@@ -4721,7 +4752,7 @@ test_one_liner_fixit_replace_non_equal_range ()
 static void
 test_one_liner_fixit_replace_equal_secondary_range ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   location_t start = linemap_position_for_column (line_table, 11);
   location_t finish = linemap_position_for_column (line_table, 15);
@@ -4762,7 +4793,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~~~~~~~~                               "
 		  "          "
@@ -4788,7 +4819,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~~~~~~~~                               "
 		  "          "
@@ -4825,7 +4856,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~~~~~~~~                               "
 		  "          "
@@ -4849,7 +4880,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
 static void
 test_one_liner_many_fixits_1 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   rich_location richloc (line_table, equals);
   for (int i = 0; i < 19; i++)
@@ -4867,7 +4898,7 @@ test_one_liner_many_fixits_1 ()
 static void
 test_one_liner_many_fixits_2 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   rich_location richloc (line_table, equals);
   for (int i = 0; i < 19; i++)
@@ -4910,7 +4941,7 @@ test_one_liner_labels ()
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
     {
-      test_diagnostic_context dc;
+      test_context dc;
       ASSERT_STREQ (" foo = bar.field;\n"
 		    " ^~~   ~~~ ~~~~~\n"
 		    " |     |   |\n"
@@ -4920,8 +4951,8 @@ test_one_liner_labels ()
 
     /* Verify that we can disable label-printing.  */
     {
-      test_diagnostic_context dc;
-      dc.m_source_printing.show_labels_p = false;
+      test_context dc;
+      dc.show_labels (false);
       ASSERT_STREQ (" foo = bar.field;\n"
 		    " ^~~   ~~~ ~~~~~\n",
 		    dc.test_show_locus (richloc));
@@ -4937,7 +4968,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ^~~   ~~~ ~~~~~\n"
 		  " |     |   |\n"
@@ -4947,9 +4978,10 @@ test_one_liner_labels ()
 		  dc.test_show_locus (richloc));
 
     {
-      test_diagnostic_context dc;
-      dc.m_source_printing.show_line_numbers_p = true;
-      auto out = make_raw_html_for_locus (richloc, DK_ERROR, dc);
+      test_context dc;
+      dc.show_line_numbers (true);
+      auto out
+	= make_raw_html_for_locus (richloc, diagnostics::kind::error, dc);
       ASSERT_STREQ
 	("<table class=\"locus\">\n"
 	 "  <tbody class=\"line-span\">\n"
@@ -4975,7 +5007,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ^~~   ~~~ ~~~~~\n"
 		  " |     |   |\n"
@@ -4993,7 +5025,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (foo, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ~~~   ~~~ ^~~~~\n"
 		  " |     |   |\n"
@@ -5011,7 +5043,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~\n"
 		  "       |\n"
@@ -5045,7 +5077,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label_1c);
     richloc.add_range (foo, SHOW_RANGE_WITHOUT_CARET, &label_2c);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ~~~   ~~~ ^~~~~\n"
 		  " |     |   |\n"
@@ -5067,7 +5099,7 @@ test_one_liner_labels ()
     text_range_label label (nullptr);
     gcc_rich_location richloc (bar, &label, nullptr);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~\n",
 		  dc.test_show_locus (richloc));
@@ -5087,7 +5119,7 @@ test_diagnostic_show_locus_one_liner (const line_table_case &case_)
      ....................1234567890123456.  */
   const char *content = "foo = bar.field;\n";
 
-  diagnostic_show_locus_fixture f (case_, content);
+  source_printing_fixture f (case_, content);
 
   location_t line_end = linemap_position_for_column (line_table, 16);
 
@@ -5117,7 +5149,7 @@ test_diagnostic_show_locus_one_liner (const line_table_case &case_)
 
 /* Version of all one-liner tests exercising multibyte awareness.
    These are all called from test_diagnostic_show_locus_one_liner,
-   which uses diagnostic_show_locus_fixture_one_liner_utf8 to create
+   which uses source_printing_fixture_one_liner_utf8 to create
    the test file; see the notes in diagnostic-show-locus-selftest.h.
 
    Note: all of the below asserts would be easier to read if we used UTF-8
@@ -5132,7 +5164,7 @@ test_diagnostic_show_locus_one_liner (const line_table_case &case_)
 static void
 test_one_liner_simple_caret_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 18);
   rich_location richloc (line_table, caret);
   ASSERT_STREQ (" \xf0\x9f\x98\x82"
@@ -5148,7 +5180,7 @@ test_one_liner_simple_caret_utf8 ()
 static void
 test_one_liner_caret_and_range_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 18);
   location_t start = linemap_position_for_column (line_table, 12);
   location_t finish = linemap_position_for_column (line_table, 30);
@@ -5168,24 +5200,24 @@ test_one_liner_caret_and_range_utf8 ()
 static void
 test_one_liner_multiple_carets_and_ranges_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t foo
     = make_location (linemap_position_for_column (line_table, 7),
 		     linemap_position_for_column (line_table, 1),
 		     linemap_position_for_column (line_table, 8));
-  dc.m_source_printing.caret_chars[0] = 'A';
+  dc.set_caret_char (0, 'A');
 
   location_t bar
     = make_location (linemap_position_for_column (line_table, 16),
 		     linemap_position_for_column (line_table, 12),
 		     linemap_position_for_column (line_table, 17));
-  dc.m_source_printing.caret_chars[1] = 'B';
+  dc.set_caret_char (1, 'B');
 
   location_t field
     = make_location (linemap_position_for_column (line_table, 26),
 		     linemap_position_for_column (line_table, 19),
 		     linemap_position_for_column (line_table, 30));
-  dc.m_source_printing.caret_chars[2] = 'C';
+  dc.set_caret_char (2, 'C');
   rich_location richloc (line_table, foo);
   richloc.add_range (bar, SHOW_RANGE_WITH_CARET);
   richloc.add_range (field, SHOW_RANGE_WITH_CARET);
@@ -5203,7 +5235,7 @@ test_one_liner_multiple_carets_and_ranges_utf8 ()
 static void
 test_one_liner_fixit_insert_before_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 12);
   rich_location richloc (line_table, caret);
   richloc.add_fixit_insert_before ("&");
@@ -5222,7 +5254,7 @@ test_one_liner_fixit_insert_before_utf8 ()
 static void
 test_one_liner_fixit_insert_after_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 1);
   location_t finish = linemap_position_for_column (line_table, 8);
   location_t foo = make_location (start, start, finish);
@@ -5243,7 +5275,7 @@ test_one_liner_fixit_insert_after_utf8 ()
 static void
 test_one_liner_fixit_remove_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 18);
   location_t finish = linemap_position_for_column (line_table, 30);
   location_t dot = make_location (start, start, finish);
@@ -5264,7 +5296,7 @@ test_one_liner_fixit_remove_utf8 ()
 static void
 test_one_liner_fixit_replace_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 19);
   location_t finish = linemap_position_for_column (line_table, 30);
   location_t field = make_location (start, start, finish);
@@ -5287,7 +5319,7 @@ test_one_liner_fixit_replace_utf8 ()
 static void
 test_one_liner_fixit_replace_non_equal_range_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   location_t start = linemap_position_for_column (line_table, 19);
   location_t finish = linemap_position_for_column (line_table, 30);
@@ -5317,7 +5349,7 @@ test_one_liner_fixit_replace_non_equal_range_utf8 ()
 static void
 test_one_liner_fixit_replace_equal_secondary_range_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   location_t start = linemap_position_for_column (line_table, 19);
   location_t finish = linemap_position_for_column (line_table, 30);
@@ -5363,7 +5395,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5393,7 +5425,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5434,7 +5466,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5462,7 +5494,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
 static void
 test_one_liner_many_fixits_1_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   rich_location richloc (line_table, equals);
   for (int i = 0; i < 19; i++)
@@ -5485,7 +5517,7 @@ test_one_liner_many_fixits_1_utf8 ()
 static void
 test_one_liner_many_fixits_2_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   rich_location richloc (line_table, equals);
   const int nlocs = 19;
@@ -5546,7 +5578,7 @@ test_one_liner_labels_utf8 ()
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
     {
-      test_diagnostic_context dc;
+      test_context dc;
       ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		       "_foo = \xcf\x80"
 			       "_bar.\xf0\x9f\x98\x82"
@@ -5572,7 +5604,7 @@ test_one_liner_labels_utf8 ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5596,7 +5628,7 @@ test_one_liner_labels_utf8 ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5621,7 +5653,7 @@ test_one_liner_labels_utf8 ()
     richloc.set_escape_on_output (true);
 
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
       ASSERT_STREQ (" <U+1F602>_foo = <U+03C0>_bar.<U+1F602>_field<U+03C0>;\n"
 		    " ^~~~~~~~~~~~~   ~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~\n"
@@ -5632,7 +5664,7 @@ test_one_liner_labels_utf8 ()
 		    dc.test_show_locus (richloc));
     }
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
       ASSERT_STREQ
 	(" <f0><9f><98><82>_foo = <cf><80>_bar.<f0><9f><98><82>_field<cf><80>;\n"
@@ -5651,8 +5683,8 @@ test_one_liner_labels_utf8 ()
 static void
 test_one_liner_colorized_utf8 ()
 {
-  test_diagnostic_context dc;
-  dc.m_source_printing.colorize_source_p = true;
+  test_context dc;
+  dc.colorize_source (true);
   diagnostic_color_init (&dc, DIAGNOSTICS_COLOR_YES);
   const location_t pi = linemap_position_for_column (line_table, 12);
   rich_location richloc (line_table, pi);
@@ -5675,9 +5707,9 @@ static const char * const one_liner_utf8_content
      1111222233334444567890122223333456789999000011112222345678999900001
      Byte columns.  */
 
-diagnostic_show_locus_fixture_one_liner_utf8::
-diagnostic_show_locus_fixture_one_liner_utf8 (const line_table_case &case_)
-: diagnostic_show_locus_fixture (case_, one_liner_utf8_content)
+source_printing_fixture_one_liner_utf8::
+source_printing_fixture_one_liner_utf8 (const line_table_case &case_)
+: source_printing_fixture (case_, one_liner_utf8_content)
 {
 }
 
@@ -5686,7 +5718,7 @@ diagnostic_show_locus_fixture_one_liner_utf8 (const line_table_case &case_)
 static void
 test_diagnostic_show_locus_one_liner_utf8 (const line_table_case &case_)
 {
-  diagnostic_show_locus_fixture_one_liner_utf8 f (case_);
+  source_printing_fixture_one_liner_utf8 f (case_);
 
   location_t line_end = linemap_position_for_column (line_table, 31);
 
@@ -5698,7 +5730,7 @@ test_diagnostic_show_locus_one_liner_utf8 (const line_table_case &case_)
   ASSERT_EQ (1, LOCATION_LINE (line_end));
   ASSERT_EQ (31, LOCATION_COLUMN (line_end));
 
-  char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
+  diagnostics::char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
   ASSERT_EQ (25, cpp_display_width (lspan.get_buffer (), lspan.length (),
 				    def_policy ()));
   ASSERT_EQ (25, location_compute_display_column (f.m_fc,
@@ -5755,7 +5787,7 @@ test_add_location_if_nearby (const line_table_case &case_)
   /* Test of add_location_if_nearby on the same line as the
      primary location.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t missing_close_brace_1_39
       = linemap_position_for_line_and_column (line_table, ord_map, 1, 39);
     const location_t matching_open_brace_1_18
@@ -5773,7 +5805,7 @@ test_add_location_if_nearby (const line_table_case &case_)
   /* Test of add_location_if_nearby on a different line to the
      primary location.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t missing_close_brace_6_1
       = linemap_position_for_line_and_column (line_table, ord_map, 6, 1);
     const location_t matching_open_brace_3_1
@@ -5822,7 +5854,7 @@ test_diagnostic_show_locus_fixit_lines (const line_table_case &case_)
 
   /* The one-liner case (line 2).  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t x
       = linemap_position_for_line_and_column (line_table, ord_map, 2, 24);
     const location_t colon
@@ -5841,7 +5873,7 @@ test_diagnostic_show_locus_fixit_lines (const line_table_case &case_)
      span starts are printed due to the gap between the span at line 3
      and that at line 6).  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t y
       = linemap_position_for_line_and_column (line_table, ord_map, 3, 24);
     const location_t colon
@@ -5869,8 +5901,8 @@ test_diagnostic_show_locus_fixit_lines (const line_table_case &case_)
     rich_location richloc (line_table, colon);
     richloc.add_fixit_insert_before (y, ".");
     richloc.add_fixit_replace (colon, "=");
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_line_numbers_p = true;
+    test_context dc;
+    dc.show_line_numbers (true);
     ASSERT_STREQ ("    3 |                        y\n"
 		  "      |                        .\n"
 		  "......\n"
@@ -6043,7 +6075,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
   const char *content
     = ("  foo *f = (foo *)ptr->field;\n");
   temp_source_file tmp (SELFTEST_LOCATION, ".C", content);
-  file_cache fc;
+  diagnostics::file_cache fc;
   line_table_test ltt (case_);
 
   const line_map_ordinary *ord_map
@@ -6075,7 +6107,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example where 3 fix-it hints are printed as one.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "const_cast<");
     richloc.add_fixit_replace (close_paren, "> (");
@@ -6144,7 +6176,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example where two are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CAST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6160,7 +6192,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example where none are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6176,7 +6208,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example of deletion fix-it hints.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(bar *)");
     source_range victim = {open_paren, close_paren};
@@ -6195,7 +6227,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example of deletion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(longer *)");
     source_range victim = {expr_start, expr_finish};
@@ -6214,7 +6246,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example of insertion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "LONGER THAN THE CAST");
     richloc.add_fixit_insert_after (close_paren, "TEST");
@@ -6280,8 +6312,8 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example where 3 fix-it hints are printed as one.  */
   {
-    test_diagnostic_context dc;
-    file_cache &fc = dc.get_file_cache ();
+    test_context dc;
+    diagnostics::file_cache &fc = dc.get_file_cache ();
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "const_cast<");
     richloc.add_fixit_replace (close_paren, "> (");
@@ -6356,7 +6388,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example where two are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CAST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6375,7 +6407,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example where none are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6394,7 +6426,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example of deletion fix-it hints.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(bar\xf0\x9f\x98\x82 *)");
     source_range victim = {open_paren, close_paren};
@@ -6417,7 +6449,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example of deletion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(long\xf0\x9f\x98\x82 *)");
     source_range victim = {expr_start, expr_finish};
@@ -6441,7 +6473,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example of insertion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before
       (open_paren, "L\xf0\x9f\x98\x82NGER THAN THE CAST");
@@ -6505,8 +6537,8 @@ test_overlapped_fixit_printing_2 (const line_table_case &case_)
 
   /* Two insertions, in the wrong order.  */
   {
-    test_diagnostic_context dc;
-    file_cache &fc = dc.get_file_cache ();
+    test_context dc;
+    diagnostics::file_cache &fc = dc.get_file_cache ();
 
     rich_location richloc (line_table, col_20);
     richloc.add_fixit_insert_before (col_23, "{");
@@ -6534,7 +6566,7 @@ test_overlapped_fixit_printing_2 (const line_table_case &case_)
   /* Various overlapping insertions, some occurring "out of order"
      (reproducing the fix-it hints from PR c/81405).  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, col_20);
 
     richloc.add_fixit_insert_before (col_20, "{{");
@@ -6588,7 +6620,7 @@ test_fixit_insert_containing_newline (const line_table_case &case_)
 
     /* Without line numbers.  */
     {
-      test_diagnostic_context dc;
+      test_context dc;
       ASSERT_STREQ ("       x = a;\n"
 		    "+      break;\n"
 		    "     case 'b':\n"
@@ -6598,8 +6630,8 @@ test_fixit_insert_containing_newline (const line_table_case &case_)
 
     /* With line numbers.  */
     {
-      test_diagnostic_context dc;
-      dc.m_source_printing.show_line_numbers_p = true;
+      test_context dc;
+      dc.show_line_numbers (true);
       ASSERT_STREQ ("    2 |       x = a;\n"
 		    "  +++ |+      break;\n"
 		    "    3 |     case 'b':\n"
@@ -6614,7 +6646,7 @@ test_fixit_insert_containing_newline (const line_table_case &case_)
     rich_location richloc (line_table, case_loc);
     richloc.add_fixit_insert_before (case_start, "break;\n");
     ASSERT_TRUE (richloc.seen_impossible_fixit_p ());
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ ("     case 'b':\n"
 		  "     ^~~~~~~~~\n",
 		  dc.test_show_locus (richloc));
@@ -6661,7 +6693,7 @@ test_fixit_insert_containing_newline_2 (const line_table_case &case_)
     return;
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ ("FILENAME:1:1:\n"
 		  "+#include <stdio.h>\n"
 		  " test (int ch)\n"
@@ -6674,8 +6706,8 @@ test_fixit_insert_containing_newline_2 (const line_table_case &case_)
   /* With line-numbering, the line spans are close enough to be
      consolidated, since it makes little sense to skip line 2.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_line_numbers_p = true;
+    test_context dc;
+    dc.show_line_numbers (true);
     ASSERT_STREQ ("  +++ |+#include <stdio.h>\n"
 		  "    1 | test (int ch)\n"
 		  "    2 | {\n"
@@ -6717,7 +6749,7 @@ test_fixit_replace_containing_newline (const line_table_case &case_)
   if (finish > LINE_MAP_MAX_LOCATION_WITH_COLS)
     return;
 
-  test_diagnostic_context dc;
+  test_context dc;
   ASSERT_STREQ (" foo = bar ();\n"
 		"             ^\n",
 		dc.test_show_locus (richloc));
@@ -6760,7 +6792,7 @@ test_fixit_deletion_affecting_newline (const line_table_case &case_)
   if (finish > LINE_MAP_MAX_LOCATION_WITH_COLS)
     return;
 
-  test_diagnostic_context dc;
+  test_context dc;
   ASSERT_STREQ (" foo = bar (\n"
 		"          ~^\n"
 		"       );\n"
@@ -6800,7 +6832,7 @@ test_tab_expansion (const line_table_case &case_)
      into 11 spaces.  Recall that print_line() also puts one space before
      everything too.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.m_tabstop = tabstop;
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
@@ -6813,7 +6845,7 @@ test_tab_expansion (const line_table_case &case_)
   /* Confirm the display width was tracked correctly across the internal tab
      as well.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.m_tabstop = tabstop;
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
@@ -6854,14 +6886,14 @@ test_escaping_bytes_1 (const line_table_case &case_)
   richloc.add_range (v_loc);
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" before \1\2\3\v\x80\xff""after\n"
 		  "       ^   ~\n",
 		  dc.test_show_locus (richloc));
   }
   richloc.set_escape_on_output (true);
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
     ASSERT_STREQ
       (" before<U+0000><U+0001><U+0002><U+0003><U+000B><80><ff>after\n"
@@ -6869,7 +6901,7 @@ test_escaping_bytes_1 (const line_table_case &case_)
        dc.test_show_locus (richloc));
   }
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
     ASSERT_STREQ (" before<00><01><02><03><0b><80><ff>after\n"
 		  "       ^~~~            ~~~~\n",
@@ -6904,21 +6936,21 @@ test_escaping_bytes_2 (const line_table_case &case_)
   gcc_rich_location richloc (nul_loc);
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ ("  after\n"
 		  " ^\n",
 		  dc.test_show_locus (richloc));
   }
   richloc.set_escape_on_output (true);
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
     ASSERT_STREQ (" <U+0000>after\n"
 		  " ^~~~~~~~\n",
 		  dc.test_show_locus (richloc));
   }
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
     ASSERT_STREQ (" <00>after\n"
 		  " ^~~~\n",
@@ -6957,9 +6989,10 @@ test_line_numbers_multiline_range ()
     = linemap_position_for_line_and_column (line_table, ord_map, 11, 4);
   location_t loc = make_location (caret, start, finish);
 
-  test_diagnostic_context dc;
-  dc.m_source_printing.show_line_numbers_p = true;
-  dc.m_source_printing.min_margin_width = 0;
+  test_context dc;
+  dc.show_line_numbers (true);
+  auto &source_printing_opts = dc.get_source_printing_options ();
+  source_printing_opts.min_margin_width = 0;
   gcc_rich_location richloc (loc);
   ASSERT_STREQ (" 9 | this is line 9\n"
 		"   |         ~~~~~~\n"
@@ -6973,7 +7006,7 @@ test_line_numbers_multiline_range ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_show_locus_cc_tests ()
+source_printing_cc_tests ()
 {
   test_line_span ();
 
@@ -7009,7 +7042,8 @@ diagnostic_show_locus_cc_tests ()
   test_line_numbers_multiline_range ();
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
diff --git a/gcc/diagnostics/state-graphs-to-dot.cc b/gcc/diagnostics/state-graphs-to-dot.cc
new file mode 100644
index 0000000..2d80e6b
--- /dev/null
+++ b/gcc/diagnostics/state-graphs-to-dot.cc
@@ -0,0 +1,551 @@
+/* Creating GraphViz .dot files from diagnostic state graphs.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#define INCLUDE_ALGORITHM
+#define INCLUDE_MAP
+#define INCLUDE_SET
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+
+#include "diagnostics/state-graphs.h"
+#include "graphviz.h"
+#include "xml.h"
+#include "xml-printer.h"
+#include "intl.h"
+
+using namespace diagnostics;
+using namespace diagnostics::state_graphs;
+
+static int
+get_depth (const digraphs::node &n)
+{
+  int deepest_child = 0;
+  for (size_t i = 0; i < n.get_num_children (); ++i)
+      deepest_child = std::max (deepest_child,
+				get_depth (n.get_child (i)));
+  return deepest_child + 1;
+}
+
+static const char *
+get_color_for_dynalloc_state (enum node_dynalloc_state dynalloc_st)
+{
+  switch (dynalloc_st)
+    {
+    default:
+      gcc_unreachable ();
+      break;
+    case node_dynalloc_state::unknown:
+    case node_dynalloc_state::nonnull:
+      return nullptr;
+
+    case node_dynalloc_state::unchecked:
+      return "#ec7a08"; // pf-orange-400
+
+    case node_dynalloc_state::freed:
+      return "#cc0000"; // pf-red-100
+    }
+}
+
+static void
+set_color_for_dynalloc_state (dot::attr_list &attrs,
+			      enum node_dynalloc_state state)
+{
+  if (const char *color = get_color_for_dynalloc_state (state))
+    attrs.add (dot::id ("color"), dot::id (color));
+}
+
+class state_diagram : public dot::graph
+{
+public:
+  state_diagram (const diagnostics::digraphs::digraph &input_state_graph,
+		 const logical_locations::manager &logical_loc_mgr)
+  : m_logical_loc_mgr (logical_loc_mgr)
+  {
+    // "node [shape=plaintext]\n"
+    {
+      auto attr_stmt
+	= std::make_unique<dot::attr_stmt> (dot::attr_stmt::kind::node);
+      attr_stmt->m_attrs.add (dot::id ("shape"), dot::id ("plaintext"));
+      add_stmt (std::move (attr_stmt));
+    }
+
+    /* Determine which nodes are involved in edges.  */
+    for (size_t i = 0; i < input_state_graph.get_num_edges (); ++i)
+      {
+	auto &edge = input_state_graph.get_edge (i);
+	m_src_nodes.insert (&edge.get_src_node ());
+	m_dst_nodes.insert (&edge.get_dst_node ());
+      }
+
+    /* Recurse down the nodes in the state graph, creating subgraphs
+       and then eventually creating nodes, and recursively
+       creating XML tables, and adding ports for the endpoints of edges
+       where needed.  */
+
+    auto root_cluster
+      = std::make_unique<dot::subgraph> (dot::id ("cluster_memory_regions"));
+    for (size_t i = 0; i < input_state_graph.get_num_nodes (); ++i)
+      on_input_state_node (*root_cluster,
+			   state_node_ref (input_state_graph.get_node (i)));
+    add_stmt (std::move (root_cluster));
+
+    /* Now create dot edges for edges in input_stage_graph.  */
+    for (size_t i = 0; i < input_state_graph.get_num_edges (); ++i)
+      {
+	auto &edge = input_state_graph.get_edge (i);
+	auto &src_node = edge.get_src_node ();
+	auto &dst_node = edge.get_dst_node ();
+
+	auto src_port_id = m_src_node_to_port_id.find (&src_node);
+	if (src_port_id == m_src_node_to_port_id.end ())
+	  continue;
+	auto dst_port_id = m_dst_node_to_port_id.find (&dst_node);
+	if (dst_port_id == m_dst_node_to_port_id.end ())
+	  continue;
+
+	auto e = std::make_unique<dot::edge_stmt> (src_port_id->second,
+						   dst_port_id->second);
+	set_color_for_dynalloc_state
+	  (e->m_attrs, state_node_ref (dst_node).get_dynalloc_state ());
+
+	add_stmt (std::move (e));
+      }
+  }
+
+private:
+  struct pending_edge
+  {
+    dot::node_id m_src_node_id;
+    std::string m_dst_region_id;
+  };
+
+  dot::id
+  get_id_for_region (const char *region_id)
+  {
+    gcc_assert (region_id);
+    return std::string ("cluster_region_") + region_id;
+  }
+
+  dot::id
+  make_id (state_node_ref state_node, bool cluster)
+  {
+    std::string input_node_id = state_node.m_node.get_id ();
+    if (cluster)
+      return std::string ("cluster_") + input_node_id;
+    else
+      return input_node_id;
+  }
+
+  bool
+  starts_node_p (state_node_ref state_node)
+  {
+    switch (state_node.get_node_kind ())
+      {
+      default:
+	return false;
+
+      case node_kind::stack:
+	/* We want all frames in the stack in the same table,
+	   so they are grouped.  */
+      case node_kind::dynalloc_buffer:
+      case node_kind::variable:
+	return true;
+      }
+  }
+
+  const char *
+  get_label_for_node (state_node_ref state_node)
+  {
+    switch (state_node.get_node_kind ())
+      {
+      default:
+	return nullptr;
+
+      case node_kind::globals:
+	return _("Globals");
+      case node_kind::code:
+	return _("Code");
+      case node_kind::stack:
+	return _("Stack");
+      case node_kind::heap_:
+	return _("Heap");
+      }
+  }
+
+  void
+  on_input_state_node (dot::subgraph &parent_subgraph,
+		       state_node_ref state_node)
+  {
+    dot::id sg_id = make_id (state_node, true);
+
+    if (starts_node_p (state_node))
+      {
+	// Create node with table
+	xml::element table ("table", false);
+	xml::printer xp (table);
+	xp.set_attr ("border", "0");
+	xp.set_attr ("cellborder", "1");
+	xp.set_attr ("cellspacing", "0");
+
+	const int max_depth = get_depth (state_node.m_node);
+	const int num_columns = max_depth + 2;
+
+	dot::id id_of_dot_node = make_id (state_node, false);
+	on_node_in_table (id_of_dot_node, xp, state_node,
+			  max_depth, 0, num_columns);
+
+	auto node = std::make_unique<dot::node_stmt> (std::move (id_of_dot_node));
+	node->m_attrs.add (dot::id ("shape"),
+			   dot::id ("plaintext"));
+
+	// xml must be done by now
+
+	node->m_attrs.add (dot::id ("label"),
+			   dot::id (table));
+
+	parent_subgraph.m_stmt_list.add_stmt (std::move (node));
+      }
+    else
+      {
+	auto child_subgraph = std::make_unique<dot::subgraph> (std::move (sg_id));
+
+	if (const char *label = get_label_for_node (state_node))
+	  child_subgraph->add_attr (dot::id ("label"), dot::id (label));
+
+	// recurse:
+	for (size_t i = 0; i < state_node.m_node.get_num_children (); ++i)
+	  on_input_state_node (*child_subgraph,
+			       state_node.m_node.get_child (i));
+	parent_subgraph.m_stmt_list.add_stmt (std::move (child_subgraph));
+      }
+  }
+
+  enum class style { h1, h2 };
+
+  void
+  add_title_tr (const dot::id &id_of_dot_node,
+		xml::printer &xp,
+		int num_columns,
+		state_node_ref state_node,
+		std::string heading,
+		enum style styl,
+		enum node_dynalloc_state dynalloc_state)
+  {
+    xp.push_tag ("tr", true);
+    xp.push_tag ("td", false);
+    xp.set_attr ("colspan", std::to_string (num_columns));
+    xp.set_attr ("cellpadding", "5");
+
+    const char *bgcolor;
+    const char *color;
+    if (const char *c = get_color_for_dynalloc_state (dynalloc_state))
+      {
+	bgcolor = c;
+	color = "white";
+      }
+    else
+      switch (styl)
+	{
+	default:
+	  gcc_unreachable ();
+	case style::h1:
+	  // from diagnostics/html-sink.cc: HTML_STYLE .linenum
+	  bgcolor = "#0088ce";
+	  color = "white";
+	  break;
+	case style::h2:
+	  // from diagnostics/html-sink.cc: HTML_STYLE .events-hdr
+	  bgcolor = "#393f44"; // pf-black-800
+	  color = "white";
+	  break;
+	}
+
+    xp.set_attr ("bgcolor", bgcolor);
+    xp.push_tag ("font", false);
+    xp.set_attr ("color", color);
+    if (heading == "")
+      heading = " ";
+    xp.add_text (std::move (heading));
+    xp.pop_tag ("font");
+
+    maybe_add_dst_port (id_of_dot_node, xp, state_node);
+
+    xp.pop_tag ("td");
+    xp.pop_tag ("tr");
+  }
+
+  /* Recursively add <TR> to XP for STATE_NODE and its descendents.  */
+  void
+  on_node_in_table (const dot::id &id_of_dot_node,
+		    xml::printer &xp,
+		    state_node_ref state_node,
+		    int max_depth,
+		    int depth,
+		    int num_columns)
+  {
+    bool recurse = true;
+    auto input_node_kind = state_node.get_node_kind ();
+
+    switch (input_node_kind)
+      {
+      case node_kind::padding:
+      case node_kind::other:
+	return;
+
+      case node_kind::stack:
+	add_title_tr (id_of_dot_node, xp, num_columns, state_node, "Stack",
+		      style::h1,
+		      node_dynalloc_state::unknown);
+	break;
+      case node_kind::stack_frame:
+	if (auto logical_loc = state_node.get_logical_loc ())
+	  if (const char *function
+		= m_logical_loc_mgr.get_short_name (logical_loc))
+	    add_title_tr (id_of_dot_node, xp, num_columns, state_node,
+			  std::string ("Frame: ") + function,
+			  style::h2,
+			  node_dynalloc_state::unknown);
+	break;
+      case node_kind::dynalloc_buffer:
+	{
+	  enum node_dynalloc_state dynalloc_st
+	    = state_node.get_dynalloc_state ();
+	  const char *extents = state_node.get_dynamic_extents ();
+	  const char *type = state_node.get_type ();
+	  pretty_printer pp;
+	  switch (dynalloc_st)
+	    {
+	    default:
+	      gcc_unreachable ();
+
+	    case node_dynalloc_state::unknown:
+	    case node_dynalloc_state::nonnull:
+	      if (type)
+		{
+		if (extents)
+		  pp_printf (&pp, "%s (%s byte allocation)",
+			     type, extents);
+		else
+		  pp_printf (&pp, "%s", type);
+		}
+	      else
+		{
+		  if (extents)
+		    pp_printf (&pp, "%s byte allocation",
+			       extents);
+		}
+	      break;
+
+	    case node_dynalloc_state::unchecked:
+	      if (type)
+		{
+		  if (extents)
+		    pp_printf (&pp, "%s (unchecked %s byte allocation)",
+			       type, extents);
+		}
+	      else
+		{
+		  if (extents)
+		    pp_printf (&pp, "Unchecked %s byte allocation",
+			       extents);
+		}
+	      break;
+
+	    case node_dynalloc_state::freed:
+	      // TODO: show deallocator
+	      // TODO: show deallocation event
+	      pp_printf (&pp, "Freed buffer");
+	      break;
+	    }
+	  maybe_add_dst_port (id_of_dot_node, xp, state_node);
+	  add_title_tr (id_of_dot_node, xp, num_columns, state_node,
+			pp_formatted_text (&pp),
+			style::h2,
+			dynalloc_st);
+	}
+	break;
+
+      default:
+	{
+	  xp.push_tag ("tr", true);
+
+	  maybe_add_dst_port (id_of_dot_node, xp, state_node);
+
+	  if (depth > 0)
+	    {
+	      /* Indent, by create a <td> spanning "depth" columns.  */
+	      xp.push_tag ("td", false);
+	      xp.set_attr ("colspan", std::to_string (depth));
+	      xp.add_text (" "); // graphviz doesn't like <td/>
+	      xp.pop_tag ("td");
+	    }
+
+	  switch (input_node_kind)
+	    {
+	    default:
+	      break;
+	    case node_kind::variable:
+	      {
+		const char *name = state_node.get_name ();
+		gcc_assert (name);
+		xp.push_tag ("td", false);
+		maybe_add_dst_port (id_of_dot_node, xp, state_node);
+		push_src_text (xp);
+		xp.add_text (name);
+		pop_src_text (xp);
+		xp.pop_tag ("td");
+	      }
+	      break;
+	    case node_kind::element:
+	      {
+		const char *index = state_node.get_index ();
+		gcc_assert (index);
+		xp.push_tag ("td", false);
+		maybe_add_dst_port (id_of_dot_node, xp, state_node);
+		push_src_text (xp);
+		xp.add_text ("[");
+		xp.add_text (index);
+		xp.add_text ("]");
+		pop_src_text (xp);
+		xp.pop_tag ("td");
+	      }
+	      break;
+	    case node_kind::field:
+	      {
+		const char *name = state_node.get_name ();
+		gcc_assert (name);
+		xp.push_tag ("td", false);
+		maybe_add_dst_port (id_of_dot_node, xp, state_node);
+		push_src_text (xp);
+		xp.add_text (".");
+		xp.add_text (name);
+		pop_src_text (xp);
+		xp.pop_tag ("td");
+	      }
+	      break;
+	    }
+
+	  if (const char *type = state_node.get_type ())
+	    {
+	      xp.push_tag ("td", false);
+	      xp.set_attr ("align", "right");
+	      push_src_text (xp);
+	      xp.add_text (type);
+	      pop_src_text (xp);
+	      xp.pop_tag ("td");
+	    }
+
+	  if (const char *value = state_node.get_value ())
+	    {
+	      xp.push_tag ("td", false);
+	      xp.set_attr ("align", "left");
+	      maybe_add_src_port (id_of_dot_node, xp, state_node);
+	      push_src_text (xp);
+	      xp.add_text (value);
+	      pop_src_text (xp);
+	      xp.pop_tag ("td");
+	      recurse = false;
+	    }
+	  xp.pop_tag ("tr");
+	}
+	break;
+      }
+
+    if (recurse)
+      for (size_t i = 0; i < state_node.m_node.get_num_children (); ++i)
+	on_node_in_table (id_of_dot_node, xp,
+			  state_node.m_node.get_child (i),
+			  max_depth, depth + 1, num_columns);
+  }
+
+  void
+  push_src_text (xml::printer &xp)
+  {
+    xp.push_tag ("font");
+    xp.set_attr ("color", "blue");
+  }
+
+  void
+  pop_src_text (xml::printer &xp)
+  {
+    xp.pop_tag ("font");
+  }
+
+  /* If STATE_NODE is in m_src_nodes, add a port to XP for possible
+     incoming edges to use.  */
+
+  void
+  maybe_add_src_port (const dot::id &id_of_dot_node,
+		      xml::printer &xp,
+		      state_node_ref state_node)
+  {
+    auto iter = m_src_nodes.find (&state_node.m_node);
+    if (iter == m_src_nodes.end ())
+      return;
+
+    dot::id src_id = make_id (state_node, false);
+    dot::node_id node_id (id_of_dot_node,
+			  dot::port (src_id,
+				     dot::compass_pt::e));
+    m_src_node_to_port_id.insert ({&state_node.m_node, node_id});
+    xp.set_attr ("port", src_id.m_str);
+  }
+
+  /* If STATE_NODE is in m_dst_nodes, add a port to XP for possible
+     incoming edges to use.  */
+
+  void
+  maybe_add_dst_port (const dot::id &id_of_dot_node,
+		      xml::printer &xp,
+		      state_node_ref state_node)
+  {
+    auto iter = m_dst_nodes.find (&state_node.m_node);
+    if (iter == m_dst_nodes.end ())
+      return;
+
+    dot::id dst_id = make_id (state_node, false);
+    dot::node_id node_id (id_of_dot_node,
+			  dot::port (dst_id/*,
+					     dot::compass_pt::w*/));
+    m_dst_node_to_port_id.insert ({&state_node.m_node, node_id});
+    xp.set_attr ("port", dst_id.m_str);
+  }
+
+private:
+  const logical_locations::manager &m_logical_loc_mgr;
+
+  /* All nodes involved in edges (and thus will need a port).  */
+  std::set<digraphs::node *> m_src_nodes;
+  std::set<digraphs::node *> m_dst_nodes;
+
+  std::map<digraphs::node *, dot::node_id> m_src_node_to_port_id;
+  std::map<digraphs::node *, dot::node_id> m_dst_node_to_port_id;
+};
+
+std::unique_ptr<dot::graph>
+state_graphs::
+make_dot_graph (const digraphs::digraph &state_graph,
+		const logical_locations::manager &logical_loc_mgr)
+{
+  return std::make_unique<state_diagram> (state_graph, logical_loc_mgr);
+}
diff --git a/gcc/diagnostics/state-graphs.cc b/gcc/diagnostics/state-graphs.cc
new file mode 100644
index 0000000..5941c41
--- /dev/null
+++ b/gcc/diagnostics/state-graphs.cc
@@ -0,0 +1,156 @@
+/* Extensions to diagnostics::digraphs to support state graphs.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#define INCLUDE_ALGORITHM
+#define INCLUDE_MAP
+#define INCLUDE_SET
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+
+#include "diagnostics/state-graphs.h"
+#include "selftest.h"
+
+using namespace diagnostics::state_graphs;
+
+const char * const node_kind_strs[] = {
+  "globals",
+  "code",
+  "function",
+  "stack",
+  "stack-frame",
+  "heap",
+  "thread-local",
+  "dynalloc-buffer",
+  "variable",
+  "field",
+  "padding",
+  "element",
+  "other",
+};
+
+const char *
+diagnostics::state_graphs::node_kind_to_str (enum node_kind k)
+{
+  return node_kind_strs[static_cast<int> (k)];
+}
+
+// struct state_node_ref
+
+enum node_kind
+state_node_ref::get_node_kind () const
+{
+  const char *value = get_attr ("kind");
+  if (!value)
+    return node_kind::other;
+
+  for (size_t i = 0; i < ARRAY_SIZE (node_kind_strs); ++i)
+    if (!strcmp (node_kind_strs[i], value))
+      return static_cast<enum node_kind> (i);
+
+  return node_kind::other;
+}
+
+void
+state_node_ref::set_node_kind (enum node_kind k)
+{
+  set_attr ("kind", node_kind_to_str (k));
+}
+
+const char * const dynalloc_state_strs[] = {
+  "unknown",
+  "nonnull",
+  "unchecked",
+  "freed"
+};
+
+enum node_dynalloc_state
+state_node_ref::get_dynalloc_state () const
+{
+  const char *value = get_attr ("dynalloc-state");
+  if (!value)
+    return node_dynalloc_state::unknown;
+
+  for (size_t i = 0; i < ARRAY_SIZE (dynalloc_state_strs); ++i)
+    if (!strcmp (dynalloc_state_strs[i], value))
+      return static_cast<enum node_dynalloc_state> (i);
+  
+  return node_dynalloc_state::unknown;
+}
+
+void
+state_node_ref::set_dynalloc_state (enum node_dynalloc_state s) const
+{
+  set_attr ("dynalloc-state",
+	    dynalloc_state_strs[static_cast <size_t> (s)]);
+}
+
+const char *
+state_node_ref::get_dynamic_extents () const
+{
+  return m_node.get_attr (STATE_NODE_PREFIX, "dynamic-extents");
+}
+
+void
+state_node_ref::set_json_attr (const char *key,
+			       std::unique_ptr<json::value> value) const
+{
+  m_node.set_json_attr (STATE_NODE_PREFIX, key, std::move (value));
+}
+
+#if CHECKING_P
+
+namespace diagnostics {
+namespace selftest {
+
+static void
+test_node_attrs ()
+{
+  digraphs::digraph g;
+  digraphs::node n (g, "a");
+  state_node_ref node_ref (n);
+
+  ASSERT_EQ (node_ref.get_node_kind (), node_kind::other);
+  node_ref.set_node_kind (node_kind::stack);
+  ASSERT_EQ (node_ref.get_node_kind (), node_kind::stack);
+
+  ASSERT_EQ (node_ref.get_dynalloc_state (), node_dynalloc_state::unknown);
+  node_ref.set_dynalloc_state (node_dynalloc_state::freed);
+  ASSERT_EQ (node_ref.get_dynalloc_state (), node_dynalloc_state::freed);
+
+  ASSERT_EQ (node_ref.get_type (), nullptr);
+  node_ref.set_type ("const char *");
+  ASSERT_STREQ (node_ref.get_type (), "const char *");
+}
+
+/* Run all of the selftests within this file.  */
+
+void
+state_graphs_cc_tests ()
+{
+  test_node_attrs ();
+}
+
+} // namespace diagnostics::selftest
+} // namespace diagnostics
+
+#endif /* CHECKING_P */
diff --git a/gcc/diagnostics/state-graphs.h b/gcc/diagnostics/state-graphs.h
new file mode 100644
index 0000000..ad18f82
--- /dev/null
+++ b/gcc/diagnostics/state-graphs.h
@@ -0,0 +1,156 @@
+/* Extensions to diagnostics::digraphs to support state graphs.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_STATE_GRAPHS_H
+#define GCC_DIAGNOSTICS_STATE_GRAPHS_H
+
+#include "diagnostics/digraphs.h"
+#include "diagnostics/logical-locations.h"
+
+/* diagnostics::digraphs provides support for directed graphs.
+
+   diagnostics::state_graphs provides a way to extend these graphs
+   for representing "state graphs" i.e. a representation of the state
+   of memory inside a program, for use e.g. by -fanalyzer.
+
+   Specifically, nodes represent memory regions, and we use property bags
+   in these nodes to stash extra properties (e.g. what kind of memory region
+   a node is e.g. stack vs heap).  */
+
+class sarif_graph;
+namespace dot { class graph; }
+
+namespace diagnostics {
+namespace state_graphs {
+
+enum class node_kind
+{
+  // Memory regions
+  globals,
+  code,
+  function, // code within a particular function
+  stack,
+  stack_frame,
+  heap_,
+  thread_local_,
+
+  /* Dynamically-allocated buffer,
+     on heap or stack (depending on parent).  */
+  dynalloc_buffer,
+
+  variable,
+
+  field, // field within a struct or union
+  padding, // padding bits in a struct or union
+  element, // element within an array
+
+  other // anything else
+};
+
+extern const char *
+node_kind_to_str (enum node_kind);
+
+enum class node_dynalloc_state
+{
+  unknown,
+  nonnull,
+  unchecked,
+  freed
+};
+
+/* Prefixes to use in SARIF property bags.  */
+#define STATE_GRAPH_PREFIX "gcc/diagnostic_state_graph/"
+#define STATE_NODE_PREFIX "gcc/diagnostic_state_node/"
+#define STATE_EDGE_PREFIX "gcc/diagnostic_state_edge/"
+
+/* A wrapper around a node that gets/sets attributes, using
+   the node's property bag for storage, so that the data roundtrips
+   through SARIF.  */
+
+struct state_node_ref
+{
+  state_node_ref (diagnostics::digraphs::node &node)
+  : m_node (node)
+  {}
+
+  enum node_kind
+  get_node_kind () const;
+  void
+  set_node_kind (enum node_kind);
+
+  // For node_kind::stack_frame, this will be the function
+  logical_locations::key
+  get_logical_loc () const
+  {
+    return m_node.get_logical_loc ();
+  }
+
+  // For node_kind::dynalloc_buffer
+  enum node_dynalloc_state
+  get_dynalloc_state () const;
+
+  void
+  set_dynalloc_state (enum node_dynalloc_state) const;
+
+  const char *
+  get_dynamic_extents () const;
+
+  const char *
+  get_name () const { return get_attr ("name"); }
+  void
+  set_name (const char *name) const { set_attr ("name", name); }
+
+  const char *
+  get_type () const { return get_attr ("type"); }
+  void
+  set_type (const char *type) const { set_attr ("type", type); }
+
+  const char *
+  get_value () const { return get_attr ("value"); }
+
+  const char *
+  get_index () const { return get_attr ("index"); }
+
+  const char *
+  get_attr (const char *key) const
+  {
+    return m_node.get_attr (STATE_NODE_PREFIX, key);
+  }
+
+  void
+  set_attr (const char *key, const char *value) const
+  {
+    return m_node.set_attr (STATE_NODE_PREFIX, key, value);
+  }
+
+  void
+  set_json_attr (const char *key, std::unique_ptr<json::value> value) const;
+
+  diagnostics::digraphs::node &m_node;
+};
+
+extern std::unique_ptr<dot::graph>
+make_dot_graph (const diagnostics::digraphs::digraph &state_graph,
+		const logical_locations::manager &logical_loc_mgr);
+
+} // namespace state_graphs
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_STATE_GRAPHS_H */
diff --git a/gcc/diagnostic-format-text.cc b/gcc/diagnostics/text-sink.cc
index 7ecbe5f..bcf91cf 100644
--- a/gcc/diagnostic-format-text.cc
+++ b/gcc/diagnostics/text-sink.cc
@@ -25,14 +25,14 @@ along with GCC; see the file COPYING3.  If not see
 #include "version.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-diagram.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-buffer.h"
+#include "diagnostics/color.h"
+#include "diagnostics/url.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/diagram.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/buffering.h"
 #include "text-art/theme.h"
 
 /* Disable warnings about quoting issues in the pp_xxx calls below
@@ -42,54 +42,55 @@ along with GCC; see the file COPYING3.  If not see
 #  pragma GCC diagnostic ignored "-Wformat-diag"
 #endif
 
-/* Concrete buffering implementation subclass for JSON output.  */
+namespace diagnostics {
 
-class diagnostic_text_format_buffer : public diagnostic_per_format_buffer
+/* Concrete buffering implementation subclass for text output.  */
+
+class text_sink_buffer : public per_sink_buffer
 {
 public:
-  friend class diagnostic_text_output_format;
+  friend class text_sink;
 
-  diagnostic_text_format_buffer (diagnostic_output_format &format);
+  text_sink_buffer (sink &sink_);
 
   void dump (FILE *out, int indent) const final override;
 
   bool empty_p () const final override;
-  void move_to (diagnostic_per_format_buffer &dest) final override;
+  void move_to (per_sink_buffer &dest) final override;
   void clear () final override;
   void flush () final override;
 
 private:
-  diagnostic_output_format &m_format;
+  sink &m_sink;
   output_buffer m_output_buffer;
 };
 
-/* class diagnostic_text_format_buffer : public diagnostic_per_format_buffer.  */
+/* class text_sink_buffer : public per_sink_buffer.  */
 
-diagnostic_text_format_buffer::
-diagnostic_text_format_buffer (diagnostic_output_format &format)
-: m_format (format)
+text_sink_buffer::text_sink_buffer (sink &sink_)
+: m_sink (sink_)
 {
   m_output_buffer.m_flush_p = false;
 }
 
 void
-diagnostic_text_format_buffer::dump (FILE *out, int indent) const
+text_sink_buffer::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_text_format_buffer:\n", indent, "");
+  fprintf (out, "%*stext_sink_buffer:\n", indent, "");
   m_output_buffer.dump (out, indent + 2);
 }
 
 bool
-diagnostic_text_format_buffer::empty_p () const
+text_sink_buffer::empty_p () const
 {
   return output_buffer_last_position_in_text (&m_output_buffer) == nullptr;
 }
 
 void
-diagnostic_text_format_buffer::move_to (diagnostic_per_format_buffer &base_dest)
+text_sink_buffer::move_to (per_sink_buffer &base_dest)
 {
-  diagnostic_text_format_buffer &dest
-    = static_cast<diagnostic_text_format_buffer &> (base_dest);
+  text_sink_buffer &dest
+    = static_cast<text_sink_buffer &> (base_dest);
   const char *str = output_buffer_formatted_text (&m_output_buffer);
   output_buffer_append_r (&dest.m_output_buffer, str, strlen (str));
 
@@ -99,9 +100,9 @@ diagnostic_text_format_buffer::move_to (diagnostic_per_format_buffer &base_dest)
 }
 
 void
-diagnostic_text_format_buffer::clear ()
+text_sink_buffer::clear ()
 {
-  pretty_printer *const pp = m_format.get_printer ();
+  pretty_printer *const pp = m_sink.get_printer ();
   output_buffer *const old_output_buffer = pp_buffer (pp);
 
   pp_buffer (pp) = &m_output_buffer;
@@ -113,9 +114,9 @@ diagnostic_text_format_buffer::clear ()
 }
 
 void
-diagnostic_text_format_buffer::flush ()
+text_sink_buffer::flush ()
 {
-  pretty_printer *const pp = m_format.get_printer ();
+  pretty_printer *const pp = m_sink.get_printer ();
   output_buffer *const old_output_buffer = pp_buffer (pp);
 
   pp_buffer (pp) = &m_output_buffer;
@@ -126,12 +127,12 @@ diagnostic_text_format_buffer::flush ()
   pp_buffer (pp) = old_output_buffer;
 }
 
-/* class diagnostic_text_output_format : public diagnostic_output_format.  */
+/* class diagnostics::text_sink : public diagnostics::sink.  */
 
-diagnostic_text_output_format::~diagnostic_text_output_format ()
+text_sink::~text_sink ()
 {
   /* Some of the errors may actually have been warnings.  */
-  if (m_context.diagnostic_count (DK_WERROR))
+  if (m_context.diagnostic_count (kind::werror))
     {
       pretty_printer *pp = get_printer ();
       /* -Werror was given.  */
@@ -155,13 +156,13 @@ diagnostic_text_output_format::~diagnostic_text_output_format ()
 }
 
 void
-diagnostic_text_output_format::dump (FILE *out, int indent) const
+text_sink::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_text_output_format\n", indent, "");
+  fprintf (out, "%*stext_sink\n", indent, "");
   fprintf (out, "%*sm_follows_reference_printer: %s\n",
 	   indent, "",
 	   m_follows_reference_printer ? "true" : "false");
-  diagnostic_output_format::dump (out, indent);
+  sink::dump (out, indent);
   fprintf (out, "%*ssaved_output_buffer:\n", indent + 2, "");
   if (m_saved_output_buffer)
     m_saved_output_buffer->dump (out, indent + 4);
@@ -170,10 +171,10 @@ diagnostic_text_output_format::dump (FILE *out, int indent) const
 }
 
 void
-diagnostic_text_output_format::set_buffer (diagnostic_per_format_buffer *base)
+text_sink::set_buffer (per_sink_buffer *base)
 {
-  diagnostic_text_format_buffer * const buffer
-    = static_cast<diagnostic_text_format_buffer *> (base);
+  text_sink_buffer * const buffer
+    = static_cast<text_sink_buffer *> (base);
 
   pretty_printer *const pp = get_printer ();
 
@@ -189,23 +190,22 @@ diagnostic_text_output_format::set_buffer (diagnostic_per_format_buffer *base)
     }
 }
 
-std::unique_ptr<diagnostic_per_format_buffer>
-diagnostic_text_output_format::make_per_format_buffer ()
+std::unique_ptr<per_sink_buffer>
+text_sink::make_per_sink_buffer ()
 {
-  return std::make_unique<diagnostic_text_format_buffer> (*this);
+  return std::make_unique<text_sink_buffer> (*this);
 }
 
-/* Implementation of diagnostic_output_format::on_report_diagnostic vfunc
+/* Implementation of diagnostics::sink::on_report_diagnostic vfunc
    for GCC's standard textual output.  */
 
 void
-diagnostic_text_output_format::
-on_report_diagnostic (const diagnostic_info &diagnostic,
-		      diagnostic_t orig_diag_kind)
+text_sink::on_report_diagnostic (const diagnostic_info &diagnostic,
+				 enum kind orig_diag_kind)
 {
   pretty_printer *pp = get_printer ();
 
-  (*diagnostic_text_starter (&m_context)) (*this, &diagnostic);
+  (*text_starter (&m_context)) (*this, &diagnostic);
 
   pp_output_formatted_text (pp, m_context.get_urlifier ());
 
@@ -242,23 +242,23 @@ on_report_diagnostic (const diagnostic_info &diagnostic,
 	}
     }
 
-  (*diagnostic_text_finalizer (&m_context)) (*this,
-					     &diagnostic,
-					     orig_diag_kind);
+  (*text_finalizer (&m_context)) (*this,
+				  &diagnostic,
+				  orig_diag_kind);
 
   if (m_show_nesting && m_show_locations_in_nesting)
     get_context ().m_last_location = diagnostic_location (&diagnostic);
 }
 
 void
-diagnostic_text_output_format::on_report_verbatim (text_info &text)
+text_sink::on_report_verbatim (text_info &text)
 {
   pp_format_verbatim (get_printer (), &text);
   pp_newline_and_flush (get_printer ());
 }
 
 void
-diagnostic_text_output_format::on_diagram (const diagnostic_diagram &diagram)
+text_sink::on_diagram (const diagnostics::diagram &d)
 {
   pretty_printer *const pp = get_printer ();
 
@@ -267,17 +267,17 @@ diagnostic_text_output_format::on_diagram (const diagnostic_diagram &diagram)
   /* Use a newline before and after and a two-space indent
      to make the diagram stand out a little from the wall of text.  */
   pp_newline (pp);
-  diagram.get_canvas ().print_to_pp (pp, "  ");
+  d.get_canvas ().print_to_pp (pp, "  ");
   pp_newline (pp);
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
 }
 
 void
-diagnostic_text_output_format::
+text_sink::
 after_diagnostic (const diagnostic_info &diagnostic)
 {
-  if (const diagnostic_path *path = diagnostic.richloc->get_path ())
+  if (const paths::path *path = diagnostic.m_richloc->get_path ())
     print_path (*path);
 }
 
@@ -289,16 +289,15 @@ after_diagnostic (const diagnostic_info &diagnostic)
 
    The caller is responsible for freeing the memory.  */
 char *
-diagnostic_text_output_format::
-build_prefix (const diagnostic_info &diagnostic) const
+text_sink::build_prefix (const diagnostic_info &diagnostic) const
 {
-  gcc_assert (diagnostic.kind < DK_LAST_DIAGNOSTIC_KIND);
+  gcc_assert (diagnostic.m_kind < kind::last_diagnostic_kind);
 
-  const char *text = _(get_diagnostic_kind_text (diagnostic.kind));
+  const char *text = _(get_text_for_kind (diagnostic.m_kind));
   const char *text_cs = "", *text_ce = "";
   pretty_printer *pp = get_printer ();
 
-  if (const char *color_name = diagnostic_get_color_for_kind (diagnostic.kind))
+  if (const char *color_name = get_color_for_kind (diagnostic.m_kind))
     {
       text_cs = colorize_start (pp_show_color (pp), color_name);
       text_ce = colorize_stop (pp_show_color (pp));
@@ -311,7 +310,7 @@ build_prefix (const diagnostic_info &diagnostic) const
 
       /* Reduce verbosity of nested diagnostics by not printing "note: "
 	 all the time.  */
-      if (diagnostic.kind == DK_NOTE)
+      if (diagnostic.m_kind == kind::note)
 	return indent_prefix;
 
       char *result = build_message_string ("%s%s%s%s", indent_prefix,
@@ -330,7 +329,7 @@ build_prefix (const diagnostic_info &diagnostic) const
 
 /* Same as build_prefix, but only the source FILE is given.  */
 char *
-diagnostic_text_output_format::file_name_as_prefix (const char *f) const
+text_sink::file_name_as_prefix (const char *f) const
 {
   pretty_printer *const pp = get_printer ();
   const char *locus_cs
@@ -354,7 +353,7 @@ get_bullet_point_unichar (bool unicode)
 /* Return true if DC's theme supports unicode characters.  */
 
 static bool
-use_unicode_p (const diagnostic_context &dc)
+use_unicode_p (const context &dc)
 {
   if (text_art::theme *theme = dc.get_diagram_theme ())
     return theme->unicode_p ();
@@ -366,7 +365,7 @@ use_unicode_p (const diagnostic_context &dc)
    nested diagnostics.  */
 
 static unsigned
-get_bullet_point_unichar (diagnostic_context &dc)
+get_bullet_point_unichar (context &dc)
 {
   return get_bullet_point_unichar (use_unicode_p (dc));
 }
@@ -382,7 +381,7 @@ get_bullet_point_unichar (diagnostic_context &dc)
    The caller is responsible for freeing the memory.  */
 
 char *
-diagnostic_text_output_format::build_indent_prefix (bool with_bullet) const
+text_sink::build_indent_prefix (bool with_bullet) const
 {
   if (!m_show_nesting)
     return xstrdup ("");
@@ -407,18 +406,18 @@ diagnostic_text_output_format::build_indent_prefix (bool with_bullet) const
 /* Add a purely textual note with text GMSGID and with LOCATION.  */
 
 void
-diagnostic_text_output_format::append_note (location_t location,
-					    const char * gmsgid, ...)
+text_sink::append_note (location_t location,
+			const char * gmsgid, ...)
 {
-  diagnostic_context *context = &get_context ();
+  context *dc = &get_context ();
 
   diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_NOTE);
-  if (context->m_inhibit_notes_p)
+  diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, kind::note);
+  if (dc->m_inhibit_notes_p)
     {
       va_end (ap);
       return;
@@ -426,25 +425,24 @@ diagnostic_text_output_format::append_note (location_t location,
   pretty_printer *pp = get_printer ();
   char *saved_prefix = pp_take_prefix (pp);
   pp_set_prefix (pp, build_prefix (diagnostic));
-  pp_format (pp, &diagnostic.message);
+  pp_format (pp, &diagnostic.m_message);
   pp_output_formatted_text (pp);
   pp_destroy_prefix (pp);
   pp_set_prefix (pp, saved_prefix);
   pp_newline (pp);
-  diagnostic_show_locus (context, get_source_printing_options (),
-			 &richloc, DK_NOTE, pp);
+  diagnostic_show_locus (dc, get_source_printing_options (),
+			 &richloc, kind::note, pp);
   va_end (ap);
 }
 
 bool
-diagnostic_text_output_format::follows_reference_printer_p () const
+text_sink::follows_reference_printer_p () const
 {
   return m_follows_reference_printer;
 }
 
 void
-diagnostic_text_output_format::
-update_printer ()
+text_sink::update_printer ()
 {
   pretty_printer *copy_from_pp
     = (m_follows_reference_printer
@@ -469,18 +467,18 @@ update_printer ()
    description of the security issue.  */
 
 void
-diagnostic_text_output_format::print_any_cwe (const diagnostic_info &diagnostic)
+text_sink::print_any_cwe (const diagnostic_info &diagnostic)
 {
-  if (!diagnostic.metadata)
+  if (!diagnostic.m_metadata)
     return;
 
-  int cwe = diagnostic.metadata->get_cwe ();
+  int cwe = diagnostic.m_metadata->get_cwe ();
   if (cwe)
     {
       pretty_printer * const pp = get_printer ();
       char *saved_prefix = pp_take_prefix (pp);
       pp_string (pp, " [");
-      const char *kind_color = diagnostic_get_color_for_kind (diagnostic.kind);
+      const char *kind_color = get_color_for_kind (diagnostic.m_kind);
       pp_string (pp, colorize_start (pp_show_color (pp), kind_color));
       if (pp->supports_urls_p ())
 	{
@@ -504,23 +502,21 @@ diagnostic_text_output_format::print_any_cwe (const diagnostic_info &diagnostic)
    with any URL provided by the rule.  */
 
 void
-diagnostic_text_output_format::
-print_any_rules (const diagnostic_info &diagnostic)
+text_sink::print_any_rules (const diagnostic_info &diagnostic)
 {
-  if (!diagnostic.metadata)
+  if (!diagnostic.m_metadata)
     return;
 
-  for (unsigned idx = 0; idx < diagnostic.metadata->get_num_rules (); idx++)
+  for (unsigned idx = 0; idx < diagnostic.m_metadata->get_num_rules (); idx++)
     {
-      const diagnostic_metadata::rule &rule
-	= diagnostic.metadata->get_rule (idx);
+      const diagnostics::metadata::rule &rule
+	= diagnostic.m_metadata->get_rule (idx);
       if (char *desc = rule.make_description ())
 	{
 	  pretty_printer * const pp = get_printer ();
 	  char *saved_prefix = pp_take_prefix (pp);
 	  pp_string (pp, " [");
-	  const char *kind_color
-	    = diagnostic_get_color_for_kind (diagnostic.kind);
+	  const char *kind_color = get_color_for_kind (diagnostic.m_kind);
 	  pp_string (pp, colorize_start (pp_show_color (pp), kind_color));
 	  char *url = nullptr;
 	  if (pp->supports_urls_p ())
@@ -542,25 +538,23 @@ print_any_rules (const diagnostic_info &diagnostic)
     }
 }
 
-/* Print any metadata about the option used to control DIAGNOSTIC to CONTEXT's
-   printer, e.g. " [-Werror=uninitialized]".
-   Subroutine of diagnostic_context::report_diagnostic.  */
+/* Print any metadata about the option used to control DIAGNOSTIC to
+   the context's printer, e.g. " [-Werror=uninitialized]".  */
 
 void
-diagnostic_text_output_format::
-print_option_information (const diagnostic_info &diagnostic,
-			  diagnostic_t orig_diag_kind)
+text_sink::print_option_information (const diagnostic_info &diagnostic,
+				     enum kind orig_diag_kind)
 {
   if (char *option_text
-      = m_context.make_option_name (diagnostic.option_id,
-				    orig_diag_kind, diagnostic.kind))
+      = m_context.make_option_name (diagnostic.m_option_id,
+				    orig_diag_kind, diagnostic.m_kind))
     {
       char *option_url = nullptr;
       pretty_printer * const pp = get_printer ();
       if (pp->supports_urls_p ())
-	option_url = m_context.make_option_url (diagnostic.option_id);
+	option_url = m_context.make_option_url (diagnostic.m_option_id);
       pp_string (pp, " [");
-      const char *kind_color = diagnostic_get_color_for_kind (diagnostic.kind);
+      const char *kind_color = get_color_for_kind (diagnostic.m_kind);
       pp_string (pp, colorize_start (pp_show_color (pp), kind_color));
       if (option_url)
 	pp_begin_url (pp, option_url);
@@ -579,7 +573,7 @@ print_option_information (const diagnostic_info &diagnostic,
 /* Only dump the "In file included from..." stack once for each file.  */
 
 bool
-diagnostic_text_output_format::includes_seen_p (const line_map_ordinary *map)
+text_sink::includes_seen_p (const line_map_ordinary *map)
 {
   /* No include path for main.  */
   if (MAIN_FILE_P (map))
@@ -602,13 +596,12 @@ diagnostic_text_output_format::includes_seen_p (const line_map_ordinary *map)
 }
 
 label_text
-diagnostic_text_output_format::
-get_location_text (const expanded_location &s) const
+text_sink::get_location_text (const expanded_location &s) const
 {
-  diagnostic_column_policy column_policy (get_context ());
-  return column_policy.get_location_text (s,
-					  show_column_p (),
-					  pp_show_color (get_printer ()));
+  column_policy column_policy_ (get_context ());
+  return column_policy_.get_location_text (s,
+					   show_column_p (),
+					   pp_show_color (get_printer ()));
 }
 
 /* Helpers for writing lang-specific starters/finalizers for text output.  */
@@ -636,7 +629,7 @@ maybe_line_and_column (int line, int col)
 }
 
 void
-diagnostic_text_output_format::report_current_module (location_t where)
+text_sink::report_current_module (location_t where)
 {
   pretty_printer *pp = get_printer ();
   const line_map_ordinary *map = nullptr;
@@ -704,8 +697,8 @@ diagnostic_text_output_format::report_current_module (location_t where)
 }
 
 void
-default_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-				 const diagnostic_info *diagnostic)
+default_text_starter (text_sink &text_output,
+		      const diagnostic_info *diagnostic)
 {
   text_output.report_current_module (diagnostic_location (diagnostic));
   pretty_printer *const pp = text_output.get_printer ();
@@ -713,9 +706,9 @@ default_diagnostic_text_starter (diagnostic_text_output_format &text_output,
 }
 
 void
-default_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-				   const diagnostic_info *diagnostic,
-				   diagnostic_t)
+default_text_finalizer (text_sink &text_output,
+			const diagnostic_info *diagnostic,
+			enum kind)
 {
   pretty_printer *const pp = text_output.get_printer ();
   char *saved_prefix = pp_take_prefix (pp);
@@ -723,7 +716,7 @@ default_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
   pp_newline (pp);
   diagnostic_show_locus (&text_output.get_context (),
 			 text_output.get_source_printing_options (),
-			 diagnostic->richloc, diagnostic->kind, pp);
+			 diagnostic->m_richloc, diagnostic->m_kind, pp);
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
 }
@@ -731,3 +724,5 @@ default_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
 #if __GNUC__ >= 10
 #  pragma GCC diagnostic pop
 #endif
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-format-text.h b/gcc/diagnostics/text-sink.h
index 64f8e13..5c60976 100644
--- a/gcc/diagnostic-format-text.h
+++ b/gcc/diagnostics/text-sink.h
@@ -18,49 +18,51 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_TEXT_H
-#define GCC_DIAGNOSTIC_FORMAT_TEXT_H
+#ifndef GCC_DIAGNOSTICS_TEXT_SINK_H
+#define GCC_DIAGNOSTICS_TEXT_SINK_H
 
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 
-/* Subclass of diagnostic_output_format for classic text-based output
+namespace diagnostics {
+
+/* Subclass of diagnostics::sink for classic text-based output
    to stderr.
 
-   Uses diagnostic_context.m_text_callbacks to provide client-specific
+   Uses diagnostics::context.m_text_callbacks to provide client-specific
    textual output (e.g. include paths, macro expansions, etc).  */
 
-class diagnostic_text_output_format : public diagnostic_output_format
+class text_sink : public sink
 {
 public:
-  diagnostic_text_output_format (diagnostic_context &context,
-				 diagnostic_source_printing_options *source_printing = nullptr,
-				 bool follows_reference_printer = false)
-  : diagnostic_output_format (context),
+  text_sink (context &dc,
+	     source_printing_options *source_printing = nullptr,
+	     bool follows_reference_printer = false)
+  : sink (dc),
     m_saved_output_buffer (nullptr),
-    m_column_policy (context),
+    m_column_policy (dc),
     m_last_module (nullptr),
     m_includes_seen (nullptr),
     m_source_printing (source_printing
 		       ? *source_printing
-		       : context.m_source_printing),
+		       : dc.get_source_printing_options ()),
     m_follows_reference_printer (follows_reference_printer),
     m_show_nesting (false),
     m_show_nesting_levels (false)
   {}
-  ~diagnostic_text_output_format ();
+  ~text_sink ();
 
   void dump (FILE *out, int indent) const override;
 
-  std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () final override;
-  void set_buffer (diagnostic_per_format_buffer *) final override;
+  std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () final override;
+  void set_buffer (per_sink_buffer *) final override;
 
   void on_begin_group () override {}
   void on_end_group () override {}
   void on_report_diagnostic (const diagnostic_info &,
-			     diagnostic_t orig_diag_kind) override;
+			     enum kind orig_diag_kind) override;
   void on_report_verbatim (text_info &) final override;
-  void on_diagram (const diagnostic_diagram &diagram) override;
+  void on_diagram (const diagram &d) override;
   void after_diagnostic (const diagnostic_info &) override;
   bool machine_readable_stderr_p () const final override
   {
@@ -70,6 +72,13 @@ public:
 
   void update_printer () override;
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &)
+    final override
+  {
+    // no-op for text
+  }
+
   /* Helpers for writing lang-specific starters/finalizers for text output.  */
   char *build_prefix (const diagnostic_info &) const;
   void report_current_module (location_t where);
@@ -81,15 +90,15 @@ public:
 
   char *build_indent_prefix (bool with_bullet) const;
 
-  void print_path (const diagnostic_path &path);
+  void print_path (const paths::path &path);
 
   bool show_column_p () const { return get_context ().m_show_column; }
 
-  const diagnostic_column_policy &get_column_policy () const
+  const column_policy &get_column_policy () const
   {
     return m_column_policy;
   }
-  diagnostic_location_print_policy get_location_print_policy () const;
+  location_print_policy get_location_print_policy () const;
 
   bool show_nesting_p () const { return m_show_nesting; }
   bool show_locations_in_nesting_p () const
@@ -109,11 +118,11 @@ public:
 
   label_text get_location_text (const expanded_location &s) const;
 
-  diagnostic_source_printing_options &get_source_printing_options ()
+  source_printing_options &get_source_printing_options ()
   {
     return m_source_printing;
   }
-  const diagnostic_source_printing_options &get_source_printing_options () const
+  const source_printing_options &get_source_printing_options () const
   {
     return m_source_printing;
   }
@@ -122,14 +131,14 @@ protected:
   void print_any_cwe (const diagnostic_info &diagnostic);
   void print_any_rules (const diagnostic_info &diagnostic);
   void print_option_information (const diagnostic_info &diagnostic,
-				 diagnostic_t orig_diag_kind);
+				 enum kind orig_diag_kind);
 
   bool includes_seen_p (const line_map_ordinary *map);
 
-  /* For handling diagnostic_buffer.  */
+  /* For handling diagnostics::buffer.  */
   output_buffer *m_saved_output_buffer;
 
-  diagnostic_column_policy m_column_policy;
+  column_policy m_column_policy;
 
   /* Used to detect when the input file stack has changed since last
      described.  */
@@ -139,10 +148,10 @@ protected:
      include path for.  */
   hash_set<location_t, false, location_hash> *m_includes_seen;
 
-  diagnostic_source_printing_options &m_source_printing;
+  source_printing_options &m_source_printing;
 
   /* If true, this is the initial default text output format created
-     when the diagnostic_context was created, and, in particular, before
+     when the diagnostics::context was created, and, in particular, before
      initializations of color and m_url_format.  Hence this should follow
      the dc's reference printer for these.
      If false, this text output was created after the dc was created, and
@@ -162,4 +171,6 @@ protected:
   bool m_show_nesting_levels;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_TEXT_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_TEXT_SINK_H */
diff --git a/gcc/diagnostic-url.h b/gcc/diagnostics/url.h
index d54775b..89efa36 100644
--- a/gcc/diagnostic-url.h
+++ b/gcc/diagnostics/url.h
@@ -17,8 +17,8 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_URL_H
-#define GCC_DIAGNOSTIC_URL_H
+#ifndef GCC_DIAGNOSTICS_URL_H
+#define GCC_DIAGNOSTICS_URL_H
 
 /* Whether to add URLs to diagnostics:
    - DIAGNOSTICS_URL_NO: never
@@ -49,4 +49,4 @@ const diagnostic_url_format URL_FORMAT_DEFAULT = URL_FORMAT_BEL;
 
 extern diagnostic_url_format determine_url_format (diagnostic_url_rule_t);
 
-#endif /* ! GCC_DIAGNOSTIC_URL_H */
+#endif /* ! GCC_DIAGNOSTICS_URL_H */
diff --git a/gcc/doc/analyzer.texi b/gcc/doc/analyzer.texi
index 4dd14c8..87a2a40 100644
--- a/gcc/doc/analyzer.texi
+++ b/gcc/doc/analyzer.texi
@@ -432,7 +432,7 @@ binop_svalue (mult_expr, initial_svalue(‘size_t’, decl_region(frame_region(â
 @subsection Analyzer Paths
 
 We need to explain to the user what the problem is, and to persuade them
-that there really is a problem.  Hence having a @code{diagnostic_path}
+that there really is a problem.  Hence having a @code{diagnostics::paths::path}
 isn't just an incidental detail of the analyzer; it's required.
 
 Paths ought to be:
@@ -491,7 +491,7 @@ and notes if it is infeasible.
 
 The above gives us a shortest feasible @code{exploded_path} through the
 @code{exploded_graph} (a list of @code{exploded_edge *}).  We use this
-@code{exploded_path} to build a @code{diagnostic_path} (a list of
+@code{exploded_path} to build a @code{diagnostics::paths::path} (a list of
 @strong{events} for the diagnostic subsystem) - specifically a
 @code{checker_path}.
 
@@ -624,17 +624,17 @@ which writes out the diagnostics in HTML form, and generates SVG state
 diagrams visualizing the state of memory at each event (inspired by the
 "ddd" debugger).  These can be seen by pressing 'j' and 'k' to single-step
 forward and backward through events.  Note that these SVG diagrams are
-created from an intermediate XML representation generated from
-@code{program_state} objects.  The XML representation can be easier to
+created from an intermediate SARIF directed graph representation generated from
+@code{program_state} objects.  The SARIF representation can be easier to
 read - for example, rather than storing the contents of memory via byte
 offsets, it uses fields for structs and element indexes for arrays,
 recursively.  However it is a different representation, and thus bugs could
 be hidden by this transformation.  Generating the SVG diagrams requires
 an invocation of "dot" per event, so it noticeably slows down diagnostic
-emission, hence the opt-in command-line flag.  The XML and ``dot''
+emission, hence the opt-in command-line flag.  The SARIF and ``dot''
 representations can be seen by @code{__analyzer_dump_xml} and
 @code{__analyzer_dump_dot} below (writing them to stderr), or by adding
-@code{show-state-diagrams-xml=yes} and
+@code{show-state-diagrams-sarif=yes} and
 @code{show-state-diagrams-dot-src=yes} to the html sink, which shows
 them within the generated HTML next to the generated SVG.
 
@@ -792,9 +792,9 @@ will emit a warning describing the state of the 2nd argument
 a name matching the 1st argument (which must be a string literal).
 This is for use when debugging, and may be of use in DejaGnu tests.
 
-@item __analyzer_dump_xml
+@item __analyzer_dump_sarif
 @smallexample
-__analyzer_dump_xml ();
+__analyzer_dump_sarif ();
 @end smallexample
 
 will dump the copious information about the analyzer's state each time it
diff --git a/gcc/doc/avr-mmcu.texi b/gcc/doc/avr-mmcu.texi
index feb7725..5efcc81 100644
--- a/gcc/doc/avr-mmcu.texi
+++ b/gcc/doc/avr-mmcu.texi
@@ -50,15 +50,15 @@
 
 @item @anchor{avrxmega2}avrxmega2
 ``XMEGA'' devices with more than 8@tie{}KiB and up to 64@tie{}KiB of program memory.
-@*@var{mcu}@tie{}= @code{atxmega8e5}, @code{atxmega16a4}, @code{atxmega16a4u}, @code{atxmega16c4}, @code{atxmega16d4}, @code{atxmega16e5}, @code{atxmega32a4}, @code{atxmega32a4u}, @code{atxmega32c3}, @code{atxmega32c4}, @code{atxmega32d3}, @code{atxmega32d4}, @code{atxmega32e5}, @code{avr64da28}, @code{avr64da32}, @code{avr64da48}, @code{avr64da64}, @code{avr64db28}, @code{avr64db32}, @code{avr64db48}, @code{avr64db64}, @code{avr64dd14}, @code{avr64dd20}, @code{avr64dd28}, @code{avr64dd32}, @code{avr64du28}, @code{avr64du32}, @code{avr64ea28}, @code{avr64ea32}, @code{avr64ea48}, @code{avr64sd28}, @code{avr64sd32}, @code{avr64sd48}.
+@*@var{mcu}@tie{}= @code{atxmega8e5}, @code{atxmega16a4}, @code{atxmega16a4u}, @code{atxmega16c4}, @code{atxmega16d4}, @code{atxmega16e5}, @code{atxmega32a4}, @code{atxmega32a4u}, @code{atxmega32c3}, @code{atxmega32c4}, @code{atxmega32d3}, @code{atxmega32d4}, @code{atxmega32e5}, @code{avr64da28}, @code{avr64da28s}, @code{avr64da32}, @code{avr64da32s}, @code{avr64da48}, @code{avr64da48s}, @code{avr64da64}, @code{avr64da64s}, @code{avr64db28}, @code{avr64db32}, @code{avr64db48}, @code{avr64db64}, @code{avr64dd14}, @code{avr64dd20}, @code{avr64dd28}, @code{avr64dd32}, @code{avr64du28}, @code{avr64du32}, @code{avr64ea28}, @code{avr64ea32}, @code{avr64ea48}, @code{avr64sd28}, @code{avr64sd32}, @code{avr64sd48}.
 
 @item @anchor{avrxmega3}avrxmega3
 ``XMEGA'' devices with up to 64@tie{}KiB of combined program memory and RAM, and with program memory visible in the RAM address space.
-@*@var{mcu}@tie{}= @code{attiny202}, @code{attiny204}, @code{attiny212}, @code{attiny214}, @code{attiny402}, @code{attiny404}, @code{attiny406}, @code{attiny412}, @code{attiny414}, @code{attiny416}, @code{attiny416auto}, @code{attiny417}, @code{attiny424}, @code{attiny426}, @code{attiny427}, @code{attiny804}, @code{attiny806}, @code{attiny807}, @code{attiny814}, @code{attiny816}, @code{attiny817}, @code{attiny824}, @code{attiny826}, @code{attiny827}, @code{attiny1604}, @code{attiny1606}, @code{attiny1607}, @code{attiny1614}, @code{attiny1616}, @code{attiny1617}, @code{attiny1624}, @code{attiny1626}, @code{attiny1627}, @code{attiny3214}, @code{attiny3216}, @code{attiny3217}, @code{attiny3224}, @code{attiny3226}, @code{attiny3227}, @code{atmega808}, @code{atmega809}, @code{atmega1608}, @code{atmega1609}, @code{atmega3208}, @code{atmega3209}, @code{atmega4808}, @code{atmega4809}, @code{avr16dd14}, @code{avr16dd20}, @code{avr16dd28}, @code{avr16dd32}, @code{avr16du14}, @code{avr16du20}, @code{avr16du28}, @code{avr16du32}, @code{avr16ea28}, @code{avr16ea32}, @code{avr16ea48}, @code{avr16eb14}, @code{avr16eb20}, @code{avr16eb28}, @code{avr16eb32}, @code{avr32da28}, @code{avr32da32}, @code{avr32da48}, @code{avr32db28}, @code{avr32db32}, @code{avr32db48}, @code{avr32dd14}, @code{avr32dd20}, @code{avr32dd28}, @code{avr32dd32}, @code{avr32du14}, @code{avr32du20}, @code{avr32du28}, @code{avr32du32}, @code{avr32ea28}, @code{avr32ea32}, @code{avr32ea48}, @code{avr32sd20}, @code{avr32sd28}, @code{avr32sd32}.
+@*@var{mcu}@tie{}= @code{attiny202}, @code{attiny204}, @code{attiny212}, @code{attiny214}, @code{attiny402}, @code{attiny404}, @code{attiny406}, @code{attiny412}, @code{attiny414}, @code{attiny416}, @code{attiny416auto}, @code{attiny417}, @code{attiny424}, @code{attiny426}, @code{attiny427}, @code{attiny804}, @code{attiny806}, @code{attiny807}, @code{attiny814}, @code{attiny816}, @code{attiny817}, @code{attiny824}, @code{attiny826}, @code{attiny827}, @code{attiny1604}, @code{attiny1606}, @code{attiny1607}, @code{attiny1614}, @code{attiny1616}, @code{attiny1617}, @code{attiny1624}, @code{attiny1626}, @code{attiny1627}, @code{attiny3214}, @code{attiny3216}, @code{attiny3217}, @code{attiny3224}, @code{attiny3226}, @code{attiny3227}, @code{atmega808}, @code{atmega809}, @code{atmega1608}, @code{atmega1609}, @code{atmega3208}, @code{atmega3209}, @code{atmega4808}, @code{atmega4809}, @code{avr16dd14}, @code{avr16dd20}, @code{avr16dd28}, @code{avr16dd32}, @code{avr16du14}, @code{avr16du20}, @code{avr16du28}, @code{avr16du32}, @code{avr16ea28}, @code{avr16ea32}, @code{avr16ea48}, @code{avr16eb14}, @code{avr16eb20}, @code{avr16eb28}, @code{avr16eb32}, @code{avr32da28}, @code{avr32da28s}, @code{avr32da32}, @code{avr32da32s}, @code{avr32da48}, @code{avr32da48s}, @code{avr32db28}, @code{avr32db32}, @code{avr32db48}, @code{avr32dd14}, @code{avr32dd20}, @code{avr32dd28}, @code{avr32dd32}, @code{avr32du14}, @code{avr32du20}, @code{avr32du28}, @code{avr32du32}, @code{avr32ea28}, @code{avr32ea32}, @code{avr32ea48}, @code{avr32sd20}, @code{avr32sd28}, @code{avr32sd32}.
 
 @item @anchor{avrxmega4}avrxmega4
 ``XMEGA'' devices with more than 64@tie{}KiB and up to 128@tie{}KiB of program memory.
-@*@var{mcu}@tie{}= @code{atxmega64a3}, @code{atxmega64a3u}, @code{atxmega64a4u}, @code{atxmega64b1}, @code{atxmega64b3}, @code{atxmega64c3}, @code{atxmega64d3}, @code{atxmega64d4}, @code{avr128da28}, @code{avr128da32}, @code{avr128da48}, @code{avr128da64}, @code{avr128db28}, @code{avr128db32}, @code{avr128db48}, @code{avr128db64}.
+@*@var{mcu}@tie{}= @code{atxmega64a3}, @code{atxmega64a3u}, @code{atxmega64a4u}, @code{atxmega64b1}, @code{atxmega64b3}, @code{atxmega64c3}, @code{atxmega64d3}, @code{atxmega64d4}, @code{avr128da28}, @code{avr128da28s}, @code{avr128da32}, @code{avr128da32s}, @code{avr128da48}, @code{avr128da48s}, @code{avr128da64}, @code{avr128da64s}, @code{avr128db28}, @code{avr128db32}, @code{avr128db48}, @code{avr128db64}.
 
 @item @anchor{avrxmega5}avrxmega5
 ``XMEGA'' devices with more than 64@tie{}KiB and up to 128@tie{}KiB of program memory and more than 64@tie{}KiB of RAM.
diff --git a/gcc/doc/cpp.texi b/gcc/doc/cpp.texi
index fc60792..cc1656f 100644
--- a/gcc/doc/cpp.texi
+++ b/gcc/doc/cpp.texi
@@ -3252,7 +3252,7 @@ helps people match the @samp{#endif} to the corresponding @samp{#ifdef}.
 Older programs sometimes put @var{MACRO} directly after the
 @samp{#endif} without enclosing it in a comment.  This is invalid code
 according to the C standard.  CPP accepts it with a warning.  It
-never affects which @samp{#ifndef} the @samp{#endif} matches.
+never affects which @samp{#ifdef} the @samp{#endif} matches.
 
 @findex #ifndef
 Sometimes you wish to use some code if a macro is @emph{not} defined.
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index 70adf2d..224d619 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -5654,12 +5654,12 @@ You can specify the kind of interrupt to be handled by adding an optional
 parameter to the interrupt attribute like this:
 
 @smallexample
-void f (void) __attribute__ ((interrupt ("user")));
+void f (void) __attribute__ ((interrupt ("supervisor")));
 @end smallexample
 
-Permissible values for this parameter are @code{user}, @code{supervisor},
-and @code{machine}.  If there is no parameter, then it defaults to
-@code{machine}.
+Permissible values for this parameter are @code{supervisor},
+@code{machine}, and @code{rnmi}.  If there is no parameter, then it
+defaults to @code{machine}.
 
 @cindex @code{riscv_vector_cc} function attribute, RISC-V
 @item riscv_vector_cc
@@ -7138,11 +7138,9 @@ The @code{aligned} attribute can also be used for functions
 @cindex @code{counted_by} variable attribute
 @item counted_by (@var{count})
 The @code{counted_by} attribute may be attached to the C99 flexible array
-member, or a pointer field of a structure.  It indicates that the number
-of the elements of the array that is held by the flexible array member
-field, or is pointed to by the pointer field, is given by the field
-"@var{count}" in the same structure as the flexible array member or the
-pointer field.
+member of a structure.  It indicates that the number of the elements of the
+array is given by the field "@var{count}" in the same structure as the
+flexible array member.
 
 This attribute is available only in C for now.
 In C++ this attribute is ignored.
@@ -7163,22 +7161,8 @@ struct P @{
 @end smallexample
 
 @noindent
-specifies that the @code{array} is a flexible array member whose number
-of elements is given by the field @code{count} in the same structure.
-
-@smallexample
-struct PP @{
-  size_t count2;
-  char other1;
-  char *array2 __attribute__ ((counted_by (count2)));
-  int other2;
-@} *pp;
-@end smallexample
-
-@noindent
-specifies that the @code{array2} is an array that is pointed by the
-pointer field, and its number of elements is given by the field
-@code{count2} in the same structure.
+specifies that the @code{array} is a flexible array member whose number of
+elements is given by the field @code{count} in the same structure.
 
 The field that represents the number of the elements should have an
 integer type.  Otherwise, the compiler reports an error and ignores
@@ -7187,12 +7171,6 @@ the attribute.
 When the field that represents the number of the elements is assigned a
 negative integer value, the compiler treats the value as zero.
 
-The @code{counted_by} attribute is not allowed for a pointer to @code{void},
-a pointer to function, or a pointer to a structure or union that includes
-a flexible array member.  However, it is allowed for a pointer to
-non-void incomplete structure or union types, as long as the type could
-be completed before the first reference to the pointer.
-
 An explicit @code{counted_by} annotation defines a relationship between
 two objects, @code{p->array} and @code{p->count}, and there are the
 following requirements on the relationship between this pair:
@@ -7208,13 +7186,6 @@ available all the time.  This relationship must hold even after any of
 these related objects are updated during the program.
 @end itemize
 
-In addition to the above requirements, there is one more requirement
-between this pair if and only if @code{p->array} is an array that is
-pointed by the pointer field:
-
-@code{p->array} and @code{p->count} can only be changed by changing the
-whole structure at the same time.
-
 It's the programmer's responsibility to make sure the above requirements to
 be kept all the time.  Otherwise the compiler reports warnings and
 the results of the array bound sanitizer and the
@@ -7236,8 +7207,6 @@ In the above, @code{ref1} uses @code{val1} as the number of the elements in
 @code{p->array}, and @code{ref2} uses @code{val2} as the number of elements
 in @code{p->array}.
 
-Note, however, the above feature is not valid for the pointer field.
-
 @cindex @code{alloc_size} variable attribute
 @item alloc_size (@var{position})
 @itemx alloc_size (@var{position-1}, @var{position-2})
@@ -12806,6 +12775,7 @@ the two, as explained in the sections below.
 @menu
 * Global Register Variables::   Variables declared at global scope.
 * Local Register Variables::    Variables declared within a function.
+* Hard Register Constraints::   Operands forced into specific machine registers.
 @end menu
 
 @node Global Register Variables
@@ -13011,6 +12981,167 @@ with slightly different characteristics (@pxref{MIPS Coprocessors,,
 Defining coprocessor specifics for MIPS targets, gccint, 
 GNU Compiler Collection (GCC) Internals}).
 
+@node Hard Register Constraints
+@subsubsection Hard Register Constraints
+
+Similar to register @code{asm} but still distinct, hard register constraints
+are another way to force operands of inline @code{asm} into specific machine
+registers.  In contrast to register @code{asm} where a variable is bound to a
+machine register, a hard register constraint binds an @code{asm} operand to a
+machine register.  Assume in the following that @code{r4} is a general-purpose
+register, @code{f5} a floating-point register, and @code{v6} a vector register
+for some target.
+
+@smallexample
+int x;
+int y __attribute__ ((vector_size (16)));
+@dots{}
+asm ("some instructions"
+     : "=@{r4@}" (x)
+     : "@{f5@}" (42.0), "@{v6@}" (y));
+@end smallexample
+
+For the inline @code{asm}, variable @code{x} is bound to register @code{r4},
+and @code{y} is loaded to @code{v6}.  Furthermore, constant @code{42.0} is
+loaded into floating-point register @code{f5}.
+
+A key difference between register @code{asm} and hard register constraints is
+that the latter are specified at the point where they are supposed to
+materialize, namely at inline @code{asm}, which may lead to more readable code.
+
+@subsubheading Usage
+
+Each input operand is loaded into the register specified by its corresponding
+hard register constraint.  Furthermore, each hard register must be used at most
+once among an alternative for inputs.  This renders hard register constraints
+more strict compared to register @code{asm} where multiple inputs may share a
+register as for example in
+
+@smallexample
+int x;
+register int y asm ("0") = @dots{};
+asm ("" : "=r" (x) : "r" (y), "r" (y));
+@end smallexample
+
+or even
+
+@smallexample
+register int x asm ("0") = 42;
+register int y asm ("0") = 24;
+asm ("" : "=r" (x) : "r" (x), "r" (y));
+@end smallexample
+
+The analogue for hard register constraints is invalid in order to prevent
+subtle bugs.
+
+Likewise, two outputs must not share a register among an alternative.  That
+means, the following example is invalid
+
+@smallexample
+int x, y;
+asm ("" : "=@{r4@}" (x), "=@{r4@}" (y));  // invalid
+@end smallexample
+
+which also aligns with register @code{asm}.  Despite that, each output must
+refer to a distinct object if a hard register constraint is involved.  For
+example, in the following, object @code{x} is assigned two registers.
+
+@smallexample
+int x;
+asm ("" : "=r" (x), "=r" (x));
+@end smallexample
+
+This is not allowed for hard register constraints in order to prevent subtle
+bugs.  Even if only one output operand has a hard register constraint, the code
+is rejected since the allocation for the object is still ambiguous.
+
+@smallexample
+int x;
+asm ("" : "=r" (x), "=@{1@}" (x));  // invalid
+@end smallexample
+
+The type of an operand must be supported by the corresponding machine register.
+
+A hard register constraint may refer to any general, floating-point, or vector
+register except a fixed register as e.g.@: the stack-pointer register.  The set
+of allowed registers is target dependent analogue to register @code{asm}.
+Furthermore, the referenced register must be a valid register name of the
+target.  Note, on some targets, a single register may be referred to by
+different names where each name specifies the length of the register.  For
+example, on x86_64 the register names @code{rcx}, @code{ecx}, and @code{cx} all
+refer to the same register but in different sizes.  If any of those names is
+used for a hard register constraint, the actual size of a register is
+determined by its corresponding operand.  For example
+
+@smallexample
+long x;
+asm ("mov\t$42, %0" : "=@{ecx@}" (x));
+@end smallexample
+
+Although the hard register constraint refers to register @code{ecx}, the actual
+register will be @code{rcx} since on x86_64 a @code{long} is 8 byte in total.
+This aligns with register @code{asm} where you could have
+
+@smallexample
+register long x asm ("ecx");
+@end smallexample
+
+@subsubheading Interaction with Register @code{asm}
+
+A mixture of both constructs as for example
+
+@smallexample
+register int x asm ("r4") = 42;
+int y;
+asm ("" : "=@{r5@}" (y) : "r" (x));
+@end smallexample
+
+is valid.
+
+If an operand is a register @code{asm} and the corresponding constraint a hard
+register, then both must refer to the same register.  That means
+
+@smallexample
+register int x asm ("r4");
+asm ("" : "=@{r4@}" (x));
+@end smallexample
+
+is valid and
+
+@smallexample
+register int x asm ("r4");
+asm ("" : "=@{r5@}" (x));  // invalid
+@end smallexample
+
+is invalid.
+
+Note, register @code{asm} may not only be clobbered by function calls but also
+by inline @code{asm} in conjunction with hard register constraints.  For
+example, in the following
+
+@smallexample
+register int x asm ("r5") = 42;
+int y;
+asm ("" : "=@{r5@}" (y));
+asm ("" : "+r" (x));
+@end smallexample
+
+variable @code{x} materializes before the very first inline @code{asm} which
+writes to register @code{r5} and therefore clobbers @code{x} which in turn is
+read by the subsequent inline @code{asm}.
+
+@subsubheading Limitations
+
+At the moment fixed registers are not supported for hard register constraints.
+Thus, idioms like
+
+@smallexample
+register void *x asm ("rsp");
+asm ("" : "=r" (x));
+@end smallexample
+
+are not supported for hard register constraints.  This might be lifted.
+
 @node Size of an asm
 @subsection Size of an @code{asm}
 
@@ -15553,7 +15684,7 @@ are 128-bit.  Only supported on targets when 128-bit types are supported.
 Returns the calculated 8-bit bit-reversed CRC using the initial CRC (8-bit),
 data (8-bit) and the polynomial (8-bit).
 @var{crc} is the initial CRC, @var{data} is the data and
-@var{poly} is the polynomial without leading 1.
+@var{poly} is the polynomial without leading 1. @var{poly} is required to be a compile-time constant.
 Table-based or clmul-based CRC may be used for the
 calculation, depending on the target architecture.
 @enddefbuiltin
@@ -15608,7 +15739,7 @@ is 32-bit.
 Returns the calculated 8-bit bit-forward CRC using the initial CRC (8-bit),
 data (8-bit) and the polynomial (8-bit).
 @var{crc} is the initial CRC, @var{data} is the data and
-@var{poly} is the polynomial without leading 1.
+@var{poly} is the polynomial without leading 1. @var{poly} is required to be a compile-time constant.
 Table-based or clmul-based CRC may be used for the
 calculation, depending on the target architecture.
 @enddefbuiltin
diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index 80ee2cd..64c1217 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -302,7 +302,7 @@ released on 2017-05-06.
 The gcobol documentation is maintained as manpages using troff
 mdoc. GNU groff is required to convert them to PDF format.  Conversion
 to HTML is done with mandoc, available at
-@uref{http://mdocml.bsd.lv/}.
+@uref{https://mandoc.bsd.lv}.
 
 Because ISO COBOL defines strict requirements for numerical precision,
 gcobol requires hardware with 128-bit computation instructions.  This
@@ -1887,8 +1887,8 @@ Produce code conforming to version 20191213.
 In the absence of this configuration option the default version is 20191213.
 
 @item --enable-__cxa_atexit
-Define if you want to use @code{__cxa_atexit}, rather than atexit, to
-register C++ destructors for local statics and global objects.
+Define if you want to use @code{__cxa_atexit}, rather than @code{atexit},
+to register C++ destructors for local statics and global objects.
 This is essential for fully standards-compliant handling of
 destructors, but requires @code{__cxa_atexit} in libc.  This option is
 currently only available on systems with GNU libc.  When enabled, this
@@ -2667,6 +2667,17 @@ target binutils supports @code{Intel CET} instructions and disabled
 otherwise.  In this case, the target libraries are configured to get
 additional @option{-fcf-protection} option.
 
+@item --enable-x86-64-mfentry
+@itemx --disable-x86-64-mfentry
+Enable @option {-mfentry} by default on x86-64 to put the profiling
+counter call, @code{__fentry__}, before the prologue so that @option{-pg}
+can be used with @option{-fshrink-wrap} which is enabled at @option{-O1}.
+This configure option is 64-bit only because @code{__fentry__} doesn't
+support PIC in 32-bit mode.
+
+@option{--enable-x86-64-mfentry=auto} is default.  @option{-mfentry} is
+enabled on Linux/x86-64 by default.
+
 @item --with-riscv-attribute=@samp{yes}, @samp{no} or @samp{default}
 Generate RISC-V attribute by default, in order to record extra build
 information in object.
@@ -5153,13 +5164,6 @@ respects, this target is the same as the
 @anchor{windows}
 @heading Microsoft Windows
 
-@subheading Intel 16-bit versions
-The 16-bit versions of Microsoft Windows, such as Windows 3.1, are not
-supported.
-
-However, the 32-bit port has limited support for Microsoft
-Windows 3.11 in the Win32s environment, as a target only.  See below.
-
 @subheading Intel 32-bit versions
 The 32-bit versions of Windows, including Windows 95, Windows NT, Windows
 XP, and Windows Vista, are supported by several different target
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 7640e7d..105a60d 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -353,7 +353,7 @@ Objective-C and Objective-C++ Dialects}.
 -Wno-builtin-macro-redefined  -Wc90-c99-compat  -Wc99-c11-compat
 -Wc11-c23-compat  -Wc23-c2y-compat
 -Wc++-compat  -Wc++11-compat  -Wc++14-compat  -Wc++17-compat
--Wc++20-compat
+-Wc++20-compat  -Wc++26-compat
 -Wno-c++11-extensions  -Wno-c++14-extensions -Wno-c++17-extensions
 -Wno-c++20-extensions  -Wno-c++23-extensions
 -Wcalloc-transposed-args
@@ -433,7 +433,8 @@ Objective-C and Objective-C++ Dialects}.
 -Wunsuffixed-float-constants
 -Wunterminated-string-initialization
 -Wunused
--Wunused-but-set-parameter  -Wunused-but-set-variable
+-Wunused-but-set-parameter  -Wunused-but-set-parameter=@var{n}
+-Wunused-but-set-variable  -Wunused-but-set-variable=@var{n}
 -Wunused-const-variable  -Wunused-const-variable=@var{n}
 -Wunused-function  -Wunused-label  -Wunused-local-typedefs
 -Wunused-macros
@@ -914,7 +915,7 @@ Objective-C and Objective-C++ Dialects}.
 @emph{AVR Options} (@ref{AVR Options})
 @gccoptlist{-mmcu=@var{mcu}  -mabsdata  -maccumulate-args  -mcvt
 -mbranch-cost=@var{cost}  -mfuse-add=@var{level}  -mfuse-move=@var{level}
--mcall-prologues  -mgas-isr-prologues  -mint8  -mflmap
+-mfuse-move2  -mcall-prologues  -mgas-isr-prologues  -mint8  -mflmap
 -mdouble=@var{bits}  -mlong-double=@var{bits}  -mno-call-main
 -mn_flash=@var{size}  -mfract-convert-truncate  -mno-interrupts
 -mmain-is-OS_task  -mrelax  -mrmw  -mstrict-X  -mtiny-stack
@@ -3816,6 +3817,23 @@ Warn when a type with an ABI tag is used in a context that does not
 have that ABI tag.  See @ref{C++ Attributes} for more information
 about ABI tags.
 
+@opindex Wabbreviated-auto-in-template-arg
+@opindex Wno-abbreviated-auto-in-template-arg
+@item -Wno-abbreviated-auto-in-template-arg
+Disable the error for an @code{auto} placeholder type used within a
+template argument list to declare a C++20 abbreviated function
+template, e.g.
+
+@smallexample
+void f(S<auto>);
+@end smallexample
+
+This feature was proposed in the Concepts TS, but was not adopted into
+C++20; in the standard, a placeholder in a parameter declaration must
+appear as a decl-specifier.  The error can also be reduced to a
+warning by @option{-fpermissive} or
+@option{-Wno-error=abbreviated-auto-in-template-arg}.
+
 @opindex Wcomma-subscript
 @opindex Wno-comma-subscript
 @item -Wcomma-subscript @r{(C++ and Objective-C++ only)}
@@ -6126,10 +6144,10 @@ end-users, and subject to change or removal without notice:
 
 @table @gcctabopt
 
-@item xml-state=@r{[}yes@r{|}no@r{]}
+@item state-graphs=@r{[}yes@r{|}no@r{]}
 This is a debugging feature and defaults to @code{no}.
-If @code{xml-state=yes}, then attempt to capture detailed state information
-from @option{-fanalyzer} in the generated SARIF.
+If @code{state-graphs=yes}, then attempt to capture detailed state
+information from @option{-fanalyzer} in the generated SARIF.
 
 @end table
 
@@ -6177,11 +6195,11 @@ then if @code{show-state-diagrams=yes},
 the generated state diagrams will also show the .dot source input to
 GraphViz used for the diagram.
 
-@item show-state-diagrams-xml=@r{[}yes@r{|}no@r{]}
+@item show-state-diagrams-sarif=@r{[}yes@r{|}no@r{]}
 This is a debugging feature and defaults to @code{no}.
-If @code{show-state-diagrams-xml=yes}
+If @code{show-state-diagrams-sarif=yes}
 then if @code{show-state-diagrams=yes}, the generated state diagrams will
-also show an XML representation of the state.
+also show a SARIF representation of the state.
 
 @end table
 
@@ -6443,6 +6461,7 @@ only by this flag, but it also downgrades some C and C++ diagnostics
 that have their own flag:
 
 @gccoptlist{
+-Wabbreviated-auto-in-template-arg @r{(C++ and Objective-C++ only)}
 -Wdeclaration-missing-parameter-type @r{(C and Objective-C only)}
 -Wimplicit-function-declaration @r{(C and Objective-C only)}
 -Wimplicit-int @r{(C and Objective-C only)}
@@ -7903,27 +7922,89 @@ statement.
 @opindex Wunused-but-set-parameter
 @opindex Wno-unused-but-set-parameter
 @item -Wunused-but-set-parameter
+@option{-Wunused-but-set-parameter} is the same as
+@option{-Wunused-but-set-parameter=3} and
+@option{-Wno-unused-but-set-parameter} is the same as
+@option{-Wunused-but-set-parameter=0}.
+
+@opindex Wunused-but-set-parameter=
+@item -Wunused-but-set-parameter=@var{n}
 Warn whenever a function parameter is assigned to, but otherwise unused
 (aside from its declaration).
 
 To suppress this warning use the @code{unused} attribute
 (@pxref{Variable Attributes}).
 
-This warning is also enabled by @option{-Wunused} together with
-@option{-Wextra}.
+@option{-Wunused-but-set-parameter=0} disables the warning.
+With @option{-Wunused-but-set-parameter=1} all uses except initialization
+and left hand side of assignment which is not further used disable the
+warning.
+With @option{-Wunused-but-set-parameter=2} additionally uses of parameter
+in @code{++} and @code{--} operators don't count as uses.
+And finally with @option{-Wunused-but-set-parameter=3} additionally
+uses in @var{parm} @code{@var{@@}=} @var{rhs} outside of @var{rhs} don't
+count as uses.  See @option{-Wunused-but-set-variable=@var{n}} option for
+examples.
+
+This @option{-Wunused-but-set-parameter=3} warning is also enabled by
+@option{-Wunused} together with @option{-Wextra}.
 
 @opindex Wunused-but-set-variable
 @opindex Wno-unused-but-set-variable
 @item -Wunused-but-set-variable
+@option{-Wunused-but-set-variable} is the same as
+@option{-Wunused-but-set-variable=3} and
+@option{-Wno-unused-but-set-variable} is the same as
+@option{-Wunused-but-set-variable=0}.
+
+@opindex Wunused-but-set-variable=
+@item -Wunused-but-set-variable=@var{n}
 Warn whenever a local variable is assigned to, but otherwise unused
 (aside from its declaration).
-This warning is enabled by @option{-Wall}.
+This @option{-Wunused-but-set-variable=3} warning is enabled by @option{-Wall}.
 
 To suppress this warning use the @code{unused} attribute
 (@pxref{Variable Attributes}).
 
-This warning is also enabled by @option{-Wunused}, which is enabled
-by @option{-Wall}.
+@option{-Wunused-but-set-variable=0} disables the warning.
+With @option{-Wunused-but-set-variable=1} all uses except initialization
+and left hand side of assignment which is not further used disable the
+warning.
+With @option{-Wunused-but-set-variable=2} additionally uses of variable
+in @code{++} and @code{--} operators don't count as uses.
+And finally with @option{-Wunused-but-set-variable=3} additionally
+uses in @var{parm} @code{@var{@@}=} @var{rhs} outside of @var{rhs} don't
+count as uses.
+
+This @option{-Wunused-but-set-variable=3} warning is also enabled by
+@option{-Wunused}, which is enabled by @option{-Wall}.
+
+@smallexample
+void foo (void)
+@{
+  int a = 1; // @option{-Wunused-variable} warning
+  int b = 0; // Warning for @var{n} >= 1
+  b = 1; b = 2;
+  int c = 0; // Warning for @var{n} >= 2
+  ++c; c--; --c; c++;
+  int d = 0; // Warning for @var{n} >= 3
+  d += 4;
+  int e = 0; // No warning, cast to void
+  (void) e;
+  int f = 0; // No warning, f used
+  int g = f = 5;
+  (void) g;
+  int h = 0; // No warning, preincrement used
+  int i = ++h;
+  (void) i;
+  int j = 0; // No warning, postdecrement used
+  int k = j--;
+  (void) k;
+  int l = 0; // No warning, l used
+  int m = l |= 2;
+  (void) m;
+@}
+@end smallexample
 
 @opindex Wunused-function
 @opindex Wno-unused-function
@@ -9672,6 +9753,12 @@ and ISO C++ 2017.  This warning is enabled by @option{-Wall}.
 Warn about C++ constructs whose meaning differs between ISO C++ 2017
 and ISO C++ 2020.  This warning is enabled by @option{-Wall}.
 
+@opindex Wc++26-compat
+@opindex Wno-c++26-compat
+@item -Wc++26-compat @r{(C++ and Objective-C++ only)}
+Warn about C++ constructs whose meaning differs between ISO C++ 2023
+and upcoming ISO C++ 2026.  This warning is enabled by @option{-Wall}.
+
 @opindex Wc++11-extensions
 @opindex Wno-c++11-extensions
 @item -Wno-c++11-extensions @r{(C++ and Objective-C++ only)}
@@ -20525,18 +20612,22 @@ LTO output files.
 @opindex fdump-rtl-@var{pass}
 @item -d@var{letters}
 @itemx -fdump-rtl-@var{pass}
-@itemx -fdump-rtl-@var{pass}=@var{filename}
+@itemx -fdump-rtl-@var{pass}-@var{options}
+@itemx -fdump-rtl-@var{pass}-@var{options}=@var{filename}
 Says to make debugging dumps during compilation at times specified by
-@var{letters}.  This is used for debugging the RTL-based passes of the
+@var{letters} when using @option{-d} or by @var{pass} when using
+@option{-fdump-rtl}.  This is used for debugging the RTL-based passes of the
 compiler.
 
 Some @option{-d@var{letters}} switches have different meaning when
 @option{-E} is used for preprocessing.  @xref{Preprocessor Options},
 for information about preprocessor-specific dump options.
 
-Debug dumps can be enabled with a @option{-fdump-rtl} switch or some
-@option{-d} option @var{letters}.  Here are the possible
-letters for use in @var{pass} and @var{letters}, and their meanings:
+The @samp{-@var{options}} form allows greater control over the details of the
+dump. See @option{-fdump-tree}.
+
+Here are actual instances of command-line options following these patterns and
+their meanings:
 
 @table @gcctabopt
 
@@ -21063,8 +21154,7 @@ GraphViz to @file{@var{file}.@var{passid}.@var{pass}.dot}.  Each function in
 the file is pretty-printed as a subgraph, so that GraphViz can render them
 all in a single plot.
 
-This option currently only works for RTL dumps, and the RTL is always
-dumped in slim form.
+RTL is always dumped in slim form.
 @item vops
 Enable showing virtual operands for every statement.
 @item lineno
@@ -25023,6 +25113,10 @@ Valid values for @var{level} are in the range @code{0} @dots{} @code{23}
 which is a 3:2:2:2 mixed radix value.  Each digit controls some
 aspect of the optimization.
 
+@opindex mfuse-move2
+@item -mfuse-move2
+Run a post combine optimization pass that tries to fuse move instructions.
+
 @opindex mstrict-X
 @item -mstrict-X
 Use address register @code{X} in a way proposed by the hardware.  This means
@@ -31283,31 +31377,14 @@ When the RISC-V specifications define an extension as depending on other
 extensions, GCC will implicitly add the dependent extensions to the enabled
 extension set if they weren't added explicitly.
 
-@opindex mcpu
-@item -mcpu=@var{processor-string}
-Use architecture of and optimize the output for the given processor, specified
-by particular CPU name.
-Permissible values for this option are: @samp{mips-p8700}, @samp{sifive-e20},
-@samp{sifive-e21}, @samp{sifive-e24}, @samp{sifive-e31}, @samp{sifive-e34},
-@samp{sifive-e76}, @samp{sifive-s21}, @samp{sifive-s51}, @samp{sifive-s54},
-@samp{sifive-s76}, @samp{sifive-u54}, @samp{sifive-u74}, @samp{sifive-x280},
-@samp{sifive-xp450}, @samp{sifive-x670}, @samp{thead-c906}, @samp{tt-ascalon-d8},
-@samp{xiangshan-nanhu}, @samp{xiangshan-kunminghu}, @samp{xt-c908}, @samp{xt-c908v},
-@samp{xt-c910}, @samp{xt-c910v2}, @samp{xt-c920}, @samp{xt-c920v2}.
+@include riscv-mcpu.texi
 
 Note that @option{-mcpu} does not override @option{-march} or @option{-mtune}.
 
-@opindex mtune
-@item -mtune=@var{processor-string}
-Optimize the output for the given processor, specified by microarchitecture or
-particular CPU name.  Permissible values for this option are:
-@samp{generic-ooo}, @samp{mips-p8700}, @samp{rocket}, @samp{sifive-3-series},
-@samp{sifive-5-series}, @samp{sifive-7-series}, @samp{size},
-@samp{sifive-p400-series}, @samp{sifive-p600-series}, and all valid options for
-@option{-mcpu=}.
+@include riscv-mtune.texi
 
 When @option{-mtune=} is not specified, use the setting from @option{-mcpu},
-the default is @samp{rocket} if both are not specified.
+the default is @samp{generic} if both are not specified.
 
 The @samp{size} choice is not intended for use by end-users.  This is used
 when @option{-Os} is specified.  It overrides the instruction cost info
@@ -34868,9 +34945,9 @@ Intel Panther Lake CPU with 64-bit extensions, MOVBE, MMX, SSE, SSE2, SSE3,
 SSSE3, SSE4.1, SSE4.2, POPCNT, AES, PREFETCHW, PCLMUL, RDRND, XSAVE, XSAVEC,
 XSAVES, XSAVEOPT, FSGSBASE, PTWRITE, RDPID, SGX, GFNI-SSE, CLWB, MOVDIRI,
 MOVDIR64B, WAITPKG, ADCX, AVX, AVX2, BMI, BMI2, F16C, FMA, LZCNT, PCONFIG, PKU,
-VAES, VPCLMULQDQ, SERIALIZE, HRESET, KL, WIDEKL, AVX-VNNI, UINTR, AVXIFMA,
-AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512, SM3, SM4 and
-PREFETCHI instruction set support.
+VAES, VPCLMULQDQ, SERIALIZE, HRESET, AVX-VNNI, UINTR, AVXIFMA, AVXVNNIINT8,
+AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512, SM3, SM4 and PREFETCHI
+instruction set support.
 
 @item sapphirerapids
 @itemx emeraldrapids
@@ -34973,9 +35050,9 @@ Intel Clearwater Forest CPU with 64-bit extensions, MOVBE, MMX, SSE, SSE2,
 SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, AES, PREFETCHW, PCLMUL, RDRND, XSAVE,
 XSAVEC, XSAVES, XSAVEOPT, FSGSBASE, PTWRITE, RDPID, SGX, GFNI-SSE, CLWB,
 MOVDIRI, MOVDIR64B, CLDEMOTE, WAITPKG, ADCX, AVX, AVX2, BMI, BMI2, F16C, FMA,
-LZCNT, PCONFIG, PKU, VAES, VPCLMULQDQ, SERIALIZE, HRESET, KL, WIDEKL, AVX-VNNI,
-ENQCMD, UINTR, AVXIFMA, AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16,
-SHA512, SM3, SM4, USER_MSR and PREFETCHI instruction set support.
+LZCNT, PCONFIG, PKU, VAES, VPCLMULQDQ, SERIALIZE, HRESET, AVX-VNNI, ENQCMD,
+UINTR, AVXIFMA, AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512,
+SM3, SM4, USER_MSR and PREFETCHI instruction set support.
 
 @item k6
 AMD K6 CPU with MMX instruction set support.
diff --git a/gcc/doc/libgdiagnostics/topics/compatibility.rst b/gcc/doc/libgdiagnostics/topics/compatibility.rst
index 10adcc5..ccf1236 100644
--- a/gcc/doc/libgdiagnostics/topics/compatibility.rst
+++ b/gcc/doc/libgdiagnostics/topics/compatibility.rst
@@ -178,6 +178,8 @@ acccessing values within a :type:`diagnostic_logical_location`:
 
   * :func:`diagnostic_logical_location_get_decorated_name`
 
+.. _LIBGDIAGNOSTICS_ABI_2:
+
 ``LIBGDIAGNOSTICS_ABI_2``
 -------------------------
 ``LIBGDIAGNOSTICS_ABI_2`` covers the addition of these functions for
@@ -186,3 +188,86 @@ supporting command-line options and SARIF playback:
   * :func:`diagnostic_manager_add_sink_from_spec`
 
   * :func:`diagnostic_manager_set_analysis_target`
+
+.. _LIBGDIAGNOSTICS_ABI_3:
+
+``LIBGDIAGNOSTICS_ABI_3``
+-------------------------
+
+``LIBGDIAGNOSTICS_ABI_3`` covers the addition of these functions for
+working with directed graphs:
+
+  * :func:`diagnostic_manager_new_graph`
+
+  * :func:`diagnostic_manager_take_global_graph`
+
+  * :func:`diagnostic_take_graph`
+
+  * :func:`diagnostic_graph_release`
+
+  * :func:`diagnostic_graph_set_description`
+
+  * :func:`diagnostic_graph_add_node`
+
+  * :func:`diagnostic_graph_add_edge`
+
+  * :func:`diagnostic_graph_get_node_by_id`
+
+  * :func:`diagnostic_graph_get_edge_by_id`
+
+  * :func:`diagnostic_node_set_label`
+
+  * :func:`diagnostic_node_set_location`
+
+  * :func:`diagnostic_node_set_logical_location`
+
+.. _LIBGDIAGNOSTICS_ABI_4:
+
+``LIBGDIAGNOSTICS_ABI_4``
+-------------------------
+
+``LIBGDIAGNOSTICS_ABI_4`` covers the addition of these functions for
+working with :type:`diagnostic_message_buffer`.
+
+  * :func:`diagnostic_message_buffer_new`
+
+  * :func:`diagnostic_message_buffer_release`
+
+  * :func:`diagnostic_message_buffer_append_str`
+
+  * :func:`diagnostic_message_buffer_append_text`
+
+  * :func:`diagnostic_message_buffer_append_byte`
+
+  * :func:`diagnostic_message_buffer_append_printf`
+
+  * :func:`diagnostic_message_buffer_append_event_id`
+
+  * :func:`diagnostic_message_buffer_begin_url`
+
+  * :func:`diagnostic_message_buffer_end_url`
+
+  * :func:`diagnostic_message_buffer_begin_quote`
+
+  * :func:`diagnostic_message_buffer_end_quote`
+
+  * :func:`diagnostic_message_buffer_begin_color`
+
+  * :func:`diagnostic_message_buffer_end_color`
+
+  * :func:`diagnostic_message_buffer_dump`
+
+  * :func:`diagnostic_finish_via_msg_buf`
+
+  * :func:`diagnostic_add_location_with_label_via_msg_buf`
+
+  * :func:`diagnostic_execution_path_add_event_via_msg_buf`
+
+.. _LIBGDIAGNOSTICS_ABI_5:
+
+``LIBGDIAGNOSTICS_ABI_5``
+-------------------------
+
+``LIBGDIAGNOSTICS_ABI_5`` covers the addition of this function:
+
+  * :func:`diagnostic_manager_set_debug_physical_locations`
diff --git a/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst b/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst
index 0390704..c94d19e 100644
--- a/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst
+++ b/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst
@@ -63,17 +63,17 @@ Responsibilities include:
 				       diagnostic_manager *control_mgr)
 
    This function can be used to support option processing similar to GCC's
-   :option:`-fdiagnostics-add-output=`.  This allows command-line tools to
-   support the same domain-specific language for specifying output sink
-   as GCC does.
-
-   The function will attempt to parse :param:`spec` as if it were
-   an argument to GCC's :option:`-fdiagnostics-add-output=OUTPUT-SPEC`.
-   If successful, it will add an output sink to :param:`affected_mgr` and return zero.
-   Otherwise, it will emit an error diagnostic to :param:`control_mgr` and
+   `-fdiagnostics-add-output= <https://gcc.gnu.org/onlinedocs/gcc/Diagnostic-Message-Formatting-Options.html#index-fdiagnostics-add-output>`_.
+   This allows command-line tools to support the same domain-specific
+   language for specifying output sinks as GCC does.
+
+   The function will attempt to parse ``spec`` as if it were
+   an argument to GCC's `-fdiagnostics-add-output= <https://gcc.gnu.org/onlinedocs/gcc/Diagnostic-Message-Formatting-Options.html#index-fdiagnostics-add-output>`_.
+   If successful, it will add an output sink to ``affected_mgr`` and return zero.
+   Otherwise, it will emit an error diagnostic to ``control_mgr`` and
    return non-zero.
 
-   :param:`affected_mgr` and :param:`control_mgr` can be the same manager,
+   ``affected_mgr`` and ``control_mgr`` can be the same manager,
    or be different managers.
 
    This function was added in :ref:`LIBGDIAGNOSTICS_ABI_2`; you can
@@ -83,14 +83,14 @@ Responsibilities include:
 
       #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_manager_add_sink_from_spec
 
-
 .. function:: void diagnostic_manager_set_analysis_target (diagnostic_manager *mgr, \
 	                                                   const diagnostic_file *file)
 
-   This function sets the "main input file" of :param:`mgr` to be
-   :param:`file`.
+   This function sets the "main input file" of ``mgr`` to be
+   ``file``.
    This affects the :code:`<title>` of generated HTML and
-   the :code:`role` of the artifact in SARIF output (SARIF v2.1.0 section 3.24.6).
+   the :code:`role` of the :code:`artifact` in SARIF output
+   (`SARIF v2.1.0 section 3.24.6 <https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/sarif-v2.1.0-errata01-os-complete.html#_Toc141790867>`_).
 
    This function was added in :ref:`LIBGDIAGNOSTICS_ABI_2`; you can
    test for its presence using
diff --git a/gcc/doc/libgdiagnostics/topics/diagnostics.rst b/gcc/doc/libgdiagnostics/topics/diagnostics.rst
index 3d24da0..7454c6e 100644
--- a/gcc/doc/libgdiagnostics/topics/diagnostics.rst
+++ b/gcc/doc/libgdiagnostics/topics/diagnostics.rst
@@ -105,6 +105,24 @@ Diagnostics are
 
    All three parameters must be non-NULL.
 
+.. function:: void diagnostic_finish_via_msg_buf (diagnostic *diag, \
+			       diagnostic_message_buffer *msg_buf)
+
+   This is equivalent to :func:`diagnostic_finish`, but using a message
+   buffer rather than a format string and variadic arguments.
+
+   ``diag`` and ``msg_buf`` must both be non-NULL.
+
+   Calling this function transfers ownership of ``msg_buf`` to the
+   diagnostic - do not call :func:`diagnostic_message_buffer_release` on
+   it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
 
 Diagnostic groups
 *****************
diff --git a/gcc/doc/libgdiagnostics/topics/execution-paths.rst b/gcc/doc/libgdiagnostics/topics/execution-paths.rst
index 321503f..8381c45 100644
--- a/gcc/doc/libgdiagnostics/topics/execution-paths.rst
+++ b/gcc/doc/libgdiagnostics/topics/execution-paths.rst
@@ -88,6 +88,28 @@ cross-references between events.  In particular FIXME
    Equivalent to :func:`diagnostic_execution_path_add_event`, but using a
    :type:`va_list` rather than directly taking variadic arguments.
 
+.. function:: diagnostic_event_id diagnostic_execution_path_add_event_via_msg_buf (diagnostic_execution_path *path, \
+						 const diagnostic_physical_location *physical_loc, \
+						 const diagnostic_logical_location *logical_loc, \
+						 unsigned stack_depth,
+						 diagnostic_message_buffer *msg_buf)
+
+   This is equivalent to :func:`diagnostic_execution_path_add_event` but
+   using a message buffer rather than a format string and variadic
+   arguments.
+
+   ``path`` and ``msg_buf`` must both be non-NULL.
+
+   Calling this function transfers ownership of ``msg_buf`` to the
+   path - do not call :func:`diagnostic_message_buffer_release` on it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
 Paths are printed to text sinks, and for SARIF sinks each path is added as
 a ``codeFlow`` object (see SARIF 2.1.0
 `§3.36 codeFlow object <https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/sarif-v2.1.0-errata01-os-complete.html#_Toc141790990>`_).
diff --git a/gcc/doc/libgdiagnostics/topics/graphs.rst b/gcc/doc/libgdiagnostics/topics/graphs.rst
new file mode 100644
index 0000000..b976013
--- /dev/null
+++ b/gcc/doc/libgdiagnostics/topics/graphs.rst
@@ -0,0 +1,197 @@
+.. Copyright (C) 2025 Free Software Foundation, Inc.
+   Originally contributed by David Malcolm <dmalcolm@redhat.com>
+
+   This is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <https://www.gnu.org/licenses/>.
+
+.. default-domain:: c
+
+Graphs
+======
+
+.. type:: diagnostic_graph
+
+SARIF has support for capturing directed graphs (such as callgraphs
+and control flow graphs), both at the level of the run as a whole,
+and at the level of individual results.
+
+libgdiagnostics supports this with the following entrypoints, allowing
+directed graphs to be
+
+* created (with :func:`diagnostic_manager_new_graph`)
+
+* reported "globally" (with :func:`diagnostic_manager_take_global_graph`)
+
+* reported as part of a :type:`diagnostic` (with :func:`diagnostic_take_graph`), or
+
+* discarded (with :func:`diagnostic_graph_release`).
+
+.. function:: diagnostic_graph * diagnostic_manager_new_graph (diagnostic_manager *manager)
+
+   Create a new directed graph.
+
+   The resulting graph is owned by the caller and must have one of
+   :func:`diagnostic_manager_take_global_graph`,
+   :func:`diagnostic_take_graph`,
+   or :func:`diagnostic_graph_release` called on it to avoid leaks.
+
+   The parameter ``manager`` must be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_manager_take_global_graph (diagnostic_manager *manager, \
+				      diagnostic_graph *graph)
+
+   Report this graph "globally", taking ownership of it.
+   This won't appear in text sinks, but in SARIF sinks the graph will be
+   added to theRun.graphs (SARIF v2.1.0 3.14.20).
+
+   Parameters ``manager`` and ``graph`` must both be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_take_graph (diagnostic *diag, \
+		      diagnostic_graph *graph)
+
+   Add this graph to ``diag``, transferring ownership of it to ``diag``.
+   This won't appear in text sinks, but in SARIF sinks the graph will be
+   added to theResult.graphs (SARIF v2.1.0 3.27.19).
+
+   Parameters ``diag`` and ``graph`` must both be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_graph_release (diagnostic_graph *graph)
+
+   Release ``graph`` which must still be owned by the caller
+   i.e. it must *not* have had
+   :func:`diagnostic_manager_take_global_graph` or
+   :func:`diagnostic_take_graph` called on it.
+
+   Parameters ``graph`` can be null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+
+.. function:: void diagnostic_graph_set_description (diagnostic_graph *graph, \
+				 const char *description)
+
+   Set the description of ``graph`` for use in the value of the
+   SARIF ``description`` property (SARIF v2.1.0 section 3.39.2).
+
+   The parameter ``graph`` must be non-null.
+   The parameter ``description`` can be null, for clearing any existing
+   description.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_node * diagnostic_graph_add_node (diagnostic_graph *graph, \
+			   const char *node_id, \
+			   diagnostic_node *parent_node)
+
+   Create and add a new node within ``graph`` with the given `id``.
+   The id must be unique within nodes in ``graph``.
+
+   The parameters ``graph`` and ``id`` must be non-null.
+
+   ``parent_node`` can be NULL (for a top-level node in the graph),
+   or non-null for a child node, allowing for arbitrary nesting of
+   nodes.
+
+   The new node is owned by ``graph``.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_edge * diagnostic_graph_add_edge (diagnostic_graph *graph, \
+			   const char *edge_id, \
+			   diagnostic_node *src_node, \
+			   diagnostic_node *dst_node, \
+			   const char *label)
+
+   Create and add a new edge within ``graph``.
+
+   The parameters ``graph``, ``src_node`` and ``dest_node``
+   must be non-null.
+
+   If non-null, then ``edge_id`` must be unique within ``graph``;
+   if ``edge_id`` is null then a unique id of the form "edge0", "edge1",
+   etc will be used automatically.
+
+   If non-null, then ``label`` will be used for the value of the
+   SARIF ``label`` property (SARIF v2.1.0 section 3.41.3).
+
+   The new edge is owned by ``graph``.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_node *diagnostic_graph_get_node_by_id (diagnostic_graph *graph, \
+				 const char *node_id)
+
+   Get the node in ``graph`` with the given id, or null.
+
+   The parameters ``graph`` and ``node_id`` must be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_edge *diagnostic_graph_get_edge_by_id (diagnostic_graph *graph, \
+				 const char *edge_id)
+
+   Get the edge in ``graph`` with the given id, or null.
+
+   The parameters ``graph`` and ``edge_id`` must be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+
+.. type:: diagnostic_node
+
+.. function:: void diagnostic_node_set_label (diagnostic_node *node, \
+			   const char *label)
+
+   Set the label of ``node`` for use in the value of the
+   SARIF ``label`` property (SARIF v2.1.0 section 3.40.3).
+
+   The parameter ``node`` must be non-null.
+   The parameter ``label`` can be null, for clearing any existing
+   label.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_node_set_location (diagnostic_node *node, \
+			      const diagnostic_physical_location *loc)
+
+   Set the physical location of ``node``, if any.
+
+   The parameter ``node`` must be non-null.
+   The parameter ``loc`` can be null, for clearing any existing
+   location.
+
+   If set, the value will be used by SARIF sinks within the
+   ``location`` property  (SARIF v2.1.0 section 3.40.4).
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_node_set_logical_location (diagnostic_node *node, \
+				      const diagnostic_logical_location *logical_loc)
+
+   Set the logical location of ``node``, if any.
+
+   The parameter ``node`` must be non-null.
+   The parameter ``logical_loc`` _can be null, for clearing any existing
+   location.
+
+   If set, the value will be used by SARIF sinks within the
+   ``location`` property  (SARIF v2.1.0 section 3.40.4).
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
diff --git a/gcc/doc/libgdiagnostics/topics/index.rst b/gcc/doc/libgdiagnostics/topics/index.rst
index 6eb3ed6..437ee05 100644
--- a/gcc/doc/libgdiagnostics/topics/index.rst
+++ b/gcc/doc/libgdiagnostics/topics/index.rst
@@ -28,6 +28,7 @@ Topic reference
    diagnostic-manager.rst
    diagnostics.rst
    message-formatting.rst
+   message-buffers.rst
    physical-locations.rst
    logical-locations.rst
    metadata.rst
@@ -35,5 +36,6 @@ Topic reference
    execution-paths.rst
    text-output.rst
    sarif.rst
+   graphs.rst
    ux.rst
    compatibility.rst
diff --git a/gcc/doc/libgdiagnostics/topics/logical-locations.rst b/gcc/doc/libgdiagnostics/topics/logical-locations.rst
index 184b563..294d396 100644
--- a/gcc/doc/libgdiagnostics/topics/logical-locations.rst
+++ b/gcc/doc/libgdiagnostics/topics/logical-locations.rst
@@ -120,7 +120,7 @@ source location
    "equal" input values on the same :type:`diagnostic_manager` will return
    the same instance of :type:`diagnostic_logical_location`.  "Equal" here
    includes different string buffers that compare as equal with
-   :func:``strcmp`.
+   :func:`strcmp`.
 
 .. function:: void diagnostic_manager_debug_dump_logical_location (const diagnostic_manager *diag_mgr, \
                                                                    const diagnostic_logical_location *loc, \
@@ -147,8 +147,9 @@ Accessors
 
 The following functions can be used to access the data that was passed to a
 :type:`diagnostic_logical_location` when it was created.  In each case, the
-``loc`` parameter must be non-NULL.  :type:`const char *` values will point
-at copies of the original buffer.
+``loc`` parameter must be non-NULL.  The return values will point
+at *copies* of the original buffer owned by the
+:type:`diagnostic_logical_location`, or be null.
 
 .. function:: enum diagnostic_logical_location_kind_t diagnostic_logical_location_get_kind (const diagnostic_logical_location *loc)
 
diff --git a/gcc/doc/libgdiagnostics/topics/message-buffers.rst b/gcc/doc/libgdiagnostics/topics/message-buffers.rst
new file mode 100644
index 0000000..c6f5851
--- /dev/null
+++ b/gcc/doc/libgdiagnostics/topics/message-buffers.rst
@@ -0,0 +1,310 @@
+.. Copyright (C) 2025 Free Software Foundation, Inc.
+   Originally contributed by David Malcolm <dmalcolm@redhat.com>
+
+   This is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <https://www.gnu.org/licenses/>.
+
+.. default-domain:: c
+
+Message buffers
+===============
+
+.. type:: diagnostic_message_buffer
+
+A :type:`diagnostic_message_buffer` is a buffer into which text can be
+accumulated, before being used:
+
+* as the message of a diagnostic, using :func:`diagnostic_finish_via_msg_buf`
+
+* as the text of a label for a :type:`diagnostic_physical_location` using
+  :func:`diagnostic_add_location_with_label_via_msg_buf`
+
+* as the text of an event within a :type:`diagnostic_execution_path` using
+  :func:`diagnostic_execution_path_add_event_via_msg_buf`
+
+This is to allow more flexible creation of messages than a "format string
+plus variadic arguments" API.
+
+.. function:: diagnostic_message_buffer * diagnostic_message_buffer_new (void)
+
+   This function creates a new :type:`diagnostic_message_buffer`.
+
+   The caller is responsible for cleaning it up, either by handing it off
+   to one of the API entrypoints that takes ownership of it (such as
+   :func:`diagnostic_finish_via_msg_buf`), or by calling
+   :func:`diagnostic_message_buffer_release` on it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_release (diagnostic_message_buffer *msg_buf)
+
+   This function releases ``msg_buf``.
+
+   Typically you don't need to call this, but instead will pass the
+   buffer to one of the API entrypoints that takes over ownership of
+   it (such as :func:`diagnostic_finish_via_msg_buf`); calling it
+   after this would lead to a double-free bug, as you no longer "own"
+   the buffer.
+
+   ``msg_buf`` must be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_str (diagnostic_message_buffer *msg_buf, \
+				      const char *p)
+
+   This function appends the null-terminated string ``p`` to the buffer.
+   The string is assumed to be UTF-8 encoded.
+
+   ``msg_buf`` and ``p`` must both be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_text (diagnostic_message_buffer *msg_buf, \
+	       const char *p, \
+	       size_t len)
+
+   This function appends ``len`` bytes from ``p`` to the buffer.
+   The bytes are assumed to be UTF-8 encoded.
+
+   ``msg_buf`` and ``p`` must both be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_byte (diagnostic_message_buffer *msg_buf,\
+				       char ch)
+
+   This function appends ``ch`` to the buffer.  This should be either
+   ASCII, or part of UTF-8 encoded text.
+
+   ``msg_buf`` must be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_printf (diagnostic_message_buffer *msg_buf, \
+					 const char *fmt, ...)
+
+   This function appends a formatted string to the buffer, using the
+   formatting rules for ``printf``.
+
+   The string is assumed to be UTF-8 encoded.
+
+   ``msg_buf`` and ``fmt`` must both be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_event_id (diagnostic_message_buffer *msg_buf, \
+					   diagnostic_event_id event_id)
+
+   This function appends a :type:`diagnostic_event_id` to the buffer.
+
+   ``msg_buf`` must be non-NULL.
+
+   For text output, the event will be printed in the form ``(1)``.
+
+   This is analogous to the
+   :doc:`%@ message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Hyperlink support
+*****************
+
+.. function:: void diagnostic_message_buffer_begin_url (diagnostic_message_buffer *msg_buf, \
+				     const char *url)
+
+   This function indicates the beginning of a run of text that should be
+   associated with the given URL.  The run of text should be closed with
+   a matching call to :func:`diagnostic_message_buffer_end_url`.
+
+   ``msg_buf`` and ``url`` must both be non-NULL.
+
+   For text output in a suitably modern terminal, the run of text will
+   be emitted as a clickable hyperlink to the URL.
+
+   For SARIF sinks, the run of text will be emitted using SARIF's
+   embedded link syntax.
+
+   This is analogous to the
+   :doc:`%{ message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+
+   This function ends a run of text within the buffer started with
+   :func:`diagnostic_message_buffer_begin_url`.
+
+   ``msg_buf`` must be non-NULL.
+
+   This is analogous to the
+   :doc:`%} message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Quoted text
+***********
+
+.. function:: void diagnostic_message_buffer_begin_quote (diagnostic_message_buffer *msg_buf)
+
+   This function indicates the beginning of a run of text that should be
+   printed in quotes.  The run of text should be closed with
+   a matching call to :func:`diagnostic_message_buffer_end_quote`.
+
+   ``msg_buf`` must be non-NULL.
+
+   For text output in a suitably modern terminal, the run of text will
+   appear in bold.
+   be emitted as a clickable hyperlink to the URL.
+
+   For SARIF sinks, the run of text will be emitted using SARIF's
+   embedded link syntax.
+
+   This is analogous to the
+   ``%<``:doc:`message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+
+   This function ends a run of text within the buffer started with
+   :func:`diagnostic_message_buffer_begin_url`.
+
+   ``msg_buf`` must be non-NULL.
+
+   This is analogous to the
+   :doc:`%> message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Color
+*****
+
+.. function:: void diagnostic_message_buffer_begin_color (diagnostic_message_buffer *msg_buf, \
+				     const char *color)
+
+   This function indicates the beginning of a run of text that should be
+   colorized as the given color.  The run of text should be closed with
+   a matching call to :func:`diagnostic_message_buffer_end_color`.
+
+   The precise set of available color names is currently undocumented.
+
+   ``msg_buf`` and ``color`` must both be non-NULL.
+
+   For text output in a suitable terminal, the run of text will
+   be colorized.
+
+   For SARIF sinks, the run of text will be emitted using SARIF's
+   embedded link syntax.
+
+   This is analogous to the
+   :doc:`%r message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_end_color (diagnostic_message_buffer *msg_buf)
+
+   This function ends a run of text within the buffer started with
+   :func:`diagnostic_message_buffer_begin_color`.
+
+   ``msg_buf`` must be non-NULL.
+
+   This is analogous to the
+   :doc:`%R message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Debugging a message buffer
+**************************
+
+.. function:: void diagnostic_message_buffer_dump (const diagnostic_message_buffer *msg_buf, \
+				FILE *outf)
+
+   This function writes a representation of the contents of ``msg_buf``
+   to ``outf``, for debugging.
+
+   ``msg_buf`` can be NULL or non-NULL.
+   ``outf`` must be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
diff --git a/gcc/doc/libgdiagnostics/topics/message-formatting.rst b/gcc/doc/libgdiagnostics/topics/message-formatting.rst
index 7064b70..803feba 100644
--- a/gcc/doc/libgdiagnostics/topics/message-formatting.rst
+++ b/gcc/doc/libgdiagnostics/topics/message-formatting.rst
@@ -23,6 +23,11 @@ Message formatting
 Various libgdiagnostics entrypoints take a format string and
 variadic arguments.
 
+.. note::
+
+   See also :type:`diagnostic_message_buffer`, which offers an
+   alternative way to build up messages.
+
 The format strings take codes prefixed by ``%``, or ``%q`` to put
 the result in quotes.  For example::
 
diff --git a/gcc/doc/libgdiagnostics/topics/physical-locations.rst b/gcc/doc/libgdiagnostics/topics/physical-locations.rst
index 099e27e..fcd81a0 100644
--- a/gcc/doc/libgdiagnostics/topics/physical-locations.rst
+++ b/gcc/doc/libgdiagnostics/topics/physical-locations.rst
@@ -284,3 +284,39 @@ This diagnostic has three locations
             |   ~~ ^ ~~~~~
             |   |    |
             |   int  const char *
+
+.. function:: void diagnostic_add_location_with_label_via_msg_buf (diagnostic *diag, \
+						const diagnostic_physical_location *loc, \
+						diagnostic_message_buffer *msg_buf)
+
+   This is equivalent to :func:`diagnostic_add_location_with_label` but
+   using a message buffer rather than a text string.
+
+   ``diag`` and ``msg_buf`` must both be non-NULL.
+
+   Calling this function transfers ownership of ``msg_buf`` to the
+   diagnostic - do not call :func:`diagnostic_message_buffer_release` on
+   it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_manager_set_debug_physical_locations (diagnostic_manager *mgr, \
+						 int value)
+
+   Calling ``diagnostic_manager_set_debug_physical_locations (mgr, 1);``
+   will lead to debugging information being printed to ``stderr`` when
+   creating :type:`diagnostic_physical_location` instances.
+
+   The precise format of these messages is subject to change.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_5`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_manager_set_debug_physical_locations
diff --git a/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst b/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst
index 95b95ca..e64b668 100644
--- a/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst
+++ b/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst
@@ -112,10 +112,10 @@ leading to output like this::
    foo.c:17: error: can't find 'foo.h'"
       17 | #include <foo.h>
 
-where libgdiagnostics will attempt to load the source file and
+where libgdiagnostics will attempt to load ``foo.c`` and
 quote the pertinent line.
 
-If libgdiagnostics cannot open the file, it will merely print::
+If libgdiagnostics cannot open ``foo.c``, it will merely print::
 
    foo.c:17: error: can't find 'foo.h'
 
@@ -160,14 +160,15 @@ On compiling and running the program, we should get this output::
       17 | #include <foo.h>
          |           ^~~~~
 
-where libgdiagnostics will attempt to load the source file and
+where libgdiagnostics will attempt to load ``foo.c`` and
 underling the pertinent part of the given line.
 
-If libgdiagnostics cannot open the file, it will merely print::
+If libgdiagnostics cannot open ``foo.c``, it will merely print::
 
    foo.c:17:8: error: can't find 'foo.h'
 
-A range can span multiple lines within the same file.
+A range can span multiple lines within the same file, but cannot
+span multiple files.
 
 As before, you can use :func:`diagnostic_manager_debug_dump_location` to
 dump the locations.  For the above example::
diff --git a/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst b/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst
index 0ac8bf0..9147171 100644
--- a/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst
+++ b/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst
@@ -134,8 +134,6 @@ Here's the above example in full:
    :end-before:  end full example
 
 
-Moving on
-*********
-
-That's the end of the tutorial.  For more information on libgdiagnostics, see
-the :doc:`topic guide <../topics/index>`.
+See the :doc:`guide to execution paths <../topics/execution-paths>`
+for more information, or go on to
+:doc:`the next section of the tutorial <08-message-buffers>`.
diff --git a/gcc/doc/libgdiagnostics/tutorial/08-message-buffers.rst b/gcc/doc/libgdiagnostics/tutorial/08-message-buffers.rst
new file mode 100644
index 0000000..a83c50c
--- /dev/null
+++ b/gcc/doc/libgdiagnostics/tutorial/08-message-buffers.rst
@@ -0,0 +1,75 @@
+.. Copyright (C) 2025 Free Software Foundation, Inc.
+   Originally contributed by David Malcolm <dmalcolm@redhat.com>
+
+   This is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <https://www.gnu.org/licenses/>.
+
+.. default-domain:: c
+
+Tutorial part 8: message buffers
+================================
+
+In previous examples, we finished a diagnostic with a call to
+:func:`diagnostic_finish`, which takes a format string and arguments
+to determine the text message of the diagnostic.
+
+Sometimes this approach is inconvenient, such as where you might want to
+build up a message programatically from a series of components.
+Additionally, you might have existing code that uses ``fprintf``, whereas
+:func:`diagnostic_finish` has its
+:doc:`own formatting conventions <../topics/message-formatting>` which are
+:strong:`not` the same as printf.
+
+For this reason libgdiagnostics (from ``LIBGDIAGNOSTICS_ABI_3`` onwards)
+supports :type:`diagnostic_message_buffer`, which can be used to accumulate a
+message before using it.
+
+You create a :type:`diagnostic_message_buffer` using
+:func:`diagnostic_message_buffer_new`.
+
+There are various API entrypoints for accumulating text into the buffer.
+
+For example:
+
+.. literalinclude:: ../../../testsuite/libgdiagnostics.dg/test-message-buffer.c
+   :language: c
+   :start-after: /* begin quoted source */
+   :end-before:  /* end quoted source */
+
+Running this will produce this text output::
+
+.. code-block:: console
+
+  $ ./test-message-buffer.c.exe
+  ./test-message-buffer.c.exe: error: this is a string; foo; int: 42 str: mostly harmless; this is a link  'this is quoted' highlight A highlight B (1).
+
+where in a suitably-capable terminal if a text sink is directly
+connected to a tty:
+
+* the ``this is a link`` will be a clickable hyperlink
+  (and the URL will be captured in SARIF output).
+
+* the quoted text will be in bold
+
+* the ``highlight A`` and ``highlight B`` text will be colorized
+
+* the event ID will be colorized (and will be a URL in SARIF output
+  if used within a :type:`diagnostic_execution_path`).
+
+
+Moving on
+*********
+
+That's the end of the tutorial.  For more information on libgdiagnostics, see
+the :doc:`topic guide <../topics/index>`.
diff --git a/gcc/doc/libgdiagnostics/tutorial/index.rst b/gcc/doc/libgdiagnostics/tutorial/index.rst
index 172a28c..09a15e9 100644
--- a/gcc/doc/libgdiagnostics/tutorial/index.rst
+++ b/gcc/doc/libgdiagnostics/tutorial/index.rst
@@ -30,3 +30,4 @@ The following tutorial gives an overview of how to use libgdiagnostics.
    05-warnings.rst
    06-fix-it-hints.rst
    07-execution-paths.rst
+   08-message-buffers.rst
diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi
index 28159b2..aba93f6 100644
--- a/gcc/doc/md.texi
+++ b/gcc/doc/md.texi
@@ -1366,6 +1366,12 @@ as for @samp{<} apply.
 A register operand is allowed provided that it is in a general
 register.
 
+@cindex hard registers in constraint
+@item @samp{@{r@}}
+An operand is bound to hard register @samp{r} which may be any general,
+floating-point, or vector register except a fixed register like a stack pointer
+register.  The set of fixed registers is target dependent.
+
 @cindex constants in constraints
 @cindex @samp{i} in constraint
 @item @samp{i}
diff --git a/gcc/doc/riscv-ext.texi b/gcc/doc/riscv-ext.texi
index c3ed1bf..572b70e 100644
--- a/gcc/doc/riscv-ext.texi
+++ b/gcc/doc/riscv-ext.texi
@@ -714,4 +714,8 @@
 @tab 1.0
 @tab Ventana integer conditional operations extension
 
+@item xmipscmov
+@tab 1.0
+@tab Mips conditional move extension
+
 @end multitable
diff --git a/gcc/doc/riscv-mcpu.texi b/gcc/doc/riscv-mcpu.texi
new file mode 100644
index 0000000..6753e51
--- /dev/null
+++ b/gcc/doc/riscv-mcpu.texi
@@ -0,0 +1,69 @@
+@c Copyright (C) 2025 Free Software Foundation, Inc.
+@c This is part of the GCC manual.
+@c For copying conditions, see the file gcc/doc/include/fdl.texi.
+
+@c This file is generated automatically using
+@c  gcc/config/riscv/gen-riscv-mcpu-texi.cc from:
+@c       gcc/config/riscv/riscv-cores.def
+
+@c Please *DO NOT* edit manually.
+
+@samp{Core Name}
+
+@opindex mcpu
+@item -mcpu=@var{processor-string}
+Use architecture of and optimize the output for the given processor, specified
+by particular CPU name. Permissible values for this option are:
+
+
+@samp{sifive-e20},
+
+@samp{sifive-e21},
+
+@samp{sifive-e24},
+
+@samp{sifive-e31},
+
+@samp{sifive-e34},
+
+@samp{sifive-e76},
+
+@samp{sifive-s21},
+
+@samp{sifive-s51},
+
+@samp{sifive-s54},
+
+@samp{sifive-s76},
+
+@samp{sifive-u54},
+
+@samp{sifive-u74},
+
+@samp{sifive-x280},
+
+@samp{sifive-p450},
+
+@samp{sifive-p670},
+
+@samp{thead-c906},
+
+@samp{xt-c908},
+
+@samp{xt-c908v},
+
+@samp{xt-c910},
+
+@samp{xt-c910v2},
+
+@samp{xt-c920},
+
+@samp{xt-c920v2},
+
+@samp{tt-ascalon-d8},
+
+@samp{xiangshan-nanhu},
+
+@samp{xiangshan-kunminghu},
+
+@samp{mips-p8700}.
diff --git a/gcc/doc/riscv-mtune.texi b/gcc/doc/riscv-mtune.texi
new file mode 100644
index 0000000..a2a4d3e
--- /dev/null
+++ b/gcc/doc/riscv-mtune.texi
@@ -0,0 +1,59 @@
+@c Copyright (C) 2025 Free Software Foundation, Inc.
+@c This is part of the GCC manual.
+@c For copying conditions, see the file gcc/doc/include/fdl.texi.
+
+@c This file is generated automatically using
+@c  gcc/config/riscv/gen-riscv-mtune-texi.cc from:
+@c       gcc/config/riscv/riscv-cores.def
+
+@c Please *DO NOT* edit manually.
+
+@samp{Tune Name}
+
+@opindex mtune
+@item -mtune=@var{processor-string}
+Optimize the output for the given processor, specified by microarchitecture or
+particular CPU name.  Permissible values for this option are:
+
+
+@samp{generic},
+
+@samp{rocket},
+
+@samp{sifive-3-series},
+
+@samp{sifive-5-series},
+
+@samp{sifive-7-series},
+
+@samp{sifive-p400-series},
+
+@samp{sifive-p600-series},
+
+@samp{tt-ascalon-d8},
+
+@samp{thead-c906},
+
+@samp{xt-c908},
+
+@samp{xt-c908v},
+
+@samp{xt-c910},
+
+@samp{xt-c910v2},
+
+@samp{xt-c920},
+
+@samp{xt-c920v2},
+
+@samp{xiangshan-nanhu},
+
+@samp{xiangshan-kunminghu},
+
+@samp{generic-ooo},
+
+@samp{size},
+
+@samp{mips-p8700},
+
+and all valid options for @option{-mcpu=}.
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 6c5586e..a919304 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -2373,6 +2373,15 @@ whether it does so by default).
 @itemx aarch64_sve1024_hw
 @itemx aarch64_sve2048_hw
 Like @code{aarch64_sve_hw}, but also test for an exact hardware vector length.
+@item aarch64_sve2_hw
+AArch64 target that is able to generate and execute SVE2 code (regardless of
+whether it does so by default).
+@item aarch64_sve2p1_hw
+AArch64 target that is able to generate and execute SVE2.1 code (regardless of
+whether it does so by default).
+@item aarch64_sme_hw
+AArch64 target that is able to generate and execute SME code (regardless of
+whether it does so by default).
 
 @item aarch64_fjcvtzs_hw
 AArch64 target that is able to generate and execute armv8.3-a FJCVTZS
diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi
index 5e30564..4d4e676 100644
--- a/gcc/doc/tm.texi
+++ b/gcc/doc/tm.texi
@@ -1424,10 +1424,10 @@ the smaller of @var{computed} and @code{BIGGEST_ALIGNMENT}
 
 @defmac MAX_FIXED_MODE_SIZE
 An integer expression for the size in bits of the largest integer
-machine mode that should actually be used.  All integer machine modes of
-this size or smaller can be used for structures and unions with the
-appropriate sizes.  If this macro is undefined, @code{GET_MODE_BITSIZE
-(DImode)} is assumed.
+machine mode that should actually be used by GCC internally.
+All integer machine modes of this size or smaller can be used for
+structures and unions with the appropriate sizes.  If this macro is
+undefined, @code{MAX (BITS_PER_WORD * 2, 64)} is assumed.
 @end defmac
 
 @defmac STACK_SAVEAREA_MODE (@var{save_level})
@@ -6382,12 +6382,14 @@ return type of the vectorized function shall be of vector type
 @var{vec_type_out} and the argument types should be @var{vec_type_in}.
 @end deftypefn
 
-@deftypefn {Target Hook} bool TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT (machine_mode @var{mode}, const_tree @var{type}, int @var{misalignment}, bool @var{is_packed})
+@deftypefn {Target Hook} bool TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT (machine_mode @var{mode}, const_tree @var{type}, int @var{misalignment}, bool @var{is_packed}, bool @var{is_gather_scatter})
 This hook should return true if the target supports misaligned vector
 store/load of a specific factor denoted in the @var{misalignment}
 parameter.  The vector store/load should be of machine mode @var{mode} and
-the elements in the vectors should be of type @var{type}.  @var{is_packed}
-parameter is true if the memory access is defined in a packed struct.
+the elements in the vectors should be of type @var{type}.  The
+@var{is_packed} parameter is true if the misalignment is unknown and the
+memory access is defined in a packed struct.  @var{is_gather_scatter} is true
+if the load/store is a gather or scatter.
 @end deftypefn
 
 @deftypefn {Target Hook} machine_mode TARGET_VECTORIZE_PREFERRED_SIMD_MODE (scalar_mode @var{mode})
@@ -6511,6 +6513,15 @@ The default is @code{NULL_TREE} which means to not vectorize scatter
 stores.
 @end deftypefn
 
+@deftypefn {Target Hook} bool TARGET_VECTORIZE_PREFER_GATHER_SCATTER (machine_mode @var{mode}, int @var{scale}, unsigned int @var{group_size})
+This hook returns TRUE if gather loads or scatter stores are cheaper on
+this target than a sequence of elementwise loads or stores.  The @var{mode}
+and @var{scale} correspond to the @code{gather_load} and
+@code{scatter_store} instruction patterns.  The @var{group_size} is the
+number of scalar elements in each scalar loop iteration that are to be
+combined into the vector.
+@end deftypefn
+
 @deftypefn {Target Hook} int TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN (struct cgraph_node *@var{}, struct cgraph_simd_clone *@var{}, @var{tree}, @var{int}, @var{bool})
 This hook should set @var{vecsize_mangle}, @var{vecsize_int}, @var{vecsize_float}
 fields in @var{simd_clone} structure pointed by @var{clone_info} argument and also
diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in
index eccc4d8..1a51ad5 100644
--- a/gcc/doc/tm.texi.in
+++ b/gcc/doc/tm.texi.in
@@ -1251,10 +1251,10 @@ the smaller of @var{computed} and @code{BIGGEST_ALIGNMENT}
 
 @defmac MAX_FIXED_MODE_SIZE
 An integer expression for the size in bits of the largest integer
-machine mode that should actually be used.  All integer machine modes of
-this size or smaller can be used for structures and unions with the
-appropriate sizes.  If this macro is undefined, @code{GET_MODE_BITSIZE
-(DImode)} is assumed.
+machine mode that should actually be used by GCC internally.
+All integer machine modes of this size or smaller can be used for
+structures and unions with the appropriate sizes.  If this macro is
+undefined, @code{MAX (BITS_PER_WORD * 2, 64)} is assumed.
 @end defmac
 
 @defmac STACK_SAVEAREA_MODE (@var{save_level})
@@ -4311,6 +4311,8 @@ address;  but often a machine-dependent strategy can generate better code.
 
 @hook TARGET_VECTORIZE_BUILTIN_SCATTER
 
+@hook TARGET_VECTORIZE_PREFER_GATHER_SCATTER
+
 @hook TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
 
 @hook TARGET_SIMD_CLONE_ADJUST
diff --git a/gcc/dump-context.h b/gcc/dump-context.h
index 9c52f03..336cb23 100644
--- a/gcc/dump-context.h
+++ b/gcc/dump-context.h
@@ -267,7 +267,7 @@ extern void verify_dumped_text (const location &loc,
 void
 verify_item (const location &loc,
 	     const optinfo_item *item,
-	     enum optinfo_item_kind expected_kind,
+	     enum optinfo_item::kind expected_kind,
 	     location_t expected_location,
 	     const char *expected_text);
 
@@ -275,7 +275,7 @@ verify_item (const location &loc,
 
 #define ASSERT_IS_TEXT(ITEM, EXPECTED_TEXT) \
   SELFTEST_BEGIN_STMT						    \
-    verify_item (SELFTEST_LOCATION, (ITEM), OPTINFO_ITEM_KIND_TEXT, \
+    verify_item (SELFTEST_LOCATION, (ITEM), optinfo_item::kind::text, \
 		 UNKNOWN_LOCATION, (EXPECTED_TEXT));		    \
   SELFTEST_END_STMT
 
@@ -283,7 +283,7 @@ verify_item (const location &loc,
 
 #define ASSERT_IS_TREE(ITEM, EXPECTED_LOCATION, EXPECTED_TEXT) \
   SELFTEST_BEGIN_STMT						    \
-    verify_item (SELFTEST_LOCATION, (ITEM), OPTINFO_ITEM_KIND_TREE, \
+    verify_item (SELFTEST_LOCATION, (ITEM), optinfo_item::kind::tree, \
 		 (EXPECTED_LOCATION), (EXPECTED_TEXT));	    \
   SELFTEST_END_STMT
 
@@ -291,7 +291,7 @@ verify_item (const location &loc,
 
 #define ASSERT_IS_GIMPLE(ITEM, EXPECTED_LOCATION, EXPECTED_TEXT) \
   SELFTEST_BEGIN_STMT						    \
-    verify_item (SELFTEST_LOCATION, (ITEM), OPTINFO_ITEM_KIND_GIMPLE, \
+    verify_item (SELFTEST_LOCATION, (ITEM), optinfo_item::kind::gimple,	\
 		 (EXPECTED_LOCATION), (EXPECTED_TEXT));	    \
   SELFTEST_END_STMT
 
@@ -299,7 +299,7 @@ verify_item (const location &loc,
 
 #define ASSERT_IS_SYMTAB_NODE(ITEM, EXPECTED_LOCATION, EXPECTED_TEXT) \
   SELFTEST_BEGIN_STMT						    \
-    verify_item (SELFTEST_LOCATION, (ITEM), OPTINFO_ITEM_KIND_SYMTAB_NODE, \
+    verify_item (SELFTEST_LOCATION, (ITEM), optinfo_item::kind::symtab_node, \
 		 (EXPECTED_LOCATION), (EXPECTED_TEXT));	    \
   SELFTEST_END_STMT
 
diff --git a/gcc/dumpfile.cc b/gcc/dumpfile.cc
index ebee8e5..bca8b38 100644
--- a/gcc/dumpfile.cc
+++ b/gcc/dumpfile.cc
@@ -636,8 +636,8 @@ make_item_for_dump_gimple_stmt (gimple *stmt, int spc, dump_flags_t dump_flags)
   pp_gimple_stmt_1 (&pp, stmt, spc, dump_flags);
   pp_newline (&pp);
 
-  std::unique_ptr<optinfo_item> item
-    = std::make_unique<optinfo_item> (OPTINFO_ITEM_KIND_GIMPLE,
+  auto item
+    = std::make_unique<optinfo_item> (optinfo_item::kind::gimple,
 				      gimple_location (stmt),
 				      xstrdup (pp_formatted_text (&pp)));
   return item;
@@ -684,8 +684,8 @@ make_item_for_dump_gimple_expr (gimple *stmt, int spc, dump_flags_t dump_flags)
   pp_needs_newline (&pp) = true;
   pp_gimple_stmt_1 (&pp, stmt, spc, dump_flags);
 
-  std::unique_ptr<optinfo_item> item
-    = std::make_unique<optinfo_item> (OPTINFO_ITEM_KIND_GIMPLE,
+  auto item
+    = std::make_unique<optinfo_item> (optinfo_item::kind::gimple,
 				      gimple_location (stmt),
 				      xstrdup (pp_formatted_text (&pp)));
   return item;
@@ -738,8 +738,8 @@ make_item_for_dump_generic_expr (tree node, dump_flags_t dump_flags)
   if (EXPR_HAS_LOCATION (node))
     loc = EXPR_LOCATION (node);
 
-  std::unique_ptr<optinfo_item> item
-    = std::make_unique<optinfo_item> (OPTINFO_ITEM_KIND_TREE, loc,
+  auto item
+    = std::make_unique<optinfo_item> (optinfo_item::kind::tree, loc,
 				      xstrdup (pp_formatted_text (&pp)));
   return item;
 }
@@ -783,8 +783,8 @@ static std::unique_ptr<optinfo_item>
 make_item_for_dump_symtab_node (symtab_node *node)
 {
   location_t loc = DECL_SOURCE_LOCATION (node->decl);
-  std::unique_ptr<optinfo_item> item
-    = std::make_unique<optinfo_item> (OPTINFO_ITEM_KIND_SYMTAB_NODE, loc,
+  auto item
+    = std::make_unique<optinfo_item> (optinfo_item::kind::symtab_node, loc,
 				      xstrdup (node->dump_name ()));
   return item;
 }
@@ -1011,8 +1011,9 @@ emit_any_pending_textual_chunks ()
     return;
 
   char *formatted_text = xstrdup (pp_formatted_text (pp));
-  std::unique_ptr<optinfo_item> item
-    = std::make_unique<optinfo_item> (OPTINFO_ITEM_KIND_TEXT, UNKNOWN_LOCATION,
+  auto item
+    = std::make_unique<optinfo_item> (optinfo_item::kind::text,
+				      UNKNOWN_LOCATION,
 				      formatted_text);
   pp->emit_item (std::move (item), m_optinfo);
 
@@ -1084,7 +1085,8 @@ make_item_for_dump_dec (const poly_int<N, C> &value)
     }
 
   auto item
-    = std::make_unique<optinfo_item> (OPTINFO_ITEM_KIND_TEXT, UNKNOWN_LOCATION,
+    = std::make_unique<optinfo_item> (optinfo_item::kind::text,
+				      UNKNOWN_LOCATION,
 				      xstrdup (pp_formatted_text (&pp)));
   return item;
 }
@@ -1162,8 +1164,9 @@ dump_context::begin_scope (const char *name,
   pretty_printer pp;
   pp_printf (&pp, "%s %s %s", "===", name, "===");
   pp_newline (&pp);
-  std::unique_ptr<optinfo_item> item
-    = std::make_unique<optinfo_item> (OPTINFO_ITEM_KIND_TEXT, UNKNOWN_LOCATION,
+  auto item
+    = std::make_unique<optinfo_item> (optinfo_item::kind::text,
+				      UNKNOWN_LOCATION,
 				      xstrdup (pp_formatted_text (&pp)));
   emit_item (*item.get (), MSG_NOTE);
 
@@ -1172,7 +1175,7 @@ dump_context::begin_scope (const char *name,
       optinfo &info
 	= begin_next_optinfo (dump_metadata_t (MSG_NOTE, impl_location),
 			      user_location);
-      info.m_kind = OPTINFO_KIND_SCOPE;
+      info.m_kind = optinfo::kind::scope;
       info.add_item (std::move (item));
       end_any_optinfo ();
     }
@@ -1221,7 +1224,7 @@ dump_context::begin_next_optinfo (const dump_metadata_t &metadata,
   end_any_optinfo ();
   gcc_assert (m_pending == NULL);
   dump_location_t loc (user_loc, metadata.get_impl_location ());
-  m_pending = new optinfo (loc, OPTINFO_KIND_NOTE, current_pass);
+  m_pending = new optinfo (loc, optinfo::kind::note, current_pass);
   m_pending->handle_dump_file_kind (metadata.get_dump_flags ());
   return *m_pending;
 }
@@ -2265,7 +2268,7 @@ verify_dumped_text (const location &loc,
 void
 verify_item (const location &loc,
 	     const optinfo_item *item,
-	     enum optinfo_item_kind expected_kind,
+	     enum optinfo_item::kind expected_kind,
 	     location_t expected_location,
 	     const char *expected_text)
 {
@@ -2324,7 +2327,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	  {
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 1);
 	    ASSERT_IS_TEXT (info->get_item (0), "int: 42 str: foo");
 	    ASSERT_IMPL_LOCATION_EQ (info->get_impl_location (),
@@ -2345,7 +2348,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	  {
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 2);
 	    ASSERT_IS_TEXT (info->get_item (0), "tree: ");
 	    ASSERT_IS_TREE (info->get_item (1), UNKNOWN_LOCATION, "0");
@@ -2367,7 +2370,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	  {
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 2);
 	    ASSERT_IS_TEXT (info->get_item (0), "gimple: ");
 	    ASSERT_IS_GIMPLE (info->get_item (1), stmt_loc, "return;");
@@ -2389,7 +2392,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	  {
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 2);
 	    ASSERT_IS_TEXT (info->get_item (0), "gimple: ");
 	    ASSERT_IS_GIMPLE (info->get_item (1), stmt_loc, "return;\n");
@@ -2411,7 +2414,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	  {
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 2);
 	    ASSERT_IS_TEXT (info->get_item (0), "node: ");
 	    ASSERT_IS_SYMTAB_NODE (info->get_item (1), decl_loc, "test_decl/1");
@@ -2441,7 +2444,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	  {
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 8);
 	    ASSERT_IS_TEXT (info->get_item (0), "before ");
 	    ASSERT_IS_TREE (info->get_item (1), UNKNOWN_LOCATION, "0");
@@ -2471,7 +2474,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
 	    ASSERT_EQ (info->get_location_t (), stmt_loc);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 2);
 	    ASSERT_IS_TEXT (info->get_item (0), "test of tree: ");
 	    ASSERT_IS_TREE (info->get_item (1), UNKNOWN_LOCATION, "0");
@@ -2494,7 +2497,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
 	    ASSERT_EQ (info->get_location_t (), stmt_loc);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 1);
 	    ASSERT_IS_TREE (info->get_item (0), UNKNOWN_LOCATION, "1");
 	    ASSERT_IMPL_LOCATION_EQ (info->get_impl_location (),
@@ -2598,7 +2601,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
 	  {
 	    optinfo *info = tmp.get_pending_optinfo ();
 	    ASSERT_TRUE (info != NULL);
-	    ASSERT_EQ (info->get_kind (), OPTINFO_KIND_NOTE);
+	    ASSERT_EQ (info->get_kind (), optinfo::kind::note);
 	    ASSERT_EQ (info->num_items (), 1);
 	    ASSERT_IS_SYMTAB_NODE (info->get_item (0), decl_loc, "test_decl/1");
 	    ASSERT_IMPL_LOCATION_EQ (info->get_impl_location (),
@@ -2727,7 +2730,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
       temp_dump_context tmp (true, true, MSG_ALL_KINDS);
       dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc, "test");
       ASSERT_EQ (tmp.get_pending_optinfo ()->get_kind (),
-		 OPTINFO_KIND_SUCCESS);
+		 optinfo::kind::success);
     }
 
     /* MSG_MISSED_OPTIMIZATION.  */
@@ -2735,7 +2738,7 @@ test_capture_of_dump_calls (const line_table_case &case_)
       temp_dump_context tmp (true, true, MSG_ALL_KINDS);
       dump_printf_loc (MSG_MISSED_OPTIMIZATION, loc, "test");
       ASSERT_EQ (tmp.get_pending_optinfo ()->get_kind (),
-		 OPTINFO_KIND_FAILURE);
+		 optinfo::kind::failure);
     }
   }
 
diff --git a/gcc/errors.cc b/gcc/errors.cc
index 6d1ecc7..fb6b19d 100644
--- a/gcc/errors.cc
+++ b/gcc/errors.cc
@@ -19,7 +19,8 @@ along with GCC; see the file COPYING3.  If not see
 
 /* warning, error, and fatal.  These definitions are suitable for use
    in the generator programs; the compiler has a more elaborate suite
-   of diagnostic printers, found in diagnostic.cc.  */
+   of diagnostic printers, found in diagnostic-global-context.cc using
+   the diagnostics/ subdirectory.  */
 
 #ifdef HOST_GENERATOR_FILE
 #include "config.h"
@@ -126,7 +127,7 @@ trim_filename (const char *name)
 
 /* "Fancy" abort.  Reports where in the compiler someone gave up.
    This file is used only by build programs, so we're not as polite as
-   the version in diagnostic.cc.  */
+   the version in diagnostics/context.cc.  */
 void
 fancy_abort (const char *file, int line, const char *func)
 {
diff --git a/gcc/explow.cc b/gcc/explow.cc
index 7799a98..8f8ca7f 100644
--- a/gcc/explow.cc
+++ b/gcc/explow.cc
@@ -854,6 +854,18 @@ promote_function_mode (const_tree type, machine_mode mode, int *punsignedp,
 
   switch (TREE_CODE (type))
     {
+    case BITINT_TYPE:
+      if (TYPE_MODE (type) == BLKmode)
+	return mode;
+
+      struct bitint_info info;
+      bool ok;
+      ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
+      gcc_assert (ok);
+
+      if (!info.extended)
+	return mode;
+      /* FALLTHRU */
     case INTEGER_TYPE:   case ENUMERAL_TYPE:   case BOOLEAN_TYPE:
     case REAL_TYPE:      case OFFSET_TYPE:     case FIXED_POINT_TYPE:
     case POINTER_TYPE:   case REFERENCE_TYPE:
@@ -893,6 +905,18 @@ promote_mode (const_tree type ATTRIBUTE_UNUSED, machine_mode mode,
 
   switch (code)
     {
+    case BITINT_TYPE:
+      if (TYPE_MODE (type) == BLKmode)
+	return mode;
+
+      struct bitint_info info;
+      bool ok;
+      ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
+      gcc_assert (ok);
+
+      if (!info.extended)
+	return mode;
+      /* FALLTHRU */
     case INTEGER_TYPE:   case ENUMERAL_TYPE:   case BOOLEAN_TYPE:
     case REAL_TYPE:      case OFFSET_TYPE:     case FIXED_POINT_TYPE:
       /* Values of these types always have scalar mode.  */
diff --git a/gcc/expr.cc b/gcc/expr.cc
index ac4fdfa..3d2b253 100644
--- a/gcc/expr.cc
+++ b/gcc/expr.cc
@@ -76,6 +76,10 @@ along with GCC; see the file COPYING3.  If not see
    the same indirect address eventually.  */
 int cse_not_expected;
 
+/* Cache of the "extended" flag in the target's _BitInt description
+   for use during expand.  */
+int bitint_extended = -1;
+
 static bool block_move_libcall_safe_for_call_parm (void);
 static bool emit_block_move_via_pattern (rtx, rtx, rtx, unsigned, unsigned,
 					 HOST_WIDE_INT, unsigned HOST_WIDE_INT,
@@ -11280,6 +11284,7 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
      when reading from SSA_NAMEs of vars.  */
 #define EXTEND_BITINT(expr) \
   ((TREE_CODE (type) == BITINT_TYPE					\
+    && !bitint_extended							\
     && reduce_bit_field							\
     && mode != BLKmode							\
     && modifier != EXPAND_MEMORY					\
@@ -11291,6 +11296,13 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
   type = TREE_TYPE (exp);
   mode = TYPE_MODE (type);
   unsignedp = TYPE_UNSIGNED (type);
+  if (TREE_CODE (type) == BITINT_TYPE && bitint_extended == -1)
+    {
+      struct bitint_info info;
+      bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
+      gcc_assert (ok);
+      bitint_extended = info.extended;
+    }
 
   treeop0 = treeop1 = treeop2 = NULL_TREE;
   if (!VL_EXP_CLASS_P (exp))
@@ -13206,6 +13218,8 @@ constant_byte_string (tree arg, tree *ptr_offset, tree *mem_size, tree *decl,
 	     of the expected type and size.  */
 	  if (!initsize)
 	    initsize = integer_zero_node;
+	  else if (!tree_fits_uhwi_p (initsize))
+	    return NULL_TREE;
 
 	  unsigned HOST_WIDE_INT size = tree_to_uhwi (initsize);
 	  if (size > (unsigned HOST_WIDE_INT) INT_MAX)
diff --git a/gcc/expr.h b/gcc/expr.h
index 53ab625..060151d 100644
--- a/gcc/expr.h
+++ b/gcc/expr.h
@@ -388,4 +388,8 @@ extern void expand_crc_table_based (rtx, rtx, rtx, rtx, machine_mode);
 extern void expand_reversed_crc_table_based (rtx, rtx, rtx, rtx, machine_mode,
 					     void (*) (rtx *));
 
+/* Cache of the "extended" flag in the target's _BitInt description
+   for use during expand.  */
+extern int bitint_extended;
+
 #endif /* GCC_EXPR_H */
diff --git a/gcc/ext-dce.cc b/gcc/ext-dce.cc
index afe7afe..67ec92a 100644
--- a/gcc/ext-dce.cc
+++ b/gcc/ext-dce.cc
@@ -651,9 +651,8 @@ ext_dce_process_uses (rtx_insn *insn, rtx obj,
 
 	  /* ?!? How much of this should mirror SET handling, potentially
 	     being shared?   */
-	  if (SUBREG_P (dst) && SUBREG_BYTE (dst).is_constant ())
+	  if (SUBREG_P (dst) && subreg_lsb (dst).is_constant (&bit))
 	    {
-	      bit = subreg_lsb (dst).to_constant ();
 	      if (bit >= HOST_BITS_PER_WIDE_INT)
 		bit = HOST_BITS_PER_WIDE_INT - 1;
 	      dst = SUBREG_REG (dst);
@@ -758,7 +757,7 @@ ext_dce_process_uses (rtx_insn *insn, rtx obj,
 		     and process the inner object.  */
 		  if (paradoxical_subreg_p (y))
 		    y = XEXP (y, 0);
-		  else if (SUBREG_P (y) && SUBREG_BYTE (y).is_constant ())
+		  else if (SUBREG_P (y) && subreg_lsb (y).is_constant (&bit))
 		    {
 		      /* If !TRULY_NOOP_TRUNCATION_MODES_P, the mode
 			 change performed by Y would normally need to be a
@@ -775,8 +774,6 @@ ext_dce_process_uses (rtx_insn *insn, rtx obj,
 				    GET_MODE (SUBREG_REG (y))))))
 			break;
 
-		      bit = subreg_lsb (y).to_constant ();
-
 		      /* If this is a wide object (more bits than we can fit
 			 in a HOST_WIDE_INT), then just break from the SET
 			 context.   That will cause the iterator to walk down
diff --git a/gcc/final.cc b/gcc/final.cc
index a4dbab7..afcb0bb 100644
--- a/gcc/final.cc
+++ b/gcc/final.cc
@@ -83,6 +83,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "function-abi.h"
 #include "common/common-target.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 
 #include "dwarf2out.h"
 
@@ -2116,7 +2117,7 @@ asm_show_source (const char *filename, int linenum)
   if (!filename)
     return;
 
-  char_span line
+  diagnostics::char_span line
     = global_dc->get_file_cache ().get_source_line (filename, linenum);
   if (!line)
     return;
diff --git a/gcc/fold-const.cc b/gcc/fold-const.cc
index 4749257..8867540 100644
--- a/gcc/fold-const.cc
+++ b/gcc/fold-const.cc
@@ -15224,7 +15224,7 @@ bool
 tree_expr_nonnegative_warnv_p (tree t, bool *strict_overflow_p, int depth)
 {
   enum tree_code code;
-  if (t == error_mark_node)
+  if (error_operand_p (t))
     return false;
 
   code = TREE_CODE (t);
diff --git a/gcc/fortran/ChangeLog b/gcc/fortran/ChangeLog
index 0ea9c39..d75d64f 100644
--- a/gcc/fortran/ChangeLog
+++ b/gcc/fortran/ChangeLog
@@ -1,3 +1,272 @@
+2025-08-01  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-decl.cc (gfc_trans_deferred_vars): Fix closing brace in
+	a comment.
+
+2025-07-31  Mikael Morin  <morin-mikael@orange.fr>
+
+	PR fortran/121342
+	* trans-expr.cc (gfc_conv_subref_array_arg): Remove offset
+	update.
+	(gfc_conv_procedure_call): For polymorphic functions, move the
+	scalarizer descriptor information...
+	* trans-array.cc (gfc_add_loop_ss_code): ... here, and evaluate
+	the bounds to fresh variables.
+	(get_class_info_from_ss): Remove offset update.
+	(gfc_conv_ss_startstride): Don't set a zero value for function
+	result upper bounds.
+	(late_set_loop_bounds): New.
+	(gfc_conv_loop_setup): If the bounds of a function result have
+	been set, and no other array provided loop bounds for a
+	dimension, use the function result bounds as loop bounds for
+	that dimension.
+	(gfc_set_delta): Don't skip delta setting for polymorphic
+	function results.
+
+2025-07-30  Mikael Morin  <morin-mikael@orange.fr>
+
+	* trans-array.cc (gfc_array_init_size): Remove the nelems
+	argument.
+	(gfc_array_allocate): Update caller.  Remove the nelems
+	argument.
+	* trans-stmt.cc (gfc_trans_allocate): Update caller.  Remove the
+	nelems variable.
+	* trans-array.h (gfc_array_allocate): Update prototype.
+
+2025-07-30  Yuao Ma  <c8ef@outlook.com>
+
+	* check.cc (gfc_check_split): Argument check for SPLIT.
+	* gfortran.h (enum gfc_isym_id): Define GFC_ISYM_SPLIT.
+	* intrinsic.cc (add_subroutines): Register SPLIT intrinsic.
+	* intrinsic.h (gfc_check_split): New decl.
+	(gfc_resolve_split): Ditto.
+	* intrinsic.texi: SPLIT documentation.
+	* iresolve.cc (gfc_resolve_split): Add resolved_sym for SPLIT.
+	* trans-decl.cc (gfc_build_intrinsic_function_decls): Add decl for
+	SPLIT in libgfortran.
+	* trans-intrinsic.cc (conv_intrinsic_split): SPLIT codegen.
+	(gfc_conv_intrinsic_subroutine): Handle SPLIT case.
+	* trans.h (GTY): Declare gfor_fndecl_string_split{, _char4}.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* trans-expr.cc (gfc_trans_assignment_1): Use the same condition
+	to set the is_alloc_lhs flag and to decide to generate
+	reallocation code.  Add explicit call to gfc_fix_class_refs
+	before evaluating the condition.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* trans-array.cc (set_factored_descriptor_value): Also trigger
+	the saving of the previously selected reference on encountering
+	an INDIRECT_REF.  Extract the saving code...
+	(save_ref): ... here as a new function.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* trans-expr.cc (gfc_get_class_from_expr): Give up class
+	container lookup on the second COMPONENT_REF after an array
+	descriptor.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Make diagnostics::context::m_source_printing private.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cpp.cc: Update usage of "diagnostic_info" to explicitly refer to
+	"diagnostics::diagnostic_info".
+	* error.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cpp.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* error.cc: Likewise.
+	* options.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cpp.cc: Update for renaming of
+	diagnostic_option_id to diagnostics::option_id.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for move of diagnostics output formats into
+	namespace "diagnostics" as "sinks".
+	* gfortran.h: Likewise.
+
+2025-07-23  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121203
+	* trans-expr.cc (gfc_conv_procedure_call): Obtain the character
+	length of an assumed character length procedure from the typespec
+	of the actual argument even if there is no explicit interface.
+
+2025-07-21  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-decl.cc (gfc_trans_deferred_vars): Fix indentation.
+
+2025-07-21  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/119106
+	* expr.cc (simplify_constructor): Do not simplify constants.
+	(gfc_simplify_expr): Continue to simplify expression when an
+	iterator is present.
+
+2025-07-21  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-array.cc (gfc_conv_ss_descriptor): Move the descriptor
+	expression initialisation...
+	(set_factored_descriptor_value): ... to this new function.
+	Before initialisation, walk the reference expression passed as
+	argument and save some of its subexpressions to a variable.
+	(substitute_t): New struct.
+	(maybe_substitute_expr): New function.
+	(substitute_subexpr_in_expr): New function.
+
+2025-07-18  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121145
+	* trans-expr.cc (gfc_conv_procedure_call): Do not create pointer
+	check for proc-pointer actual passed to optional dummy.
+
+2025-07-16  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/121060
+	* interface.cc (matching_typebound_op): Defer determination of
+	specific procedure until resolution by returning NULL.
+
+2025-07-16  Steve Kargl  <sgk@troutmask.apl.washington.edu>
+
+	* decl.cc (gfc_match_import): Correct minor whitespace snafu
+	and fix NULL pointer dereferences in two places.
+
+2025-07-15  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	PR fortran/104428
+	* trans-openmp.cc (gfc_trans_omp_declare_variant): Check that proc_st
+	is non-NULL before dereferencing.  Add line number to error message.
+
+2025-07-15  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* gfortran.h (gfc_symbol): Remove field allocated_in_scope.
+	* trans-array.cc (gfc_array_allocate): Don't set it.
+	(gfc_alloc_allocatable_for_assignment): Likewise.
+	Generate the unallocated descriptor bounds initialisation
+	before the opening of the reallocation code block.  Create a
+	variable and use it as additional condition to the unallocated
+	descriptor bounds initialisation.
+
+2025-07-15  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-array.cc (gfc_conv_ss_descriptor): Don't evaluate
+	offset and data to a variable if is_alloc_lhs is set.  Move the
+	existing evaluation decision condition for data...
+	(save_descriptor_data): ... here as a new predicate.
+	(evaluate_bound): Add argument save_value.  Omit the evaluation
+	of the value to a variable if that argument isn't set.
+	(gfc_conv_expr_descriptor): Update caller.
+	(gfc_conv_section_startstride): Update caller.  Set save_value
+	if is_alloc_lhs is not set.  Omit the evaluation of stride to a
+	variable if save_value isn't set.
+	(gfc_set_delta): Omit the evaluation of delta to a variable
+	if is_alloc_lhs is set.
+	(gfc_is_reallocatable_lhs): Return false if flag_realloc_lhs
+	isn't set.
+	(gfc_alloc_allocatable_for_assignment): Don't update
+	the variables that may be stored in saved_offset, delta, and
+	data.  Call instead...
+	(update_reallocated_descriptor): ... this new procedure.
+	* trans-expr.cc (gfc_trans_assignment_1): Don't omit setting the
+	is_alloc_lhs flag if the right hand side is an intrinsic
+	function.  Clear the flag if the right hand side is scalar.
+
+2025-07-15  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-expr.cc (gfc_trans_assignment_1): Generate array
+	reallocation code before entering the scalarisation loops.
+
+2025-07-15  Filip Kastl  <fkastl@suse.cz>
+
+	* resolve.cc (resolve_select_type): Fix indentation.
+
+2025-07-12  Tobias Burnus  <tburnus@baylibre.com>
+
+	* invoke.texi (-Wsurprising): Note about OpenACC warning
+	related to PARAMATER.
+	* openmp.cc (resolve_omp_clauses, gfc_resolve_oacc_declare):
+	Accept PARAMETER for OpenACC but add surprising warning.
+	* trans-openmp.cc (gfc_trans_omp_variable_list,
+	gfc_trans_omp_clauses): Ignore PARAMETER inside clauses.
+
+2025-07-11  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/106135
+	* decl.cc (build_sym): Emit an error if a symbol associated by
+	an IMPORT, ONLY or IMPORT, all statement is being redeclared.
+	(gfc_match_import): Parse and check the F2018 versions of the
+	IMPORT statement. For scopes other than and interface body, if
+	the symbol cannot be found in the host scope, generate it and
+	set it up such that gfc_fixup_sibling_symbols can transfer its
+	'imported attribute' if it turnes out to be a not yet parsed
+	procedure. Test for violations of C897-8100.
+	* gfortran.h : Add 'import_only' to the gfc_symtree structure.
+	Add the enum, 'importstate', which is used for values the new
+	field 'import_state' in gfc_namespace.
+	* parse.cc (gfc_fixup_sibling_symbols): Transfer the attribute
+	'imported' to the new symbol.
+	* resolve.cc (check_sym_import_status, check_import_status):
+	New functions to test symbols and expressions for violations of
+	F2018:C8102.
+	(resolve_call): Test the 'resolved_sym' against C8102 by a call
+	to 'check_sym_import_status'.
+	(gfc_resolve_expr): If the expression is OK and an IMPORT
+	statement has been registered in the current scope, test C102
+	by calling 'check_import_status'.
+	(resolve_select_type): Test the declared derived type in TYPE
+	IS and CLASS IS statements.
+
+2025-07-08  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120637
+	* class.cc (finalize_component): Return true, when a finalizable
+	component was detect and do not free it.
+
+2025-07-07  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-intrinsic.cc (conv_intrinsic_move_alloc): Add pre and
+	post code for the FROM and TO arguments.
+
+2025-07-04  Martin Jambor  <mjambor@suse.cz>
+
+	* io.cc (format_asterisk): Add a brace around static initialization
+	location part of the field locus.
+
+2025-07-03  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120843
+	* resolve.cc (resolve_operator): Remove conformability check,
+	because it is not in the standard.
+
 2025-07-01  Harald Anlauf  <anlauf@gmx.de>
 
 	* coarray.cc (check_add_new_component): Treat pure and elemental
diff --git a/gcc/fortran/check.cc b/gcc/fortran/check.cc
index 838d523..8626526 100644
--- a/gcc/fortran/check.cc
+++ b/gcc/fortran/check.cc
@@ -5559,6 +5559,27 @@ gfc_check_scan (gfc_expr *x, gfc_expr *y, gfc_expr *z, gfc_expr *kind)
   return true;
 }
 
+bool
+gfc_check_split (gfc_expr *string, gfc_expr *set, gfc_expr *pos, gfc_expr *back)
+{
+  if (!type_check (string, 0, BT_CHARACTER))
+    return false;
+
+  if (!type_check (set, 1, BT_CHARACTER))
+    return false;
+
+  if (!type_check (pos, 2, BT_INTEGER) || !scalar_check (pos, 2))
+    return false;
+
+  if (back != NULL
+      && (!type_check (back, 3, BT_LOGICAL) || !scalar_check (back, 3)))
+    return false;
+
+  if (!same_type_check (string, 0, set, 1))
+    return false;
+
+  return true;
+}
 
 bool
 gfc_check_secnds (gfc_expr *r)
diff --git a/gcc/fortran/class.cc b/gcc/fortran/class.cc
index df18601..a1c6faf 100644
--- a/gcc/fortran/class.cc
+++ b/gcc/fortran/class.cc
@@ -1034,7 +1034,7 @@ comp_is_finalizable (gfc_component *comp)
    of calling the appropriate finalizers, coarray deregistering, and
    deallocation of allocatable subcomponents.  */
 
-static void
+static bool
 finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
 		    gfc_symbol *stat, gfc_symbol *fini_coarray, gfc_code **code,
 		    gfc_namespace *sub_ns)
@@ -1044,14 +1044,14 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
   gfc_was_finalized *f;
 
   if (!comp_is_finalizable (comp))
-    return;
+    return false;
 
   /* If this expression with this component has been finalized
      already in this namespace, there is nothing to do.  */
   for (f = sub_ns->was_finalized; f; f = f->next)
     {
       if (f->e == expr && f->c == comp)
-	return;
+	return false;
     }
 
   e = gfc_copy_expr (expr);
@@ -1208,8 +1208,6 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
       final_wrap->ext.actual->next->next = gfc_get_actual_arglist ();
       final_wrap->ext.actual->next->next->expr = fini_coarray_expr;
 
-
-
       if (*code)
 	{
 	  (*code)->next = final_wrap;
@@ -1221,11 +1219,14 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
   else
     {
       gfc_component *c;
+      bool ret = false;
 
       for (c = comp->ts.u.derived->components; c; c = c->next)
-	finalize_component (e, comp->ts.u.derived, c, stat, fini_coarray, code,
-			    sub_ns);
-      gfc_free_expr (e);
+	ret |= finalize_component (e, comp->ts.u.derived, c, stat, fini_coarray,
+				   code, sub_ns);
+      /* Only free the expression, if it has never been used.  */
+      if (!ret)
+	gfc_free_expr (e);
     }
 
   /* Record that this was finalized already in this namespace.  */
@@ -1234,6 +1235,7 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
   sub_ns->was_finalized->e = expr;
   sub_ns->was_finalized->c = comp;
   sub_ns->was_finalized->next = f;
+  return true;
 }
 
 
@@ -2314,6 +2316,7 @@ finish_assumed_rank:
     {
       gfc_symbol *stat;
       gfc_code *block = NULL;
+      gfc_expr *ptr_expr;
 
       if (!ptr)
 	{
@@ -2359,14 +2362,15 @@ finish_assumed_rank:
 					     sub_ns);
       block = block->next;
 
+      ptr_expr = gfc_lval_expr_from_sym (ptr);
       for (comp = derived->components; comp; comp = comp->next)
 	{
 	  if (comp == derived->components && derived->attr.extension
 	      && ancestor_wrapper && ancestor_wrapper->expr_type != EXPR_NULL)
 	    continue;
 
-	  finalize_component (gfc_lval_expr_from_sym (ptr), derived, comp,
-			      stat, fini_coarray, &block, sub_ns);
+	  finalize_component (ptr_expr, derived, comp, stat, fini_coarray,
+			      &block, sub_ns);
 	  if (!last_code->block->next)
 	    last_code->block->next = block;
 	}
diff --git a/gcc/fortran/cpp.cc b/gcc/fortran/cpp.cc
index 1b70420..15ecc7d 100644
--- a/gcc/fortran/cpp.cc
+++ b/gcc/fortran/cpp.cc
@@ -1063,7 +1063,7 @@ cb_used_define (cpp_reader *pfile, location_t line ATTRIBUTE_UNUSED,
 /* Return the gcc option code associated with the reason for a cpp
    message, or 0 if none.  */
 
-static diagnostic_option_id
+static diagnostics::option_id
 cb_cpp_diagnostic_cpp_option (enum cpp_warning_reason reason)
 {
   const struct cpp_reason_option_codes_t *entry;
@@ -1088,8 +1088,8 @@ cb_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
 		   rich_location *richloc,
 		   const char *msg, va_list *ap)
 {
-  diagnostic_info diagnostic;
-  diagnostic_t dlevel;
+  diagnostics::diagnostic_info diagnostic;
+  enum diagnostics::kind dlevel;
   bool save_warn_system_headers = global_dc->m_warn_system_headers;
   bool ret;
 
@@ -1099,22 +1099,22 @@ cb_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
       global_dc->m_warn_system_headers = 1;
       /* Fall through.  */
     case CPP_DL_WARNING:
-      dlevel = DK_WARNING;
+      dlevel = diagnostics::kind::warning;
       break;
     case CPP_DL_PEDWARN:
-      dlevel = DK_PEDWARN;
+      dlevel = diagnostics::kind::pedwarn;
       break;
     case CPP_DL_ERROR:
-      dlevel = DK_ERROR;
+      dlevel = diagnostics::kind::error;
       break;
     case CPP_DL_ICE:
-      dlevel = DK_ICE;
+      dlevel = diagnostics::kind::ice;
       break;
     case CPP_DL_NOTE:
-      dlevel = DK_NOTE;
+      dlevel = diagnostics::kind::note;
       break;
     case CPP_DL_FATAL:
-      dlevel = DK_FATAL;
+      dlevel = diagnostics::kind::fatal;
       break;
     default:
       gcc_unreachable ();
diff --git a/gcc/fortran/decl.cc b/gcc/fortran/decl.cc
index 69acd2d..af42575 100644
--- a/gcc/fortran/decl.cc
+++ b/gcc/fortran/decl.cc
@@ -1723,13 +1723,17 @@ build_sym (const char *name, int elem, gfc_charlen *cl, bool cl_deferred,
   symbol_attribute attr;
   gfc_symbol *sym;
   int upper;
-  gfc_symtree *st;
+  gfc_symtree *st, *host_st = NULL;
 
   /* Symbols in a submodule are host associated from the parent module or
      submodules. Therefore, they can be overridden by declarations in the
      submodule scope. Deal with this by attaching the existing symbol to
      a new symtree and recycling the old symtree with a new symbol...  */
   st = gfc_find_symtree (gfc_current_ns->sym_root, name);
+  if (((st && st->import_only) || (gfc_current_ns->import_state == IMPORT_ALL))
+      && gfc_current_ns->parent)
+    host_st = gfc_find_symtree (gfc_current_ns->parent->sym_root, name);
+
   if (st != NULL && gfc_state_stack->state == COMP_SUBMODULE
       && st->n.sym != NULL
       && st->n.sym->attr.host_assoc && st->n.sym->attr.used_in_submodule)
@@ -1742,6 +1746,20 @@ build_sym (const char *name, int elem, gfc_charlen *cl, bool cl_deferred,
       sym->refs++;
       gfc_set_sym_referenced (sym);
     }
+  /* ...Check that F2018 IMPORT, ONLY and IMPORT, ALL statements, within the
+     current scope are not violated by local redeclarations. Note that there is
+     no need to guard for std >= F2018 because import_only and IMPORT_ALL are
+     only set for these standards.  */
+  else if (host_st && host_st->n.sym
+	   && host_st->n.sym != gfc_current_ns->proc_name
+	   && !(st && st->n.sym
+		&& (st->n.sym->attr.dummy || st->n.sym->attr.result)))
+    {
+      gfc_error ("F2018: C8102 %s at %L is already imported by an %s "
+		 "statement and must not be re-declared", name, var_locus,
+		 (st && st->import_only) ? "IMPORT, ONLY" : "IMPORT, ALL");
+      return false;
+    }
   /* ...Otherwise generate a new symtree and new symbol.  */
   else if (gfc_get_symbol (name, NULL, &sym, var_locus))
     return false;
@@ -5100,6 +5118,54 @@ error:
 }
 
 
+/* Match the IMPORT statement.  IMPORT was added to F2003 as
+
+   R1209 import-stmt  is IMPORT [[ :: ] import-name-list ]
+
+   C1210 (R1209) The IMPORT statement is allowed only in an interface-body.
+
+   C1211 (R1209) Each import-name shall be the name of an entity in the
+		 host scoping unit.
+
+   under the description of an interface block. Under F2008, IMPORT was
+   split out of the interface block description to 12.4.3.3 and C1210
+   became
+
+   C1210 (R1209) The IMPORT statement is allowed only in an interface-body
+		 that is not a module procedure interface body.
+
+   Finally, F2018, section 8.8, has changed the IMPORT statement to
+
+   R867 import-stmt  is IMPORT [[ :: ] import-name-list ]
+		     or IMPORT, ONLY : import-name-list
+		     or IMPORT, NONE
+		     or IMPORT, ALL
+
+   C896 (R867) An IMPORT statement shall not appear in the scoping unit of
+		a main-program, external-subprogram, module, or block-data.
+
+   C897 (R867) Each import-name shall be the name of an entity in the host
+		scoping unit.
+
+   C898  If any IMPORT statement in a scoping unit has an ONLY specifier,
+	 all IMPORT statements in that scoping unit shall have an ONLY
+	 specifier.
+
+   C899  IMPORT, NONE shall not appear in the scoping unit of a submodule.
+
+   C8100 If an IMPORT, NONE or IMPORT, ALL statement appears in a scoping
+	 unit, no other IMPORT statement shall appear in that scoping unit.
+
+   C8101 Within an interface body, an entity that is accessed by host
+	 association shall be accessible by host or use association within
+	 the host scoping unit, or explicitly declared prior to the interface
+	 body.
+
+   C8102 An entity whose name appears as an import-name or which is made
+	 accessible by an IMPORT, ALL statement shall not appear in any
+	 context described in 19.5.1.4 that would cause the host entity
+	 of that name to be inaccessible.  */
+
 match
 gfc_match_import (void)
 {
@@ -5107,16 +5173,28 @@ gfc_match_import (void)
   match m;
   gfc_symbol *sym;
   gfc_symtree *st;
+  bool f2018_allowed = gfc_option.allow_std & ~GFC_STD_OPT_F08;;
+  importstate current_import_state = gfc_current_ns->import_state;
 
-  if (gfc_current_ns->proc_name == NULL
-      || gfc_current_ns->proc_name->attr.if_source != IFSRC_IFBODY)
+  if (!f2018_allowed
+      && (gfc_current_ns->proc_name == NULL
+	  || gfc_current_ns->proc_name->attr.if_source != IFSRC_IFBODY))
     {
       gfc_error ("IMPORT statement at %C only permitted in "
 		 "an INTERFACE body");
       return MATCH_ERROR;
     }
+  else if (f2018_allowed
+	   && (!gfc_current_ns->parent || gfc_current_ns->is_block_data))
+    goto C897;
+
+  if (f2018_allowed
+      && (current_import_state == IMPORT_ALL
+	  || current_import_state == IMPORT_NONE))
+    goto C8100;
 
-  if (gfc_current_ns->proc_name->attr.module_procedure)
+  if (gfc_current_ns->proc_name
+      && gfc_current_ns->proc_name->attr.module_procedure)
     {
       gfc_error ("F2008: C1210 IMPORT statement at %C is not permitted "
 		 "in a module procedure interface body");
@@ -5126,20 +5204,65 @@ gfc_match_import (void)
   if (!gfc_notify_std (GFC_STD_F2003, "IMPORT statement at %C"))
     return MATCH_ERROR;
 
+  gfc_current_ns->import_state = IMPORT_NOT_SET;
+  if (f2018_allowed)
+    {
+      if (gfc_match (" , none") == MATCH_YES)
+	{
+	  if (current_import_state == IMPORT_ONLY)
+	    goto C898;
+	  if (gfc_current_state () == COMP_SUBMODULE)
+	    goto C899;
+	  gfc_current_ns->import_state = IMPORT_NONE;
+	}
+      else if (gfc_match (" , only :") == MATCH_YES)
+	{
+	  if (current_import_state != IMPORT_NOT_SET
+	      && current_import_state != IMPORT_ONLY)
+	    goto C898;
+	  gfc_current_ns->import_state = IMPORT_ONLY;
+	}
+      else if (gfc_match (" , all") == MATCH_YES)
+	{
+	  if (current_import_state == IMPORT_ONLY)
+	    goto C898;
+	  gfc_current_ns->import_state = IMPORT_ALL;
+	}
+
+      if (current_import_state != IMPORT_NOT_SET
+	  && (gfc_current_ns->import_state == IMPORT_NONE
+	      || gfc_current_ns->import_state == IMPORT_ALL))
+	goto C8100;
+    }
+
+  /* F2008 IMPORT<eos> is distinct from F2018 IMPORT, ALL.  */
   if (gfc_match_eos () == MATCH_YES)
     {
-      /* All host variables should be imported.  */
-      gfc_current_ns->has_import_set = 1;
+      /* This is the F2008 variant.  */
+      if (gfc_current_ns->import_state == IMPORT_NOT_SET)
+	{
+	  if (current_import_state == IMPORT_ONLY)
+	    goto C898;
+	  gfc_current_ns->import_state = IMPORT_F2008;
+	}
+
+      /* Host variables should be imported.  */
+      if (gfc_current_ns->import_state != IMPORT_NONE)
+	gfc_current_ns->has_import_set = 1;
       return MATCH_YES;
     }
 
-  if (gfc_match (" ::") == MATCH_YES)
+  if (gfc_match (" ::") == MATCH_YES
+      && gfc_current_ns->import_state != IMPORT_ONLY)
     {
       if (gfc_match_eos () == MATCH_YES)
-	{
-	   gfc_error ("Expecting list of named entities at %C");
-	   return MATCH_ERROR;
-	}
+	goto expecting_list;
+      gfc_current_ns->import_state = IMPORT_F2008;
+    }
+  else if (gfc_current_ns->import_state == IMPORT_ONLY)
+    {
+      if (gfc_match_eos () == MATCH_YES)
+	goto expecting_list;
     }
 
   for(;;)
@@ -5149,13 +5272,15 @@ gfc_match_import (void)
       switch (m)
 	{
 	case MATCH_YES:
-	  if (gfc_current_ns->parent !=  NULL
+	  if (gfc_current_ns->parent != NULL
 	      && gfc_find_symbol (name, gfc_current_ns->parent, 1, &sym))
 	    {
 	       gfc_error ("Type name %qs at %C is ambiguous", name);
 	       return MATCH_ERROR;
 	    }
-	  else if (!sym && gfc_current_ns->proc_name->ns->parent !=  NULL
+	  else if (!sym
+		   && gfc_current_ns->proc_name
+		   && gfc_current_ns->proc_name->ns->parent
 		   && gfc_find_symbol (name,
 				       gfc_current_ns->proc_name->ns->parent,
 				       1, &sym))
@@ -5166,12 +5291,29 @@ gfc_match_import (void)
 
 	  if (sym == NULL)
 	    {
-	      gfc_error ("Cannot IMPORT %qs from host scoping unit "
-			 "at %C - does not exist.", name);
-	      return MATCH_ERROR;
+	      if (gfc_current_ns->proc_name
+		  && gfc_current_ns->proc_name->attr.if_source == IFSRC_IFBODY)
+		{
+		  gfc_error ("Cannot IMPORT %qs from host scoping unit "
+			     "at %C - does not exist.", name);
+		  return MATCH_ERROR;
+		}
+	      else
+		{
+		  /* This might be a procedure that has not yet been parsed. If
+		     so gfc_fixup_sibling_symbols will replace this symbol with
+		     that of the procedure.  */
+		  gfc_get_sym_tree (name, gfc_current_ns, &st, false,
+				    &gfc_current_locus);
+		  st->n.sym->refs++;
+		  st->n.sym->attr.imported = 1;
+		  st->import_only = 1;
+		  goto next_item;
+		}
 	    }
 
-	  if (gfc_find_symtree (gfc_current_ns->sym_root, name))
+	  st = gfc_find_symtree (gfc_current_ns->sym_root, name);
+	  if (st && st->n.sym && st->n.sym->attr.imported)
 	    {
 	      gfc_warning (0, "%qs is already IMPORTed from host scoping unit "
 			   "at %C", name);
@@ -5182,6 +5324,7 @@ gfc_match_import (void)
 	  st->n.sym = sym;
 	  sym->refs++;
 	  sym->attr.imported = 1;
+	  st->import_only = 1;
 
 	  if (sym->attr.generic && (sym = gfc_find_dt_in_generic (sym)))
 	    {
@@ -5193,6 +5336,7 @@ gfc_match_import (void)
 	      st->n.sym = sym;
 	      sym->refs++;
 	      sym->attr.imported = 1;
+	      st->import_only = 1;
 	    }
 
 	  goto next_item;
@@ -5216,6 +5360,34 @@ gfc_match_import (void)
 syntax:
   gfc_error ("Syntax error in IMPORT statement at %C");
   return MATCH_ERROR;
+
+C897:
+  gfc_error ("F2018: C897 IMPORT statement at %C cannot appear in a main "
+	     "program, an external subprogram, a module or block data");
+  return MATCH_ERROR;
+
+C898:
+  gfc_error ("F2018: C898 IMPORT statement at %C is not permitted because "
+	     "a scoping unit has an ONLY specifier, can only have IMPORT "
+	     "with an ONLY specifier");
+  return MATCH_ERROR;
+
+C899:
+  gfc_error ("F2018: C899 IMPORT, NONE shall not appear in the scoping unit"
+	     " of a submodule as at %C");
+  return MATCH_ERROR;
+
+C8100:
+  gfc_error ("F2018: C8100 IMPORT statement at %C is not permitted because "
+	     "%s has already been declared, which must be unique in the "
+	     "scoping unit",
+	     gfc_current_ns->import_state == IMPORT_ALL ? "IMPORT, ALL" :
+							  "IMPORT, NONE");
+  return MATCH_ERROR;
+
+expecting_list:
+  gfc_error ("Expecting list of named entities at %C");
+  return MATCH_ERROR;
 }
 
 
diff --git a/gcc/fortran/error.cc b/gcc/fortran/error.cc
index 004a4b2..ebf9e61 100644
--- a/gcc/fortran/error.cc
+++ b/gcc/fortran/error.cc
@@ -31,9 +31,9 @@ along with GCC; see the file COPYING3.  If not see
 #include "gfortran.h"
 
 #include "diagnostic.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 #include "tree-diagnostic.h" /* tree_diagnostics_defaults */
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 static int suppress_errors = 0;
 
@@ -43,7 +43,7 @@ static bool warnings_not_errors = false;
 static bool buffered_p;
 
 static gfc_error_buffer *error_buffer;
-static diagnostic_buffer *pp_error_buffer, *pp_warning_buffer;
+static diagnostics::buffer *pp_error_buffer, *pp_warning_buffer;
 
 gfc_error_buffer::gfc_error_buffer ()
 : flag (false), buffer (*global_dc)
@@ -228,7 +228,7 @@ gfc_print_wide_char (gfc_char_t c)
    it to global_dc.  */
 
 static void
-gfc_clear_diagnostic_buffer (diagnostic_buffer *this_buffer)
+gfc_clear_diagnostic_buffer (diagnostics::buffer *this_buffer)
 {
   gcc_assert (this_buffer);
   global_dc->clear_diagnostic_buffer (*this_buffer);
@@ -237,13 +237,13 @@ gfc_clear_diagnostic_buffer (diagnostic_buffer *this_buffer)
 /* The currently-printing diagnostic, for use by gfc_format_decoder,
    for colorizing %C and %L.  */
 
-static diagnostic_info *curr_diagnostic;
+static diagnostics::diagnostic_info *curr_diagnostic;
 
 /* A helper function to call diagnostic_report_diagnostic, while setting
    curr_diagnostic for the duration of the call.  */
 
 static bool
-gfc_report_diagnostic (diagnostic_info *diagnostic)
+gfc_report_diagnostic (diagnostics::diagnostic_info *diagnostic)
 {
   gcc_assert (diagnostic != NULL);
   curr_diagnostic = diagnostic;
@@ -261,9 +261,9 @@ gfc_warning (int opt, const char *gmsgid, va_list ap)
   va_list argp;
   va_copy (argp, ap);
 
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
-  diagnostic_buffer *old_buffer = global_dc->get_diagnostic_buffer ();
+  diagnostics::buffer *old_buffer = global_dc->get_diagnostic_buffer ();
   gcc_assert (!old_buffer);
 
   gfc_clear_diagnostic_buffer (pp_warning_buffer);
@@ -272,8 +272,8 @@ gfc_warning (int opt, const char *gmsgid, va_list ap)
     global_dc->set_diagnostic_buffer (pp_warning_buffer);
 
   diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
-		       DK_WARNING);
-  diagnostic.option_id = opt;
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   bool ret = gfc_report_diagnostic (&diagnostic);
 
   if (buffered_p)
@@ -441,7 +441,7 @@ gfc_format_decoder (pretty_printer *pp, text_info *text, const char *spec,
 	const char *color
 	  = (loc_num
 	     ? "range1"
-	     : diagnostic_get_color_for_kind (curr_diagnostic->kind));
+	     : diagnostics::get_color_for_kind (curr_diagnostic->m_kind));
 	pp_string (pp, colorize_start (pp_show_color (pp), color));
 	pp_string (pp, result[loc_num]);
 	pp_string (pp, colorize_stop (pp_show_color (pp)));
@@ -460,8 +460,8 @@ gfc_format_decoder (pretty_printer *pp, text_info *text, const char *spec,
 /* Return a malloc'd string describing the kind of diagnostic.  The
    caller is responsible for freeing the memory.  */
 static char *
-gfc_diagnostic_build_kind_prefix (diagnostic_context *context,
-				  const diagnostic_info *diagnostic)
+gfc_diagnostic_build_kind_prefix (diagnostics::context *context,
+				  const diagnostics::diagnostic_info *diagnostic)
 {
   static const char *const diagnostic_kind_text[] = {
 #define DEFINE_DIAGNOSTIC_KIND(K, T, C) (T),
@@ -475,15 +475,16 @@ gfc_diagnostic_build_kind_prefix (diagnostic_context *context,
 #undef DEFINE_DIAGNOSTIC_KIND
     NULL
   };
-  gcc_assert (diagnostic->kind < DK_LAST_DIAGNOSTIC_KIND);
-  const char *text = _(diagnostic_kind_text[diagnostic->kind]);
+  const int diag_kind_idx = static_cast<int> (diagnostic->m_kind);
+  gcc_assert (diagnostic->m_kind < diagnostics::kind::last_diagnostic_kind);
+  const char *text = _(diagnostic_kind_text[diag_kind_idx]);
   const char *text_cs = "", *text_ce = "";
   pretty_printer *const pp = context->get_reference_printer ();
 
-  if (diagnostic_kind_color[diagnostic->kind])
+if (diagnostic_kind_color[diag_kind_idx])
     {
       text_cs = colorize_start (pp_show_color (pp),
-				diagnostic_kind_color[diagnostic->kind]);
+				diagnostic_kind_color[diag_kind_idx]);
       text_ce = colorize_stop (pp_show_color (pp));
     }
   return build_message_string ("%s%s:%s ", text_cs, text, text_ce);
@@ -492,7 +493,7 @@ gfc_diagnostic_build_kind_prefix (diagnostic_context *context,
 /* Return a malloc'd string describing a location.  The caller is
    responsible for freeing the memory.  */
 static char *
-gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_policy,
+gfc_diagnostic_build_locus_prefix (const diagnostics::location_print_policy &loc_policy,
 				   expanded_location s,
 				   bool colorize)
 {
@@ -511,7 +512,7 @@ gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_p
 /* Return a malloc'd string describing two locations.  The caller is
    responsible for freeing the memory.  */
 static char *
-gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_policy,
+gfc_diagnostic_build_locus_prefix (const diagnostics::location_print_policy &loc_policy,
 				   expanded_location s, expanded_location s2,
 				   bool colorize)
 {
@@ -548,16 +549,16 @@ gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_p
        [locus of primary range]: Error: Some error at (1) and (2)
 */
 static void
-gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-			     const diagnostic_info *diagnostic)
+gfc_diagnostic_text_starter (diagnostics::text_sink &text_output,
+			     const diagnostics::diagnostic_info *diagnostic)
 {
-  diagnostic_context *const context = &text_output.get_context ();
+  diagnostics::context *const context = &text_output.get_context ();
   pretty_printer *const pp = text_output.get_printer ();
   char * kind_prefix = gfc_diagnostic_build_kind_prefix (context, diagnostic);
 
   expanded_location s1 = diagnostic_expand_location (diagnostic);
   expanded_location s2;
-  bool one_locus = diagnostic->richloc->get_num_locations () < 2;
+  bool one_locus = diagnostic->m_richloc->get_num_locations () < 2;
   bool same_locus = false;
 
   if (!one_locus)
@@ -566,13 +567,13 @@ gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
       same_locus = diagnostic_same_line (context, s1, s2);
     }
 
-  diagnostic_location_print_policy loc_policy (text_output);
+  diagnostics::location_print_policy loc_policy (text_output);
   const bool colorize = pp_show_color (pp);
   char * locus_prefix = (one_locus || !same_locus)
     ? gfc_diagnostic_build_locus_prefix (loc_policy, s1, colorize)
     : gfc_diagnostic_build_locus_prefix (loc_policy, s1, s2, colorize);
 
-  if (!context->m_source_printing.enabled
+  if (!context->get_source_printing_options ().enabled
       || diagnostic_location (diagnostic, 0) <= BUILTINS_LOCATION
       || diagnostic_location (diagnostic, 0) == context->m_last_location)
     {
@@ -608,7 +609,7 @@ gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
       pp_newline (pp);
       diagnostic_show_locus (context,
 			     text_output.get_source_printing_options (),
-			     diagnostic->richloc, diagnostic->kind,
+			     diagnostic->m_richloc, diagnostic->m_kind,
 			     pp);
       /* If the caret line was shown, the prefix does not contain the
 	 locus.  */
@@ -617,11 +618,11 @@ gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
 }
 
 static void
-gfc_diagnostic_start_span (const diagnostic_location_print_policy &loc_policy,
-			   to_text &sink,
+gfc_diagnostic_start_span (const diagnostics::location_print_policy &loc_policy,
+			   diagnostics::to_text &sink,
 			   expanded_location exploc)
 {
-  pretty_printer *pp = get_printer (sink);
+  pretty_printer *pp = diagnostics::get_printer (sink);
   const bool colorize = pp_show_color (pp);
   char *locus_prefix
     = gfc_diagnostic_build_locus_prefix (loc_policy, exploc, colorize);
@@ -634,9 +635,9 @@ gfc_diagnostic_start_span (const diagnostic_location_print_policy &loc_policy,
 
 
 static void
-gfc_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-			       const diagnostic_info *diagnostic ATTRIBUTE_UNUSED,
-			       diagnostic_t orig_diag_kind ATTRIBUTE_UNUSED)
+gfc_diagnostic_text_finalizer (diagnostics::text_sink &text_output,
+			       const diagnostics::diagnostic_info *,
+			       enum diagnostics::kind orig_diag_kind ATTRIBUTE_UNUSED)
 {
   pretty_printer *const pp = text_output.get_printer ();
   pp_destroy_prefix (pp);
@@ -650,13 +651,14 @@ bool
 gfc_warning_now_at (location_t loc, int opt, const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, loc);
   bool ret;
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_WARNING);
-  diagnostic.option_id = opt;
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   ret = gfc_report_diagnostic (&diagnostic);
   va_end (argp);
   return ret;
@@ -668,14 +670,14 @@ bool
 gfc_warning_now (int opt, const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
   bool ret;
 
   va_start (argp, gmsgid);
   diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
-		       DK_WARNING);
-  diagnostic.option_id = opt;
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   ret = gfc_report_diagnostic (&diagnostic);
   va_end (argp);
   return ret;
@@ -687,14 +689,14 @@ bool
 gfc_warning_internal (int opt, const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
   bool ret;
 
   va_start (argp, gmsgid);
   diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
-		       DK_WARNING);
-  diagnostic.option_id = opt;
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   ret = gfc_report_diagnostic (&diagnostic);
   va_end (argp);
   return ret;
@@ -706,13 +708,14 @@ void
 gfc_error_now (const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
   error_buffer->flag = true;
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_ERROR);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::error);
   gfc_report_diagnostic (&diagnostic);
   va_end (argp);
 }
@@ -724,11 +727,12 @@ void
 gfc_fatal_error (const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_FATAL);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::fatal);
   gfc_report_diagnostic (&diagnostic);
   va_end (argp);
 
@@ -776,9 +780,9 @@ gfc_error_opt (int opt, const char *gmsgid, va_list ap)
       return;
     }
 
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  diagnostic_buffer *old_buffer = global_dc->get_diagnostic_buffer ();
+  diagnostics::buffer *old_buffer = global_dc->get_diagnostic_buffer ();
   gcc_assert (!old_buffer);
 
   gfc_clear_diagnostic_buffer (pp_error_buffer);
@@ -786,7 +790,8 @@ gfc_error_opt (int opt, const char *gmsgid, va_list ap)
   if (buffered_p)
     global_dc->set_diagnostic_buffer (pp_error_buffer);
 
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &richloc, DK_ERROR);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &richloc,
+		       diagnostics::kind::error);
   gfc_report_diagnostic (&diagnostic);
 
   if (buffered_p)
@@ -823,7 +828,7 @@ gfc_internal_error (const char *gmsgid, ...)
 {
   int e, w;
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
   gfc_get_errors (&w, &e);
@@ -831,7 +836,8 @@ gfc_internal_error (const char *gmsgid, ...)
     exit(EXIT_FAILURE);
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_ICE);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::ice);
   gfc_report_diagnostic (&diagnostic);
   va_end (argp);
 
@@ -885,8 +891,8 @@ static void
 gfc_move_error_buffer_from_to (gfc_error_buffer * buffer_from,
 			       gfc_error_buffer * buffer_to)
 {
-  diagnostic_buffer * from = &(buffer_from->buffer);
-  diagnostic_buffer * to =  &(buffer_to->buffer);
+  diagnostics::buffer * from = &(buffer_from->buffer);
+  diagnostics::buffer * to =  &(buffer_to->buffer);
 
   buffer_to->flag = buffer_from->flag;
   buffer_from->flag = false;
@@ -950,13 +956,14 @@ gfc_errors_to_warnings (bool f)
 void
 gfc_diagnostics_init (void)
 {
-  diagnostic_text_starter (global_dc) = gfc_diagnostic_text_starter;
-  diagnostic_start_span (global_dc) = gfc_diagnostic_start_span;
-  diagnostic_text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
+  diagnostics::text_starter (global_dc) = gfc_diagnostic_text_starter;
+  diagnostics::start_span (global_dc) = gfc_diagnostic_start_span;
+  diagnostics::text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
   global_dc->set_format_decoder (gfc_format_decoder);
-  global_dc->m_source_printing.caret_chars[0] = '1';
-  global_dc->m_source_printing.caret_chars[1] = '2';
-  pp_warning_buffer = new diagnostic_buffer (*global_dc);
+  auto &source_printing_opts = global_dc->get_source_printing_options ();
+  source_printing_opts.caret_chars[0] = '1';
+  source_printing_opts.caret_chars[1] = '2';
+  pp_warning_buffer = new diagnostics::buffer (*global_dc);
   error_buffer = new gfc_error_buffer ();
   pp_error_buffer = &(error_buffer->buffer);
 }
@@ -967,10 +974,11 @@ gfc_diagnostics_finish (void)
   tree_diagnostics_defaults (global_dc);
   /* We still want to use the gfc starter and finalizer, not the tree
      defaults.  */
-  diagnostic_text_starter (global_dc) = gfc_diagnostic_text_starter;
-  diagnostic_text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
-  global_dc->m_source_printing.caret_chars[0] = '^';
-  global_dc->m_source_printing.caret_chars[1] = '^';
+  diagnostics::text_starter (global_dc) = gfc_diagnostic_text_starter;
+  diagnostics::text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
+  auto &source_printing_opts = global_dc->get_source_printing_options ();
+  source_printing_opts.caret_chars[0] = '^';
+  source_printing_opts.caret_chars[1] = '^';
   delete error_buffer;
   error_buffer = nullptr;
   pp_error_buffer = nullptr;
diff --git a/gcc/fortran/expr.cc b/gcc/fortran/expr.cc
index b0495b7..b8d04ff 100644
--- a/gcc/fortran/expr.cc
+++ b/gcc/fortran/expr.cc
@@ -1372,7 +1372,7 @@ simplify_constructor (gfc_constructor_base base, int type)
 	      || !gfc_simplify_expr (c->iterator->step, type)))
 	return false;
 
-      if (c->expr)
+      if (c->expr && c->expr->expr_type != EXPR_CONSTANT)
 	{
 	  /* Try and simplify a copy.  Replace the original if successful
 	     but keep going through the constructor at all costs.  Not
@@ -2469,7 +2469,8 @@ gfc_simplify_expr (gfc_expr *p, int type)
 	{
 	  if (!simplify_parameter_variable (p, type))
 	    return false;
-	  break;
+	  if (!iter_stack)
+	    break;
 	}
 
       if (type == 1)
diff --git a/gcc/fortran/gfortran.h b/gcc/fortran/gfortran.h
index 6848bd1..d9dcd1b 100644
--- a/gcc/fortran/gfortran.h
+++ b/gcc/fortran/gfortran.h
@@ -729,6 +729,8 @@ enum gfc_isym_id
   GFC_ISYM_COSPI,
   GFC_ISYM_SINPI,
   GFC_ISYM_TANPI,
+
+  GFC_ISYM_SPLIT,
 };
 
 enum init_local_logical
@@ -2028,10 +2030,6 @@ typedef struct gfc_symbol
   /* Set if this should be passed by value, but is not a VALUE argument
      according to the Fortran standard.  */
   unsigned pass_as_value:1;
-  /* Set if an allocatable array variable has been allocated in the current
-     scope. Used in the suppression of uninitialized warnings in reallocation
-     on assignment.  */
-  unsigned allocated_in_scope:1;
   /* Set if an external dummy argument is called with different argument lists.
      This is legal in Fortran, but can cause problems with autogenerated
      C prototypes for C23.  */
@@ -2188,6 +2186,7 @@ typedef struct gfc_symtree
     gfc_omp_udr *omp_udr;
   }
   n;
+  unsigned import_only:1;
 }
 gfc_symtree;
 
@@ -2215,6 +2214,17 @@ typedef struct gfc_was_finalized {
 }
 gfc_was_finalized;
 
+
+  /* Flag F2018 import status */
+enum importstate
+{ IMPORT_NOT_SET = 0,	/* Default condition.  */
+  IMPORT_F2008,		/* Old style IMPORT.  */
+  IMPORT_ONLY,		/* Import list used.  */
+  IMPORT_NONE,		/* No host association.  Unique in scoping unit.  */
+  IMPORT_ALL		/* Must be unique in the scoping unit.  */
+};
+
+
 /* A namespace describes the contents of procedure, module, interface block
    or BLOCK construct.  */
 /* ??? Anything else use these?  */
@@ -2328,6 +2338,10 @@ typedef struct gfc_namespace
   /* Set to 1 if namespace is an interface body with "IMPORT" used.  */
   unsigned has_import_set:1;
 
+  /* Flag F2018 import status */
+  ENUM_BITFIELD (importstate) import_state :3;
+
+
   /* Set to 1 if the namespace uses "IMPLICIT NONE (export)".  */
   unsigned has_implicit_none_export:1;
 
@@ -3582,11 +3596,11 @@ bool gfc_notify_std (int, const char *, ...) ATTRIBUTE_GCC_GFC(2,3);
 #define gfc_syntax_error(ST)	\
   gfc_error ("Syntax error in %s statement at %C", gfc_ascii_statement (ST));
 
-#include "diagnostic-buffer.h"  /* For diagnostic_buffer.  */
+#include "diagnostics/buffering.h"  /* For diagnostics::buffer.  */
 struct gfc_error_buffer
 {
   bool flag;
-  diagnostic_buffer buffer;
+  diagnostics::buffer buffer;
 
   gfc_error_buffer();
 };
diff --git a/gcc/fortran/interface.cc b/gcc/fortran/interface.cc
index f74fbf0..d08f683 100644
--- a/gcc/fortran/interface.cc
+++ b/gcc/fortran/interface.cc
@@ -4781,6 +4781,13 @@ matching_typebound_op (gfc_expr** tb_base,
 		gfc_actual_arglist* argcopy;
 		bool matches;
 
+		/* If expression matching comes here during parsing, eg. when
+		   parsing ASSOCIATE, generic TBPs have not yet been resolved
+		   and g->specific will not have been set. Wait for expression
+		   resolution by returning NULL.  */
+		if (!g->specific && !gfc_current_ns->resolved)
+		  return NULL;
+
 		gcc_assert (g->specific);
 		if (g->specific->error)
 		  continue;
diff --git a/gcc/fortran/intrinsic.cc b/gcc/fortran/intrinsic.cc
index 9e07627..c99a7a8 100644
--- a/gcc/fortran/intrinsic.cc
+++ b/gcc/fortran/intrinsic.cc
@@ -3933,6 +3933,14 @@ add_subroutines (void)
 	      pt, BT_INTEGER, di, OPTIONAL, INTENT_IN,
 	      gt, BT_INTEGER, di, OPTIONAL, INTENT_OUT);
 
+  add_sym_4s ("split", GFC_ISYM_SPLIT, CLASS_PURE,
+	      BT_UNKNOWN, 0, GFC_STD_F2023,
+	      gfc_check_split, NULL, gfc_resolve_split,
+	      "string", BT_CHARACTER, dc, REQUIRED, INTENT_IN,
+	      "set", BT_CHARACTER, dc, REQUIRED, INTENT_IN,
+	      "pos", BT_INTEGER, di, REQUIRED, INTENT_INOUT,
+	      "back", BT_LOGICAL, dl, OPTIONAL, INTENT_IN);
+
   /* The following subroutines are part of ISO_C_BINDING.  */
 
   add_sym_3s ("c_f_pointer", GFC_ISYM_C_F_POINTER, CLASS_IMPURE, BT_UNKNOWN, 0,
diff --git a/gcc/fortran/intrinsic.h b/gcc/fortran/intrinsic.h
index fd54588..8a0ab93 100644
--- a/gcc/fortran/intrinsic.h
+++ b/gcc/fortran/intrinsic.h
@@ -215,6 +215,7 @@ bool gfc_check_mvbits (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *,
 bool gfc_check_random_init (gfc_expr *, gfc_expr *);
 bool gfc_check_random_number (gfc_expr *);
 bool gfc_check_random_seed (gfc_expr *, gfc_expr *, gfc_expr *);
+bool gfc_check_split (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *);
 bool gfc_check_dtime_etime_sub (gfc_expr *, gfc_expr *);
 bool gfc_check_fgetputc_sub (gfc_expr *, gfc_expr *, gfc_expr *);
 bool gfc_check_fgetput_sub (gfc_expr *, gfc_expr *);
@@ -693,6 +694,7 @@ void gfc_resolve_link_sub (gfc_code *);
 void gfc_resolve_symlnk_sub (gfc_code *);
 void gfc_resolve_signal_sub (gfc_code *);
 void gfc_resolve_sleep_sub (gfc_code *);
+void gfc_resolve_split (gfc_code *);
 void gfc_resolve_stat_sub (gfc_code *);
 void gfc_resolve_system_clock (gfc_code *);
 void gfc_resolve_system_sub (gfc_code *);
diff --git a/gcc/fortran/intrinsic.texi b/gcc/fortran/intrinsic.texi
index 3103da3..a24b234 100644
--- a/gcc/fortran/intrinsic.texi
+++ b/gcc/fortran/intrinsic.texi
@@ -313,6 +313,7 @@ Some basic guidelines for editing this document:
 * @code{SIZEOF}:        SIZEOF,    Determine the size in bytes of an expression
 * @code{SLEEP}:         SLEEP,     Sleep for the specified number of seconds
 * @code{SPACING}:       SPACING,   Smallest distance between two numbers of a given type
+* @code{SPLIT}:         SPLIT,     Parse a string into tokens, one at a time.
 * @code{SPREAD}:        SPREAD,    Add a dimension to an array 
 * @code{SQRT}:          SQRT,      Square-root function
 * @code{SRAND}:         SRAND,     Reinitialize the random number generator
@@ -14203,6 +14204,69 @@ Fortran 90 and later
 
 
 
+@node SPLIT
+@section @code{SPLIT} --- Parse a string into tokens, one at a time
+@fnindex SPLIT
+@cindex string, split
+
+@table @asis
+@item @emph{Synopsis}:
+@code{RESULT = SPLIT(STRING, SET, POS [, BACK])}
+
+@item @emph{Description}:
+Updates the integer @var{POS} to the position of the next (or previous)
+separator in @var{STRING}.
+
+If @var{BACK} is absent or is present with the value false, @var{POS} is
+assigned the position of the leftmost token delimiter in @var{STRING} whose
+position is greater than @var{POS}, or if there is no such character, it is
+assigned a value one greater than the length of @var{STRING}. This identifies
+a token with starting position one greater than the value of @var{POS} on
+invocation, and ending position one less than the value of @var{POS} on return.
+
+If @var{BACK} is present with the value true, @var{POS} is assigned the
+position of the rightmost token delimiter in @var{STRING} whose position is
+less than @var{POS}, or if there is no such character, it is assigned the value
+zero. This identifies a token with ending position one less than the value of
+@var{POS} on invocation, and starting position one greater than the value of
+@var{POS} on return.
+
+@item @emph{Class}:
+Subroutine
+
+@item @emph{Arguments}:
+@multitable @columnfractions .15 .70
+@item @var{STRING} @tab Shall be of type @code{CHARACTER}.
+@item @var{SET}    @tab Shall be of type @code{CHARACTER}.
+@item @var{POS}    @tab Shall be of type @code{INTEGER}.
+@item @var{BACK}   @tab (Optional) Shall be of type @code{LOGICAL}.
+@end multitable
+
+@item @emph{Example}:
+@smallexample
+character(len=:), allocatable :: input
+character(len=2) :: set = ', '
+integer :: p
+input = "one,last example"
+p = 0
+do
+  if (p > len(input)) exit
+  istart = p + 1
+  call split(input, set, p)
+  iend = p - 1
+  print '(t7, a)', input(istart:iend)
+end do
+@end smallexample
+
+@item @emph{Standard}:
+Fortran 2023
+
+@item @emph{See also}:
+@ref{SCAN}
+@end table
+
+
+
 @node SPREAD
 @section @code{SPREAD} --- Add a dimension to an array
 @fnindex SPREAD
diff --git a/gcc/fortran/invoke.texi b/gcc/fortran/invoke.texi
index da085d1..0b893e8 100644
--- a/gcc/fortran/invoke.texi
+++ b/gcc/fortran/invoke.texi
@@ -1170,6 +1170,10 @@ A @code{CHARACTER} variable is declared with negative length.
 With @option{-fopenmp}, for fixed-form source code, when an @code{omx}
 vendor-extension sentinel is encountered. (The equivalent @code{ompx},
 used in free-form source code, is diagnosed by default.)
+
+@item
+With @option{-fopenacc}, when using named constances with clauses that
+take a variable as doing so has no effect.
 @end itemize
 
 @opindex Wtabs
diff --git a/gcc/fortran/io.cc b/gcc/fortran/io.cc
index 7466d8f..4d28c2c 100644
--- a/gcc/fortran/io.cc
+++ b/gcc/fortran/io.cc
@@ -29,7 +29,7 @@ along with GCC; see the file COPYING3.  If not see
 
 gfc_st_label
 format_asterisk = {0, NULL, NULL, -1, ST_LABEL_FORMAT, ST_LABEL_FORMAT, NULL,
-		   0, {NULL, NULL}, NULL, 0};
+		   0, {NULL, {NULL}}, NULL, 0};
 
 typedef struct
 {
diff --git a/gcc/fortran/iresolve.cc b/gcc/fortran/iresolve.cc
index 1001309..da354ab 100644
--- a/gcc/fortran/iresolve.cc
+++ b/gcc/fortran/iresolve.cc
@@ -3863,6 +3863,19 @@ gfc_resolve_sleep_sub (gfc_code *c)
   c->resolved_sym = gfc_get_intrinsic_sub_symbol (name);
 }
 
+void
+gfc_resolve_split (gfc_code *c)
+{
+  const char *name;
+  gfc_expr *string;
+
+  string = c->ext.actual->expr;
+  if (string->ts.type == BT_CHARACTER && string->ts.kind == 4)
+    name = "__split_char4";
+  else
+    name = "__split";
+  c->resolved_sym = gfc_get_intrinsic_sub_symbol (name);
+}
 
 /* G77 compatibility function srand().  */
 
diff --git a/gcc/fortran/openmp.cc b/gcc/fortran/openmp.cc
index fe0a47a..f1acc00 100644
--- a/gcc/fortran/openmp.cc
+++ b/gcc/fortran/openmp.cc
@@ -8895,15 +8895,21 @@ resolve_omp_clauses (gfc_code *code, gfc_omp_clauses *omp_clauses,
 	if (list == OMP_LIST_MAP
 	    && n->sym->attr.flavor == FL_PARAMETER)
 	  {
+	    /* OpenACC since 3.4 permits for Fortran named constants, but
+	       permits removing then as optimization is not needed and such
+	       ignore them. Likewise below for FIRSTPRIVATE.  */
 	    if (openacc)
-	      gfc_error ("Object %qs is not a variable at %L; parameters"
-			 " cannot be and need not be copied", n->sym->name,
-			 &n->where);
+	      gfc_warning (OPT_Wsurprising, "Clause for object %qs at %L is "
+			   "ignored as parameters need not be copied",
+			   n->sym->name, &n->where);
 	    else
 	      gfc_error ("Object %qs is not a variable at %L; parameters"
 			 " cannot be and need not be mapped", n->sym->name,
 			 &n->where);
 	  }
+	else if (openacc && n->sym->attr.flavor == FL_PARAMETER)
+	  gfc_warning (OPT_Wsurprising, "Clause for object %qs at %L is ignored"
+		       " as it is a parameter", n->sym->name, &n->where);
 	else if (list != OMP_LIST_USES_ALLOCATORS)
 	  gfc_error ("Object %qs is not a variable at %L", n->sym->name,
 		     &n->where);
@@ -12756,9 +12762,21 @@ gfc_resolve_oacc_declare (gfc_namespace *ns)
 		&& (n->sym->attr.flavor != FL_PROCEDURE
 		    || n->sym->result != n->sym))
 	      {
-		gfc_error ("Object %qs is not a variable at %L",
-			   n->sym->name, &oc->loc);
-		continue;
+		if (n->sym->attr.flavor != FL_PARAMETER)
+		  {
+		    gfc_error ("Object %qs is not a variable at %L",
+			       n->sym->name, &oc->loc);
+		    continue;
+		  }
+		/* Note that OpenACC 3.4 permits name constants, but the
+		   implementation is permitted to ignore the clause;
+		   as semantically, device_resident kind of makes sense
+		   (and the wording with it is a bit odd), the warning
+		   is suppressed.  */
+		if (list != OMP_LIST_DEVICE_RESIDENT)
+		  gfc_warning (OPT_Wsurprising, "Object %qs at %L is ignored as"
+			       " parameters need not be copied", n->sym->name,
+			       &oc->loc);
 	      }
 
 	    if (n->expr && n->expr->ref->type == REF_ARRAY)
diff --git a/gcc/fortran/options.cc b/gcc/fortran/options.cc
index d3c9066..821a8c8 100644
--- a/gcc/fortran/options.cc
+++ b/gcc/fortran/options.cc
@@ -406,7 +406,8 @@ gfc_post_options (const char **pfilename)
       if (warn_line_truncation && !OPTION_SET_P (warnings_are_errors)
 	  && option_unspecified_p (OPT_Wline_truncation))
 	diagnostic_classify_diagnostic (global_dc, OPT_Wline_truncation,
-					DK_ERROR, UNKNOWN_LOCATION);
+					diagnostics::kind::error,
+					UNKNOWN_LOCATION);
     }
   else
     {
diff --git a/gcc/fortran/parse.cc b/gcc/fortran/parse.cc
index 8d4ca39..847ff37 100644
--- a/gcc/fortran/parse.cc
+++ b/gcc/fortran/parse.cc
@@ -6793,6 +6793,7 @@ gfc_fixup_sibling_symbols (gfc_symbol *sym, gfc_namespace *siblings)
   gfc_namespace *ns;
   gfc_symtree *st;
   gfc_symbol *old_sym;
+  bool imported;
 
   for (ns = siblings; ns; ns = ns->sibling)
     {
@@ -6808,6 +6809,7 @@ gfc_fixup_sibling_symbols (gfc_symbol *sym, gfc_namespace *siblings)
 	goto fixup_contained;
 
       old_sym = st->n.sym;
+      imported = old_sym->attr.imported == 1;
       if (old_sym->ns == ns
 	    && !old_sym->attr.contained
 
@@ -6834,7 +6836,8 @@ gfc_fixup_sibling_symbols (gfc_symbol *sym, gfc_namespace *siblings)
 	  /* Replace it with the symbol from the parent namespace.  */
 	  st->n.sym = sym;
 	  sym->refs++;
-
+	  if (imported)
+	    sym->attr.imported = 1;
 	  gfc_release_symbol (old_sym);
 	}
 
diff --git a/gcc/fortran/resolve.cc b/gcc/fortran/resolve.cc
index 4a6e951..c33bd17 100644
--- a/gcc/fortran/resolve.cc
+++ b/gcc/fortran/resolve.cc
@@ -3919,10 +3919,153 @@ found:
 }
 
 
+
+static bool
+check_sym_import_status (gfc_symbol *sym, gfc_symtree *s, gfc_expr *e,
+			 gfc_code *c, gfc_namespace *ns)
+{
+  locus *here;
+
+  /* If the type has been imported then its vtype functions are OK.  */
+  if (e && e->expr_type == EXPR_FUNCTION && sym->attr.vtype)
+    return true;
+
+  if (e)
+    here = &e->where;
+  else
+    here = &c->loc;
+
+  if (s && !s->import_only)
+    s = gfc_find_symtree (ns->sym_root, sym->name);
+
+  if (ns->import_state == IMPORT_ONLY
+      && sym->ns != ns
+      && (!s || !s->import_only))
+    {
+      gfc_error ("F2018: C8102 %qs at %L is host associated but does not "
+		 "appear in an IMPORT or IMPORT, ONLY list", sym->name, here);
+      return false;
+    }
+  else if (ns->import_state == IMPORT_NONE
+	   && sym->ns != ns)
+    {
+      gfc_error ("F2018: C8102 %qs at %L is host associated in a scope that "
+		 "has IMPORT, NONE", sym->name, here);
+      return false;
+    }
+  return true;
+}
+
+
+static bool
+check_import_status (gfc_expr *e)
+{
+  gfc_symtree *st;
+  gfc_ref *ref;
+  gfc_symbol *sym, *der;
+  gfc_namespace *ns = gfc_current_ns;
+
+  switch (e->expr_type)
+    {
+      case EXPR_VARIABLE:
+      case EXPR_FUNCTION:
+      case EXPR_SUBSTRING:
+	sym = e->symtree ? e->symtree->n.sym : NULL;
+
+	/* Check the symbol itself.  */
+	if (sym
+	    && !(ns->proc_name
+		 && (sym == ns->proc_name))
+	    && !check_sym_import_status (sym, e->symtree, e, NULL, ns))
+	  return false;
+
+	/* Check the declared derived type.  */
+	if (sym->ts.type == BT_DERIVED)
+	  {
+	    der = sym->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+	else if (sym->ts.type == BT_CLASS && !UNLIMITED_POLY (sym))
+	  {
+	    der = CLASS_DATA (sym) ? CLASS_DATA (sym)->ts.u.derived
+				   : sym->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+
+	/* Check the declared derived types of component references.  */
+	for (ref = e->ref; ref; ref = ref->next)
+	  if (ref->type == REF_COMPONENT)
+	    {
+	      gfc_component *c = ref->u.c.component;
+	      if (c->ts.type == BT_DERIVED)
+		{
+		  der = c->ts.u.derived;
+		  st = gfc_find_symtree (ns->sym_root, der->name);
+		  if (!check_sym_import_status (der, st, e, NULL, ns))
+		    return false;
+		}
+	      else if (c->ts.type == BT_CLASS && !UNLIMITED_POLY (c))
+		{
+		  der = CLASS_DATA (c) ? CLASS_DATA (c)->ts.u.derived
+				       : c->ts.u.derived;
+		  st = gfc_find_symtree (ns->sym_root, der->name);
+		  if (!check_sym_import_status (der, st, e, NULL, ns))
+		    return false;
+		}
+	    }
+
+	break;
+
+      case EXPR_ARRAY:
+      case EXPR_STRUCTURE:
+	/* Check the declared derived type.  */
+	if (e->ts.type == BT_DERIVED)
+	  {
+	    der = e->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+	else if (e->ts.type == BT_CLASS && !UNLIMITED_POLY (e))
+	  {
+	    der = CLASS_DATA (e) ? CLASS_DATA (e)->ts.u.derived
+				   : e->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+
+	break;
+
+/* Either not applicable or resolved away
+      case EXPR_OP:
+      case EXPR_UNKNOWN:
+      case EXPR_CONSTANT:
+      case EXPR_NULL:
+      case EXPR_COMPCALL:
+      case EXPR_PPC: */
+
+      default:
+	break;
+    }
+
+  return true;
+}
+
+
 /* Resolve a subroutine call.  Although it was tempting to use the same code
    for functions, subroutines and functions are stored differently and this
    makes things awkward.  */
 
+
 static bool
 resolve_call (gfc_code *c)
 {
@@ -4080,6 +4223,11 @@ resolve_call (gfc_code *c)
 		 "Using subroutine %qs at %L is deprecated",
 		 c->resolved_sym->name, &c->loc);
 
+  csym = c->resolved_sym ? c->resolved_sym : csym;
+  if (t && gfc_current_ns->import_state != IMPORT_NOT_SET && !c->resolved_isym
+      && csym != gfc_current_ns->proc_name)
+    return check_sym_import_status (csym, c->symtree, NULL, c, gfc_current_ns);
+
   return t;
 }
 
@@ -7792,6 +7940,7 @@ fixup_unique_dummy (gfc_expr *e)
     e->symtree = st;
 }
 
+
 /* Resolve an expression.  That is, make sure that types of operands agree
    with their operators, intrinsic operators are converted to function calls
    for overloaded types and unresolved function references are resolved.  */
@@ -7919,6 +8068,9 @@ gfc_resolve_expr (gfc_expr *e)
       && UNLIMITED_POLY (e->symtree->n.sym))
     e->do_not_resolve_again = 1;
 
+  if (t && gfc_current_ns->import_state != IMPORT_NOT_SET)
+    t = check_import_status (e);
+
   return t;
 }
 
@@ -10572,6 +10724,7 @@ resolve_select_type (gfc_code *code, gfc_namespace *old_ns)
   int rank = 0, corank = 0;
   gfc_ref* ref = NULL;
   gfc_expr *selector_expr = NULL;
+  gfc_code *old_code = code;
 
   ns = code->ext.block.ns;
   if (code->expr2)
@@ -10860,6 +11013,18 @@ resolve_select_type (gfc_code *code, gfc_namespace *old_ns)
 	 when this case is actually true, so build a new ASSOCIATE
 	 that does precisely this here (instead of using the
 	 'global' one).  */
+
+      /* First check the derived type import status.  */
+      if (gfc_current_ns->import_state != IMPORT_NOT_SET
+	  && (c->ts.type == BT_DERIVED || c->ts.type == BT_CLASS))
+	{
+	  st = gfc_find_symtree (gfc_current_ns->sym_root,
+				 c->ts.u.derived->name);
+	  if (!check_sym_import_status (c->ts.u.derived, st, NULL, old_code,
+					gfc_current_ns))
+	    error++;
+	}
+
       const char * var_name = gfc_var_name_for_select_type_temp (orig_expr1);
       if (c->ts.type == BT_CLASS)
 	snprintf (name, sizeof (name), "__tmp_class_%s_%s",
diff --git a/gcc/fortran/trans-array.cc b/gcc/fortran/trans-array.cc
index 7be2d7b..45980d6 100644
--- a/gcc/fortran/trans-array.cc
+++ b/gcc/fortran/trans-array.cc
@@ -284,16 +284,6 @@ gfc_conv_descriptor_data_set (stmtblock_t *block, tree desc, tree value)
 }
 
 
-/* This provides address access to the data field.  This should only be
-   used by array allocation, passing this on to the runtime.  */
-
-tree
-gfc_conv_descriptor_data_addr (tree desc)
-{
-  tree field = gfc_get_descriptor_field (desc, DATA_FIELD);
-  return gfc_build_addr_expr (NULL_TREE, field);
-}
-
 static tree
 gfc_conv_descriptor_offset (tree desc)
 {
@@ -1426,12 +1416,6 @@ get_class_info_from_ss (stmtblock_t * pre, gfc_ss *ss, tree *eltype,
 	  tmp2 = gfc_class_len_get (class_expr);
 	  gfc_add_modify (pre, tmp, tmp2);
 	}
-
-      if (rhs_function)
-	{
-	  tmp = gfc_class_data_get (class_expr);
-	  gfc_conv_descriptor_offset_set (pre, tmp, gfc_index_zero_node);
-	}
     }
   else if (rhs_ss->info->data.array.descriptor)
    {
@@ -3372,18 +3356,51 @@ gfc_add_loop_ss_code (gfc_loopinfo * loop, gfc_ss * ss, bool subscript,
 	  break;
 
 	case GFC_SS_FUNCTION:
-	  /* Array function return value.  We call the function and save its
-	     result in a temporary for use inside the loop.  */
-	  gfc_init_se (&se, NULL);
-	  se.loop = loop;
-	  se.ss = ss;
-	  if (gfc_is_class_array_function (expr))
-	    expr->must_finalize = 1;
-	  gfc_conv_expr (&se, expr);
-	  gfc_add_block_to_block (&outer_loop->pre, &se.pre);
-	  gfc_add_block_to_block (&outer_loop->post, &se.post);
-	  gfc_add_block_to_block (&outer_loop->post, &se.finalblock);
-	  ss_info->string_length = se.string_length;
+	  {
+	    /* Array function return value.  We call the function and save its
+	       result in a temporary for use inside the loop.  */
+	    gfc_init_se (&se, NULL);
+	    se.loop = loop;
+	    se.ss = ss;
+	    bool class_func = gfc_is_class_array_function (expr);
+	    if (class_func)
+	      expr->must_finalize = 1;
+	    gfc_conv_expr (&se, expr);
+	    gfc_add_block_to_block (&outer_loop->pre, &se.pre);
+	    if (class_func
+		&& se.expr
+		&& GFC_CLASS_TYPE_P (TREE_TYPE (se.expr)))
+	      {
+		tree tmp = gfc_class_data_get (se.expr);
+		info->descriptor = tmp;
+		info->data = gfc_conv_descriptor_data_get (tmp);
+		info->offset = gfc_conv_descriptor_offset_get (tmp);
+		for (gfc_ss *s = ss; s; s = s->parent)
+		  for (int n = 0; n < s->dimen; n++)
+		    {
+		      int dim = s->dim[n];
+		      tree tree_dim = gfc_rank_cst[dim];
+
+		      tree start;
+		      start = gfc_conv_descriptor_lbound_get (tmp, tree_dim);
+		      start = gfc_evaluate_now (start, &outer_loop->pre);
+		      info->start[dim] = start;
+
+		      tree end;
+		      end = gfc_conv_descriptor_ubound_get (tmp, tree_dim);
+		      end = gfc_evaluate_now (end, &outer_loop->pre);
+		      info->end[dim] = end;
+
+		      tree stride;
+		      stride = gfc_conv_descriptor_stride_get (tmp, tree_dim);
+		      stride = gfc_evaluate_now (stride, &outer_loop->pre);
+		      info->stride[dim] = stride;
+		    }
+	      }
+	    gfc_add_block_to_block (&outer_loop->post, &se.post);
+	    gfc_add_block_to_block (&outer_loop->post, &se.finalblock);
+	    ss_info->string_length = se.string_length;
+	  }
 	  break;
 
 	case GFC_SS_CONSTRUCTOR:
@@ -3420,6 +3437,183 @@ gfc_add_loop_ss_code (gfc_loopinfo * loop, gfc_ss * ss, bool subscript,
 }
 
 
+/* Given an array descriptor expression DESCR and its data pointer DATA, decide
+   whether to either save the data pointer to a variable and use the variable or
+   use the data pointer expression directly without any intermediary variable.
+   */
+
+static bool
+save_descriptor_data (tree descr, tree data)
+{
+  return !(DECL_P (data)
+	   || (TREE_CODE (data) == ADDR_EXPR
+	       && DECL_P (TREE_OPERAND (data, 0)))
+	   || (GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (descr))
+	       && TREE_CODE (descr) == COMPONENT_REF
+	       && GFC_CLASS_TYPE_P (TREE_TYPE (TREE_OPERAND (descr, 0)))));
+}
+
+
+/* Type of the DATA argument passed to walk_tree by substitute_subexpr_in_expr
+   and used by maybe_substitute_expr.  */
+
+typedef struct
+{
+  tree target, repl;
+}
+substitute_t;
+
+
+/* Check if the expression in *TP is equal to the substitution target provided
+   in DATA->TARGET and replace it with DATA->REPL in that case.   This is a
+   callback function for use with walk_tree.  */
+
+static tree
+maybe_substitute_expr (tree *tp, int *walk_subtree, void *data)
+{
+  substitute_t *subst = (substitute_t *) data;
+  if (*tp == subst->target)
+    {
+      *tp = subst->repl;
+      *walk_subtree = 0;
+    }
+
+  return NULL_TREE;
+}
+
+
+/* Substitute in EXPR any occurence of TARGET with REPLACEMENT.  */
+
+static void
+substitute_subexpr_in_expr (tree target, tree replacement, tree expr)
+{
+  substitute_t subst;
+  subst.target = target;
+  subst.repl = replacement;
+
+  walk_tree (&expr, maybe_substitute_expr, &subst, nullptr);
+}
+
+
+/* Save REF to a fresh variable in all of REPLACEMENT_ROOTS, appending extra
+   code to CODE.  Before returning, add REF to REPLACEMENT_ROOTS and clear
+   REF.  */
+
+static void
+save_ref (tree &code, tree &ref, vec<tree> &replacement_roots)
+{
+  stmtblock_t tmp_block;
+  gfc_init_block (&tmp_block);
+  tree var = gfc_evaluate_now (ref, &tmp_block);
+  gfc_add_expr_to_block (&tmp_block, code);
+  code = gfc_finish_block (&tmp_block);
+
+  unsigned i;
+  tree repl_root;
+  FOR_EACH_VEC_ELT (replacement_roots, i, repl_root)
+    substitute_subexpr_in_expr (ref, var, repl_root);
+
+  replacement_roots.safe_push (ref);
+  ref = NULL_TREE;
+}
+
+
+/* Save the descriptor reference VALUE to storage pointed by DESC_PTR.  Before
+   that, try to factor subexpressions of VALUE to variables, adding extra code
+   to BLOCK.
+
+   The candidate references to factoring are dereferenced pointers because they
+   are cheap to copy and array descriptors because they are often the base of
+   multiple subreferences.  */
+
+static void
+set_factored_descriptor_value (tree *desc_ptr, tree value, stmtblock_t *block)
+{
+  /* As the reference is processed from outer to inner, variable definitions
+     will be generated in reversed order, so can't be put directly in BLOCK.
+     We use TMP_BLOCK instead.  */
+  tree accumulated_code = NULL_TREE;
+
+  /* The current candidate to factoring.  */
+  tree saveable_ref = NULL_TREE;
+
+  /* The root expressions in which we look for subexpressions to replace with
+     variables.  */
+  auto_vec<tree> replacement_roots;
+  replacement_roots.safe_push (value);
+
+  tree data_ref = value;
+  tree next_ref = NULL_TREE;
+
+  /* If the candidate reference is not followed by a subreference, it can't be
+     saved to a variable as it may be reallocatable, and we have to keep the
+     parent reference to be able to store the new pointer value in case of
+     reallocation.  */
+  bool maybe_reallocatable = true;
+
+  while (true)
+    {
+      if (!maybe_reallocatable
+	  && GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (data_ref)))
+	saveable_ref = data_ref;
+
+      if (TREE_CODE (data_ref) == INDIRECT_REF)
+	{
+	  next_ref = TREE_OPERAND (data_ref, 0);
+
+	  if (!maybe_reallocatable)
+	    {
+	      if (saveable_ref != NULL_TREE && saveable_ref != data_ref)
+		{
+		  /* A reference worth saving has been seen, and now the pointer
+		     to the current reference is also worth saving.  If the
+		     previous reference to save wasn't the current one, do save
+		     it now.  Otherwise drop it as we prefer saving the
+		     pointer.  */
+		  save_ref (accumulated_code, saveable_ref, replacement_roots);
+		}
+
+	      /* Don't evaluate the pointer to a variable yet; do it only if the
+		 variable would be significantly more simple than the reference
+		 it replaces.  That is if the reference contains anything
+		 different from NOPs, COMPONENTs and DECLs.  */
+	      saveable_ref = next_ref;
+	    }
+	}
+      else if (TREE_CODE (data_ref) == COMPONENT_REF)
+	{
+	  maybe_reallocatable = false;
+	  next_ref = TREE_OPERAND (data_ref, 0);
+	}
+      else if (TREE_CODE (data_ref) == NOP_EXPR)
+	next_ref = TREE_OPERAND (data_ref, 0);
+      else
+	{
+	  if (DECL_P (data_ref))
+	    break;
+
+	  if (TREE_CODE (data_ref) == ARRAY_REF)
+	    {
+	      maybe_reallocatable = false;
+	      next_ref = TREE_OPERAND (data_ref, 0);
+	    }
+
+	  if (saveable_ref != NULL_TREE)
+	    /* We have seen a reference worth saving.  Do it now.  */
+	    save_ref (accumulated_code, saveable_ref, replacement_roots);
+
+	  if (TREE_CODE (data_ref) != ARRAY_REF)
+	    break;
+	}
+
+      data_ref = next_ref;
+    }
+
+  *desc_ptr = value;
+  gfc_add_expr_to_block (block, accumulated_code);
+}
+
+
 /* Translate expressions for the descriptor and data pointer of a SS.  */
 /*GCC ARRAYS*/
 
@@ -3440,7 +3634,7 @@ gfc_conv_ss_descriptor (stmtblock_t * block, gfc_ss * ss, int base)
   se.descriptor_only = 1;
   gfc_conv_expr_lhs (&se, ss_info->expr);
   gfc_add_block_to_block (block, &se.pre);
-  info->descriptor = se.expr;
+  set_factored_descriptor_value (&info->descriptor, se.expr, block);
   ss_info->string_length = se.string_length;
   ss_info->class_container = se.class_container;
 
@@ -3466,17 +3660,14 @@ gfc_conv_ss_descriptor (stmtblock_t * block, gfc_ss * ss, int base)
          Otherwise we must evaluate it now to avoid breaking dependency
 	 analysis by pulling the expressions for elemental array indices
 	 inside the loop.  */
-      if (!(DECL_P (tmp)
-	    || (TREE_CODE (tmp) == ADDR_EXPR
-		&& DECL_P (TREE_OPERAND (tmp, 0)))
-	    || (GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (se.expr))
-		&& TREE_CODE (se.expr) == COMPONENT_REF
-		&& GFC_CLASS_TYPE_P (TREE_TYPE (TREE_OPERAND (se.expr, 0))))))
+      if (save_descriptor_data (se.expr, tmp) && !ss->is_alloc_lhs)
 	tmp = gfc_evaluate_now (tmp, block);
       info->data = tmp;
 
       tmp = gfc_conv_array_offset (se.expr);
-      info->offset = gfc_evaluate_now (tmp, block);
+      if (!ss->is_alloc_lhs)
+	tmp = gfc_evaluate_now (tmp, block);
+      info->offset = tmp;
 
       /* Make absolutely sure that the saved_offset is indeed saved
 	 so that the variable is still accessible after the loops
@@ -4769,13 +4960,12 @@ gfc_trans_scalarized_loop_boundary (gfc_loopinfo * loop, stmtblock_t * body)
 
 static void
 evaluate_bound (stmtblock_t *block, tree *bounds, gfc_expr ** values,
-		tree desc, int dim, bool lbound, bool deferred)
+		tree desc, int dim, bool lbound, bool deferred, bool save_value)
 {
   gfc_se se;
   gfc_expr * input_val = values[dim];
   tree *output = &bounds[dim];
 
-
   if (input_val)
     {
       /* Specified section bound.  */
@@ -4801,7 +4991,8 @@ evaluate_bound (stmtblock_t *block, tree *bounds, gfc_expr ** values,
       *output = lbound ? gfc_conv_array_lbound (desc, dim) :
 			 gfc_conv_array_ubound (desc, dim);
     }
-  *output = gfc_evaluate_now (*output, block);
+  if (save_value)
+    *output = gfc_evaluate_now (*output, block);
 }
 
 
@@ -4834,18 +5025,18 @@ gfc_conv_section_startstride (stmtblock_t * block, gfc_ss * ss, int dim)
 	      || ar->dimen_type[dim] == DIMEN_THIS_IMAGE);
   desc = info->descriptor;
   stride = ar->stride[dim];
-
+  bool save_value = !ss->is_alloc_lhs;
 
   /* Calculate the start of the range.  For vector subscripts this will
      be the range of the vector.  */
   evaluate_bound (block, info->start, ar->start, desc, dim, true,
-		  ar->as->type == AS_DEFERRED);
+		  ar->as->type == AS_DEFERRED, save_value);
 
   /* Similarly calculate the end.  Although this is not used in the
      scalarizer, it is needed when checking bounds and where the end
      is an expression with side-effects.  */
   evaluate_bound (block, info->end, ar->end, desc, dim, false,
-		  ar->as->type == AS_DEFERRED);
+		  ar->as->type == AS_DEFERRED, save_value);
 
 
   /* Calculate the stride.  */
@@ -4856,7 +5047,11 @@ gfc_conv_section_startstride (stmtblock_t * block, gfc_ss * ss, int dim)
       gfc_init_se (&se, NULL);
       gfc_conv_expr_type (&se, stride, gfc_array_index_type);
       gfc_add_block_to_block (block, &se.pre);
-      info->stride[dim] = gfc_evaluate_now (se.expr, block);
+      tree value = se.expr;
+      if (save_value)
+	info->stride[dim] = gfc_evaluate_now (value, block);
+      else
+	info->stride[dim] = value;
     }
 }
 
@@ -5205,7 +5400,8 @@ done:
 	      int dim = ss->dim[n];
 
 	      info->start[dim]  = gfc_index_zero_node;
-	      info->end[dim]    = gfc_index_zero_node;
+	      if (ss_info->type != GFC_SS_FUNCTION)
+		info->end[dim]    = gfc_index_zero_node;
 	      info->stride[dim] = gfc_index_one_node;
 	    }
 	  break;
@@ -5890,6 +6086,46 @@ set_loop_bounds (gfc_loopinfo *loop)
 }
 
 
+/* Last attempt to set the loop bounds, in case they depend on an allocatable
+   function result.  */
+
+static void
+late_set_loop_bounds (gfc_loopinfo *loop)
+{
+  int n, dim;
+  gfc_array_info *info;
+  gfc_ss **loopspec;
+
+  loopspec = loop->specloop;
+
+  for (n = 0; n < loop->dimen; n++)
+    {
+      /* Set the extents of this range.  */
+      if (loop->from[n] == NULL_TREE
+	  || loop->to[n] == NULL_TREE)
+	{
+	  /* We should have found the scalarization loop specifier.  If not,
+	     that's bad news.  */
+	  gcc_assert (loopspec[n]);
+
+	  info = &loopspec[n]->info->data.array;
+	  dim = loopspec[n]->dim[n];
+
+	  if (loopspec[n]->info->type == GFC_SS_FUNCTION
+	      && info->start[dim]
+	      && info->end[dim])
+	    {
+	      loop->from[n] = info->start[dim];
+	      loop->to[n] = info->end[dim];
+	    }
+	}
+    }
+
+  for (loop = loop->nested; loop; loop = loop->next)
+    late_set_loop_bounds (loop);
+}
+
+
 /* Initialize the scalarization loop.  Creates the loop variables.  Determines
    the range of the loop variables.  Creates a temporary if required.
    Also generates code for scalar expressions which have been
@@ -5908,6 +6144,8 @@ gfc_conv_loop_setup (gfc_loopinfo * loop, locus * where)
      allocating the temporary.  */
   gfc_add_loop_ss_code (loop, loop->ss, false, where);
 
+  late_set_loop_bounds (loop);
+
   tmp_ss = loop->temp_ss;
   /* If we want a temporary then create it.  */
   if (tmp_ss != NULL)
@@ -5964,9 +6202,11 @@ gfc_set_delta (gfc_loopinfo *loop)
       gfc_ss_type ss_type;
 
       ss_type = ss->info->type;
-      if (ss_type != GFC_SS_SECTION
-	  && ss_type != GFC_SS_COMPONENT
-	  && ss_type != GFC_SS_CONSTRUCTOR)
+      if (!(ss_type == GFC_SS_SECTION
+	    || ss_type == GFC_SS_COMPONENT
+	    || ss_type == GFC_SS_CONSTRUCTOR
+	    || (ss_type == GFC_SS_FUNCTION
+		&& gfc_is_class_array_function (ss->info->expr))))
 	continue;
 
       info = &ss->info->data.array;
@@ -5991,7 +6231,10 @@ gfc_set_delta (gfc_loopinfo *loop)
 				     gfc_array_index_type,
 				     info->start[dim], tmp);
 
-	      info->delta[dim] = gfc_evaluate_now (tmp, &outer_loop->pre);
+	      if (ss->is_alloc_lhs)
+		info->delta[dim] = tmp;
+	      else
+		info->delta[dim] = gfc_evaluate_now (tmp, &outer_loop->pre);
 	    }
 	}
     }
@@ -6115,8 +6358,8 @@ static tree
 gfc_array_init_size (tree descriptor, int rank, int corank, tree * poffset,
 		     gfc_expr ** lower, gfc_expr ** upper, stmtblock_t * pblock,
 		     stmtblock_t * descriptor_block, tree * overflow,
-		     tree expr3_elem_size, tree *nelems, gfc_expr *expr3,
-		     tree expr3_desc, bool e3_has_nodescriptor, gfc_expr *expr,
+		     tree expr3_elem_size, gfc_expr *expr3, tree expr3_desc,
+		     bool e3_has_nodescriptor, gfc_expr *expr,
 		     tree *element_size, bool explicit_ts)
 {
   tree type;
@@ -6392,7 +6635,6 @@ gfc_array_init_size (tree descriptor, int rank, int corank, tree * poffset,
   if (rank == 0)
     return *element_size;
 
-  *nelems = gfc_evaluate_now (stride, pblock);
   stride = fold_convert (size_type_node, stride);
 
   /* First check for overflow. Since an array of type character can
@@ -6481,9 +6723,8 @@ retrieve_last_ref (gfc_ref **ref_in, gfc_ref **prev_ref_in)
 bool
 gfc_array_allocate (gfc_se * se, gfc_expr * expr, tree status, tree errmsg,
 		    tree errlen, tree label_finish, tree expr3_elem_size,
-		    tree *nelems, gfc_expr *expr3, tree e3_arr_desc,
-		    bool e3_has_nodescriptor, gfc_omp_namelist *omp_alloc,
-		    bool explicit_ts)
+		    gfc_expr *expr3, tree e3_arr_desc, bool e3_has_nodescriptor,
+		    gfc_omp_namelist *omp_alloc, bool explicit_ts)
 {
   tree tmp;
   tree pointer;
@@ -6614,7 +6855,7 @@ gfc_array_allocate (gfc_se * se, gfc_expr * expr, tree status, tree errmsg,
 			      coarray ? ref->u.ar.as->corank : 0,
 			      &offset, lower, upper,
 			      &se->pre, &set_descriptor_block, &overflow,
-			      expr3_elem_size, nelems, expr3, e3_arr_desc,
+			      expr3_elem_size, expr3, e3_arr_desc,
 			      e3_has_nodescriptor, expr, &element_size,
 			      explicit_ts);
 
@@ -6779,8 +7020,6 @@ gfc_array_allocate (gfc_se * se, gfc_expr * expr, tree status, tree errmsg,
   else
       gfc_add_expr_to_block (&se->pre, set_descriptor);
 
-  expr->symtree->n.sym->allocated_in_scope = 1;
-
   return true;
 }
 
@@ -8470,7 +8709,7 @@ gfc_conv_expr_descriptor (gfc_se *se, gfc_expr *expr)
 	  gcc_assert (n == codim - 1);
 	  evaluate_bound (&loop.pre, info->start, ar->start,
 			  info->descriptor, n + ndim, true,
-			  ar->as->type == AS_DEFERRED);
+			  ar->as->type == AS_DEFERRED, true);
 	  loop.from[n + loop.dimen] = info->start[n + ndim];
 	}
       else
@@ -9339,9 +9578,8 @@ gfc_conv_array_parameter (gfc_se *se, gfc_expr *expr, bool g77,
 	      new_field = gfc_conv_descriptor_dtype (new_desc);
 	      gfc_add_modify (&se->pre, new_field, old_field);
 
-	      old_field = gfc_conv_descriptor_offset (old_desc);
-	      new_field = gfc_conv_descriptor_offset (new_desc);
-	      gfc_add_modify (&se->pre, new_field, old_field);
+	      old_field = gfc_conv_descriptor_offset_get (old_desc);
+	      gfc_conv_descriptor_offset_set (&se->pre, new_desc, old_field);
 
 	      for (int i = 0; i < expr->rank; i++)
 		{
@@ -11206,6 +11444,9 @@ gfc_is_reallocatable_lhs (gfc_expr *expr)
   gfc_ref * ref;
   gfc_symbol *sym;
 
+  if (!flag_realloc_lhs)
+    return false;
+
   if (!expr->ref)
     return false;
 
@@ -11330,6 +11571,55 @@ concat_str_length (gfc_expr* expr)
 }
 
 
+/* Among the scalarization chain of LOOP, find the element associated with an
+   allocatable array on the lhs of an assignment and evaluate its fields
+   (bounds, offset, etc) to new variables, putting the new code in BLOCK.  This
+   function is to be called after putting the reallocation code in BLOCK and
+   before the beginning of the scalarization loop body.
+
+   The fields to be saved are expected to hold on entry to the function
+   expressions referencing the array descriptor.  Especially the expressions
+   shouldn't be already temporary variable references as the value saved before
+   reallocation would be incorrect after reallocation.
+   At the end of the function, the expressions have been replaced with variable
+   references.  */
+
+static void
+update_reallocated_descriptor (stmtblock_t *block, gfc_loopinfo *loop)
+{
+  for (gfc_ss *s = loop->ss; s != gfc_ss_terminator; s = s->loop_chain)
+    {
+      if (!s->is_alloc_lhs)
+	continue;
+
+      gcc_assert (s->info->type == GFC_SS_SECTION);
+      gfc_array_info *info = &s->info->data.array;
+
+#define SAVE_VALUE(value) \
+	      do \
+		{ \
+		  value = gfc_evaluate_now (value, block); \
+		} \
+	      while (0)
+
+      if (save_descriptor_data (info->descriptor, info->data))
+	SAVE_VALUE (info->data);
+      SAVE_VALUE (info->offset);
+      info->saved_offset = info->offset;
+      for (int i = 0; i < s->dimen; i++)
+	{
+	  int dim = s->dim[i];
+	  SAVE_VALUE (info->start[dim]);
+	  SAVE_VALUE (info->end[dim]);
+	  SAVE_VALUE (info->stride[dim]);
+	  SAVE_VALUE (info->delta[dim]);
+	}
+
+#undef SAVE_VALUE
+    }
+}
+
+
 /* Allocate the lhs of an assignment to an allocatable array, otherwise
    reallocate it.  */
 
@@ -11368,7 +11658,6 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
   tree lbd;
   tree class_expr2 = NULL_TREE;
   int n;
-  int dim;
   gfc_array_spec * as;
   bool coarray = (flag_coarray == GFC_FCOARRAY_LIB
 		  && gfc_caf_attr (expr1, true).codimension);
@@ -11423,14 +11712,61 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
       && !expr2->value.function.isym)
     expr2->ts.u.cl->backend_decl = rss->info->string_length;
 
-  gfc_start_block (&fblock);
-
   /* Since the lhs is allocatable, this must be a descriptor type.
      Get the data and array size.  */
   desc = linfo->descriptor;
   gcc_assert (GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (desc)));
   array1 = gfc_conv_descriptor_data_get (desc);
 
+  /* If the data is null, set the descriptor bounds and offset.  This suppresses
+     the maybe used uninitialized warning.  Note that the always false variable
+     prevents this block from ever being executed, and makes sure that the
+     optimizers are able to remove it.  Component references are not subject to
+     the warnings, so we don't uselessly complicate the generated code for them.
+     */
+  for (ref = expr1->ref; ref; ref = ref->next)
+    if (ref->type == REF_COMPONENT)
+      break;
+
+  if (!ref)
+    {
+      stmtblock_t unalloc_init_block;
+      gfc_init_block (&unalloc_init_block);
+      tree guard = gfc_create_var (logical_type_node, "unallocated_init_guard");
+      gfc_add_modify (&unalloc_init_block, guard, logical_false_node);
+
+      gfc_start_block (&loop_pre_block);
+      for (n = 0; n < expr1->rank; n++)
+	{
+	  gfc_conv_descriptor_lbound_set (&loop_pre_block, desc,
+					  gfc_rank_cst[n],
+					  gfc_index_one_node);
+	  gfc_conv_descriptor_ubound_set (&loop_pre_block, desc,
+					  gfc_rank_cst[n],
+					  gfc_index_zero_node);
+	  gfc_conv_descriptor_stride_set (&loop_pre_block, desc,
+					  gfc_rank_cst[n],
+					  gfc_index_zero_node);
+	}
+
+      gfc_conv_descriptor_offset_set (&loop_pre_block, desc,
+				      gfc_index_zero_node);
+
+      tmp = fold_build2_loc (input_location, EQ_EXPR,
+			     logical_type_node, array1,
+			     build_int_cst (TREE_TYPE (array1), 0));
+      tmp = fold_build2_loc (input_location, TRUTH_ANDIF_EXPR,
+			     logical_type_node, tmp, guard);
+      tmp = build3_v (COND_EXPR, tmp,
+		      gfc_finish_block (&loop_pre_block),
+		      build_empty_stmt (input_location));
+      gfc_prepend_expr_to_block (&loop->pre, tmp);
+      gfc_prepend_expr_to_block (&loop->pre,
+				 gfc_finish_block (&unalloc_init_block));
+    }
+
+  gfc_start_block (&fblock);
+
   if (expr2)
     desc2 = rss->info->data.array.descriptor;
   else
@@ -11543,45 +11879,6 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
 			 array1, build_int_cst (TREE_TYPE (array1), 0));
   cond_null= gfc_evaluate_now (cond_null, &fblock);
 
-  /* If the data is null, set the descriptor bounds and offset. This suppresses
-     the maybe used uninitialized warning and forces the use of malloc because
-     the size is zero in all dimensions. Note that this block is only executed
-     if the lhs is unallocated and is only applied once in any namespace.
-     Component references are not subject to the warnings.  */
-  for (ref = expr1->ref; ref; ref = ref->next)
-    if (ref->type == REF_COMPONENT)
-      break;
-
-  if (!expr1->symtree->n.sym->allocated_in_scope && !ref)
-    {
-      gfc_start_block (&loop_pre_block);
-      for (n = 0; n < expr1->rank; n++)
-	{
-	  gfc_conv_descriptor_lbound_set (&loop_pre_block, desc,
-					  gfc_rank_cst[n],
-					  gfc_index_one_node);
-	  gfc_conv_descriptor_ubound_set (&loop_pre_block, desc,
-					  gfc_rank_cst[n],
-					  gfc_index_zero_node);
-	  gfc_conv_descriptor_stride_set (&loop_pre_block, desc,
-					  gfc_rank_cst[n],
-					  gfc_index_zero_node);
-	}
-
-      tmp = gfc_conv_descriptor_offset (desc);
-      gfc_add_modify (&loop_pre_block, tmp, gfc_index_zero_node);
-
-      tmp = fold_build2_loc (input_location, EQ_EXPR,
-			     logical_type_node, array1,
-			     build_int_cst (TREE_TYPE (array1), 0));
-      tmp = build3_v (COND_EXPR, tmp,
-		      gfc_finish_block (&loop_pre_block),
-		      build_empty_stmt (input_location));
-      gfc_prepend_expr_to_block (&loop->pre, tmp);
-
-      expr1->symtree->n.sym->allocated_in_scope = 1;
-    }
-
   tmp = build3_v (COND_EXPR, cond_null,
 		  build1_v (GOTO_EXPR, jump_label1),
 		  build_empty_stmt (input_location));
@@ -11736,21 +12033,6 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
      running offset.  Use the saved_offset instead.  */
   tmp = gfc_conv_descriptor_offset (desc);
   gfc_add_modify (&fblock, tmp, offset);
-  if (linfo->saved_offset
-      && VAR_P (linfo->saved_offset))
-    gfc_add_modify (&fblock, linfo->saved_offset, tmp);
-
-  /* Now set the deltas for the lhs.  */
-  for (n = 0; n < expr1->rank; n++)
-    {
-      tmp = gfc_conv_descriptor_lbound_get (desc, gfc_rank_cst[n]);
-      dim = lss->dim[n];
-      tmp = fold_build2_loc (input_location, MINUS_EXPR,
-			     gfc_array_index_type, tmp,
-			     loop->from[dim]);
-      if (linfo->delta[dim] && VAR_P (linfo->delta[dim]))
-	gfc_add_modify (&fblock, linfo->delta[dim], tmp);
-    }
 
   /* Take into account _len of unlimited polymorphic entities, so that span
      for array descriptors and allocation sizes are computed correctly.  */
@@ -11972,18 +12254,18 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
   tmp = build3_v (COND_EXPR, cond_null, alloc_expr, realloc_expr);
   gfc_add_expr_to_block (&fblock, tmp);
 
-  /* Make sure that the scalarizer data pointer is updated.  */
-  if (linfo->data && VAR_P (linfo->data))
-    {
-      tmp = gfc_conv_descriptor_data_get (desc);
-      gfc_add_modify (&fblock, linfo->data, tmp);
-    }
-
   /* Add the label for same shape lhs and rhs.  */
   tmp = build1_v (LABEL_EXPR, jump_label2);
   gfc_add_expr_to_block (&fblock, tmp);
 
-  return gfc_finish_block (&fblock);
+  tree realloc_code = gfc_finish_block (&fblock);
+
+  stmtblock_t result_block;
+  gfc_init_block (&result_block);
+  gfc_add_expr_to_block (&result_block, realloc_code);
+  update_reallocated_descriptor (&result_block, loop);
+
+  return gfc_finish_block (&result_block);
 }
 
 
diff --git a/gcc/fortran/trans-array.h b/gcc/fortran/trans-array.h
index 1bb3294..345a975 100644
--- a/gcc/fortran/trans-array.h
+++ b/gcc/fortran/trans-array.h
@@ -20,9 +20,8 @@ along with GCC; see the file COPYING3.  If not see
 
 /* Generate code to initialize and allocate an array.  Statements are added to
    se, which should contain an expression for the array descriptor.  */
-bool gfc_array_allocate (gfc_se *, gfc_expr *, tree, tree, tree, tree,
-			 tree, tree *, gfc_expr *, tree, bool,
-			 gfc_omp_namelist *, bool);
+bool gfc_array_allocate (gfc_se *, gfc_expr *, tree, tree, tree, tree, tree,
+			 gfc_expr *, tree, bool, gfc_omp_namelist *, bool);
 
 /* Allow the bounds of a loop to be set from a callee's array spec.  */
 void gfc_set_loop_bounds_from_array_spec (gfc_interface_mapping *,
@@ -174,7 +173,6 @@ void gfc_get_descriptor_offsets_for_info (const_tree, tree *, tree *, tree *, tr
 					  tree *, tree *, tree *, tree *);
 
 tree gfc_conv_descriptor_data_get (tree);
-tree gfc_conv_descriptor_data_addr (tree);
 tree gfc_conv_descriptor_offset_get (tree);
 tree gfc_conv_descriptor_span_get (tree);
 tree gfc_conv_descriptor_dtype (tree);
diff --git a/gcc/fortran/trans-decl.cc b/gcc/fortran/trans-decl.cc
index 43bd7be..b495f43 100644
--- a/gcc/fortran/trans-decl.cc
+++ b/gcc/fortran/trans-decl.cc
@@ -197,6 +197,7 @@ tree gfor_fndecl_string_scan;
 tree gfor_fndecl_string_verify;
 tree gfor_fndecl_string_trim;
 tree gfor_fndecl_string_minmax;
+tree gfor_fndecl_string_split;
 tree gfor_fndecl_adjustl;
 tree gfor_fndecl_adjustr;
 tree gfor_fndecl_select_string;
@@ -208,6 +209,7 @@ tree gfor_fndecl_string_scan_char4;
 tree gfor_fndecl_string_verify_char4;
 tree gfor_fndecl_string_trim_char4;
 tree gfor_fndecl_string_minmax_char4;
+tree gfor_fndecl_string_split_char4;
 tree gfor_fndecl_adjustl_char4;
 tree gfor_fndecl_adjustr_char4;
 tree gfor_fndecl_select_string_char4;
@@ -3569,6 +3571,12 @@ gfc_build_intrinsic_function_decls (void)
 	build_pointer_type (pchar1_type_node), integer_type_node,
 	integer_type_node);
 
+  gfor_fndecl_string_split = gfc_build_library_function_decl_with_spec (
+    get_identifier (PREFIX ("string_split")), ". . R . R . . ",
+    gfc_charlen_type_node, 6, gfc_charlen_type_node, pchar1_type_node,
+    gfc_charlen_type_node, pchar1_type_node, gfc_charlen_type_node,
+    gfc_logical4_type_node);
+
   gfor_fndecl_adjustl = gfc_build_library_function_decl_with_spec (
 	get_identifier (PREFIX("adjustl")), ". W . R ",
 	void_type_node, 3, pchar1_type_node, gfc_charlen_type_node,
@@ -3641,6 +3649,12 @@ gfc_build_intrinsic_function_decls (void)
 	build_pointer_type (pchar4_type_node), integer_type_node,
 	integer_type_node);
 
+  gfor_fndecl_string_split_char4 = gfc_build_library_function_decl_with_spec (
+    get_identifier (PREFIX ("string_split_char4")), ". . R . R . . ",
+    gfc_charlen_type_node, 6, gfc_charlen_type_node, pchar4_type_node,
+    gfc_charlen_type_node, pchar4_type_node, gfc_charlen_type_node,
+    gfc_logical4_type_node);
+
   gfor_fndecl_adjustl_char4 = gfc_build_library_function_decl_with_spec (
 	get_identifier (PREFIX("adjustl_char4")), ". W . R ",
 	void_type_node, 3, pchar4_type_node, gfc_charlen_type_node,
@@ -4773,14 +4787,14 @@ gfc_trans_deferred_vars (gfc_symbol * proc_sym, gfc_wrapped_block * block)
       /* Nullify explicit return class arrays on entry.  */
       tree type;
       tmp = get_proc_result (proc_sym);
-	if (tmp && GFC_CLASS_TYPE_P (TREE_TYPE (tmp)))
-	  {
-	    gfc_start_block (&init);
-	    tmp = gfc_class_data_get (tmp);
-	    type = TREE_TYPE (gfc_conv_descriptor_data_get (tmp));
-	    gfc_conv_descriptor_data_set (&init, tmp, build_int_cst (type, 0));
-	    gfc_add_init_cleanup (block, gfc_finish_block (&init), NULL_TREE);
-	  }
+      if (tmp && GFC_CLASS_TYPE_P (TREE_TYPE (tmp)))
+	{
+	  gfc_start_block (&init);
+	  tmp = gfc_class_data_get (tmp);
+	  type = TREE_TYPE (gfc_conv_descriptor_data_get (tmp));
+	  gfc_conv_descriptor_data_set (&init, tmp, build_int_cst (type, 0));
+	  gfc_add_init_cleanup (block, gfc_finish_block (&init), NULL_TREE);
+	}
     }
 
 
@@ -5134,18 +5148,31 @@ gfc_trans_deferred_vars (gfc_symbol * proc_sym, gfc_wrapped_block * block)
 		      se.descriptor_only = 1;
 		      gfc_conv_expr (&se, e);
 		      descriptor = se.expr;
-		      se.expr = gfc_conv_descriptor_data_addr (se.expr);
-		      se.expr = build_fold_indirect_ref_loc (input_location, se.expr);
+		      se.expr = gfc_conv_descriptor_data_get (se.expr);
 		    }
 		  gfc_free_expr (e);
 
 		  if (!sym->attr.dummy || sym->attr.intent == INTENT_OUT)
 		    {
 		      /* Nullify when entering the scope.  */
-		      tmp = fold_build2_loc (input_location, MODIFY_EXPR,
-					     TREE_TYPE (se.expr), se.expr,
-					     fold_convert (TREE_TYPE (se.expr),
-							   null_pointer_node));
+		      if (sym->ts.type == BT_CLASS
+			  && (CLASS_DATA (sym)->attr.dimension
+			      || CLASS_DATA (sym)->attr.codimension))
+			{
+			  stmtblock_t nullify;
+			  gfc_init_block (&nullify);
+			  gfc_conv_descriptor_data_set (&nullify, descriptor,
+							null_pointer_node);
+			  tmp = gfc_finish_block (&nullify);
+			}
+		      else
+			{
+			  tree typed_null = fold_convert (TREE_TYPE (se.expr),
+							  null_pointer_node);
+			  tmp = fold_build2_loc (input_location, MODIFY_EXPR,
+						 TREE_TYPE (se.expr), se.expr,
+						 typed_null);
+			}
 		      if (sym->attr.optional)
 			{
 			  tree present = gfc_conv_expr_present (sym);
@@ -5326,7 +5353,7 @@ gfc_trans_deferred_vars (gfc_symbol * proc_sym, gfc_wrapped_block * block)
 	continue;
       /* 'omp allocate( {purpose: allocator, value: align},
 			{purpose: init-stmtlist, value: cleanup-stmtlist},
-			{purpose: size-var, value: last-size-expr}}
+			{purpose: size-var, value: last-size-expr} )
 	  where init-stmt/cleanup-stmt is the STATEMENT list to find the
 	  try-final block; last-size-expr is to find the location after
 	  which to add the code and 'size-var' is for the proper size, cf.
diff --git a/gcc/fortran/trans-expr.cc b/gcc/fortran/trans-expr.cc
index 3e0d763..2dd0936 100644
--- a/gcc/fortran/trans-expr.cc
+++ b/gcc/fortran/trans-expr.cc
@@ -714,6 +714,8 @@ gfc_get_class_from_expr (tree expr)
 {
   tree tmp;
   tree type;
+  bool array_descr_found = false;
+  bool comp_after_descr_found = false;
 
   for (tmp = expr; tmp; tmp = TREE_OPERAND (tmp, 0))
     {
@@ -725,6 +727,8 @@ gfc_get_class_from_expr (tree expr)
 	{
 	  if (GFC_CLASS_TYPE_P (type))
 	    return tmp;
+	  if (GFC_DESCRIPTOR_TYPE_P (type))
+	    array_descr_found = true;
 	  if (type != TYPE_CANONICAL (type))
 	    type = TYPE_CANONICAL (type);
 	  else
@@ -732,6 +736,23 @@ gfc_get_class_from_expr (tree expr)
 	}
       if (VAR_P (tmp) || TREE_CODE (tmp) == PARM_DECL)
 	break;
+
+      /* Avoid walking up the reference chain too far.  For class arrays, the
+	 array descriptor is a direct component (through a pointer) of the class
+	 container.  So there is exactly one COMPONENT_REF between a class
+	 container and its child array descriptor.  After seeing an array
+	 descriptor, we can give up on the second COMPONENT_REF we see, if no
+	 class container was found until that point.  */
+      if (array_descr_found)
+	{
+	  if (comp_after_descr_found)
+	    {
+	      if (TREE_CODE (tmp) == COMPONENT_REF)
+		return NULL_TREE;
+	    }
+	  else if (TREE_CODE (tmp) == COMPONENT_REF)
+	    comp_after_descr_found = true;
+	}
     }
 
   if (POINTER_TYPE_P (TREE_TYPE (tmp)))
@@ -1147,7 +1168,6 @@ gfc_conv_intrinsic_to_class (gfc_se *parmse, gfc_expr *e,
       else
 	{
 	  parmse->ss = ss;
-	  parmse->use_offset = 1;
 	  gfc_conv_expr_descriptor (parmse, e);
 
 	  /* Array references with vector subscripts and non-variable expressions
@@ -5464,16 +5484,6 @@ gfc_conv_subref_array_arg (gfc_se *se, gfc_expr * expr, int g77,
   /* Translate the expression.  */
   gfc_conv_expr (&rse, expr);
 
-  /* Reset the offset for the function call since the loop
-     is zero based on the data pointer.  Note that the temp
-     comes first in the loop chain since it is added second.  */
-  if (gfc_is_class_array_function (expr))
-    {
-      tmp = loop.ss->loop_chain->info->data.array.descriptor;
-      gfc_conv_descriptor_offset_set (&loop.pre, tmp,
-				      gfc_index_zero_node);
-    }
-
   gfc_conv_tmp_array_ref (&lse);
 
   if (intent != INTENT_OUT)
@@ -7531,7 +7541,6 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 			|| CLASS_DATA (fsym)->attr.codimension))
 	    {
 	      /* Pass a class array.  */
-	      parmse.use_offset = 1;
 	      gfc_conv_expr_descriptor (&parmse, e);
 	      bool defer_to_dealloc_blk = false;
 
@@ -7909,21 +7918,21 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 	  se->ss->info->class_container = arg1_cntnr;
 	}
 
-      if (fsym && e)
+      /* Obtain the character length of an assumed character length procedure
+	 from the typespec of the actual argument.  */
+      if (e
+	  && parmse.string_length == NULL_TREE
+	  && e->ts.type == BT_PROCEDURE
+	  && e->symtree->n.sym->ts.type == BT_CHARACTER
+	  && e->symtree->n.sym->ts.u.cl->length != NULL
+	  && e->symtree->n.sym->ts.u.cl->length->expr_type == EXPR_CONSTANT)
 	{
-	  /* Obtain the character length of an assumed character length
-	     length procedure from the typespec.  */
-	  if (fsym->ts.type == BT_CHARACTER
-	      && parmse.string_length == NULL_TREE
-	      && e->ts.type == BT_PROCEDURE
-	      && e->symtree->n.sym->ts.type == BT_CHARACTER
-	      && e->symtree->n.sym->ts.u.cl->length != NULL
-	      && e->symtree->n.sym->ts.u.cl->length->expr_type == EXPR_CONSTANT)
-	    {
-	      gfc_conv_const_charlen (e->symtree->n.sym->ts.u.cl);
-	      parmse.string_length = e->symtree->n.sym->ts.u.cl->backend_decl;
-	    }
+	  gfc_conv_const_charlen (e->symtree->n.sym->ts.u.cl);
+	  parmse.string_length = e->symtree->n.sym->ts.u.cl->backend_decl;
+	}
 
+      if (fsym && e)
+	{
 	  /* Obtain the character length for a NULL() actual with a character
 	     MOLD argument.  Otherwise substitute a suitable dummy length.
 	     Here we handle non-optional dummies of non-bind(c) procedures.  */
@@ -8159,7 +8168,8 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 		msg = xasprintf ("Pointer actual argument '%s' is not "
 				 "associated", e->symtree->n.sym->name);
 	      else if (attr.proc_pointer && !e->value.function.actual
-		       && (fsym == NULL || !fsym_attr.proc_pointer))
+		       && (fsym == NULL
+			   || (!fsym_attr.proc_pointer && !fsym_attr.optional)))
 		msg = xasprintf ("Proc-pointer actual argument '%s' is not "
 				 "associated", e->symtree->n.sym->name);
 	      else
@@ -8842,28 +8852,9 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 	  && se->expr && GFC_CLASS_TYPE_P (TREE_TYPE (se->expr))
 	  && expr->must_finalize)
 	{
-	  int n;
-	  if (se->ss && se->ss->loop)
-	    {
-	      gfc_add_block_to_block (&se->ss->loop->pre, &se->pre);
-	      se->expr = gfc_evaluate_now (se->expr, &se->ss->loop->pre);
-	      tmp = gfc_class_data_get (se->expr);
-	      info->descriptor = tmp;
-	      info->data = gfc_conv_descriptor_data_get (tmp);
-	      info->offset = gfc_conv_descriptor_offset_get (tmp);
-	      for (n = 0; n < se->ss->loop->dimen; n++)
-		{
-		  tree dim = gfc_rank_cst[n];
-		  se->ss->loop->to[n] = gfc_conv_descriptor_ubound_get (tmp, dim);
-		  se->ss->loop->from[n] = gfc_conv_descriptor_lbound_get (tmp, dim);
-		}
-	    }
-	  else
-	    {
-	      /* TODO Eliminate the doubling of temporaries. This
-		 one is necessary to ensure no memory leakage.  */
-	      se->expr = gfc_evaluate_now (se->expr, &se->pre);
-	    }
+	  /* TODO Eliminate the doubling of temporaries.  This
+	     one is necessary to ensure no memory leakage.  */
+	  se->expr = gfc_evaluate_now (se->expr, &se->pre);
 
 	  /* Finalize the result, if necessary.  */
 	  attr = expr->value.function.esym
@@ -9590,8 +9581,8 @@ gfc_trans_alloc_subarray_assign (tree dest, gfc_component * cm,
 
   /* Shift the lbound and ubound of temporaries to being unity,
      rather than zero, based. Always calculate the offset.  */
+  gfc_conv_descriptor_offset_set (&block, dest, gfc_index_zero_node);
   offset = gfc_conv_descriptor_offset_get (dest);
-  gfc_add_modify (&block, offset, gfc_index_zero_node);
   tmp2 =gfc_create_var (gfc_array_index_type, NULL);
 
   for (n = 0; n < expr->rank; n++)
@@ -12870,16 +12861,19 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
   gfc_init_se (&lse, NULL);
   gfc_init_se (&rse, NULL);
 
+  gfc_fix_class_refs (expr1);
+
+  realloc_flag = flag_realloc_lhs
+		 && gfc_is_reallocatable_lhs (expr1)
+		 && expr2->rank
+		 && !is_runtime_conformable (expr1, expr2);
+
   /* Walk the lhs.  */
   lss = gfc_walk_expr (expr1);
-  if (gfc_is_reallocatable_lhs (expr1))
+  if (realloc_flag)
     {
       lss->no_bounds_check = 1;
-      if (!(expr2->expr_type == EXPR_FUNCTION
-	    && expr2->value.function.isym != NULL
-	    && !(expr2->value.function.isym->elemental
-		 || expr2->value.function.isym->conversion)))
-	lss->is_alloc_lhs = 1;
+      lss->is_alloc_lhs = 1;
     }
   else
     lss->no_bounds_check = expr1->no_bounds_check;
@@ -12927,11 +12921,6 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
 
   assoc_assign = is_assoc_assign (expr1, expr2);
 
-  realloc_flag = flag_realloc_lhs
-		 && gfc_is_reallocatable_lhs (expr1)
-		 && expr2->rank
-		 && !is_runtime_conformable (expr1, expr2);
-
   /* Only analyze the expressions for coarray properties, when in coarray-lib
      mode.  Avoid false-positive uninitialized diagnostics with initializing
      the codimension flag unconditionally.  */
@@ -12943,6 +12932,7 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
       rhs_caf_attr = gfc_caf_attr (expr2, false, &rhs_refs_comp);
     }
 
+  tree reallocation = NULL_TREE;
   if (lss != gfc_ss_terminator)
     {
       /* The assignment needs scalarization.  */
@@ -12961,8 +12951,12 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
       /* Walk the rhs.  */
       rss = gfc_walk_expr (expr2);
       if (rss == gfc_ss_terminator)
-	/* The rhs is scalar.  Add a ss for the expression.  */
-	rss = gfc_get_scalar_ss (gfc_ss_terminator, expr2);
+	{
+	  /* The rhs is scalar.  Add a ss for the expression.  */
+	  rss = gfc_get_scalar_ss (gfc_ss_terminator, expr2);
+	  lss->is_alloc_lhs = 0;
+	}
+
       /* When doing a class assign, then the handle to the rhs needs to be a
 	 pointer to allow for polymorphism.  */
       if (is_poly_assign && expr2->rank == 0 && !UNLIMITED_POLY (expr2))
@@ -13011,6 +13005,15 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
 	  ompws_flags |= OMPWS_SCALARIZER_WS | OMPWS_SCALARIZER_BODY;
 	}
 
+      /* F2003: Allocate or reallocate lhs of allocatable array.  */
+      if (realloc_flag)
+	{
+	  realloc_lhs_warning (expr1->ts.type, true, &expr1->where);
+	  ompws_flags &= ~OMPWS_SCALARIZER_WS;
+	  reallocation = gfc_alloc_allocatable_for_assignment (&loop, expr1,
+							       expr2);
+	}
+
       /* Start the scalarized loop body.  */
       gfc_start_scalarized_body (&loop, &body);
     }
@@ -13319,15 +13322,8 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
 	  gfc_add_expr_to_block (&body, tmp);
 	}
 
-      /* F2003: Allocate or reallocate lhs of allocatable array.  */
-      if (realloc_flag)
-	{
-	  realloc_lhs_warning (expr1->ts.type, true, &expr1->where);
-	  ompws_flags &= ~OMPWS_SCALARIZER_WS;
-	  tmp = gfc_alloc_allocatable_for_assignment (&loop, expr1, expr2);
-	  if (tmp != NULL_TREE)
-	    gfc_add_expr_to_block (&loop.code[expr1->rank - 1], tmp);
-	}
+      if (reallocation != NULL_TREE)
+	gfc_add_expr_to_block (&loop.code[loop.dimen - 1], reallocation);
 
       if (maybe_workshare)
 	ompws_flags &= ~OMPWS_SCALARIZER_BODY;
diff --git a/gcc/fortran/trans-intrinsic.cc b/gcc/fortran/trans-intrinsic.cc
index f1bfd3e..f68ceb1 100644
--- a/gcc/fortran/trans-intrinsic.cc
+++ b/gcc/fortran/trans-intrinsic.cc
@@ -3466,6 +3466,74 @@ else
   return gfc_finish_block (&block);
 }
 
+static tree
+conv_intrinsic_split (gfc_code *code)
+{
+  stmtblock_t block, post_block;
+  gfc_se se;
+  gfc_expr *string_expr, *set_expr, *pos_expr, *back_expr;
+  tree string, string_len;
+  tree set, set_len;
+  tree pos, pos_for_call;
+  tree back;
+  tree fndecl, call;
+
+  string_expr = code->ext.actual->expr;
+  set_expr = code->ext.actual->next->expr;
+  pos_expr = code->ext.actual->next->next->expr;
+  back_expr = code->ext.actual->next->next->next->expr;
+
+  gfc_start_block (&block);
+  gfc_init_block (&post_block);
+
+  gfc_init_se (&se, NULL);
+  gfc_conv_expr (&se, string_expr);
+  gfc_conv_string_parameter (&se);
+  gfc_add_block_to_block (&block, &se.pre);
+  gfc_add_block_to_block (&post_block, &se.post);
+  string = se.expr;
+  string_len = se.string_length;
+
+  gfc_init_se (&se, NULL);
+  gfc_conv_expr (&se, set_expr);
+  gfc_conv_string_parameter (&se);
+  gfc_add_block_to_block (&block, &se.pre);
+  gfc_add_block_to_block (&post_block, &se.post);
+  set = se.expr;
+  set_len = se.string_length;
+
+  gfc_init_se (&se, NULL);
+  gfc_conv_expr (&se, pos_expr);
+  gfc_add_block_to_block (&block, &se.pre);
+  gfc_add_block_to_block (&post_block, &se.post);
+  pos = se.expr;
+  pos_for_call = fold_convert (gfc_charlen_type_node, pos);
+
+  if (back_expr)
+    {
+      gfc_init_se (&se, NULL);
+      gfc_conv_expr (&se, back_expr);
+      gfc_add_block_to_block (&block, &se.pre);
+      gfc_add_block_to_block (&post_block, &se.post);
+      back = se.expr;
+    }
+  else
+    back = logical_false_node;
+
+  if (string_expr->ts.kind == 1)
+    fndecl = gfor_fndecl_string_split;
+  else if (string_expr->ts.kind == 4)
+    fndecl = gfor_fndecl_string_split_char4;
+  else
+    gcc_unreachable ();
+
+  call = build_call_expr_loc (input_location, fndecl, 6, string_len, string,
+			      set_len, set, pos_for_call, back);
+  gfc_add_modify (&block, pos, fold_convert (TREE_TYPE (pos), call));
+
+  gfc_add_block_to_block (&block, &post_block);
+  return gfc_finish_block (&block);
+}
 
 /* Return a character string containing the tty name.  */
 
@@ -13101,6 +13169,8 @@ conv_intrinsic_move_alloc (gfc_code *code)
     }
   gfc_conv_expr_descriptor (&to_se, to_expr);
   gfc_conv_expr_descriptor (&from_se, from_expr);
+  gfc_add_block_to_block (&block, &to_se.pre);
+  gfc_add_block_to_block (&block, &from_se.pre);
 
   /* For coarrays, call SYNC ALL if TO is already deallocated as MOVE_ALLOC
      is an image control "statement", cf. IR F08/0040 in 12-006A.  */
@@ -13174,6 +13244,9 @@ conv_intrinsic_move_alloc (gfc_code *code)
   if (fin_label)
     gfc_add_expr_to_block (&block, build1_v (LABEL_EXPR, fin_label));
 
+  gfc_add_block_to_block (&block, &to_se.post);
+  gfc_add_block_to_block (&block, &from_se.post);
+
   return gfc_finish_block (&block);
 }
 
@@ -13256,6 +13329,10 @@ gfc_conv_intrinsic_subroutine (gfc_code *code)
       res = conv_intrinsic_system_clock (code);
       break;
 
+    case GFC_ISYM_SPLIT:
+      res = conv_intrinsic_split (code);
+      break;
+
     default:
       res = NULL_TREE;
       break;
diff --git a/gcc/fortran/trans-openmp.cc b/gcc/fortran/trans-openmp.cc
index a2e70fc..278e91c 100644
--- a/gcc/fortran/trans-openmp.cc
+++ b/gcc/fortran/trans-openmp.cc
@@ -2792,8 +2792,13 @@ gfc_trans_omp_variable_list (enum omp_clause_code code,
 			     gfc_omp_namelist *namelist, tree list,
 			     bool declare_simd)
 {
+  /* PARAMETER (named constants) are excluded as OpenACC 3.4 permits them now
+     as 'var' but permits compilers to ignore them.  In expressions, it should
+     have been replaced by the value (and this function should not be called
+     anyway) and for var-using clauses, they should just be skipped.  */
   for (; namelist != NULL; namelist = namelist->next)
-    if (namelist->sym->attr.referenced || declare_simd)
+    if ((namelist->sym->attr.referenced || declare_simd)
+	&& namelist->sym->attr.flavor != FL_PARAMETER)
       {
 	tree t = gfc_trans_omp_variable (namelist->sym, declare_simd);
 	if (t != error_mark_node)
@@ -4029,7 +4034,8 @@ gfc_trans_omp_clauses (stmtblock_t *block, gfc_omp_clauses *clauses,
 	case OMP_LIST_MAP:
 	  for (; n != NULL; n = n->next)
 	    {
-	      if (!n->sym->attr.referenced)
+	      if (!n->sym->attr.referenced
+		  || n->sym->attr.flavor == FL_PARAMETER)
 		continue;
 
 	      location_t map_loc = gfc_get_location (&n->where);
@@ -4986,7 +4992,8 @@ gfc_trans_omp_clauses (stmtblock_t *block, gfc_omp_clauses *clauses,
 	case OMP_LIST_CACHE:
 	  for (; n != NULL; n = n->next)
 	    {
-	      if (!n->sym->attr.referenced)
+	      if (!n->sym->attr.referenced
+		  && n->sym->attr.flavor != FL_PARAMETER)
 		continue;
 
 	      switch (list)
@@ -9707,11 +9714,12 @@ gfc_trans_omp_declare_variant (gfc_namespace *ns, gfc_namespace *parent_ns)
 	{
 	  gfc_symtree *proc_st;
 	  gfc_find_sym_tree (variant_proc_name, gfc_current_ns, 1, &proc_st);
-	  variant_proc_sym = proc_st->n.sym;
+	  variant_proc_sym = proc_st ? proc_st->n.sym : NULL;
 	}
       if (variant_proc_sym == NULL)
 	{
-	  gfc_error ("Cannot find symbol %qs", variant_proc_name);
+	  gfc_error ("Cannot find symbol %qs at %L", variant_proc_name,
+						     &odv->where);
 	  continue;
 	}
       set_selectors = omp_check_context_selector
diff --git a/gcc/fortran/trans-stmt.cc b/gcc/fortran/trans-stmt.cc
index f105401..198acee 100644
--- a/gcc/fortran/trans-stmt.cc
+++ b/gcc/fortran/trans-stmt.cc
@@ -2116,7 +2116,6 @@ trans_associate_var (gfc_symbol *sym, gfc_wrapped_block *block)
       if (sym->assoc->variable || cst_array_ctor)
 	{
 	  se.direct_byref = 1;
-	  se.use_offset = 1;
 	  se.expr = desc;
 	  GFC_DECL_PTR_ARRAY_P (sym->backend_decl) = 1;
 	}
@@ -2494,9 +2493,10 @@ trans_associate_var (gfc_symbol *sym, gfc_wrapped_block *block)
 	{
 	  tmp = sym->backend_decl;
 	  if (GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (tmp)))
-	    tmp = gfc_conv_descriptor_data_get (tmp);
-	  gfc_add_modify (&se.pre, tmp, fold_convert (TREE_TYPE (tmp),
-						    null_pointer_node));
+	    gfc_conv_descriptor_data_set (&se.pre, tmp, null_pointer_node);
+	  else
+	    gfc_add_modify (&se.pre, tmp,
+			    fold_convert (TREE_TYPE (tmp), null_pointer_node));
 	}
 
       lhs = gfc_lval_expr_from_sym (sym);
@@ -6710,7 +6710,6 @@ gfc_trans_allocate (gfc_code * code, gfc_omp_namelist *omp_allocate)
   stmtblock_t block;
   stmtblock_t post;
   stmtblock_t final_block;
-  tree nelems;
   bool upoly_expr, tmp_expr3_len_flag = false, al_len_needs_set, is_coarray;
   bool needs_caf_sync, caf_refs_comp;
   bool e3_has_nodescriptor = false;
@@ -7242,7 +7241,6 @@ gfc_trans_allocate (gfc_code * code, gfc_omp_namelist *omp_allocate)
 	 to handle the complete array allocation.  Only the element size
 	 needs to be provided, which is done most of the time by the
 	 pre-evaluation step.  */
-      nelems = NULL_TREE;
       if (expr3_len && (code->expr3->ts.type == BT_CHARACTER
 			|| code->expr3->ts.type == BT_CLASS))
 	{
@@ -7313,9 +7311,8 @@ gfc_trans_allocate (gfc_code * code, gfc_omp_namelist *omp_allocate)
 
 	}
 
-      if (!gfc_array_allocate (&se, expr, stat, errmsg, errlen,
-			       label_finish, tmp, &nelems,
-			       e3rhs ? e3rhs : code->expr3,
+      if (!gfc_array_allocate (&se, expr, stat, errmsg, errlen, label_finish,
+			       tmp, e3rhs ? e3rhs : code->expr3,
 			       e3_is == E3_DESC ? expr3 : NULL_TREE,
 			       e3_has_nodescriptor, omp_alloc_item,
 			       code->ext.alloc.ts.type != BT_UNKNOWN))
diff --git a/gcc/fortran/trans.cc b/gcc/fortran/trans.cc
index 13fd5ad..47396c3 100644
--- a/gcc/fortran/trans.cc
+++ b/gcc/fortran/trans.cc
@@ -1740,7 +1740,7 @@ gfc_finalize_tree_expr (gfc_se *se, gfc_symbol *derived,
 			     gfc_call_free (data_ptr),
 			     build_empty_stmt (input_location));
       gfc_add_expr_to_block (&se->loop->post, tmp);
-      gfc_add_modify (&se->loop->post, data_ptr, data_null);
+      gfc_conv_descriptor_data_set (&se->loop->post, desc, data_null);
     }
   else
     {
@@ -1754,7 +1754,7 @@ gfc_finalize_tree_expr (gfc_se *se, gfc_symbol *derived,
 				 gfc_call_free (data_ptr),
 				 build_empty_stmt (input_location));
 	  gfc_add_expr_to_block (&se->finalblock, tmp);
-	  gfc_add_modify (&se->finalblock, data_ptr, data_null);
+	  gfc_conv_descriptor_data_set (&se->finalblock, desc, data_null);
 	}
     }
 }
diff --git a/gcc/fortran/trans.h b/gcc/fortran/trans.h
index 461b0cd..5554184 100644
--- a/gcc/fortran/trans.h
+++ b/gcc/fortran/trans.h
@@ -105,10 +105,6 @@ typedef struct gfc_se
   /* If set, will pass subref descriptors without a temporary.  */
   unsigned force_no_tmp:1;
 
-  /* Unconditionally calculate offset for array segments and constant
-     arrays in gfc_conv_expr_descriptor.  */
-  unsigned use_offset:1;
-
   unsigned want_coarray:1;
 
   /* Scalarization parameters.  */
@@ -961,6 +957,7 @@ extern GTY(()) tree gfor_fndecl_string_scan;
 extern GTY(()) tree gfor_fndecl_string_verify;
 extern GTY(()) tree gfor_fndecl_string_trim;
 extern GTY(()) tree gfor_fndecl_string_minmax;
+extern GTY(()) tree gfor_fndecl_string_split;
 extern GTY(()) tree gfor_fndecl_adjustl;
 extern GTY(()) tree gfor_fndecl_adjustr;
 extern GTY(()) tree gfor_fndecl_select_string;
@@ -972,6 +969,7 @@ extern GTY(()) tree gfor_fndecl_string_scan_char4;
 extern GTY(()) tree gfor_fndecl_string_verify_char4;
 extern GTY(()) tree gfor_fndecl_string_trim_char4;
 extern GTY(()) tree gfor_fndecl_string_minmax_char4;
+extern GTY(()) tree gfor_fndecl_string_split_char4;
 extern GTY(()) tree gfor_fndecl_adjustl_char4;
 extern GTY(()) tree gfor_fndecl_adjustr_char4;
 extern GTY(()) tree gfor_fndecl_select_string_char4;
diff --git a/gcc/function.cc b/gcc/function.cc
index 48167b0c..5a054a9 100644
--- a/gcc/function.cc
+++ b/gcc/function.cc
@@ -4965,6 +4965,10 @@ prepare_function_start (void)
 
   /* Indicate we have no need of a frame pointer yet.  */
   frame_pointer_needed = 0;
+
+  /* Reset the cache of the "extended" flag in the target's
+     _BitInt info struct.  */
+  bitint_extended = -1;
 }
 
 void
@@ -7009,6 +7013,115 @@ match_asm_constraints_1 (rtx_insn *insn, rtx *p_sets, int noutputs)
     df_insn_rescan (insn);
 }
 
+/* It is expected and desired that optimizations coalesce multiple pseudos into
+   one whenever possible.  However, in case of hard register constraints we may
+   have to undo this and introduce copies since otherwise we could constraint a
+   single pseudo to different hard registers.  For example, during register
+   allocation the following insn would be unsatisfiable since pseudo 60 is
+   constrained to hard register r5 and r6 at the same time.
+
+   (insn 7 5 0 2 (asm_operands/v ("foo") ("") 0 [
+	       (reg:DI 60) repeated x2
+	   ]
+	    [
+	       (asm_input:DI ("{r5}") t.c:4)
+	       (asm_input:DI ("{r6}") t.c:4)
+	   ]
+	    [] t.c:4) "t.c":4:3 -1
+	(expr_list:REG_DEAD (reg:DI 60)
+	   (nil)))
+
+   Therefore, introduce a copy of pseudo 60 and transform it into
+
+   (insn 10 5 7 2 (set (reg:DI 62)
+	   (reg:DI 60)) "t.c":4:3 1503 {*movdi_64}
+	(nil))
+   (insn 7 10 11 2 (asm_operands/v ("foo") ("") 0 [
+	       (reg:DI 60)
+	       (reg:DI 62)
+	   ]
+	    [
+	       (asm_input:DI ("{r5}") t.c:4)
+	       (asm_input:DI ("{r6}") t.c:4)
+	   ]
+	    [] t.c:4) "t.c":4:3 -1
+	(expr_list:REG_DEAD (reg:DI 62)
+	   (expr_list:REG_DEAD (reg:DI 60)
+	       (nil))))
+
+   Now, LRA can assign pseudo 60 to r5, and pseudo 62 to r6.
+
+   TODO: The current implementation is conservative and we could do a bit
+   better in case of alternatives.  For example
+
+   (insn 7 5 0 2 (asm_operands/v ("foo") ("") 0 [
+	       (reg:DI 60) repeated x2
+	   ]
+	    [
+	       (asm_input:DI ("r,{r5}") t.c:4)
+	       (asm_input:DI ("{r6},r") t.c:4)
+	   ]
+	    [] t.c:4) "t.c":4:3 -1
+	(expr_list:REG_DEAD (reg:DI 60)
+	   (nil)))
+
+   For this insn we wouldn't need to come up with a copy of pseudo 60 since in
+   each alternative pseudo 60 is constrained exactly one time.  */
+
+static void
+match_asm_constraints_2 (rtx_insn *insn, rtx pat)
+{
+  rtx op;
+  if (GET_CODE (pat) == SET && GET_CODE (SET_SRC (pat)) == ASM_OPERANDS)
+    op = SET_SRC (pat);
+  else if (GET_CODE (pat) == ASM_OPERANDS)
+    op = pat;
+  else
+    return;
+  int ninputs = ASM_OPERANDS_INPUT_LENGTH (op);
+  rtvec inputs = ASM_OPERANDS_INPUT_VEC (op);
+  bool changed = false;
+  auto_bitmap constrained_regs;
+
+  for (int i = 0; i < ninputs; ++i)
+    {
+      rtx input = RTVEC_ELT (inputs, i);
+      const char *constraint = ASM_OPERANDS_INPUT_CONSTRAINT (op, i);
+      if ((!REG_P (input) && !SUBREG_P (input))
+	  || (REG_P (input) && HARD_REGISTER_P (input))
+	  || strchr (constraint, '{') == nullptr)
+	continue;
+      int regno;
+      if (SUBREG_P (input))
+	{
+	  if (REG_P (SUBREG_REG (input)))
+	    regno = REGNO (SUBREG_REG (input));
+	  else
+	    continue;
+	}
+      else
+	regno = REGNO (input);
+      /* Keep the first usage of a constrained pseudo as is and only
+	 introduce copies for subsequent usages.  */
+      if (! bitmap_bit_p (constrained_regs, regno))
+	{
+	  bitmap_set_bit (constrained_regs, regno);
+	  continue;
+	}
+      rtx tmp = gen_reg_rtx (GET_MODE (input));
+      start_sequence ();
+      emit_move_insn (tmp, input);
+      rtx_insn *insns = get_insns ();
+      end_sequence ();
+      emit_insn_before (insns, insn);
+      RTVEC_ELT (inputs, i) = tmp;
+      changed = true;
+    }
+
+  if (changed)
+    df_insn_rescan (insn);
+}
+
 /* Add the decl D to the local_decls list of FUN.  */
 
 void
@@ -7065,6 +7178,13 @@ pass_match_asm_constraints::execute (function *fun)
 	    continue;
 
 	  pat = PATTERN (insn);
+
+	  if (GET_CODE (pat) == PARALLEL)
+	    for (int i = XVECLEN (pat, 0) - 1; i >= 0; --i)
+	      match_asm_constraints_2 (insn, XVECEXP (pat, 0, i));
+	  else
+	    match_asm_constraints_2 (insn, pat);
+
 	  if (GET_CODE (pat) == PARALLEL)
 	    p_sets = &XVECEXP (pat, 0, 0), noutputs = XVECLEN (pat, 0);
 	  else if (GET_CODE (pat) == SET)
diff --git a/gcc/diagnostic-spec.cc b/gcc/gcc-diagnostic-spec.cc
index df78579..99ca6a0 100644
--- a/gcc/diagnostic-spec.cc
+++ b/gcc/gcc-diagnostic-spec.cc
@@ -27,7 +27,7 @@
 #include "tree.h"
 #include "cgraph.h"
 #include "hash-map.h"
-#include "diagnostic-spec.h"
+#include "gcc-diagnostic-spec.h"
 #include "pretty-print.h"
 #include "options.h"
 
@@ -72,9 +72,9 @@ nowarn_spec_t::nowarn_spec_t (opt_code opt)
     case OPT_Wstrict_aliasing:
     case OPT_Wunused:
     case OPT_Wunused_function:
-    case OPT_Wunused_but_set_variable:
+    case OPT_Wunused_but_set_variable_:
     case OPT_Wunused_variable:
-    case OPT_Wunused_but_set_parameter:
+    case OPT_Wunused_but_set_parameter_:
       m_bits = NW_LEXICAL;
       break;
 
diff --git a/gcc/diagnostic-spec.h b/gcc/gcc-diagnostic-spec.h
index 13455ce..2b5e784 100644
--- a/gcc/diagnostic-spec.h
+++ b/gcc/gcc-diagnostic-spec.h
@@ -19,8 +19,8 @@
    along with GCC; see the file COPYING3.  If not see
    <http://www.gnu.org/licenses/>.  */
 
-#ifndef DIAGNOSTIC_SPEC_H_INCLUDED
-#define DIAGNOSTIC_SPEC_H_INCLUDED
+#ifndef GCC_DIAGNOSTIC_SPEC_H_INCLUDED
+#define GCC_DIAGNOSTIC_SPEC_H_INCLUDED
 
 #include "hash-map.h"
 
@@ -146,4 +146,4 @@ typedef hash_map<location_hash, nowarn_spec_t> nowarn_map_t;
 /* A mapping from a 'location_t' to the warning spec set for it.  */
 extern GTY(()) nowarn_map_t *nowarn_map;
 
-#endif // DIAGNOSTIC_SPEC_H_INCLUDED
+#endif // GCC_DIAGNOSTIC_SPEC_H_INCLUDED
diff --git a/gcc/gcc-rich-location.cc b/gcc/gcc-rich-location.cc
index 2095987..beb7b6c 100644
--- a/gcc/gcc-rich-location.cc
+++ b/gcc/gcc-rich-location.cc
@@ -37,6 +37,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "intl.h"
 #include "cpplib.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 
 /* Add a range to the rich_location, covering expression EXPR,
    using LABEL if non-NULL. */
@@ -79,11 +80,11 @@ gcc_rich_location::add_fixit_misspelled_id (location_t misspelled_token_loc,
 /* Return true if there is nothing on LOC's line before LOC.  */
 
 static bool
-blank_line_before_p (file_cache &fc,
+blank_line_before_p (diagnostics::file_cache &fc,
 		     location_t loc)
 {
   expanded_location exploc = expand_location (loc);
-  char_span line = fc.get_source_line (exploc.file, exploc.line);
+  diagnostics::char_span line = fc.get_source_line (exploc.file, exploc.line);
   if (!line)
     return false;
   if (line.length () < (size_t)exploc.column)
@@ -101,7 +102,7 @@ blank_line_before_p (file_cache &fc,
    If true is returned then *OUT_START_OF_LINE is written to.  */
 
 static bool
-use_new_line (file_cache &fc,
+use_new_line (diagnostics::file_cache &fc,
 	      location_t insertion_point, location_t indent,
 	      location_t *out_start_of_line)
 {
diff --git a/gcc/gcc-rich-location.h b/gcc/gcc-rich-location.h
index cbf59b7..69cf074 100644
--- a/gcc/gcc-rich-location.h
+++ b/gcc/gcc-rich-location.h
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "rich-location.h"
 
-class diagnostic_source_print_policy;
+namespace diagnostics { class source_print_policy; }
 
 /* A gcc_rich_location is libcpp's rich_location with additional
    helper methods for working with gcc's types.  The class is not
@@ -79,14 +79,14 @@ class gcc_rich_location : public rich_location
 	if (!added secondary)
 	  inform (secondary_loc, "message for secondary");
 
-     Implemented in diagnostic-show-locus.cc.  */
+     Implemented in diagnostics/source-printing.cc.  */
 
-  bool add_location_if_nearby (const diagnostic_source_print_policy &policy,
+  bool add_location_if_nearby (const diagnostics::source_print_policy &policy,
 			       location_t loc,
 			       bool restrict_to_current_line_spans = true,
 			       const range_label *label = NULL);
 
-  bool add_location_if_nearby (const diagnostic_context &dc,
+  bool add_location_if_nearby (const diagnostics::context &dc,
 			       location_t loc,
 			       bool restrict_to_current_line_spans = true,
 			       const range_label *label = NULL);
diff --git a/gcc/gcc.cc b/gcc/gcc.cc
index 235fe80..bfa588e 100644
--- a/gcc/gcc.cc
+++ b/gcc/gcc.cc
@@ -43,7 +43,7 @@ compilation is specified by a string called a "spec".  */
 #include "opt-suggestions.h"
 #include "gcc.h"
 #include "diagnostic.h"
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 #include "pretty-print-urlifier.h"
 #include "flags.h"
 #include "opts.h"
@@ -4195,7 +4195,7 @@ driver_handle_option (struct gcc_options *opts,
 		      unsigned int lang_mask ATTRIBUTE_UNUSED, int kind,
 		      location_t loc,
 		      const struct cl_option_handlers *handlers ATTRIBUTE_UNUSED,
-		      diagnostic_context *dc,
+		      diagnostics::context *dc,
 		      void (*) (void))
 {
   size_t opt_index = decoded->opt_index;
@@ -4207,7 +4207,8 @@ driver_handle_option (struct gcc_options *opts,
 
   gcc_assert (opts == &global_options);
   gcc_assert (opts_set == &global_options_set);
-  gcc_assert (kind == DK_UNSPECIFIED);
+  gcc_assert (static_cast<diagnostics::kind> (kind)
+	      == diagnostics::kind::unspecified);
   gcc_assert (loc == UNKNOWN_LOCATION);
   gcc_assert (dc == global_dc);
 
@@ -4367,10 +4368,10 @@ driver_handle_option (struct gcc_options *opts,
 	  const char *basename = (opts->x_dump_base_name ? opts->x_dump_base_name
 				  : opts->x_main_input_basename);
 	  gcc_assert (dc);
-	  diagnostic_output_format_init (*dc,
-					 opts->x_main_input_filename, basename,
-					 (enum diagnostics_output_format)value,
-					 opts->x_flag_diagnostics_json_formatting);
+	  diagnostics::output_format_init (*dc,
+					   opts->x_main_input_filename, basename,
+					   (enum diagnostics_output_format)value,
+					   opts->x_flag_diagnostics_json_formatting);
 	  break;
 	}
 
diff --git a/gcc/gcov-io.cc b/gcc/gcov-io.cc
index f39b4bd..dd3fc88 100644
--- a/gcc/gcov-io.cc
+++ b/gcc/gcov-io.cc
@@ -69,7 +69,7 @@ gcov_position (void)
 
 /* Return nonzero if the error flag is set.  */
 /* We need to expose this function when compiling for gcov-tool.  */
-#ifndef IN_GCOV_TOOL
+#if !defined (IN_GCOV_TOOL) && !defined (IN_GCC)
 static inline
 #endif
 int
diff --git a/gcc/gcov-io.h b/gcc/gcov-io.h
index d48291c..313c15c 100644
--- a/gcc/gcov-io.h
+++ b/gcc/gcov-io.h
@@ -349,6 +349,11 @@ struct gcov_summary
 {
   gcov_unsigned_t runs;		/* Number of program runs.  */
   gcov_type sum_max;    	/* Sum of individual run max values.  */
+  gcov_type cutoff;		/* Values smaller than this value are not
+				   reliable (0 may mean non-zero).
+				   For read profile cutoff is typically 1
+				   however when we scale up or use auto-fdo
+				   it may become bigger value.  */
 };
 
 #if !defined(inhibit_libc)
@@ -382,6 +387,7 @@ char *mangle_path (char const *base);
 /* Available outside gcov */
 GCOV_LINKAGE void gcov_write (const void *, unsigned) ATTRIBUTE_HIDDEN;
 GCOV_LINKAGE void gcov_write_unsigned (gcov_unsigned_t) ATTRIBUTE_HIDDEN;
+GCOV_LINKAGE int gcov_is_error (void);
 #endif
 
 #if !IN_GCOV && !IN_LIBGCOV
diff --git a/gcc/gcse.cc b/gcc/gcse.cc
index 96aae0e..ea57c55 100644
--- a/gcc/gcse.cc
+++ b/gcc/gcse.cc
@@ -4238,8 +4238,15 @@ execute_hardreg_pre (void)
     {
       int changed;
       current_hardreg_regno = regnos[i];
+      if (!df_regs_ever_live_p (current_hardreg_regno))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Skipping hardreg PRE for regno %d, which is never live\n",
+		     current_hardreg_regno);
+	  continue;
+	}
       if (dump_file)
-	fprintf(dump_file, "Entering hardreg PRE for regno %d\n",
+	fprintf (dump_file, "Entering hardreg PRE for regno %d\n",
 		current_hardreg_regno);
       delete_unreachable_blocks ();
       df_analyze ();
diff --git a/gcc/gdbinit.in b/gcc/gdbinit.in
index 417571f..1ee3d92 100644
--- a/gcc/gdbinit.in
+++ b/gcc/gdbinit.in
@@ -293,12 +293,12 @@ See also 'help-gcc-hooks'.
 end
 
 define break-on-diagnostic
-break diagnostic_context::maybe_show_locus
+break diagnostics::context::maybe_show_locus
 end
 
 document break-on-diagnostic
-Put a breakpoint on diagnostic_show_locus, called whenever a diagnostic
-is emitted (as opposed to those warnings that are suppressed by
+Put a breakpoint on diagnostics::context::show_locus, called whenever
+a diagnostic is emitted (as opposed to those warnings that are suppressed by
 command-line options).
 end
 
diff --git a/gcc/gengtype.cc b/gcc/gengtype.cc
index 6a3621c..84c7636 100644
--- a/gcc/gengtype.cc
+++ b/gcc/gengtype.cc
@@ -1722,7 +1722,7 @@ open_base_files (void)
       "target-globals.h", "ipa-ref.h", "cgraph.h", "symbol-summary.h",
       "sreal.h", "ipa-cp.h", "ipa-prop.h", "ipa-fnsummary.h", "dwarf2out.h",
       "omp-general.h", "omp-offload.h", "ipa-modref-tree.h", "ipa-modref.h",
-      "symtab-thunks.h", "symtab-clones.h", "diagnostic-spec.h", "ctfc.h",
+      "symtab-thunks.h", "symtab-clones.h", "gcc-diagnostic-spec.h", "ctfc.h",
       NULL
     };
     const char *const *ifp;
diff --git a/gcc/genoutput.cc b/gcc/genoutput.cc
index 25d0b8b..183ffc1 100644
--- a/gcc/genoutput.cc
+++ b/gcc/genoutput.cc
@@ -200,6 +200,8 @@ static const char indep_constraints[] = ",=+%*?!^$#&g";
 static class constraint_data *
 constraints_by_letter_table[1 << CHAR_BIT];
 
+static hash_set<free_string_hash> used_reg_names;
+
 static int mdep_constraint_len (const char *, file_location, int);
 static void note_constraint (md_rtx_info *);
 
@@ -1156,6 +1158,45 @@ main (int argc, const char **argv)
   output_insn_data ();
   output_get_insn_name ();
 
+  /* Since genoutput has no information about hard register names we cannot
+     statically verify hard register names in constraints of the machine
+     description.  Therefore, we have to do it at runtime.  Although
+     verification shouldn't be too expensive, restrict it to checking builds.
+   */
+  printf ("\n\n#if CHECKING_P\n");
+  if (used_reg_names.is_empty ())
+    printf ("void verify_reg_names_in_constraints () { }\n");
+  else
+    {
+      size_t max_len = 0;
+      for (auto it = used_reg_names.begin (); it != used_reg_names.end (); ++it)
+	{
+	  size_t len = strlen (*it);
+	  if (len > max_len)
+	    max_len = len;
+	}
+      printf ("void\nverify_reg_names_in_constraints ()\n{\n");
+      printf ("  static const char hregnames[%zu][%zu] = {\n",
+	      used_reg_names.elements (), max_len + 1);
+      auto it = used_reg_names.begin ();
+      while (it != used_reg_names.end ())
+	{
+	  printf ("    \"%s\"", *it);
+	  ++it;
+	  if (it != used_reg_names.end ())
+	    printf (",");
+	  printf ("\n");
+	}
+      printf ("  };\n");
+      printf ("  for (size_t i = 0; i < %zu; ++i)\n",
+	      used_reg_names.elements ());
+      printf ("    if (decode_reg_name (hregnames[i]) < 0)\n");
+      printf ("      internal_error (\"invalid register %%qs used in "
+	      "constraint of machine description\", hregnames[i]);\n");
+      printf ("}\n");
+    }
+  printf ("#endif\n");
+
   fflush (stdout);
   return (ferror (stdout) != 0 || have_error
 	? FATAL_EXIT_CODE : SUCCESS_EXIT_CODE);
@@ -1284,6 +1325,25 @@ mdep_constraint_len (const char *s, file_location loc, int opno)
       if (!strncmp (s, p->name, p->namelen))
 	return p->namelen;
 
+  if (*s == '{')
+    {
+      const char *end = s + 1;
+      while (*end != '}' && *end != '"' && *end != '\0')
+	++end;
+      /* Similarly as in decode_hreg_constraint(), consider any hard register
+	 name longer than a few characters as an error.  */
+      ptrdiff_t len = end - s;
+      if (*end == '}' && len > 1 && len < 31)
+	{
+	  char *regname = new char[len];
+	  memcpy (regname, s + 1, len - 1);
+	  regname[len - 1] = '\0';
+	  if (used_reg_names.add (regname))
+	    delete[] regname;
+	  return len + 1;
+	}
+    }
+
   error_at (loc, "error: undefined machine-specific constraint "
 	    "at this point: \"%s\"", s);
   message_at (loc, "note:  in operand %d", opno);
diff --git a/gcc/genpreds.cc b/gcc/genpreds.cc
index d09d769..7ee68c9 100644
--- a/gcc/genpreds.cc
+++ b/gcc/genpreds.cc
@@ -719,7 +719,6 @@ static const char const_dbl_constraints[] = "GH";
 
 /* Summary data used to decide whether to output various functions and
    macro definitions.  */
-static unsigned int constraint_max_namelen;
 static bool have_register_constraints;
 static bool have_memory_constraints;
 static bool have_special_memory_constraints;
@@ -942,7 +941,6 @@ add_constraint (const char *name, const char *regclass,
   *last_constraint_ptr = c;
   last_constraint_ptr = &c->next_textual;
 
-  constraint_max_namelen = MAX (constraint_max_namelen, strlen (name));
   have_register_constraints |= c->is_register;
   have_const_int_constraints |= c->is_const_int;
   have_extra_constraints |= c->is_extra;
@@ -1150,7 +1148,7 @@ write_insn_constraint_len (void)
   unsigned int i;
 
   puts ("static inline size_t\n"
-	"insn_constraint_len (char fc, const char *str ATTRIBUTE_UNUSED)\n"
+	"insn_constraint_len (char fc, const char *str)\n"
 	"{\n"
 	"  switch (fc)\n"
 	"    {");
@@ -1183,6 +1181,13 @@ write_insn_constraint_len (void)
 
   puts ("    default: break;\n"
 	"    }\n"
+	"  if (str[0] == '{')\n"
+	"    {\n"
+	"      size_t len = 1;\n"
+	"      while (str[len] != '}' && str[len] != '\\0')\n"
+	"        ++len;\n"
+	"      return len + 1;\n"
+	"    }\n"
 	"  return 1;\n"
 	"}\n");
 }
@@ -1556,133 +1561,124 @@ write_tm_preds_h (void)
 	  "#endif\n"
 	  "\n");
 
-  if (constraint_max_namelen > 0)
-    {
-      write_enum_constraint_num ();
-      puts ("extern enum constraint_num lookup_constraint_1 (const char *);\n"
-	    "extern const unsigned char lookup_constraint_array[];\n"
+  write_enum_constraint_num ();
+  puts ("extern enum constraint_num lookup_constraint_1 (const char *);\n"
+	"extern const unsigned char lookup_constraint_array[];\n"
+	"\n"
+	"/* Return the constraint at the beginning of P, or"
+	" CONSTRAINT__UNKNOWN if it\n"
+	"   isn't recognized.  */\n"
+	"\n"
+	"static inline enum constraint_num\n"
+	"lookup_constraint (const char *p)\n"
+	"{\n"
+	"  unsigned int index = lookup_constraint_array"
+	"[(unsigned char) *p];\n"
+	"  return (index == UCHAR_MAX\n"
+	"          ? lookup_constraint_1 (p)\n"
+	"          : (enum constraint_num) index);\n"
+	"}\n");
+  if (satisfied_start == num_constraints)
+    puts ("/* Return true if X satisfies constraint C.  */\n"
+	  "\n"
+	  "static inline bool\n"
+	  "constraint_satisfied_p (rtx, enum constraint_num)\n"
+	  "{\n"
+	  "  return false;\n"
+	  "}\n");
+  else
+    printf ("extern bool (*constraint_satisfied_p_array[]) (rtx);\n"
 	    "\n"
-	    "/* Return the constraint at the beginning of P, or"
-	    " CONSTRAINT__UNKNOWN if it\n"
-	    "   isn't recognized.  */\n"
+	    "/* Return true if X satisfies constraint C.  */\n"
 	    "\n"
-	    "static inline enum constraint_num\n"
-	    "lookup_constraint (const char *p)\n"
-	    "{\n"
-	    "  unsigned int index = lookup_constraint_array"
-	    "[(unsigned char) *p];\n"
-	    "  return (index == UCHAR_MAX\n"
-	    "          ? lookup_constraint_1 (p)\n"
-	    "          : (enum constraint_num) index);\n"
-	    "}\n");
-      if (satisfied_start == num_constraints)
-	puts ("/* Return true if X satisfies constraint C.  */\n"
-	      "\n"
-	      "static inline bool\n"
-	      "constraint_satisfied_p (rtx, enum constraint_num)\n"
-	      "{\n"
-	      "  return false;\n"
-	      "}\n");
-      else
-	printf ("extern bool (*constraint_satisfied_p_array[]) (rtx);\n"
-		"\n"
-		"/* Return true if X satisfies constraint C.  */\n"
-		"\n"
-		"static inline bool\n"
-		"constraint_satisfied_p (rtx x, enum constraint_num c)\n"
-		"{\n"
-		"  int i = (int) c - (int) CONSTRAINT_%s;\n"
-		"  return i >= 0 && constraint_satisfied_p_array[i] (x);\n"
-		"}\n"
-		"\n",
-		enum_order[satisfied_start]->name);
-
-      write_range_function ("insn_extra_register_constraint",
-			    register_start, register_end);
-      write_range_function ("insn_extra_memory_constraint",
-			    memory_start, memory_end);
-      write_range_function ("insn_extra_special_memory_constraint",
-			    special_memory_start, special_memory_end);
-      write_range_function ("insn_extra_relaxed_memory_constraint",
-			    relaxed_memory_start, relaxed_memory_end);
-      write_range_function ("insn_extra_address_constraint",
-			    address_start, address_end);
-      write_allows_reg_mem_function ();
-
-      if (constraint_max_namelen > 1)
-        {
-	  write_insn_constraint_len ();
-	  puts ("#define CONSTRAINT_LEN(c_,s_) "
-		"insn_constraint_len (c_,s_)\n");
-	}
-      else
-	puts ("#define CONSTRAINT_LEN(c_,s_) 1\n");
-      if (have_register_constraints)
-	puts ("extern enum reg_class reg_class_for_constraint_1 "
-	      "(enum constraint_num);\n"
-	      "\n"
-	      "static inline enum reg_class\n"
-	      "reg_class_for_constraint (enum constraint_num c)\n"
-	      "{\n"
-	      "  if (insn_extra_register_constraint (c))\n"
-	      "    return reg_class_for_constraint_1 (c);\n"
-	      "  return NO_REGS;\n"
-	      "}\n");
-      else
-	puts ("static inline enum reg_class\n"
-	      "reg_class_for_constraint (enum constraint_num)\n"
-	      "{\n"
-	      "  return NO_REGS;\n"
-	      "}\n");
-      if (have_const_int_constraints)
-	puts ("extern bool insn_const_int_ok_for_constraint "
-	      "(HOST_WIDE_INT, enum constraint_num);\n"
-	      "#define CONST_OK_FOR_CONSTRAINT_P(v_,c_,s_) \\\n"
-	      "    insn_const_int_ok_for_constraint (v_, "
-	      "lookup_constraint (s_))\n");
-      else
-	puts ("static inline bool\n"
-	      "insn_const_int_ok_for_constraint (HOST_WIDE_INT,"
-	      " enum constraint_num)\n"
-	      "{\n"
-	      "  return false;\n"
-	      "}\n");
-
-      puts ("enum constraint_type\n"
+	    "static inline bool\n"
+	    "constraint_satisfied_p (rtx x, enum constraint_num c)\n"
 	    "{\n"
-	    "  CT_REGISTER,\n"
-	    "  CT_CONST_INT,\n"
-	    "  CT_MEMORY,\n"
-	    "  CT_SPECIAL_MEMORY,\n"
-	    "  CT_RELAXED_MEMORY,\n"
-	    "  CT_ADDRESS,\n"
-	    "  CT_FIXED_FORM\n"
-	    "};\n"
-	    "\n"
-	    "static inline enum constraint_type\n"
-	    "get_constraint_type (enum constraint_num c)\n"
-	    "{");
-      auto_vec <std::pair <unsigned int, const char *>, 4> values;
-      if (const_int_start != const_int_end)
-	values.safe_push (std::make_pair (const_int_start, "CT_CONST_INT"));
-      if (memory_start != memory_end)
-	values.safe_push (std::make_pair (memory_start, "CT_MEMORY"));
-      if (special_memory_start != special_memory_end)
-	values.safe_push (std::make_pair (special_memory_start,
-					  "CT_SPECIAL_MEMORY"));
-      if (relaxed_memory_start != relaxed_memory_end)
-	values.safe_push (std::make_pair (relaxed_memory_start,
-					  "CT_RELAXED_MEMORY"));
-      if (address_start != address_end)
-	values.safe_push (std::make_pair (address_start, "CT_ADDRESS"));
-      if (address_end != num_constraints)
-	values.safe_push (std::make_pair (address_end, "CT_FIXED_FORM"));
-      print_type_tree (values, 0, values.length (), "CT_REGISTER", 2);
-      puts ("}");
-
-      write_get_register_filter ();
-      write_get_register_filter_id ();
-    }
+	    "  int i = (int) c - (int) CONSTRAINT_%s;\n"
+	    "  return i >= 0 && constraint_satisfied_p_array[i] (x);\n"
+	    "}\n"
+	    "\n",
+	    enum_order[satisfied_start]->name);
+
+  write_range_function ("insn_extra_register_constraint",
+			register_start, register_end);
+  write_range_function ("insn_extra_memory_constraint",
+			memory_start, memory_end);
+  write_range_function ("insn_extra_special_memory_constraint",
+			special_memory_start, special_memory_end);
+  write_range_function ("insn_extra_relaxed_memory_constraint",
+			relaxed_memory_start, relaxed_memory_end);
+  write_range_function ("insn_extra_address_constraint",
+			address_start, address_end);
+  write_allows_reg_mem_function ();
+
+  write_insn_constraint_len ();
+  puts ("#define CONSTRAINT_LEN(c_,s_) insn_constraint_len (c_,s_)\n");
+  if (have_register_constraints)
+    puts ("extern enum reg_class reg_class_for_constraint_1 "
+	  "(enum constraint_num);\n"
+	  "\n"
+	  "static inline enum reg_class\n"
+	  "reg_class_for_constraint (enum constraint_num c)\n"
+	  "{\n"
+	  "  if (insn_extra_register_constraint (c))\n"
+	  "    return reg_class_for_constraint_1 (c);\n"
+	  "  return NO_REGS;\n"
+	  "}\n");
+  else
+    puts ("static inline enum reg_class\n"
+	  "reg_class_for_constraint (enum constraint_num)\n"
+	  "{\n"
+	  "  return NO_REGS;\n"
+	  "}\n");
+  if (have_const_int_constraints)
+    puts ("extern bool insn_const_int_ok_for_constraint "
+	  "(HOST_WIDE_INT, enum constraint_num);\n"
+	  "#define CONST_OK_FOR_CONSTRAINT_P(v_,c_,s_) \\\n"
+	  "    insn_const_int_ok_for_constraint (v_, "
+	  "lookup_constraint (s_))\n");
+  else
+    puts ("static inline bool\n"
+	  "insn_const_int_ok_for_constraint (HOST_WIDE_INT,"
+	  " enum constraint_num)\n"
+	  "{\n"
+	  "  return false;\n"
+	  "}\n");
+
+  puts ("enum constraint_type\n"
+	"{\n"
+	"  CT_REGISTER,\n"
+	"  CT_CONST_INT,\n"
+	"  CT_MEMORY,\n"
+	"  CT_SPECIAL_MEMORY,\n"
+	"  CT_RELAXED_MEMORY,\n"
+	"  CT_ADDRESS,\n"
+	"  CT_FIXED_FORM\n"
+	"};\n"
+	"\n"
+	"static inline enum constraint_type\n"
+	"get_constraint_type (enum constraint_num c)\n"
+	"{");
+  auto_vec <std::pair <unsigned int, const char *>, 4> values;
+  if (const_int_start != const_int_end)
+    values.safe_push (std::make_pair (const_int_start, "CT_CONST_INT"));
+  if (memory_start != memory_end)
+    values.safe_push (std::make_pair (memory_start, "CT_MEMORY"));
+  if (special_memory_start != special_memory_end)
+    values.safe_push (std::make_pair (special_memory_start,
+				      "CT_SPECIAL_MEMORY"));
+  if (relaxed_memory_start != relaxed_memory_end)
+    values.safe_push (std::make_pair (relaxed_memory_start,
+				      "CT_RELAXED_MEMORY"));
+  if (address_start != address_end)
+    values.safe_push (std::make_pair (address_start, "CT_ADDRESS"));
+  if (address_end != num_constraints)
+    values.safe_push (std::make_pair (address_end, "CT_FIXED_FORM"));
+  print_type_tree (values, 0, values.length (), "CT_REGISTER", 2);
+  puts ("}");
+
+  write_get_register_filter ();
+  write_get_register_filter_id ();
 
   puts ("#endif /* tm-preds.h */");
 }
@@ -1743,17 +1739,14 @@ write_insn_preds_c (void)
   FOR_ALL_PREDICATES (p)
     write_one_predicate_function (p);
 
-  if (constraint_max_namelen > 0)
-    {
-      write_lookup_constraint_1 ();
-      write_lookup_constraint_array ();
-      if (have_register_constraints)
-	write_reg_class_for_constraint_1 ();
-      write_constraint_satisfied_p_array ();
-
-      if (have_const_int_constraints)
-	write_insn_const_int_ok_for_constraint ();
-    }
+  write_lookup_constraint_1 ();
+  write_lookup_constraint_array ();
+  if (have_register_constraints)
+    write_reg_class_for_constraint_1 ();
+  write_constraint_satisfied_p_array ();
+
+  if (have_const_int_constraints)
+    write_insn_const_int_ok_for_constraint ();
 
   write_init_reg_class_start_regs ();
 }
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index c5c8e22..85319b3 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -6762,10 +6762,10 @@ fold_stmt (gimple_stmt_iterator *gsi, tree (*valueize) (tree), bitmap dce_bitmap
    which can produce *&x = 0.  */
 
 bool
-fold_stmt_inplace (gimple_stmt_iterator *gsi)
+fold_stmt_inplace (gimple_stmt_iterator *gsi, tree (*valueize) (tree))
 {
   gimple *stmt = gsi_stmt (*gsi);
-  bool changed = fold_stmt_1 (gsi, true, no_follow_ssa_edges);
+  bool changed = fold_stmt_1 (gsi, true, valueize);
   gcc_assert (gsi_stmt (*gsi) == stmt);
   return changed;
 }
@@ -9916,10 +9916,17 @@ fold_nonarray_ctor_reference (tree type, tree ctor,
 	    {
 	      if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
 		return NULL_TREE;
-	      const unsigned int encoding_size
-		= GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (TREE_TYPE (cfield)));
 	      if (BYTES_BIG_ENDIAN)
-		inner_offset += encoding_size - wi::to_offset (field_size);
+		{
+		  tree ctype = TREE_TYPE (cfield);
+		  unsigned int encoding_size;
+		  if (TYPE_MODE (ctype) != BLKmode)
+		    encoding_size
+		      = GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (ctype));
+		  else
+		    encoding_size = TREE_INT_CST_LOW (TYPE_SIZE (ctype));
+		  inner_offset += encoding_size - wi::to_offset (field_size);
+		}
 	    }
 
 	  return fold_ctor_reference (type, cval,
diff --git a/gcc/gimple-fold.h b/gcc/gimple-fold.h
index e3cf1f6..b678502 100644
--- a/gcc/gimple-fold.h
+++ b/gcc/gimple-fold.h
@@ -28,9 +28,12 @@ struct c_strlen_data;
 extern bool get_range_strlen (tree, c_strlen_data *, unsigned eltsize);
 extern void gimplify_and_update_call_from_tree (gimple_stmt_iterator *, tree);
 extern bool update_gimple_call (gimple_stmt_iterator *, tree, int, ...);
+extern tree no_follow_ssa_edges (tree);
+extern tree follow_single_use_edges (tree);
+extern tree follow_all_ssa_edges (tree);
 extern bool fold_stmt (gimple_stmt_iterator *, bitmap = nullptr);
 extern bool fold_stmt (gimple_stmt_iterator *, tree (*) (tree), bitmap = nullptr);
-extern bool fold_stmt_inplace (gimple_stmt_iterator *);
+extern bool fold_stmt_inplace (gimple_stmt_iterator *, tree (*) (tree) = no_follow_ssa_edges);
 extern tree maybe_fold_and_comparisons (tree, enum tree_code, tree, tree,
 					enum tree_code, tree, tree,
 					basic_block = nullptr);
@@ -39,9 +42,6 @@ extern tree maybe_fold_or_comparisons (tree, enum tree_code, tree, tree,
 				       basic_block = nullptr);
 extern bool optimize_atomic_compare_exchange_p (gimple *);
 extern void fold_builtin_atomic_compare_exchange (gimple_stmt_iterator *);
-extern tree no_follow_ssa_edges (tree);
-extern tree follow_single_use_edges (tree);
-extern tree follow_all_ssa_edges (tree);
 extern tree gimple_fold_stmt_to_constant_1 (gimple *, tree (*) (tree),
 					    tree (*) (tree) = no_follow_ssa_edges);
 extern tree gimple_fold_stmt_to_constant (gimple *, tree (*) (tree));
diff --git a/gcc/gimple-ssa-sccopy.cc b/gcc/gimple-ssa-sccopy.cc
index 341bae4..e65f532 100644
--- a/gcc/gimple-ssa-sccopy.cc
+++ b/gcc/gimple-ssa-sccopy.cc
@@ -78,7 +78,7 @@ along with GCC; see the file COPYING3.  If not see
 
    To find all three types of copy statements we use an algorithm based on
    strongly-connected components (SCCs) in dataflow graph.  The algorithm was
-   introduced in an article from 2013[1]. We describe the algorithm bellow.
+   introduced in an article from 2013[1]. We describe the algorithm below.
 
    To identify SCCs we implement the Robert Tarjan's SCC algorithm.  For the
    SCC computation we wrap potential copy statements in the 'vertex' struct.
diff --git a/gcc/gimple-ssa-store-merging.cc b/gcc/gimple-ssa-store-merging.cc
index 90f449c..d8075ca 100644
--- a/gcc/gimple-ssa-store-merging.cc
+++ b/gcc/gimple-ssa-store-merging.cc
@@ -644,9 +644,7 @@ find_bswap_or_nop_1 (gimple *stmt, struct symbolic_number *n, int limit)
 
 	  /* Mask.  */
 	  uint64_t mask = 0;
-	  uint64_t tmp = (1 << BITS_PER_UNIT) - 1;
-	  for (unsigned i = 0; i < bitsize / BITS_PER_UNIT;
-	       i++, tmp <<= BITS_PER_UNIT)
+	  for (unsigned i = 0; i < bitsize / BITS_PER_UNIT; i++)
 	    mask |= (uint64_t) MARKER_MASK << (i * BITS_PER_MARKER);
 	  n->n &= mask;
 
@@ -1035,7 +1033,7 @@ find_bswap_or_nop (gimple *stmt, struct symbolic_number *n, bool *bswap,
 	 source when rsize < range.  */
       if (n->range == orig_range
 	  /* There're case like 0x300000200 for uint32->uint64 cast,
-	     Don't hanlde this.  */
+	     Don't handle this.  */
 	  && n->range == TYPE_PRECISION (n->type)
 	  && ((orig_range == 32
 	       && optab_handler (rotl_optab, SImode) != CODE_FOR_nothing)
@@ -1045,7 +1043,7 @@ find_bswap_or_nop (gimple *stmt, struct symbolic_number *n, bool *bswap,
 	{
 	  uint64_t range = (orig_range / BITS_PER_UNIT) * BITS_PER_MARKER;
 	  uint64_t count = (tmp_n & MARKER_MASK) * BITS_PER_MARKER;
-	  /* .i.e. hanlde 0x203040506070800 when lower byte is zero.  */
+	  /* .i.e. handle 0x203040506070800 when lower byte is zero.  */
 	  if (!count)
 	    {
 	      for (uint64_t i = 1; i != range / BITS_PER_MARKER; i++)
@@ -1057,6 +1055,8 @@ find_bswap_or_nop (gimple *stmt, struct symbolic_number *n, bool *bswap,
 		      if (count <= range / BITS_PER_MARKER)
 			{
 			  count = (count + i) * BITS_PER_MARKER % range;
+			  if (!count)
+			    return NULL;
 			  break;
 			}
 		      else
diff --git a/gcc/gimple-walk.cc b/gcc/gimple-walk.cc
index eceda41..b8b1077 100644
--- a/gcc/gimple-walk.cc
+++ b/gcc/gimple-walk.cc
@@ -118,7 +118,7 @@ walk_gimple_asm (gasm *stmt, walk_tree_fn callback_op,
       if (wi)
 	{
 	  if (parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-				       &allows_reg, &is_inout))
+				       &allows_reg, &is_inout, nullptr))
 	    wi->val_only = (allows_reg || !allows_mem);
 	}
       if (wi)
@@ -137,7 +137,8 @@ walk_gimple_asm (gasm *stmt, walk_tree_fn callback_op,
       if (wi)
 	{
 	  if (parse_input_constraint (&constraint, 0, 0, noutputs, 0,
-				      oconstraints, &allows_mem, &allows_reg))
+				      oconstraints, &allows_mem, &allows_reg,
+				      nullptr))
 	    {
 	      wi->val_only = (allows_reg || !allows_mem);
 	      /* Although input "m" is not really a LHS, we need a lvalue.  */
@@ -897,7 +898,7 @@ walk_stmt_load_store_addr_ops (gimple *stmt, void *data,
 		    (TREE_VALUE (TREE_PURPOSE (link)));
 		oconstraints[i] = constraint;
 		parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-					 &allows_reg, &is_inout);
+					 &allows_reg, &is_inout, nullptr);
 		if (op && !allows_reg && allows_mem)
 		  ret |= visit_addr (stmt, op, TREE_VALUE (link), data);
 	      }
@@ -922,8 +923,8 @@ walk_stmt_load_store_addr_ops (gimple *stmt, void *data,
 			constraint = TREE_STRING_POINTER
 			    (TREE_VALUE (TREE_PURPOSE (link)));
 			parse_input_constraint (&constraint, 0, 0, noutputs,
-						0, oconstraints,
-						&allows_mem, &allows_reg);
+						0, oconstraints, &allows_mem,
+						&allows_reg, nullptr);
 			if (!allows_reg && allows_mem)
 			  ret |= visit_addr (stmt, op, TREE_VALUE (link),
 					     data);
diff --git a/gcc/gimple-warn-recursion.cc b/gcc/gimple-warn-recursion.cc
index e6a0c4c..d061e7e 100644
--- a/gcc/gimple-warn-recursion.cc
+++ b/gcc/gimple-warn-recursion.cc
@@ -190,6 +190,7 @@ pass_warn_recursion::execute (function *func)
   if (find_function_exit (entry_bb) || m_calls->length () == 0)
     return 0;
 
+  auto_diagnostic_group d;
   if (warning_at (DECL_SOURCE_LOCATION (func->decl),
 		  OPT_Winfinite_recursion,
 		  "infinite recursion detected"))
diff --git a/gcc/gimple.h b/gcc/gimple.h
index 2688846..5c970ce 100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -865,7 +865,7 @@ struct GTY((tag("GSS_ASSUME")))
    tell the runtime that it should begin the transaction in
    serial-irrevocable mode.  */
 #define GTMA_DOES_GO_IRREVOCABLE	(1u << 6)
-/* The transaction contains no instrumentation code whatsover, most
+/* The transaction contains no instrumentation code whatsoever, most
    likely because it is guaranteed to go irrevocable upon entry.  */
 #define GTMA_HAS_NO_INSTRUMENTATION	(1u << 7)
 
diff --git a/gcc/gimplify-me.cc b/gcc/gimplify-me.cc
index 7407822..5b4b987 100644
--- a/gcc/gimplify-me.cc
+++ b/gcc/gimplify-me.cc
@@ -194,7 +194,7 @@ gimple_regimplify_operands (gimple *stmt, gimple_stmt_iterator *gsi_p)
 	    constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (op)));
 	    oconstraints[i] = constraint;
 	    parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-				     &allows_reg, &is_inout);
+				     &allows_reg, &is_inout, nullptr);
 	    gimplify_expr (&TREE_VALUE (op), &pre, NULL,
 			   is_inout ? is_gimple_min_lval : is_gimple_lvalue,
 			   fb_lvalue | fb_mayfail);
@@ -204,7 +204,8 @@ gimple_regimplify_operands (gimple *stmt, gimple_stmt_iterator *gsi_p)
 	    tree op = gimple_asm_input_op (asm_stmt, i);
 	    constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (op)));
 	    parse_input_constraint (&constraint, 0, 0, noutputs, 0,
-				    oconstraints, &allows_mem, &allows_reg);
+				    oconstraints, &allows_mem, &allows_reg,
+				    nullptr);
 	    if (TREE_ADDRESSABLE (TREE_TYPE (TREE_VALUE (op))) && allows_mem)
 	      allows_reg = 0;
 	    if (!allows_reg && allows_mem)
diff --git a/gcc/gimplify.cc b/gcc/gimplify.cc
index 9f9ff92..fbf47dd 100644
--- a/gcc/gimplify.cc
+++ b/gcc/gimplify.cc
@@ -71,6 +71,10 @@ along with GCC; see the file COPYING3.  If not see
 #include "context.h"
 #include "tree-nested.h"
 #include "gcc-urlifier.h"
+#include "insn-config.h"
+#include "recog.h"
+#include "output.h"
+#include "gimplify_reg_info.h"
 
 /* Identifier for a basic condition, mapping it to other basic conditions of
    its Boolean expression.  Basic conditions given the same uid (in the same
@@ -7305,6 +7309,7 @@ gimplify_modify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
       && !(TREE_ADDRESSABLE (TREE_TYPE (*from_p))
 	   && TREE_CODE (*from_p) == CALL_EXPR))
     {
+      suppress_warning (*from_p, OPT_Wunused_result);
       gimplify_stmt (from_p, pre_p);
       gimplify_stmt (to_p, pre_p);
       *expr_p = NULL_TREE;
@@ -7822,6 +7827,42 @@ gimplify_addr_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
   return ret;
 }
 
+/* Return the number of times character C occurs in string S.  */
+
+static int
+num_occurrences (int c, const char *s)
+{
+  int n = 0;
+  while (*s)
+    n += (*s++ == c);
+  return n;
+}
+
+/* A subroutine of gimplify_asm_expr.  Check that all operands have
+   the same number of alternatives.  Return -1 if this is violated.  Otherwise
+   return the number of alternatives.  */
+
+static int
+num_alternatives (const_tree link)
+{
+  if (link == nullptr)
+    return 0;
+
+  const char *constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
+  int num = num_occurrences (',', constraint);
+
+  if (num + 1 > MAX_RECOG_ALTERNATIVES)
+    return -1;
+
+  for (link = TREE_CHAIN (link); link; link = TREE_CHAIN (link))
+    {
+      constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
+      if (num_occurrences (',', constraint) != num)
+	return -1;
+    }
+  return num + 1;
+}
+
 /* Gimplify the operands of an ASM_EXPR.  Input operands should be a gimple
    value; output operands should be a gimple lvalue.  */
 
@@ -7852,6 +7893,36 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
   clobbers = NULL;
   labels = NULL;
 
+  int num_alternatives_out = num_alternatives (ASM_OUTPUTS (expr));
+  int num_alternatives_in = num_alternatives (ASM_INPUTS (expr));
+  if (num_alternatives_out == -1 || num_alternatives_in == -1
+      || (num_alternatives_out > 0 && num_alternatives_in > 0
+	  && num_alternatives_out != num_alternatives_in))
+    {
+      error ("operand constraints for %<asm%> differ "
+	     "in number of alternatives");
+      return GS_ERROR;
+    }
+  int num_alternatives = MAX (num_alternatives_out, num_alternatives_in);
+
+  gimplify_reg_info reg_info (num_alternatives, noutputs);
+
+  link_next = NULL_TREE;
+  for (link = ASM_CLOBBERS (expr); link; link = link_next)
+    {
+      /* The clobber entry could also be an error marker.  */
+      if (TREE_CODE (TREE_VALUE (link)) == STRING_CST)
+	{
+	  const char *regname= TREE_STRING_POINTER (TREE_VALUE (link));
+	  int regno = decode_reg_name (regname);
+	  if (regno >= 0)
+	    reg_info.set_clobbered (regno);
+	}
+      link_next = TREE_CHAIN (link);
+      TREE_CHAIN (link) = NULL_TREE;
+      vec_safe_push (clobbers, link);
+    }
+
   ret = GS_ALL_DONE;
   link_next = NULL_TREE;
   for (i = 0, link = ASM_OUTPUTS (expr); link; ++i, link = link_next)
@@ -7868,8 +7939,9 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
       if (constraint_len == 0)
         continue;
 
-      ok = parse_output_constraint (&constraint, i, 0, 0,
-				    &allows_mem, &allows_reg, &is_inout);
+      reg_info.operand = TREE_VALUE (link);
+      ok = parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
+				    &allows_reg, &is_inout, &reg_info);
       if (!ok)
 	{
 	  ret = GS_ERROR;
@@ -8000,8 +8072,8 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 			*end = '\0';
 		      beg[-1] = '=';
 		      tem = beg - 1;
-		      parse_output_constraint (&tem, i, 0, 0,
-					       &mem_p, &reg_p, &inout_p);
+		      parse_output_constraint (&tem, i, 0, 0, &mem_p, &reg_p,
+					       &inout_p, nullptr);
 		      if (dst != str)
 			*dst++ = ',';
 		      if (reg_p)
@@ -8040,13 +8112,60 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 	}
     }
 
+  /* After all output operands have been gimplified, verify that each output
+     operand is used at most once in case of hard register constraints.  Thus,
+     error out in cases like
+       asm ("" : "={0}" (x), "={1}" (x));
+     or even for
+       asm ("" : "=r" (x), "={1}" (x));
+
+     FIXME: Ideally we would also error out for cases like
+       int x;
+       asm ("" : "=r" (x), "=r" (x));
+     However, since code like that was previously accepted, erroring out now might
+     break existing code.  On the other hand, we already error out for register
+     asm like
+       register int x asm ("0");
+       asm ("" : "=r" (x), "=r" (x));
+     Thus, maybe it wouldn't be too bad to also error out in the former
+     non-register-asm case.
+  */
+  for (unsigned i = 0; i < vec_safe_length (outputs); ++i)
+    {
+      tree link = (*outputs)[i];
+      tree op1 = TREE_VALUE (link);
+      const char *constraint
+	= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
+      if (strchr (constraint, '{') != nullptr)
+	for (unsigned j = 0; j < vec_safe_length (outputs); ++j)
+	  {
+	    if (i == j)
+	      continue;
+	    tree link2 = (*outputs)[j];
+	    tree op2 = TREE_VALUE (link2);
+	    if (op1 == op2)
+	      {
+		error ("multiple outputs to lvalue %qE", op2);
+		return GS_ERROR;
+	      }
+	  }
+    }
+
   link_next = NULL_TREE;
-  for (link = ASM_INPUTS (expr); link; ++i, link = link_next)
+  int input_num = 0;
+  for (link = ASM_INPUTS (expr); link; ++input_num, ++i, link = link_next)
     {
       link_next = TREE_CHAIN (link);
       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
-      parse_input_constraint (&constraint, 0, 0, noutputs, 0,
-			      oconstraints, &allows_mem, &allows_reg);
+      reg_info.operand = TREE_VALUE (link);
+      bool ok = parse_input_constraint (&constraint, input_num, 0, noutputs, 0,
+					oconstraints, &allows_mem, &allows_reg,
+					&reg_info);
+      if (!ok)
+	{
+	  ret = GS_ERROR;
+	  is_inout = false;
+	}
 
       /* If we can't make copies, we can only accept memory.  */
       tree intype = TREE_TYPE (TREE_VALUE (link));
@@ -8128,15 +8247,7 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
     }
 
   link_next = NULL_TREE;
-  for (link = ASM_CLOBBERS (expr); link; ++i, link = link_next)
-    {
-      link_next = TREE_CHAIN (link);
-      TREE_CHAIN (link) = NULL_TREE;
-      vec_safe_push (clobbers, link);
-    }
-
-  link_next = NULL_TREE;
-  for (link = ASM_LABELS (expr); link; ++i, link = link_next)
+  for (link = ASM_LABELS (expr); link; link = link_next)
     {
       link_next = TREE_CHAIN (link);
       TREE_CHAIN (link) = NULL_TREE;
@@ -9526,9 +9637,7 @@ gimplify_omp_affinity (tree *list_p, gimple_seq *pre_p)
     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_AFFINITY)
       {
 	tree t = OMP_CLAUSE_DECL (c);
-	if (TREE_CODE (t) == TREE_LIST
-		    && TREE_PURPOSE (t)
-		    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    if (TREE_VALUE (t) == null_pointer_node)
 	      continue;
@@ -9633,6 +9742,155 @@ gimplify_omp_affinity (tree *list_p, gimple_seq *pre_p)
   return;
 }
 
+/* Returns a tree expression containing the total iteration count of the
+   OpenMP iterator IT.  */
+
+static tree
+compute_omp_iterator_count (tree it, gimple_seq *pre_p)
+{
+  tree tcnt = size_one_node;
+  for (; it; it = TREE_CHAIN (it))
+    {
+      if (gimplify_expr (&TREE_VEC_ELT (it, 1), pre_p, NULL,
+			 is_gimple_val, fb_rvalue) == GS_ERROR
+	  || gimplify_expr (&TREE_VEC_ELT (it, 2), pre_p, NULL,
+			    is_gimple_val, fb_rvalue) == GS_ERROR
+	  || gimplify_expr (&TREE_VEC_ELT (it, 3), pre_p, NULL,
+			    is_gimple_val, fb_rvalue) == GS_ERROR
+	  || (gimplify_expr (&TREE_VEC_ELT (it, 4), pre_p, NULL,
+			     is_gimple_val, fb_rvalue) == GS_ERROR))
+	return NULL_TREE;
+      tree var = TREE_VEC_ELT (it, 0);
+      tree begin = TREE_VEC_ELT (it, 1);
+      tree end = TREE_VEC_ELT (it, 2);
+      tree step = TREE_VEC_ELT (it, 3);
+      tree orig_step = TREE_VEC_ELT (it, 4);
+      tree type = TREE_TYPE (var);
+      tree stype = TREE_TYPE (step);
+      location_t loc = DECL_SOURCE_LOCATION (var);
+      tree endmbegin;
+      /* Compute count for this iterator as
+	 orig_step > 0
+	 ? (begin < end ? (end - begin + (step - 1)) / step : 0)
+	 : (begin > end ? (end - begin + (step + 1)) / step : 0)
+	 and compute product of those for the entire clause.  */
+      if (POINTER_TYPE_P (type))
+	endmbegin = fold_build2_loc (loc, POINTER_DIFF_EXPR, stype, end, begin);
+      else
+	endmbegin = fold_build2_loc (loc, MINUS_EXPR, type, end, begin);
+      tree stepm1 = fold_build2_loc (loc, MINUS_EXPR, stype, step,
+				     build_int_cst (stype, 1));
+      tree stepp1 = fold_build2_loc (loc, PLUS_EXPR, stype, step,
+				     build_int_cst (stype, 1));
+      tree pos = fold_build2_loc (loc, PLUS_EXPR, stype,
+				  unshare_expr (endmbegin), stepm1);
+      pos = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype, pos, step);
+      tree neg = fold_build2_loc (loc, PLUS_EXPR, stype, endmbegin, stepp1);
+      if (TYPE_UNSIGNED (stype))
+	{
+	  neg = fold_build1_loc (loc, NEGATE_EXPR, stype, neg);
+	  step = fold_build1_loc (loc, NEGATE_EXPR, stype, step);
+	}
+      neg = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype, neg, step);
+      step = NULL_TREE;
+      tree cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node, begin, end);
+      pos = fold_build3_loc (loc, COND_EXPR, stype, cond, pos,
+			     build_int_cst (stype, 0));
+      cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node, end, begin);
+      neg = fold_build3_loc (loc, COND_EXPR, stype, cond, neg,
+			     build_int_cst (stype, 0));
+      tree osteptype = TREE_TYPE (orig_step);
+      cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, orig_step,
+			      build_int_cst (osteptype, 0));
+      tree cnt = fold_build3_loc (loc, COND_EXPR, stype, cond, pos, neg);
+      cnt = fold_convert_loc (loc, sizetype, cnt);
+      if (gimplify_expr (&cnt, pre_p, NULL, is_gimple_val,
+			 fb_rvalue) == GS_ERROR)
+	return NULL_TREE;
+      tcnt = size_binop_loc (loc, MULT_EXPR, tcnt, cnt);
+    }
+  if (gimplify_expr (&tcnt, pre_p, NULL, is_gimple_val, fb_rvalue) == GS_ERROR)
+    return NULL_TREE;
+
+  return tcnt;
+}
+
+/* Build loops iterating over the space defined by the OpenMP iterator IT.
+   Returns a pointer to the BIND_EXPR_BODY in the innermost loop body.
+   LAST_BIND is set to point to the BIND_EXPR containing the whole loop.  */
+
+static tree *
+build_omp_iterator_loop (tree it, gimple_seq *pre_p, tree *last_bind)
+{
+  if (*last_bind)
+    gimplify_and_add (*last_bind, pre_p);
+  tree block = TREE_VEC_ELT (it, 5);
+  *last_bind = build3 (BIND_EXPR, void_type_node,
+		       BLOCK_VARS (block), NULL, block);
+  TREE_SIDE_EFFECTS (*last_bind) = 1;
+  tree *p = &BIND_EXPR_BODY (*last_bind);
+  for (; it; it = TREE_CHAIN (it))
+    {
+      tree var = TREE_VEC_ELT (it, 0);
+      tree begin = TREE_VEC_ELT (it, 1);
+      tree end = TREE_VEC_ELT (it, 2);
+      tree step = TREE_VEC_ELT (it, 3);
+      tree orig_step = TREE_VEC_ELT (it, 4);
+      tree type = TREE_TYPE (var);
+      location_t loc = DECL_SOURCE_LOCATION (var);
+      /* Emit:
+	 var = begin;
+	 goto cond_label;
+	 beg_label:
+	 ...
+	 var = var + step;
+	 cond_label:
+	 if (orig_step > 0) {
+	   if (var < end) goto beg_label;
+	 } else {
+	   if (var > end) goto beg_label;
+	 }
+	 for each iterator, with inner iterators added to
+	 the ... above.  */
+      tree beg_label = create_artificial_label (loc);
+      tree cond_label = NULL_TREE;
+      tree tem = build2_loc (loc, MODIFY_EXPR, void_type_node, var, begin);
+      append_to_statement_list_force (tem, p);
+      tem = build_and_jump (&cond_label);
+      append_to_statement_list_force (tem, p);
+      tem = build1 (LABEL_EXPR, void_type_node, beg_label);
+      append_to_statement_list (tem, p);
+      tree bind = build3 (BIND_EXPR, void_type_node, NULL_TREE,
+			  NULL_TREE, NULL_TREE);
+      TREE_SIDE_EFFECTS (bind) = 1;
+      SET_EXPR_LOCATION (bind, loc);
+      append_to_statement_list_force (bind, p);
+      if (POINTER_TYPE_P (type))
+	tem = build2_loc (loc, POINTER_PLUS_EXPR, type,
+			  var, fold_convert_loc (loc, sizetype, step));
+      else
+	tem = build2_loc (loc, PLUS_EXPR, type, var, step);
+      tem = build2_loc (loc, MODIFY_EXPR, void_type_node, var, tem);
+      append_to_statement_list_force (tem, p);
+      tem = build1 (LABEL_EXPR, void_type_node, cond_label);
+      append_to_statement_list (tem, p);
+      tree cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node, var, end);
+      tree pos = fold_build3_loc (loc, COND_EXPR, void_type_node, cond,
+				  build_and_jump (&beg_label), void_node);
+      cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, var, end);
+      tree neg = fold_build3_loc (loc, COND_EXPR, void_type_node, cond,
+				  build_and_jump (&beg_label), void_node);
+      tree osteptype = TREE_TYPE (orig_step);
+      cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, orig_step,
+			      build_int_cst (osteptype, 0));
+      tem = fold_build3_loc (loc, COND_EXPR, void_type_node, cond, pos, neg);
+      append_to_statement_list_force (tem, p);
+      p = &BIND_EXPR_BODY (bind);
+    }
+
+  return p;
+}
+
 /* If *LIST_P contains any OpenMP depend clauses with iterators,
    lower all the depend clauses by populating corresponding depend
    array.  Returns 0 if there are no such depend clauses, or
@@ -9677,89 +9935,13 @@ gimplify_omp_depend (tree *list_p, gimple_seq *pre_p)
 	tree t = OMP_CLAUSE_DECL (c);
 	if (first_loc == UNKNOWN_LOCATION)
 	  first_loc = OMP_CLAUSE_LOCATION (c);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    if (TREE_PURPOSE (t) != last_iter)
 	      {
-		tree tcnt = size_one_node;
-		for (tree it = TREE_PURPOSE (t); it; it = TREE_CHAIN (it))
-		  {
-		    if (gimplify_expr (&TREE_VEC_ELT (it, 1), pre_p, NULL,
-				       is_gimple_val, fb_rvalue) == GS_ERROR
-			|| gimplify_expr (&TREE_VEC_ELT (it, 2), pre_p, NULL,
-					  is_gimple_val, fb_rvalue) == GS_ERROR
-			|| gimplify_expr (&TREE_VEC_ELT (it, 3), pre_p, NULL,
-					  is_gimple_val, fb_rvalue) == GS_ERROR
-			|| (gimplify_expr (&TREE_VEC_ELT (it, 4), pre_p, NULL,
-					   is_gimple_val, fb_rvalue)
-			    == GS_ERROR))
-		      return 2;
-		    tree var = TREE_VEC_ELT (it, 0);
-		    tree begin = TREE_VEC_ELT (it, 1);
-		    tree end = TREE_VEC_ELT (it, 2);
-		    tree step = TREE_VEC_ELT (it, 3);
-		    tree orig_step = TREE_VEC_ELT (it, 4);
-		    tree type = TREE_TYPE (var);
-		    tree stype = TREE_TYPE (step);
-		    location_t loc = DECL_SOURCE_LOCATION (var);
-		    tree endmbegin;
-		    /* Compute count for this iterator as
-		       orig_step > 0
-		       ? (begin < end ? (end - begin + (step - 1)) / step : 0)
-		       : (begin > end ? (end - begin + (step + 1)) / step : 0)
-		       and compute product of those for the entire depend
-		       clause.  */
-		    if (POINTER_TYPE_P (type))
-		      endmbegin = fold_build2_loc (loc, POINTER_DIFF_EXPR,
-						   stype, end, begin);
-		    else
-		      endmbegin = fold_build2_loc (loc, MINUS_EXPR, type,
-						   end, begin);
-		    tree stepm1 = fold_build2_loc (loc, MINUS_EXPR, stype,
-						   step,
-						   build_int_cst (stype, 1));
-		    tree stepp1 = fold_build2_loc (loc, PLUS_EXPR, stype, step,
-						   build_int_cst (stype, 1));
-		    tree pos = fold_build2_loc (loc, PLUS_EXPR, stype,
-						unshare_expr (endmbegin),
-						stepm1);
-		    pos = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype,
-					   pos, step);
-		    tree neg = fold_build2_loc (loc, PLUS_EXPR, stype,
-						endmbegin, stepp1);
-		    if (TYPE_UNSIGNED (stype))
-		      {
-			neg = fold_build1_loc (loc, NEGATE_EXPR, stype, neg);
-			step = fold_build1_loc (loc, NEGATE_EXPR, stype, step);
-		      }
-		    neg = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype,
-					   neg, step);
-		    step = NULL_TREE;
-		    tree cond = fold_build2_loc (loc, LT_EXPR,
-						 boolean_type_node,
-						 begin, end);
-		    pos = fold_build3_loc (loc, COND_EXPR, stype, cond, pos,
-					   build_int_cst (stype, 0));
-		    cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node,
-					    end, begin);
-		    neg = fold_build3_loc (loc, COND_EXPR, stype, cond, neg,
-					   build_int_cst (stype, 0));
-		    tree osteptype = TREE_TYPE (orig_step);
-		    cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
-					    orig_step,
-					    build_int_cst (osteptype, 0));
-		    tree cnt = fold_build3_loc (loc, COND_EXPR, stype,
-						cond, pos, neg);
-		    cnt = fold_convert_loc (loc, sizetype, cnt);
-		    if (gimplify_expr (&cnt, pre_p, NULL, is_gimple_val,
-				       fb_rvalue) == GS_ERROR)
-		      return 2;
-		    tcnt = size_binop_loc (loc, MULT_EXPR, tcnt, cnt);
-		  }
-		if (gimplify_expr (&tcnt, pre_p, NULL, is_gimple_val,
-				   fb_rvalue) == GS_ERROR)
+		tree tcnt = compute_omp_iterator_count (TREE_PURPOSE (t),
+							pre_p);
+		if (!tcnt)
 		  return 2;
 		last_iter = TREE_PURPOSE (t);
 		last_count = tcnt;
@@ -9913,91 +10095,13 @@ gimplify_omp_depend (tree *list_p, gimple_seq *pre_p)
 	    gcc_unreachable ();
 	  }
 	tree t = OMP_CLAUSE_DECL (c);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    if (TREE_PURPOSE (t) != last_iter)
 	      {
-		if (last_bind)
-		  gimplify_and_add (last_bind, pre_p);
-		tree block = TREE_VEC_ELT (TREE_PURPOSE (t), 5);
-		last_bind = build3 (BIND_EXPR, void_type_node,
-				    BLOCK_VARS (block), NULL, block);
-		TREE_SIDE_EFFECTS (last_bind) = 1;
+		last_body = build_omp_iterator_loop (TREE_PURPOSE (t), pre_p,
+						     &last_bind);
 		SET_EXPR_LOCATION (last_bind, OMP_CLAUSE_LOCATION (c));
-		tree *p = &BIND_EXPR_BODY (last_bind);
-		for (tree it = TREE_PURPOSE (t); it; it = TREE_CHAIN (it))
-		  {
-		    tree var = TREE_VEC_ELT (it, 0);
-		    tree begin = TREE_VEC_ELT (it, 1);
-		    tree end = TREE_VEC_ELT (it, 2);
-		    tree step = TREE_VEC_ELT (it, 3);
-		    tree orig_step = TREE_VEC_ELT (it, 4);
-		    tree type = TREE_TYPE (var);
-		    location_t loc = DECL_SOURCE_LOCATION (var);
-		    /* Emit:
-		       var = begin;
-		       goto cond_label;
-		       beg_label:
-		       ...
-		       var = var + step;
-		       cond_label:
-		       if (orig_step > 0) {
-			 if (var < end) goto beg_label;
-		       } else {
-			 if (var > end) goto beg_label;
-		       }
-		       for each iterator, with inner iterators added to
-		       the ... above.  */
-		    tree beg_label = create_artificial_label (loc);
-		    tree cond_label = NULL_TREE;
-		    tem = build2_loc (loc, MODIFY_EXPR, void_type_node,
-				      var, begin);
-		    append_to_statement_list_force (tem, p);
-		    tem = build_and_jump (&cond_label);
-		    append_to_statement_list_force (tem, p);
-		    tem = build1 (LABEL_EXPR, void_type_node, beg_label);
-		    append_to_statement_list (tem, p);
-		    tree bind = build3 (BIND_EXPR, void_type_node, NULL_TREE,
-					NULL_TREE, NULL_TREE);
-		    TREE_SIDE_EFFECTS (bind) = 1;
-		    SET_EXPR_LOCATION (bind, loc);
-		    append_to_statement_list_force (bind, p);
-		    if (POINTER_TYPE_P (type))
-		      tem = build2_loc (loc, POINTER_PLUS_EXPR, type,
-					var, fold_convert_loc (loc, sizetype,
-							       step));
-		    else
-		      tem = build2_loc (loc, PLUS_EXPR, type, var, step);
-		    tem = build2_loc (loc, MODIFY_EXPR, void_type_node,
-				      var, tem);
-		    append_to_statement_list_force (tem, p);
-		    tem = build1 (LABEL_EXPR, void_type_node, cond_label);
-		    append_to_statement_list (tem, p);
-		    tree cond = fold_build2_loc (loc, LT_EXPR,
-						 boolean_type_node,
-						 var, end);
-		    tree pos
-		      = fold_build3_loc (loc, COND_EXPR, void_type_node,
-					 cond, build_and_jump (&beg_label),
-					 void_node);
-		    cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
-					    var, end);
-		    tree neg
-		      = fold_build3_loc (loc, COND_EXPR, void_type_node,
-					 cond, build_and_jump (&beg_label),
-					 void_node);
-		    tree osteptype = TREE_TYPE (orig_step);
-		    cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
-					    orig_step,
-					    build_int_cst (osteptype, 0));
-		    tem = fold_build3_loc (loc, COND_EXPR, void_type_node,
-					   cond, pos, neg);
-		    append_to_statement_list_force (tem, p);
-		    p = &BIND_EXPR_BODY (bind);
-		  }
-		last_body = p;
 	      }
 	    last_iter = TREE_PURPOSE (t);
 	    if (TREE_CODE (TREE_VALUE (t)) == COMPOUND_EXPR)
diff --git a/gcc/gimplify_reg_info.h b/gcc/gimplify_reg_info.h
new file mode 100644
index 0000000..b56b225
--- /dev/null
+++ b/gcc/gimplify_reg_info.h
@@ -0,0 +1,182 @@
+/* gimplify_reg_info is used during gimplification while walking over
+   operands and their corresponding constraints of asm statements in order to
+   detect errors.
+
+   m_alt_output is a mapping describing which registers are potentially used in
+   which alternative over all outputs.  Similarly for m_alt_input but over all
+   inputs.
+
+   m_early_clobbered_alt is a mapping describing which register is early
+   clobbered in which alternative over all outputs.
+
+   m_early_clobbered_output is the counter part to the prior one, i.e., it
+   is a mapping describing which register is early clobbered in which operand
+   over all alternatives.
+
+   m_reg_asm_output is the set of registers (including register pairs) used for
+   register asm output operands.
+
+   m_reg_asm_input similar as m_reg_asm_output but for inputs.  */
+
+#include "regs.h"
+
+class gimplify_reg_info
+{
+  HARD_REG_SET *m_buf;
+  HARD_REG_SET *m_alt_output;
+  HARD_REG_SET *m_alt_input;
+  HARD_REG_SET *m_early_clobbered_alt;
+  HARD_REG_SET *m_early_clobbered_output;
+  HARD_REG_SET m_reg_asm_output;
+  HARD_REG_SET m_reg_asm_input;
+  const unsigned m_num_alternatives;
+  const unsigned m_num_outputs;
+  /* Member m_clobbered describes all the registers marked as clobbered in an
+     asm statement, i.e., this is the clobbers list of an extended asm
+
+     asm asm-qualifiers ( AssemblerTemplate
+		 : OutputOperands
+		 [ : InputOperands
+		 [ : Clobbers ] ])
+
+     and is not to be confused with the early clobbers sets.  */
+  HARD_REG_SET m_clobbered;
+
+  /* Return the first overlapping register of REGS and REGNO:MODE or -1.  */
+  int test (const HARD_REG_SET &regs, int regno) const
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+
+    if (TEST_HARD_REG_BIT (regs, regno))
+      return regno;
+
+    int end_regno = end_hard_regno (mode, regno);
+    while (++regno < end_regno)
+      if (TEST_HARD_REG_BIT (regs, regno))
+	return regno;
+
+    return -1;
+  }
+
+public:
+  tree operand;
+
+  gimplify_reg_info (unsigned num_alternatives,
+		     unsigned num_outputs)
+    : m_num_alternatives{num_alternatives}
+    , m_num_outputs{num_outputs}
+  {
+    CLEAR_HARD_REG_SET (m_reg_asm_output);
+    CLEAR_HARD_REG_SET (m_reg_asm_input);
+    CLEAR_HARD_REG_SET (m_clobbered);
+
+    /* If there are no alternatives, then there are no outputs/inputs and there
+       is nothing to do on our end.  Thus, we are dealing most likely with a
+       basic asm.  */
+    if (num_alternatives == 0)
+      return;
+
+    unsigned buf_size = num_alternatives * 3 + num_outputs;
+    m_buf = new HARD_REG_SET[buf_size];
+    for (unsigned i = 0; i < buf_size; ++i)
+      CLEAR_HARD_REG_SET (m_buf[i]);
+    m_alt_output = &m_buf[0];
+    m_alt_input = &m_buf[num_alternatives];
+    m_early_clobbered_alt = &m_buf[num_alternatives * 2];
+    if (num_outputs > 0)
+      m_early_clobbered_output = &m_buf[num_alternatives * 3];
+    else
+      m_early_clobbered_output = nullptr;
+  }
+
+  ~gimplify_reg_info ()
+  {
+    if (m_num_alternatives > 0)
+      delete[] m_buf;
+  }
+
+  void set_output (unsigned alt, int regno)
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_alt_output[alt], mode, regno);
+  }
+
+  void set_input (unsigned alt, int regno)
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_alt_input[alt], mode, regno);
+  }
+
+  int test_alt_output (unsigned alt, int regno) const
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    return test (m_alt_output[alt], regno);
+  }
+
+  int test_alt_input (unsigned alt, int regno) const
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    return test (m_alt_input[alt], regno);
+  }
+
+  void set_reg_asm_output (int regno)
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_reg_asm_output, mode, regno);
+  }
+
+  int test_reg_asm_output (int regno) const
+  {
+    return test (m_reg_asm_output, regno);
+  }
+
+  void set_reg_asm_input (int regno)
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_reg_asm_input, mode, regno);
+  }
+
+  int test_reg_asm_input (int regno) const
+  {
+    return test (m_reg_asm_input, regno);
+  }
+
+  void set_early_clobbered (unsigned alt, unsigned output, int regno)
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    gcc_checking_assert (output < m_num_outputs);
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_early_clobbered_alt[alt], mode, regno);
+    add_to_hard_reg_set (&m_early_clobbered_output[output], mode, regno);
+  }
+
+  bool test_early_clobbered_alt (unsigned alt, int regno) const
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    return TEST_HARD_REG_BIT (m_early_clobbered_alt[alt], regno);
+  }
+
+  bool is_early_clobbered_in_any_output_unequal (unsigned operand,
+						 int regno) const
+  {
+    gcc_checking_assert (operand < m_num_outputs);
+    for (unsigned op = 0; op < m_num_outputs; ++op)
+      if (op != operand
+	  && TEST_HARD_REG_BIT (m_early_clobbered_output[op], regno))
+	return true;
+    return false;
+  }
+
+  void set_clobbered (int regno)
+  {
+    SET_HARD_REG_BIT (m_clobbered, regno);
+  }
+
+  bool is_clobbered (int regno) const
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    return overlaps_hard_reg_set_p (m_clobbered, mode, regno);
+  }
+};
diff --git a/gcc/go/ChangeLog b/gcc/go/ChangeLog
index 83829c5..ed189f9 100644
--- a/gcc/go/ChangeLog
+++ b/gcc/go/ChangeLog
@@ -1,3 +1,10 @@
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* gofrontend/gogo.cc (Function::export_func_with_type): Remove
+	unused but set variable i.
+
 2025-02-05  Ian Lance Taylor  <iant@golang.org>
 
 	PR go/118746
diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE
index b6fdf72..99bf24c 100644
--- a/gcc/go/gofrontend/MERGE
+++ b/gcc/go/gofrontend/MERGE
@@ -1,4 +1,4 @@
-f5c453aa726ebb509e7b8cb20df7734f0e411404
+c4d7bfb9895efc196b04f18f5da77fd99b39212a
 
 The first line of this file holds the git revision number of the last
 merge done from the gofrontend repository.
diff --git a/gcc/go/gofrontend/gogo.cc b/gcc/go/gofrontend/gogo.cc
index 3aad419..566eca8 100644
--- a/gcc/go/gofrontend/gogo.cc
+++ b/gcc/go/gofrontend/gogo.cc
@@ -5987,12 +5987,11 @@ Function::export_func_with_type(Export* exp, const Named_object* no,
   const Typed_identifier_list* parameters = fntype->parameters();
   if (parameters != NULL)
     {
-      size_t i = 0;
       bool is_varargs = fntype->is_varargs();
       bool first = true;
       for (Typed_identifier_list::const_iterator p = parameters->begin();
 	   p != parameters->end();
-	   ++p, ++i)
+	   ++p)
 	{
 	  if (first)
 	    first = false;
diff --git a/gcc/graphite.h b/gcc/graphite.h
index 66d4db6..9cb3513 100644
--- a/gcc/graphite.h
+++ b/gcc/graphite.h
@@ -155,7 +155,7 @@ struct poly_dr
      The OpenScop access function is printed as follows:
 
      | 1  # The number of disjunct components in a union of access functions.
-     | R C O I L P  # Described bellow.
+     | R C O I L P  # Described below.
      | a  s0  s1   i   j   k   1
      | 1   0   0   0   0   0  -5     =  0
      | 0   1   0  -1   0   0   0     =  0
diff --git a/gcc/graphviz.h b/gcc/graphviz.h
index 5943589..9a0fe6f 100644
--- a/gcc/graphviz.h
+++ b/gcc/graphviz.h
@@ -311,6 +311,16 @@ struct node_id : public ast_node
       m_port = std::make_unique<port> (*other.m_port);
   }
 
+  node_id &operator= (const node_id &other)
+  {
+    m_id = other.m_id;
+    if (other.m_port)
+      m_port = std::make_unique<port> (*other.m_port);
+    else
+      m_port = nullptr;
+    return *this;
+  }
+
   void print (writer &w) const final override;
 
   id m_id;
diff --git a/gcc/hooks.cc b/gcc/hooks.cc
index 951825d..76cb5931 100644
--- a/gcc/hooks.cc
+++ b/gcc/hooks.cc
@@ -117,6 +117,13 @@ hook_bool_mode_const_rtx_true (machine_mode, const_rtx)
   return true;
 }
 
+/* Generic hook that takes (machine_mode, int, unsigned) and returns false.  */
+bool
+hook_bool_mode_int_unsigned_false (machine_mode, int, unsigned)
+{
+  return false;
+}
+
 /* Generic hook that takes (machine_mode, rtx) and returns false.  */
 bool
 hook_bool_mode_rtx_false (machine_mode, rtx)
diff --git a/gcc/hooks.h b/gcc/hooks.h
index c0663bf..e95bd11 100644
--- a/gcc/hooks.h
+++ b/gcc/hooks.h
@@ -36,6 +36,7 @@ extern bool hook_bool_mode_true (machine_mode);
 extern bool hook_bool_mode_mode_true (machine_mode, machine_mode);
 extern bool hook_bool_mode_const_rtx_false (machine_mode, const_rtx);
 extern bool hook_bool_mode_const_rtx_true (machine_mode, const_rtx);
+extern bool hook_bool_mode_int_unsigned_false (machine_mode, int, unsigned);
 extern bool hook_bool_mode_rtx_false (machine_mode, rtx);
 extern bool hook_bool_mode_rtx_true (machine_mode, rtx);
 extern bool hook_bool_const_rtx_insn_const_rtx_insn_true (const rtx_insn *,
diff --git a/gcc/input.cc b/gcc/input.cc
index fabfbfb..b9a5539 100644
--- a/gcc/input.cc
+++ b/gcc/input.cc
@@ -22,6 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "intl.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 #include "selftest.h"
 #include "cpplib.h"
 
@@ -35,188 +36,6 @@ special_fname_builtin ()
   return _("<built-in>");
 }
 
-/* Input charset configuration.  */
-static const char *default_charset_callback (const char *)
-{
-  return nullptr;
-}
-
-void
-file_cache::initialize_input_context (diagnostic_input_charset_callback ccb,
-				      bool should_skip_bom)
-{
-  m_input_context.ccb = (ccb ? ccb : default_charset_callback);
-  m_input_context.should_skip_bom = should_skip_bom;
-}
-
-/* This is a cache used by get_next_line to store the content of a
-   file to be searched for file lines.  */
-class file_cache_slot
-{
-public:
-  file_cache_slot ();
-  ~file_cache_slot ();
-
-  void dump (FILE *out, int indent) const;
-  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
-
-  bool read_line_num (size_t line_num,
-		      char ** line, ssize_t *line_len);
-
-  /* Accessors.  */
-  const char *get_file_path () const { return m_file_path; }
-  unsigned get_use_count () const { return m_use_count; }
-  bool missing_trailing_newline_p () const
-  {
-    return m_missing_trailing_newline;
-  }
-  char_span get_full_file_content ();
-
-  void inc_use_count () { m_use_count++; }
-
-  bool create (const file_cache::input_context &in_context,
-	       const char *file_path, FILE *fp, unsigned highest_use_count);
-  void evict ();
-  void set_content (const char *buf, size_t sz);
-
-  static size_t tune (size_t line_record_size_)
-  {
-    size_t ret = line_record_size;
-    line_record_size = line_record_size_;
-    return ret;
-  }
-
- private:
-  /* These are information used to store a line boundary.  */
-  class line_info
-  {
-  public:
-    /* The line number.  It starts from 1.  */
-    size_t line_num;
-
-    /* The position (byte count) of the beginning of the line,
-       relative to the file data pointer.  This starts at zero.  */
-    size_t start_pos;
-
-    /* The position (byte count) of the last byte of the line.  This
-       normally points to the '\n' character, or to one byte after the
-       last byte of the file, if the file doesn't contain a '\n'
-       character.  */
-    size_t end_pos;
-
-    line_info (size_t l, size_t s, size_t e)
-      : line_num (l), start_pos (s), end_pos (e)
-    {}
-
-    line_info ()
-      :line_num (0), start_pos (0), end_pos (0)
-    {}
-
-    static bool less_than(const line_info &a, const line_info &b)
-    {
-      return a.line_num < b.line_num;
-    }
-  };
-
-  bool needs_read_p () const;
-  bool needs_grow_p () const;
-  void maybe_grow ();
-  bool read_data ();
-  bool maybe_read_data ();
-  bool get_next_line (char **line, ssize_t *line_len);
-  bool read_next_line (char ** line, ssize_t *line_len);
-  bool goto_next_line ();
-
-  static const size_t buffer_size = 4 * 1024;
-  static size_t line_record_size;
-  static size_t recent_cached_lines_shift;
-
-  /* The number of time this file has been accessed.  This is used
-     to designate which file cache to evict from the cache
-     array.  */
-  unsigned m_use_count;
-
-  /* The file_path is the key for identifying a particular file in
-     the cache.  This copy is owned by the slot.  */
-  char *m_file_path;
-
-  FILE *m_fp;
-
-  /* True when an read error happened.  */
-  bool m_error;
-
-  /* This points to the content of the file that we've read so
-     far.  */
-  char *m_data;
-
-  /* The allocated buffer to be freed may start a little earlier than DATA,
-     e.g. if a UTF8 BOM was skipped at the beginning.  */
-  int m_alloc_offset;
-
-  /*  The size of the DATA array above.*/
-  size_t m_size;
-
-  /* The number of bytes read from the underlying file so far.  This
-     must be less (or equal) than SIZE above.  */
-  size_t m_nb_read;
-
-  /* The index of the beginning of the current line.  */
-  size_t m_line_start_idx;
-
-  /* The number of the previous line read.  This starts at 1.  Zero
-     means we've read no line so far.  */
-  size_t m_line_num;
-
-  /* Could this file be missing a trailing newline on its final line?
-     Initially true (to cope with empty files), set to true/false
-     as each line is read.  */
-  bool m_missing_trailing_newline;
-
-  /* This is a record of the beginning and end of the lines we've seen
-     while reading the file.  This is useful to avoid walking the data
-     from the beginning when we are asked to read a line that is
-     before LINE_START_IDX above.  When the lines exceed line_record_size
-     this is scaled down dynamically, with the line_info becoming anchors.  */
-  vec<line_info, va_heap> m_line_record;
-
-  /* A cache of the recently seen lines. This is maintained as a ring
-     buffer. */
-  vec<line_info, va_heap> m_line_recent;
-
-  /* First and last valid entry in m_line_recent.  */
-  size_t m_line_recent_last, m_line_recent_first;
-
-  void offset_buffer (int offset)
-  {
-    gcc_assert (offset < 0 ? m_alloc_offset + offset >= 0
-		: (size_t) offset <= m_size);
-    gcc_assert (m_data);
-    m_alloc_offset += offset;
-    m_data += offset;
-    m_size -= offset;
-  }
-
-};
-
-size_t file_cache_slot::line_record_size = 0;
-size_t file_cache_slot::recent_cached_lines_shift = 8;
-
-/* Tune file_cache.  */
-void
-file_cache::tune (size_t num_file_slots, size_t lines)
-{
-  if (file_cache_slot::tune (lines) != lines
-      || m_num_file_slots != num_file_slots)
-    {
-      delete[] m_file_slots;
-      m_file_slots = new file_cache_slot[num_file_slots];
-    }
-  m_num_file_slots = num_file_slots;
-}
-
-static const char *
-find_end_of_line (const char *s, size_t len);
-
 /* Current position in real source file.  */
 
 location_t input_location = UNKNOWN_LOCATION;
@@ -320,738 +139,13 @@ expand_location_1 (const line_maps *set,
   return xloc;
 }
 
-/* Lookup the cache used for the content of a given file accessed by
-   caret diagnostic.  Return the found cached file, or NULL if no
-   cached file was found.  */
-
-file_cache_slot *
-file_cache::lookup_file (const char *file_path)
-{
-  gcc_assert (file_path);
-
-  /* This will contain the found cached file.  */
-  file_cache_slot *r = NULL;
-  for (unsigned i = 0; i < m_num_file_slots; ++i)
-    {
-      file_cache_slot *c = &m_file_slots[i];
-      if (c->get_file_path () && !strcmp (c->get_file_path (), file_path))
-	{
-	  c->inc_use_count ();
-	  r = c;
-	}
-    }
-
-  if (r)
-    r->inc_use_count ();
-
-  return r;
-}
-
-/* Purge any mention of FILENAME from the cache of files used for
-   printing source code.  For use in selftests when working
-   with tempfiles.  */
-
-void
-file_cache::forcibly_evict_file (const char *file_path)
-{
-  gcc_assert (file_path);
-
-  file_cache_slot *r = lookup_file (file_path);
-  if (!r)
-    /* Not found.  */
-    return;
-
-  r->evict ();
-}
-
-/* Determine if FILE_PATH missing a trailing newline on its final line.
-   Only valid to call once all of the file has been loaded, by
-   requesting a line number beyond the end of the file.  */
-
-bool
-file_cache::missing_trailing_newline_p (const char *file_path)
-{
-  gcc_assert (file_path);
-
-  file_cache_slot *r = lookup_or_add_file (file_path);
-  return r->missing_trailing_newline_p ();
-}
-
-void
-file_cache::add_buffered_content (const char *file_path,
-				  const char *buffer,
-				  size_t sz)
-{
-  gcc_assert (file_path);
-
-  file_cache_slot *r = lookup_file (file_path);
-  if (!r)
-    {
-      unsigned highest_use_count = 0;
-      r = evicted_cache_tab_entry (&highest_use_count);
-      if (!r->create (m_input_context, file_path, nullptr, highest_use_count))
-	return;
-    }
-
-  r->set_content (buffer, sz);
-}
-
-void
-file_cache_slot::evict ()
-{
-  free (m_file_path);
-  m_file_path = NULL;
-  if (m_fp)
-    fclose (m_fp);
-  m_error = false;
-  m_fp = NULL;
-  m_nb_read = 0;
-  m_line_start_idx = 0;
-  m_line_num = 0;
-  m_line_record.truncate (0);
-  m_line_recent_first = 0;
-  m_line_recent_last = 0;
-  m_use_count = 0;
-  m_missing_trailing_newline = true;
-}
-
-/* Return the file cache that has been less used, recently, or the
-   first empty one.  If HIGHEST_USE_COUNT is non-null,
-   *HIGHEST_USE_COUNT is set to the highest use count of the entries
-   in the cache table.  */
-
-file_cache_slot*
-file_cache::evicted_cache_tab_entry (unsigned *highest_use_count)
-{
-  file_cache_slot *to_evict = &m_file_slots[0];
-  unsigned huc = to_evict->get_use_count ();
-  for (unsigned i = 1; i < m_num_file_slots; ++i)
-    {
-      file_cache_slot *c = &m_file_slots[i];
-      bool c_is_empty = (c->get_file_path () == NULL);
-
-      if (c->get_use_count () < to_evict->get_use_count ()
-	  || (to_evict->get_file_path () && c_is_empty))
-	/* We evict C because it's either an entry with a lower use
-	   count or one that is empty.  */
-	to_evict = c;
-
-      if (huc < c->get_use_count ())
-	huc = c->get_use_count ();
-
-      if (c_is_empty)
-	/* We've reached the end of the cache; subsequent elements are
-	   all empty.  */
-	break;
-    }
-
-  if (highest_use_count)
-    *highest_use_count = huc;
-
-  return to_evict;
-}
-
-/* Create the cache used for the content of a given file to be
-   accessed by caret diagnostic.  This cache is added to an array of
-   cache and can be retrieved by lookup_file_in_cache_tab.  This
-   function returns the created cache.  Note that only the last
-   m_num_file_slots files are cached.
-
-   This can return nullptr if the FILE_PATH can't be opened for
-   reading, or if the content can't be converted to the input_charset.  */
-
-file_cache_slot*
-file_cache::add_file (const char *file_path)
-{
-
-  FILE *fp = fopen (file_path, "r");
-  if (fp == NULL)
-    return NULL;
-
-  unsigned highest_use_count = 0;
-  file_cache_slot *r = evicted_cache_tab_entry (&highest_use_count);
-  if (!r->create (m_input_context, file_path, fp, highest_use_count))
-    return NULL;
-  return r;
-}
-
-/* Get a borrowed char_span to the full content of this file
-   as decoded according to the input charset, encoded as UTF-8.  */
-
-char_span
-file_cache_slot::get_full_file_content ()
-{
-  char *line;
-  ssize_t line_len;
-  while (get_next_line (&line, &line_len))
-    {
-    }
-  return char_span (m_data, m_nb_read);
-}
-
-/* Populate this slot for use on FILE_PATH and FP, dropping any
-   existing cached content within it.  */
-
-bool
-file_cache_slot::create (const file_cache::input_context &in_context,
-			 const char *file_path, FILE *fp,
-			 unsigned highest_use_count)
-{
-  m_file_path = file_path ? xstrdup (file_path) : nullptr;
-  if (m_fp)
-    fclose (m_fp);
-  m_error = false;
-  m_fp = fp;
-  if (m_alloc_offset)
-    offset_buffer (-m_alloc_offset);
-  m_nb_read = 0;
-  m_line_start_idx = 0;
-  m_line_num = 0;
-  m_line_recent_first = 0;
-  m_line_recent_last = 0;
-  m_line_record.truncate (0);
-  /* Ensure that this cache entry doesn't get evicted next time
-     add_file_to_cache_tab is called.  */
-  m_use_count = ++highest_use_count;
-  m_missing_trailing_newline = true;
-
-
-  /* Check the input configuration to determine if we need to do any
-     transformations, such as charset conversion or BOM skipping.  */
-  if (const char *input_charset = in_context.ccb (file_path))
-    {
-      /* Need a full-blown conversion of the input charset.  */
-      fclose (m_fp);
-      m_fp = NULL;
-      const cpp_converted_source cs
-	= cpp_get_converted_source (file_path, input_charset);
-      if (!cs.data)
-	return false;
-      if (m_data)
-	XDELETEVEC (m_data);
-      m_data = cs.data;
-      m_nb_read = m_size = cs.len;
-      m_alloc_offset = cs.data - cs.to_free;
-    }
-  else if (in_context.should_skip_bom)
-    {
-      if (read_data ())
-	{
-	  const int offset = cpp_check_utf8_bom (m_data, m_nb_read);
-	  offset_buffer (offset);
-	  m_nb_read -= offset;
-	}
-    }
-
-  return true;
-}
-
-void
-file_cache_slot::set_content (const char *buf, size_t sz)
-{
-  m_data = (char *)xmalloc (sz);
-  memcpy (m_data, buf, sz);
-  m_nb_read = m_size = sz;
-  m_alloc_offset = 0;
-
-  if (m_fp)
-    {
-      fclose (m_fp);
-      m_fp = nullptr;
-    }
-}
-
-/* file_cache's ctor.  */
-
-file_cache::file_cache ()
-: m_num_file_slots (16), m_file_slots (new file_cache_slot[m_num_file_slots])
-{
-  initialize_input_context (nullptr, false);
-}
-
-/* file_cache's dtor.  */
-
-file_cache::~file_cache ()
-{
-  delete[] m_file_slots;
-}
-
-void
-file_cache::dump (FILE *out, int indent) const
-{
-  for (size_t i = 0; i < m_num_file_slots; ++i)
-    {
-      fprintf (out, "%*sslot[%i]:\n", indent, "", (int)i);
-      m_file_slots[i].dump (out, indent + 2);
-    }
-}
-
-void
-file_cache::dump () const
-{
-  dump (stderr, 0);
-}
-
-/* Lookup the cache used for the content of a given file accessed by
-   caret diagnostic.  If no cached file was found, create a new cache
-   for this file, add it to the array of cached file and return
-   it.
-
-   This can return nullptr on a cache miss if FILE_PATH can't be opened for
-   reading, or if the content can't be converted to the input_charset.  */
-
-file_cache_slot*
-file_cache::lookup_or_add_file (const char *file_path)
-{
-  file_cache_slot *r = lookup_file (file_path);
-  if (r == NULL)
-    r = add_file (file_path);
-  return r;
-}
-
-/* Default constructor for a cache of file used by caret
-   diagnostic.  */
-
-file_cache_slot::file_cache_slot ()
-: m_use_count (0), m_file_path (NULL), m_fp (NULL), m_error (false), m_data (0),
-  m_alloc_offset (0), m_size (0), m_nb_read (0), m_line_start_idx (0),
-  m_line_num (0), m_missing_trailing_newline (true),
-  m_line_recent_last (0), m_line_recent_first (0)
-{
-  m_line_record.create (0);
-  m_line_recent.create (1U << recent_cached_lines_shift);
-  for (int i = 0; i < 1 << recent_cached_lines_shift; i++)
-    m_line_recent.quick_push (file_cache_slot::line_info (0, 0, 0));
-}
-
-/* Destructor for a cache of file used by caret diagnostic.  */
-
-file_cache_slot::~file_cache_slot ()
-{
-  free (m_file_path);
-  if (m_fp)
-    {
-      fclose (m_fp);
-      m_fp = NULL;
-    }
-  if (m_data)
-    {
-      offset_buffer (-m_alloc_offset);
-      XDELETEVEC (m_data);
-      m_data = 0;
-    }
-  m_line_record.release ();
-  m_line_recent.release ();
-}
-
-void
-file_cache_slot::dump (FILE *out, int indent) const
-{
-  if (!m_file_path)
-    {
-      fprintf (out, "%*s(unused)\n", indent, "");
-      return;
-    }
-  fprintf (out, "%*sfile_path: %s\n", indent, "", m_file_path);
-  fprintf (out, "%*sfp: %p\n", indent, "", (void *)m_fp);
-  fprintf (out, "%*sneeds_read_p: %i\n", indent, "", (int)needs_read_p ());
-  fprintf (out, "%*sneeds_grow_p: %i\n", indent, "", (int)needs_grow_p ());
-  fprintf (out, "%*suse_count: %i\n", indent, "", m_use_count);
-  fprintf (out, "%*ssize: %zi\n", indent, "", m_size);
-  fprintf (out, "%*snb_read: %zi\n", indent, "", m_nb_read);
-  fprintf (out, "%*sstart_line_idx: %zi\n", indent, "", m_line_start_idx);
-  fprintf (out, "%*sline_num: %zi\n", indent, "", m_line_num);
-  fprintf (out, "%*smissing_trailing_newline: %i\n",
-	   indent, "", (int)m_missing_trailing_newline);
-  fprintf (out, "%*sline records (%i):\n",
-	   indent, "", m_line_record.length ());
-  int idx = 0;
-  for (auto &line : m_line_record)
-    fprintf (out, "%*s[%i]: line %zi: byte offsets: %zi-%zi\n",
-	     indent + 2, "",
-	     idx++, line.line_num, line.start_pos, line.end_pos);
-}
-
-/* Returns TRUE iff the cache would need to be filled with data coming
-   from the file.  That is, either the cache is empty or full or the
-   current line is empty.  Note that if the cache is full, it would
-   need to be extended and filled again.  */
-
-bool
-file_cache_slot::needs_read_p () const
-{
-  return m_fp && (m_nb_read == 0
-	  || m_nb_read == m_size
-	  || (m_line_start_idx >= m_nb_read - 1));
-}
-
-/*  Return TRUE iff the cache is full and thus needs to be
-    extended.  */
-
-bool
-file_cache_slot::needs_grow_p () const
-{
-  return m_nb_read == m_size;
-}
-
-/* Grow the cache if it needs to be extended.  */
-
-void
-file_cache_slot::maybe_grow ()
-{
-  if (!needs_grow_p ())
-    return;
-
-  if (!m_data)
-    {
-      gcc_assert (m_size == 0 && m_alloc_offset == 0);
-      m_size = buffer_size;
-      m_data = XNEWVEC (char, m_size);
-    }
-  else
-    {
-      const int offset = m_alloc_offset;
-      offset_buffer (-offset);
-      m_size *= 2;
-      m_data = XRESIZEVEC (char, m_data, m_size);
-      offset_buffer (offset);
-    }
-}
-
-/*  Read more data into the cache.  Extends the cache if need be.
-    Returns TRUE iff new data could be read.  */
-
-bool
-file_cache_slot::read_data ()
-{
-  if (feof (m_fp) || ferror (m_fp))
-    return false;
-
-  maybe_grow ();
-
-  char * from = m_data + m_nb_read;
-  size_t to_read = m_size - m_nb_read;
-  size_t nb_read = fread (from, 1, to_read, m_fp);
-
-  if (ferror (m_fp))
-    {
-      m_error = true;
-      return false;
-    }
-
-  m_nb_read += nb_read;
-  return !!nb_read;
-}
-
-/* Read new data iff the cache needs to be filled with more data
-   coming from the file FP.  Return TRUE iff the cache was filled with
-   mode data.  */
-
-bool
-file_cache_slot::maybe_read_data ()
-{
-  if (!needs_read_p ())
-    return false;
-  return read_data ();
-}
-
-/* Helper function for file_cache_slot::get_next_line (), to find the end of
-   the next line.  Returns with the memchr convention, i.e. nullptr if a line
-   terminator was not found.  We need to determine line endings in the same
-   manner that libcpp does: any of \n, \r\n, or \r is a line ending.  */
-
-static const char *
-find_end_of_line (const char *s, size_t len)
-{
-  for (const auto end = s + len; s != end; ++s)
-    {
-      if (*s == '\n')
-	return s;
-      if (*s == '\r')
-	{
-	  const auto next = s + 1;
-	  if (next == end)
-	    {
-	      /* Don't find the line ending if \r is the very last character
-		 in the buffer; we do not know if it's the end of the file or
-		 just the end of what has been read so far, and we wouldn't
-		 want to break in the middle of what's actually a \r\n
-		 sequence.  Instead, we will handle the case of a file ending
-		 in a \r later.  */
-	      break;
-	    }
-	  return (*next == '\n' ? next : s);
-	}
-    }
-  return nullptr;
-}
-
-/* Read a new line from file FP, using C as a cache for the data
-   coming from the file.  Upon successful completion, *LINE is set to
-   the beginning of the line found.  *LINE points directly in the
-   line cache and is only valid until the next call of get_next_line.
-   *LINE_LEN is set to the length of the line.  Note that the line
-   does not contain any terminal delimiter.  This function returns
-   true if some data was read or process from the cache, false
-   otherwise.  Note that subsequent calls to get_next_line might
-   make the content of *LINE invalid.  */
-
-bool
-file_cache_slot::get_next_line (char **line, ssize_t *line_len)
-{
-  /* Fill the cache with data to process.  */
-  maybe_read_data ();
-
-  size_t remaining_size = m_nb_read - m_line_start_idx;
-  if (remaining_size == 0)
-    /* There is no more data to process.  */
-    return false;
-
-  const char *line_start = m_data + m_line_start_idx;
-
-  const char *next_line_start = NULL;
-  size_t len = 0;
-  const char *line_end = find_end_of_line (line_start, remaining_size);
-  if (line_end == NULL)
-    {
-      /* We haven't found an end-of-line delimiter in the cache.
-	 Fill the cache with more data from the file and look again.  */
-      while (maybe_read_data ())
-	{
-	  line_start = m_data + m_line_start_idx;
-	  remaining_size = m_nb_read - m_line_start_idx;
-	  line_end = find_end_of_line (line_start, remaining_size);
-	  if (line_end != NULL)
-	    {
-	      next_line_start = line_end + 1;
-	      break;
-	    }
-	}
-      if (line_end == NULL)
-	{
-	  /* We've loaded all the file into the cache and still no
-	     terminator.  Let's say the line ends up at one byte past the
-	     end of the file.  This is to stay consistent with the case
-	     of when the line ends up with a terminator and line_end points to
-	     that.  That consistency is useful below in the len calculation.
-
-	     If the file ends in a \r, we didn't identify it as a line
-	     terminator above, so do that now instead.  */
-	  line_end = m_data + m_nb_read;
-	  if (m_nb_read && line_end[-1] == '\r')
-	    {
-	      --line_end;
-	      m_missing_trailing_newline = false;
-	    }
-	  else
-	    m_missing_trailing_newline = true;
-	}
-      else
-	m_missing_trailing_newline = false;
-    }
-  else
-    {
-      next_line_start = line_end + 1;
-      m_missing_trailing_newline = false;
-    }
-
-  if (m_error)
-    return false;
-
-  /* At this point, we've found the end of the of line.  It either points to
-     the line terminator or to one byte after the last byte of the file.  */
-  gcc_assert (line_end != NULL);
-
-  len = line_end - line_start;
-
-  if (m_line_start_idx < m_nb_read)
-    *line = const_cast<char *> (line_start);
-
-  ++m_line_num;
-
-  /* Now update our line record so that re-reading lines from the
-     before m_line_start_idx is faster.  */
-  size_t rlen = m_line_record.length ();
-  /* Only update when beyond the previously cached region.  */
-  if (rlen == 0 || m_line_record[rlen - 1].line_num < m_line_num)
-    {
-      size_t spacing
-	= (rlen >= 2
-	   ? (m_line_record[rlen - 1].line_num
-	      - m_line_record[rlen - 2].line_num) : 1);
-      size_t delta
-	= rlen >= 1 ? m_line_num - m_line_record[rlen - 1].line_num : 1;
-
-      size_t max_size = line_record_size;
-      /* One anchor per hundred input lines.  */
-      if (max_size == 0)
-	max_size = m_line_num / 100;
-
-      /* If we're too far beyond drop half of the lines to rebalance.  */
-      if (rlen == max_size && delta >= spacing * 2)
-	{
-	  size_t j = 0;
-	  for (size_t i = 1; i < rlen; i += 2)
-	    m_line_record[j++] = m_line_record[i];
-	  m_line_record.truncate (j);
-	  rlen = j;
-	  spacing *= 2;
-	}
-
-      if (rlen < max_size && delta >= spacing)
-	{
-	  file_cache_slot::line_info li (m_line_num, m_line_start_idx,
-					 line_end - m_data);
-	  m_line_record.safe_push (li);
-	}
-    }
-
-  /* Cache recent tail lines separately for fast access. This assumes
-     most accesses do not skip backwards.  */
-  if (m_line_recent_last == m_line_recent_first
-      || m_line_recent[m_line_recent_last].line_num == m_line_num - 1)
-    {
-      size_t mask = ((size_t) 1 << recent_cached_lines_shift) - 1;
-      m_line_recent_last = (m_line_recent_last + 1) & mask;
-      if (m_line_recent_last == m_line_recent_first)
-	m_line_recent_first = (m_line_recent_first + 1) & mask;
-      m_line_recent[m_line_recent_last]
-	= file_cache_slot::line_info (m_line_num, m_line_start_idx,
-				      line_end - m_data);
-    }
-
-  /* Update m_line_start_idx so that it points to the next line to be
-     read.  */
-  if (next_line_start)
-    m_line_start_idx = next_line_start - m_data;
-  else
-    /* We didn't find any terminal '\n'.  Let's consider that the end
-       of line is the end of the data in the cache.  The next
-       invocation of get_next_line will either read more data from the
-       underlying file or return false early because we've reached the
-       end of the file.  */
-    m_line_start_idx = m_nb_read;
-
-  *line_len = len;
-
-  return true;
-}
-
-/* Consume the next bytes coming from the cache (or from its
-   underlying file if there are remaining unread bytes in the file)
-   until we reach the next end-of-line (or end-of-file).  There is no
-   copying from the cache involved.  Return TRUE upon successful
-   completion.  */
-
-bool
-file_cache_slot::goto_next_line ()
-{
-  char *l;
-  ssize_t len;
-
-  return get_next_line (&l, &len);
-}
-
-/* Read an arbitrary line number LINE_NUM from the file cached in C.
-   If the line was read successfully, *LINE points to the beginning
-   of the line in the file cache and *LINE_LEN is the length of the
-   line.  *LINE is not nul-terminated, but may contain zero bytes.
-   *LINE is only valid until the next call of read_line_num.
-   This function returns bool if a line was read.  */
-
-bool
-file_cache_slot::read_line_num (size_t line_num,
-				char ** line, ssize_t *line_len)
-{
-  gcc_assert (line_num > 0);
-
-  /* Is the line in the recent line cache?
-     This assumes the main file processing is only using
-     a single contiguous cursor with only temporary excursions.  */
-  if (m_line_recent_first != m_line_recent_last
-	&& m_line_recent[m_line_recent_first].line_num <= line_num
-	&& m_line_recent[m_line_recent_last].line_num >= line_num)
-    {
-      line_info &last = m_line_recent[m_line_recent_last];
-      size_t mask = (1U << recent_cached_lines_shift) - 1;
-      size_t idx = (m_line_recent_last - (last.line_num - line_num)) & mask;
-      line_info &recent = m_line_recent[idx];
-      gcc_assert (recent.line_num == line_num);
-      *line = m_data + recent.start_pos;
-      *line_len = recent.end_pos - recent.start_pos;
-      return true;
-    }
-
-  if (line_num <= m_line_num)
-    {
-      line_info l (line_num, 0, 0);
-      int i = m_line_record.lower_bound (l, line_info::less_than);
-      if (i == 0)
-	{
-	  m_line_start_idx = 0;
-	  m_line_num = 0;
-	}
-      else if (m_line_record[i - 1].line_num == line_num)
-	{
-	  /* We have the start/end of the line.  */
-	  *line = m_data + m_line_record[i - 1].start_pos;
-	  *line_len = m_line_record[i - 1].end_pos - m_line_record[i - 1].start_pos;
-	  return true;
-	}
-      else
-       {
-	 gcc_assert (m_line_record[i - 1].line_num < m_line_num);
-	 m_line_start_idx = m_line_record[i - 1].start_pos;
-	 m_line_num = m_line_record[i - 1].line_num - 1;
-       }
-    }
-
-  /*  Let's walk from line m_line_num up to line_num - 1, without
-      copying any line.  */
-  while (m_line_num < line_num - 1)
-    if (!goto_next_line ())
-      return false;
-
-  /* The line we want is the next one.  Let's read it.  */
-  return get_next_line (line, line_len);
-}
-
-/* Return the physical source line that corresponds to FILE_PATH/LINE.
-   The line is not nul-terminated.  The returned pointer is only
-   valid until the next call of location_get_source_line.
-   Note that the line can contain several null characters,
-   so the returned value's length has the actual length of the line.
-   If the function fails, a NULL char_span is returned.  */
-
-char_span
-file_cache::get_source_line (const char *file_path, int line)
-{
-  char *buffer = NULL;
-  ssize_t len;
-
-  if (line == 0)
-    return char_span (NULL, 0);
-
-  if (file_path == NULL)
-    return char_span (NULL, 0);
-
-  file_cache_slot *c = lookup_or_add_file (file_path);
-  if (c == NULL)
-    return char_span (NULL, 0);
-
-  bool read = c->read_line_num (line, &buffer, &len);
-  if (!read)
-    return char_span (NULL, 0);
-
-  return char_span (buffer, len);
-}
-
 /* Return a NUL-terminated copy of the source text between two locations, or
    NULL if the arguments are invalid.  The caller is responsible for freeing
    the return value.  */
 
 char *
-get_source_text_between (file_cache &fc, location_t start, location_t end)
+get_source_text_between (diagnostics::file_cache &fc,
+			 location_t start, location_t end)
 {
   expanded_location expstart
     = expand_location_to_spelling_point (start, LOCATION_ASPECT_START);
@@ -1076,7 +170,8 @@ get_source_text_between (file_cache &fc, location_t start, location_t end)
   /* For a single line we need to trim both edges.  */
   if (expstart.line == expend.line)
     {
-      char_span line = fc.get_source_line (expstart.file, expstart.line);
+      diagnostics::char_span line
+	= fc.get_source_line (expstart.file, expstart.line);
       if (line.length () < 1)
 	return NULL;
       int s = expstart.column - 1;
@@ -1093,7 +188,7 @@ get_source_text_between (file_cache &fc, location_t start, location_t end)
      parts of the start and end lines off depending on column values.  */
   for (int lnum = expstart.line; lnum <= expend.line; ++lnum)
     {
-      char_span line = fc.get_source_line (expstart.file, lnum);
+      diagnostics::char_span line = fc.get_source_line (expstart.file, lnum);
       if (line.length () < 1 && (lnum != expstart.line && lnum != expend.line))
 	continue;
 
@@ -1138,16 +233,6 @@ get_source_text_between (file_cache &fc, location_t start, location_t end)
   return xstrdup (buf);
 }
 
-
-char_span
-file_cache::get_source_file_content (const char *file_path)
-{
-  file_cache_slot *c = lookup_or_add_file (file_path);
-  if (c == nullptr)
-    return char_span (nullptr, 0);
-  return c->get_full_file_content ();
-}
-
 /* Test if the location originates from the spelling location of a
    builtin-tokens.  That is, return TRUE if LOC is a (possibly
    virtual) location of a built-in token that appears in the expansion
@@ -1280,13 +365,13 @@ make_location (location_t caret, source_range src_range)
    source line in order to calculate the display width.  If that cannot be done
    for any reason, then returns the byte column as a fallback.  */
 int
-location_compute_display_column (file_cache &fc,
+location_compute_display_column (diagnostics::file_cache &fc,
 				 expanded_location exploc,
 				 const cpp_char_column_policy &policy)
 {
   if (!(exploc.file && *exploc.file && exploc.line && exploc.column))
     return exploc.column;
-  char_span line = fc.get_source_line (exploc.file, exploc.line);
+  diagnostics::char_span line = fc.get_source_line (exploc.file, exploc.line);
   /* If line is NULL, this function returns exploc.column which is the
      desired fallback.  */
   return cpp_byte_column_to_display_column (line.get_buffer (), line.length (),
@@ -1431,7 +516,7 @@ dump_labelled_location_range (FILE *stream,
 void
 dump_location_info (FILE *stream)
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
 
   /* Visualize the reserved locations.  */
   dump_labelled_location_range (stream, "RESERVED LOCATIONS",
@@ -1506,8 +591,8 @@ dump_location_info (FILE *stream)
 	    {
 	      /* Beginning of a new source line: draw the line.  */
 
-	      char_span line_text = fc.get_source_line (exploc.file,
-							exploc.line);
+	      diagnostics::char_span line_text
+		= fc.get_source_line (exploc.file, exploc.line);
 	      if (!line_text)
 		break;
 	      fprintf (stream,
@@ -1524,10 +609,10 @@ dump_location_info (FILE *stream)
 	      if (max_col > line_text.length ())
 		max_col = line_text.length () + 1;
 
-	      int len_lnum = num_digits (exploc.line);
+	      int len_lnum = diagnostics::num_digits (exploc.line);
 	      if (len_lnum < 3)
 		len_lnum = 3;
-	      int len_loc = num_digits (loc);
+	      int len_loc = diagnostics::num_digits (loc);
 	      if (len_loc < 5)
 		len_loc = 5;
 
@@ -1757,7 +842,7 @@ class auto_cpp_string_vec :  public auto_vec <cpp_string>
 
 static const char *
 get_substring_ranges_for_loc (cpp_reader *pfile,
-			      file_cache &fc,
+			      diagnostics::file_cache &fc,
 			      string_concat_db *concats,
 			      location_t strloc,
 			      enum cpp_ttype type,
@@ -1837,7 +922,7 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
       if (start.column > finish.column)
 	return "range endpoints are reversed";
 
-      char_span line = fc.get_source_line (start.file, start.line);
+      diagnostics::char_span line = fc.get_source_line (start.file, start.line);
       if (!line)
 	return "unable to read source line";
 
@@ -1852,7 +937,8 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
       if (line.length () < (start.column - 1 + literal_length))
 	return "line is not wide enough";
 
-      char_span literal = line.subspan (start.column - 1, literal_length);
+      diagnostics::char_span literal
+	= line.subspan (start.column - 1, literal_length);
 
       cpp_string from;
       from.len = literal_length;
@@ -1925,7 +1011,7 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
 
 const char *
 get_location_within_string (cpp_reader *pfile,
-			    file_cache &fc,
+			    diagnostics::file_cache &fc,
 			    string_concat_db *concats,
 			    location_t strloc,
 			    enum cpp_ttype type,
@@ -2008,7 +1094,7 @@ namespace selftest {
 
 static const char *
 get_source_range_for_char (cpp_reader *pfile,
-			   file_cache &fc,
+			   diagnostics::file_cache &fc,
 			   string_concat_db *concats,
 			   location_t strloc,
 			   enum cpp_ttype type,
@@ -2036,7 +1122,7 @@ get_source_range_for_char (cpp_reader *pfile,
 
 static const char *
 get_num_source_ranges_for_substring (cpp_reader *pfile,
-				     file_cache &fc,
+				     diagnostics::file_cache &fc,
 				     string_concat_db *concats,
 				     location_t strloc,
 				     enum cpp_ttype type,
@@ -2359,119 +1445,6 @@ test_make_location_nonpure_range_endpoints (const line_table_case &case_)
   ASSERT_FALSE (IS_ADHOC_LOC (get_finish (not_aaa_eq_bbb)));
 }
 
-/* Verify reading of a specific line LINENUM in TMP, FC.  */
-
-static void
-check_line (temp_source_file &tmp, file_cache &fc, int linenum)
-{
-  char_span line = fc.get_source_line (tmp.get_filename (), linenum);
-  int n;
-  const char *b = line.get_buffer ();
-  size_t l = line.length ();
-  char buf[5];
-  ASSERT_LT (l, 5);
-  memcpy (buf, b, l);
-  buf[l] = '\0';
-  ASSERT_TRUE (sscanf (buf, "%d", &n) == 1);
-  ASSERT_EQ (n, linenum);
-}
-
-/* Test file cache replacement.  */
-
-static void
-test_replacement ()
-{
-  const int maxline = 1000;
-
-  char *vec = XNEWVEC (char, maxline * 5);
-  char *p = vec;
-  int i;
-  for (i = 1; i <= maxline; i++)
-    p += sprintf (p, "%d\n", i);
-
-  temp_source_file tmp (SELFTEST_LOCATION, ".txt", vec);
-  free (vec);
-  file_cache fc;
-
-  for (i = 2; i <= maxline; i++)
-    {
-      check_line (tmp, fc, i);
-      check_line (tmp, fc, i - 1);
-      if (i >= 10)
-	check_line (tmp, fc, i - 9);
-      if (i >= 350) /* Exceed the look behind cache.  */
-	check_line (tmp, fc, i - 300);
-    }
-  for (i = 5; i <= maxline; i += 100)
-    check_line (tmp, fc, i);
-  for (i = 1; i <= maxline; i++)
-    check_line (tmp, fc, i);
-}
-
-/* Verify reading of input files (e.g. for caret-based diagnostics).  */
-
-static void
-test_reading_source_line ()
-{
-  /* Create a tempfile and write some text to it.  */
-  temp_source_file tmp (SELFTEST_LOCATION, ".txt",
-			"01234567890123456789\n"
-			"This is the test text\n"
-			"This is the 3rd line");
-  file_cache fc;
-
-  /* Read back a specific line from the tempfile.  */
-  char_span source_line = fc.get_source_line (tmp.get_filename (), 3);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (20, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the 3rd line",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (tmp.get_filename (), 2);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (21, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the test text",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (tmp.get_filename (), 4);
-  ASSERT_FALSE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () == NULL);
-}
-
-/* Verify reading from buffers (e.g. for sarif-replay).  */
-
-static void
-test_reading_source_buffer ()
-{
-  const char *text = ("01234567890123456789\n"
-		      "This is the test text\n"
-		      "This is the 3rd line");
-  const char *filename = "foo.txt";
-  file_cache fc;
-  fc.add_buffered_content (filename, text, strlen (text));
-
-  /* Read back a specific line from the tempfile.  */
-  char_span source_line = fc.get_source_line (filename, 3);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (20, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the 3rd line",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (filename, 2);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (21, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the test text",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (filename, 4);
-  ASSERT_FALSE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () == NULL);
-}
-
 /* Tests of lexing.  */
 
 /* Verify that token TOK from PARSER has cpp_token_as_text
@@ -2628,7 +1601,7 @@ public:
   line_table_test m_ltt;
   cpp_reader_ptr m_parser;
   temp_source_file m_tempfile;
-  file_cache m_file_cache;
+  diagnostics::file_cache m_file_cache;
   string_concat_db m_concats;
   bool m_implicitly_expect_EOF;
 };
@@ -4356,10 +3329,6 @@ input_cc_tests ()
   for_each_line_table_case (test_lexer_string_locations_raw_string_unterminated);
   for_each_line_table_case (test_lexer_char_constants);
 
-  test_reading_source_line ();
-  test_reading_source_buffer ();
-  test_replacement ();
-
   test_line_offset_overflow ();
 
   test_cpp_utf8 ();
diff --git a/gcc/input.h b/gcc/input.h
index b0a1ca0..eeef290 100644
--- a/gcc/input.h
+++ b/gcc/input.h
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "line-map.h"
 
-class file_cache;
+namespace diagnostics { class file_cache; }
 
 extern GTY(()) class line_maps *line_table;
 extern GTY(()) class line_maps *saved_line_table;
@@ -69,110 +69,12 @@ extern expanded_location expand_location (location_t);
 class cpp_char_column_policy;
 
 extern int
-location_compute_display_column (file_cache &fc,
+location_compute_display_column (diagnostics::file_cache &fc,
 				 expanded_location exploc,
 				 const cpp_char_column_policy &policy);
 
-/* A class capturing the bounds of a buffer, to allow for run-time
-   bounds-checking in a checked build.  */
-
-class char_span
-{
- public:
-  char_span (const char *ptr, size_t n_elts) : m_ptr (ptr), m_n_elts (n_elts) {}
-
-  /* Test for a non-NULL pointer.  */
-  operator bool() const { return m_ptr; }
-
-  /* Get length, not including any 0-terminator (which may not be,
-     in fact, present).  */
-  size_t length () const { return m_n_elts; }
-
-  const char *get_buffer () const { return m_ptr; }
-
-  char operator[] (int idx) const
-  {
-    gcc_assert (idx >= 0);
-    gcc_assert ((size_t)idx < m_n_elts);
-    return m_ptr[idx];
-  }
-
-  char_span subspan (int offset, int n_elts) const
-  {
-    gcc_assert (offset >= 0);
-    gcc_assert (offset < (int)m_n_elts);
-    gcc_assert (n_elts >= 0);
-    gcc_assert (offset + n_elts <= (int)m_n_elts);
-    return char_span (m_ptr + offset, n_elts);
-  }
-
-  char *xstrdup () const
-  {
-    return ::xstrndup (m_ptr, m_n_elts);
-  }
-
- private:
-  const char *m_ptr;
-  size_t m_n_elts;
-};
-
 extern char *
-get_source_text_between (file_cache &, location_t, location_t);
-
-/* Forward decl of slot within file_cache, so that the definition doesn't
-   need to be in this header.  */
-class file_cache_slot;
-
-/* A cache of source files for use when emitting diagnostics
-   (and in a few places in the C/C++ frontends).
-
-   Results are only valid until the next call to the cache, as
-   slots can be evicted.
-
-   Filenames are stored by pointer, and so must outlive the cache
-   instance.  */
-
-class file_cache
-{
- public:
-  file_cache ();
-  ~file_cache ();
-
-  void dump (FILE *out, int indent) const;
-  void DEBUG_FUNCTION dump () const;
-
-  file_cache_slot *lookup_or_add_file (const char *file_path);
-  void forcibly_evict_file (const char *file_path);
-
-  /* See comments in diagnostic.h about the input conversion context.  */
-  struct input_context
-  {
-    diagnostic_input_charset_callback ccb;
-    bool should_skip_bom;
-  };
-  void initialize_input_context (diagnostic_input_charset_callback ccb,
-				 bool should_skip_bom);
-
-  char_span get_source_file_content (const char *file_path);
-  char_span get_source_line (const char *file_path, int line);
-  bool missing_trailing_newline_p (const char *file_path);
-
-  void add_buffered_content (const char *file_path,
-			     const char *buffer,
-			     size_t sz);
-
-  void tune (size_t num_file_slots, size_t lines);
-
- private:
-  file_cache_slot *evicted_cache_tab_entry (unsigned *highest_use_count);
-  file_cache_slot *add_file (const char *file_path);
-  file_cache_slot *lookup_file (const char *file_path);
-
- private:
-  size_t m_num_file_slots;
-  file_cache_slot *m_file_slots;
-  input_context m_input_context;
-};
+get_source_text_between (diagnostics::file_cache &, location_t, location_t);
 
 extern expanded_location
 expand_location_to_spelling_point (location_t,
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 3f4ac93..bf2fac8 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -3442,25 +3442,23 @@ expand_DEFERRED_INIT (internal_fn, gcall *stmt)
 }
 
 /* Expand the IFN_ACCESS_WITH_SIZE function:
-   ACCESS_WITH_SIZE (REF_TO_OBJ, REF_TO_SIZE, CLASS_OF_SIZE,
-		     TYPE_OF_SIZE, ACCESS_MODE)
+   ACCESS_WITH_SIZE (REF_TO_OBJ, REF_TO_SIZE,
+		     TYPE_OF_SIZE + ACCESS_MODE, TYPE_SIZE_UNIT for element)
    which returns the REF_TO_OBJ same as the 1st argument;
 
    1st argument REF_TO_OBJ: The reference to the object;
    2nd argument REF_TO_SIZE: The reference to the size of the object,
-   3rd argument CLASS_OF_SIZE: The size referenced by the REF_TO_SIZE represents
-     0: the number of bytes.
-     1: the number of the elements of the object type;
-   4th argument TYPE_OF_SIZE: A constant 0 with its TYPE being the same as the TYPE
-    of the object referenced by REF_TO_SIZE
-   5th argument ACCESS_MODE:
-    -1: Unknown access semantics
-     0: none
-     1: read_only
-     2: write_only
-     3: read_write
-   6th argument: A constant 0 with the pointer TYPE to the original flexible
-     array type.
+   3rd argument TYPE_OF_SIZE + ACCESS_MODE: An integer constant with a pointer
+     TYPE.
+     The pointee TYPE of the pointer TYPE is the TYPE of the object referenced
+	by REF_TO_SIZE.
+     The integer constant value represents the ACCESS_MODE:
+	0: none
+	1: read_only
+	2: write_only
+	3: read_write
+
+   4th argument: The TYPE_SIZE_UNIT of the element TYPE of the array.
 
    Both the return type and the type of the first argument of this
    function have been converted from the incomplete array type to
@@ -3654,8 +3652,8 @@ expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
   internal_fn ifn = gimple_call_internal_fn (stmt);
   int rhs_index = internal_fn_stored_value_index (ifn);
   tree base = gimple_call_arg (stmt, 0);
-  tree offset = gimple_call_arg (stmt, 1);
-  tree scale = gimple_call_arg (stmt, 2);
+  tree offset = gimple_call_arg (stmt, internal_fn_offset_index (ifn));
+  tree scale = gimple_call_arg (stmt, internal_fn_scale_index (ifn));
   tree rhs = gimple_call_arg (stmt, rhs_index);
 
   rtx base_rtx = expand_normal (base);
@@ -3680,12 +3678,12 @@ expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB.  */
 
 static void
-expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
+expand_gather_load_optab_fn (internal_fn ifn, gcall *stmt, direct_optab optab)
 {
   tree lhs = gimple_call_lhs (stmt);
   tree base = gimple_call_arg (stmt, 0);
-  tree offset = gimple_call_arg (stmt, 1);
-  tree scale = gimple_call_arg (stmt, 2);
+  tree offset = gimple_call_arg (stmt, internal_fn_offset_index (ifn));
+  tree scale = gimple_call_arg (stmt, internal_fn_scale_index (ifn));
 
   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
   rtx base_rtx = expand_normal (base);
@@ -4031,9 +4029,14 @@ expand_crc_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab)
   rtx dest = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
   rtx crc = expand_normal (rhs1);
   rtx data = expand_normal (rhs2);
-  gcc_assert (TREE_CODE (rhs3) == INTEGER_CST);
-  rtx polynomial = gen_rtx_CONST_INT (TYPE_MODE (result_type),
-				      TREE_INT_CST_LOW (rhs3));
+  rtx polynomial;
+  if (TREE_CODE (rhs3) != INTEGER_CST)
+    {
+      error ("third argument to %<crc%> builtins must be a constant");
+      polynomial = const0_rtx;
+    }
+  else
+    polynomial = convert_to_mode (TYPE_MODE (result_type), expand_normal (rhs3), 0);
 
   /* Use target specific expansion if it exists.
      Otherwise, generate table-based CRC.  */
@@ -4423,6 +4426,7 @@ commutative_binary_fn_p (internal_fn fn)
     case IFN_ADD_OVERFLOW:
     case IFN_MUL_OVERFLOW:
     case IFN_SAT_ADD:
+    case IFN_SAT_MUL:
     case IFN_VEC_WIDEN_PLUS:
     case IFN_VEC_WIDEN_PLUS_LO:
     case IFN_VEC_WIDEN_PLUS_HI:
@@ -4544,6 +4548,33 @@ widening_fn_p (code_helper code)
     }
 }
 
+/* Return true if this CODE describes an internal_fn that returns a vector with
+   elements twice as wide as the element size of the input vectors and operates
+   on even/odd parts of the input.  */
+
+bool
+widening_evenodd_fn_p (code_helper code)
+{
+  if (!code.is_fn_code ())
+    return false;
+
+  if (!internal_fn_p ((combined_fn) code))
+    return false;
+
+  internal_fn fn = as_internal_fn ((combined_fn) code);
+  switch (fn)
+    {
+    #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
+    case IFN_##NAME##_EVEN:					  \
+    case IFN_##NAME##_ODD:					  \
+      return true;
+    #include "internal-fn.def"
+
+    default:
+      return false;
+    }
+}
+
 /* Return true if IFN_SET_EDOM is supported.  */
 
 bool
@@ -4936,11 +4967,13 @@ internal_fn_len_index (internal_fn fn)
       return 2;
 
     case IFN_MASK_LEN_SCATTER_STORE:
+      return 6;
+
     case IFN_MASK_LEN_STRIDED_LOAD:
       return 5;
 
     case IFN_MASK_LEN_GATHER_LOAD:
-      return 6;
+      return 7;
 
     case IFN_COND_LEN_FMA:
     case IFN_COND_LEN_FMS:
@@ -5044,7 +5077,7 @@ internal_fn_else_index (internal_fn fn)
 
     case IFN_MASK_GATHER_LOAD:
     case IFN_MASK_LEN_GATHER_LOAD:
-      return 5;
+      return 6;
 
     default:
       return -1;
@@ -5079,7 +5112,7 @@ internal_fn_mask_index (internal_fn fn)
     case IFN_MASK_SCATTER_STORE:
     case IFN_MASK_LEN_GATHER_LOAD:
     case IFN_MASK_LEN_SCATTER_STORE:
-      return 4;
+      return 5;
 
     case IFN_VCOND_MASK:
     case IFN_VCOND_MASK_LEN:
@@ -5104,10 +5137,11 @@ internal_fn_stored_value_index (internal_fn fn)
 
     case IFN_MASK_STORE:
     case IFN_MASK_STORE_LANES:
+      return 3;
     case IFN_SCATTER_STORE:
     case IFN_MASK_SCATTER_STORE:
     case IFN_MASK_LEN_SCATTER_STORE:
-      return 3;
+      return 4;
 
     case IFN_LEN_STORE:
       return 4;
@@ -5121,6 +5155,75 @@ internal_fn_stored_value_index (internal_fn fn)
     }
 }
 
+/* If FN has an alias pointer return its index, otherwise return -1.  */
+
+int
+internal_fn_alias_ptr_index (internal_fn fn)
+{
+  switch (fn)
+    {
+    case IFN_MASK_LOAD:
+    case IFN_MASK_LEN_LOAD:
+    case IFN_GATHER_LOAD:
+    case IFN_MASK_GATHER_LOAD:
+    case IFN_MASK_LEN_GATHER_LOAD:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+    case IFN_MASK_LEN_SCATTER_STORE:
+      return 1;
+
+    default:
+      return -1;
+    }
+}
+
+/* If FN is a gather/scatter return the index of its offset argument,
+   otherwise return -1.  */
+
+int
+internal_fn_offset_index (internal_fn fn)
+{
+  if (!internal_gather_scatter_fn_p (fn))
+    return -1;
+
+  switch (fn)
+    {
+    case IFN_GATHER_LOAD:
+    case IFN_MASK_GATHER_LOAD:
+    case IFN_MASK_LEN_GATHER_LOAD:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+    case IFN_MASK_LEN_SCATTER_STORE:
+      return 2;
+
+    default:
+      return -1;
+    }
+}
+
+/* If FN is a gather/scatter return the index of its scale argument,
+   otherwise return -1.  */
+
+int
+internal_fn_scale_index (internal_fn fn)
+{
+  if (!internal_gather_scatter_fn_p (fn))
+    return -1;
+
+  switch (fn)
+    {
+    case IFN_GATHER_LOAD:
+    case IFN_MASK_GATHER_LOAD:
+    case IFN_MASK_LEN_GATHER_LOAD:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+    case IFN_MASK_LEN_SCATTER_STORE:
+      return 3;
+
+    default:
+      return -1;
+    }
+}
 
 /* Store all supported else values for the optab referred to by ICODE
    in ELSE_VALS.  The index of the else operand must be specified in
@@ -5199,13 +5302,9 @@ internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
     && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
     && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale));
 
-  /* For gather the optab's operand indices do not match the IFN's because
-     the latter does not have the extension operand (operand 3).  It is
-     implicitly added during expansion so we use the IFN's else index + 1.
-     */
   if (ok && elsvals)
     get_supported_else_vals
-      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD) + 1, *elsvals);
+      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD), *elsvals);
 
   return ok;
 }
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 8edfa35..d2480a1 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -282,6 +282,7 @@ DEF_INTERNAL_SIGNED_OPTAB_FN (MULHRS, ECF_CONST | ECF_NOTHROW, first,
 
 DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_ADD, ECF_CONST, first, ssadd, usadd, binary)
 DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_SUB, ECF_CONST, first, sssub, ussub, binary)
+DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_MUL, ECF_CONST, first, ssmul, usmul, binary)
 
 DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_TRUNC, ECF_CONST, first, sstrunc, ustrunc, unary_convert)
 
@@ -524,7 +525,7 @@ DEF_INTERNAL_FN (DEFERRED_INIT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
 
 /* A function to associate the access size and access mode information
    with the corresponding reference to an object.  It only reads from the
-   2nd argument.  */
+   2nd and the 4th arguments.  */
 DEF_INTERNAL_FN (ACCESS_WITH_SIZE, ECF_PURE | ECF_LEAF | ECF_NOTHROW, NULL)
 
 /* DIM_SIZE and DIM_POS return the size of a particular compute
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index afd4f8e..fd21694 100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -219,6 +219,7 @@ extern bool commutative_ternary_fn_p (internal_fn);
 extern int first_commutative_argument (internal_fn);
 extern bool associative_binary_fn_p (internal_fn);
 extern bool widening_fn_p (code_helper);
+extern bool widening_evenodd_fn_p (code_helper);
 
 extern bool set_edom_supported_p (void);
 
@@ -239,6 +240,9 @@ extern int internal_fn_mask_index (internal_fn);
 extern int internal_fn_len_index (internal_fn);
 extern int internal_fn_else_index (internal_fn);
 extern int internal_fn_stored_value_index (internal_fn);
+extern int internal_fn_offset_index (internal_fn fn);
+extern int internal_fn_scale_index (internal_fn fn);
+extern int internal_fn_alias_ptr_index (internal_fn fn);
 extern bool internal_gather_scatter_fn_supported_p (internal_fn, tree,
 						    tree, tree, int,
 						    vec<int> * = nullptr);
diff --git a/gcc/ipa-devirt.cc b/gcc/ipa-devirt.cc
index 532e25e..18cb5a8 100644
--- a/gcc/ipa-devirt.cc
+++ b/gcc/ipa-devirt.cc
@@ -1763,7 +1763,7 @@ add_type_duplicate (odr_type val, tree type)
 	      }
 	    /* One base is polymorphic and the other not.
 	       This ought to be diagnosed earlier, but do not ICE in the
-	       checking bellow.  */
+	       checking below.  */
 	    else if (TYPE_BINFO (type1)
 		     && polymorphic_type_binfo_p (TYPE_BINFO (type1))
 		        != polymorphic_type_binfo_p (TYPE_BINFO (type2)))
diff --git a/gcc/ipa-fnsummary.cc b/gcc/ipa-fnsummary.cc
index 48343a7..924a54b 100644
--- a/gcc/ipa-fnsummary.cc
+++ b/gcc/ipa-fnsummary.cc
@@ -3421,6 +3421,21 @@ compute_fn_summary (struct cgraph_node *node, bool early)
 	 info->inlinable = tree_inlinable_function_p (node->decl);
 
        bool no_signature = false;
+
+       /* Don't allow signature changes for functions which have
+	  [[gnu::musttail]] or [[clang::musttail]] calls.  Sometimes
+	  (more often on targets which pass everything on the stack)
+	  signature changes can result in tail calls being impossible
+	  even when without the signature changes they would be ok.
+	  See PR121023.  */
+       if (cfun->has_musttail)
+	 {
+	   if (dump_file)
+	    fprintf (dump_file, "No signature change:"
+		     " function has calls with musttail attribute.\n");
+	   no_signature = true;
+	 }
+
        /* Type attributes can use parameter indices to describe them.
 	  Special case fn spec since we can safely preserve them in
 	  modref summaries.  */
diff --git a/gcc/ipa-inline-transform.cc b/gcc/ipa-inline-transform.cc
index 07a1024..9d759d2 100644
--- a/gcc/ipa-inline-transform.cc
+++ b/gcc/ipa-inline-transform.cc
@@ -331,7 +331,7 @@ inline_call (struct cgraph_edge *e, bool update_original,
   int estimated_growth = 0;
   if (! update_overall_summary)
     estimated_growth = estimate_edge_growth (e);
-  /* This is used only for assert bellow.  */
+  /* This is used only for assert below.  */
 #if 0
   bool predicated = inline_edge_summary (e)->predicate != NULL;
 #endif
diff --git a/gcc/ipa-inline.cc b/gcc/ipa-inline.cc
index ca605b0..0cf97a80 100644
--- a/gcc/ipa-inline.cc
+++ b/gcc/ipa-inline.cc
@@ -2222,6 +2222,7 @@ inline_small_functions (void)
 
   gcc_assert (in_lto_p
 	      || !(max_count > 0)
+	      || flag_auto_profile
 	      || (profile_info && flag_branch_probabilities));
 
   while (!edge_heap.empty ())
diff --git a/gcc/ipa-polymorphic-call.cc b/gcc/ipa-polymorphic-call.cc
index 7b105f1..09316de 100644
--- a/gcc/ipa-polymorphic-call.cc
+++ b/gcc/ipa-polymorphic-call.cc
@@ -1353,7 +1353,7 @@ record_known_type (struct type_change_info *tci, tree type, HOST_WIDE_INT offset
 
   /* If we found a constructor of type that is not polymorphic or
      that may contain the type in question as a field (not as base),
-     restrict to the inner class first to make type matching bellow
+     restrict to the inner class first to make type matching below
      happier.  */
   if (type
       && (offset
diff --git a/gcc/ipa-pure-const.cc b/gcc/ipa-pure-const.cc
index 93297ed..2519fb8 100644
--- a/gcc/ipa-pure-const.cc
+++ b/gcc/ipa-pure-const.cc
@@ -200,7 +200,7 @@ function_always_visible_to_compiler_p (tree decl)
    by the function.  */
 
 static hash_set<tree> *
-suggest_attribute (diagnostic_option_id option, tree decl, bool known_finite,
+suggest_attribute (diagnostics::option_id option, tree decl, bool known_finite,
 		   hash_set<tree> *warned_about,
 		   const char * attrib_name)
 {
diff --git a/gcc/ipa-reference.cc b/gcc/ipa-reference.cc
index 2d8e62f..975341c 100644
--- a/gcc/ipa-reference.cc
+++ b/gcc/ipa-reference.cc
@@ -732,7 +732,7 @@ get_read_write_all_from_node (struct cgraph_node *node,
 /* Skip edges from and to nodes without ipa_reference enabled.
    Ignore not available symbols.  This leave
    them out of strongly connected components and makes them easy to skip in the
-   propagation loop bellow.  */
+   propagation loop below.  */
 
 static bool
 ignore_edge_p (cgraph_edge *e)
diff --git a/gcc/ipa-strub.cc b/gcc/ipa-strub.cc
index d7c0a92..86270af 100644
--- a/gcc/ipa-strub.cc
+++ b/gcc/ipa-strub.cc
@@ -3059,6 +3059,8 @@ pass_ipa_strub::execute (function *)
 			 TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl)));
 	}
     }
+#else
+    (void) named_args;
 #endif
 
     {
diff --git a/gcc/ira.cc b/gcc/ira.cc
index 979a034..4eebc9c 100644
--- a/gcc/ira.cc
+++ b/gcc/ira.cc
@@ -2113,6 +2113,87 @@ ira_get_dup_out_num (int op_num, alternative_mask alts,
 
 
 
+/* Return true if a replacement of SRC by DEST does not lead to unsatisfiable
+   asm.  A replacement is valid if SRC or DEST are not constrained in asm
+   inputs of a single asm statement.  See match_asm_constraints_2() for more
+   details.  TODO: As in match_asm_constraints_2() consider alternatives more
+   precisely.  */
+
+static bool
+valid_replacement_for_asm_input_p_1 (const_rtx asmops, const_rtx src, const_rtx dest)
+{
+  int ninputs = ASM_OPERANDS_INPUT_LENGTH (asmops);
+  rtvec inputs = ASM_OPERANDS_INPUT_VEC (asmops);
+  for (int i = 0; i < ninputs; ++i)
+    {
+      rtx input_src = RTVEC_ELT (inputs, i);
+      const char *constraint_src
+	= ASM_OPERANDS_INPUT_CONSTRAINT (asmops, i);
+      if (rtx_equal_p (input_src, src)
+	  && strchr (constraint_src, '{') != nullptr)
+	for (int j = 0; j < ninputs; ++j)
+	  {
+	    rtx input_dest = RTVEC_ELT (inputs, j);
+	    const char *constraint_dest
+	      = ASM_OPERANDS_INPUT_CONSTRAINT (asmops, j);
+	    if (rtx_equal_p (input_dest, dest)
+		&& strchr (constraint_dest, '{') != nullptr)
+	      return false;
+	  }
+    }
+  return true;
+}
+
+/* Return true if a replacement of SRC by DEST does not lead to unsatisfiable
+   asm.  A replacement is valid if SRC or DEST are not constrained in asm
+   inputs of a single asm statement.  The final check is done in function
+   valid_replacement_for_asm_input_p_1.  */
+
+static bool
+valid_replacement_for_asm_input_p (const_rtx src, const_rtx dest)
+{
+  /* Bail out early if there is no asm statement.  */
+  if (!crtl->has_asm_statement)
+    return true;
+  for (df_ref use = DF_REG_USE_CHAIN (REGNO (src));
+       use;
+       use = DF_REF_NEXT_REG (use))
+    {
+      struct df_insn_info *use_info = DF_REF_INSN_INFO (use);
+      /* Only check real uses, not artificial ones.  */
+      if (use_info)
+	{
+	  rtx_insn *insn = DF_REF_INSN (use);
+	  rtx pat = PATTERN (insn);
+	  if (asm_noperands (pat) <= 0)
+	    continue;
+	  if (GET_CODE (pat) == SET)
+	    {
+	      if (!valid_replacement_for_asm_input_p_1 (SET_SRC (pat), src, dest))
+		return false;
+	    }
+	  else if (GET_CODE (pat) == PARALLEL)
+	    for (int i = 0, len = XVECLEN (pat, 0); i < len; ++i)
+	      {
+		rtx asmops = XVECEXP (pat, 0, i);
+		if (GET_CODE (asmops) == SET)
+		  asmops = SET_SRC (asmops);
+		if (GET_CODE (asmops) == ASM_OPERANDS
+		    && !valid_replacement_for_asm_input_p_1 (asmops, src, dest))
+		  return false;
+	      }
+	  else if (GET_CODE (pat) == ASM_OPERANDS)
+	    {
+	      if (!valid_replacement_for_asm_input_p_1 (pat, src, dest))
+		return false;
+	    }
+	  else
+	    gcc_unreachable ();
+	}
+    }
+  return true;
+}
+
 /* Search forward to see if the source register of a copy insn dies
    before either it or the destination register is modified, but don't
    scan past the end of the basic block.  If so, we can replace the
@@ -2162,7 +2243,8 @@ decrease_live_ranges_number (void)
 	       auto-inc memory reference, so we must disallow this
 	       optimization on them.  */
 	    || sregno == STACK_POINTER_REGNUM
-	    || dregno == STACK_POINTER_REGNUM)
+	    || dregno == STACK_POINTER_REGNUM
+	    || !valid_replacement_for_asm_input_p (src, dest))
 	  continue;
 
 	dest_death = NULL_RTX;
diff --git a/gcc/jit/ChangeLog b/gcc/jit/ChangeLog
index adee3de..da88cc6 100644
--- a/gcc/jit/ChangeLog
+++ b/gcc/jit/ChangeLog
@@ -1,3 +1,30 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update usage of "diagnostic_info" to
+	explicitly refer to "diagnostics::diagnostic_info".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+	* jit-playback.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* jit-playback.cc: Update for diagnostic_info moving into
+	namespace diagnostics.
+	* jit-playback.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update for move of diagnostics output formats
+	into namespace "diagnostics" as "sinks".
+
 2025-04-28  David Malcolm  <dmalcolm@redhat.com>
 
 	* dummy-frontend.cc: Drop include of "make-unique.h".
diff --git a/gcc/jit/dummy-frontend.cc b/gcc/jit/dummy-frontend.cc
index bf31a9d..3ffca92 100644
--- a/gcc/jit/dummy-frontend.cc
+++ b/gcc/jit/dummy-frontend.cc
@@ -32,7 +32,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "attribs.h"
 #include "cgraph.h"
 #include "target.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 #include "print-tree.h"
 
 #include <mpfr.h>
@@ -993,16 +993,16 @@ struct ggc_root_tab jit_root_tab[] =
     LAST_GGC_ROOT_TAB
   };
 
-/* Subclass of diagnostic_output_format for libgccjit: like text
+/* Subclass of diagnostics::text_sink for libgccjit: like text
    output, but capture the message and call add_diagnostic with it
    on the active playback context.  */
 
-class jit_diagnostic_listener : public diagnostic_text_output_format
+class jit_diagnostic_listener : public diagnostics::text_sink
 {
 public:
-  jit_diagnostic_listener (diagnostic_context &dc,
+  jit_diagnostic_listener (diagnostics::context &dc,
 			   gcc::jit::playback::context &playback_ctxt)
-  : diagnostic_text_output_format (dc),
+  : diagnostics::text_sink (dc),
     m_playback_ctxt (playback_ctxt)
   {
   }
@@ -1015,13 +1015,13 @@ public:
 	     (void *)&m_playback_ctxt);
   }
 
-  void on_report_diagnostic (const diagnostic_info &info,
-			     diagnostic_t orig_diag_kind) final override
+  void on_report_diagnostic (const diagnostics::diagnostic_info &info,
+			     enum diagnostics::kind orig_diag_kind) final override
   {
     JIT_LOG_SCOPE (gcc::jit::active_playback_ctxt->get_logger ());
 
     /* Let the text output format do most of the work.  */
-    diagnostic_text_output_format::on_report_diagnostic (info, orig_diag_kind);
+    diagnostics::text_sink::on_report_diagnostic (info, orig_diag_kind);
 
     const char *text = pp_formatted_text (get_printer ());
 
@@ -1041,8 +1041,8 @@ private:
 /* Implementation of "begin_diagnostic".  */
 
 static void
-jit_begin_diagnostic (diagnostic_text_output_format &,
-		      const diagnostic_info */*diagnostic*/)
+jit_begin_diagnostic (diagnostics::text_sink &,
+		      const diagnostics::diagnostic_info */*diagnostic*/)
 {
   gcc_assert (gcc::jit::active_playback_ctxt);
   JIT_LOG_SCOPE (gcc::jit::active_playback_ctxt->get_logger ());
@@ -1054,9 +1054,9 @@ jit_begin_diagnostic (diagnostic_text_output_format &,
 /* Implementation of "end_diagnostic".  */
 
 static void
-jit_end_diagnostic (diagnostic_text_output_format &,
-		    const diagnostic_info *,
-		    diagnostic_t)
+jit_end_diagnostic (diagnostics::text_sink &,
+		    const diagnostics::diagnostic_info *,
+		    enum diagnostics::kind)
 {
   gcc_assert (gcc::jit::active_playback_ctxt);
   JIT_LOG_SCOPE (gcc::jit::active_playback_ctxt->get_logger ());
@@ -1081,13 +1081,13 @@ jit_langhook_init (void)
     }
 
   gcc_assert (global_dc);
-  diagnostic_text_starter (global_dc) = jit_begin_diagnostic;
-  diagnostic_text_finalizer (global_dc) = jit_end_diagnostic;
+  diagnostics::text_starter (global_dc) = jit_begin_diagnostic;
+  diagnostics::text_finalizer (global_dc) = jit_end_diagnostic;
   auto sink
     = std::make_unique<jit_diagnostic_listener>
 	(*global_dc,
 	 *gcc::jit::active_playback_ctxt);
-  global_dc->set_output_format (std::move (sink));
+  global_dc->set_sink (std::move (sink));
 
   build_common_tree_nodes (flag_signed_char);
 
diff --git a/gcc/jit/jit-playback.cc b/gcc/jit/jit-playback.cc
index 6946f10..4473d8b 100644
--- a/gcc/jit/jit-playback.cc
+++ b/gcc/jit/jit-playback.cc
@@ -3892,7 +3892,7 @@ add_error_va (location *loc, const char *fmt, va_list ap)
 void
 playback::context::
 add_diagnostic (const char *text,
-		const diagnostic_info &diagnostic)
+		const diagnostics::diagnostic_info &diagnostic)
 {
   /* Get location information (if any) from the diagnostic.
      The recording::context::add_error[_va] methods require a
diff --git a/gcc/jit/jit-playback.h b/gcc/jit/jit-playback.h
index e9a5c38..b0625dc 100644
--- a/gcc/jit/jit-playback.h
+++ b/gcc/jit/jit-playback.h
@@ -30,8 +30,11 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "jit-recording.h"
 
-class diagnostic_context;
-struct diagnostic_info;
+namespace diagnostics
+{
+  class context;
+  struct diagnostic_info;
+}
 
 namespace gcc {
 
@@ -298,7 +301,7 @@ public:
 
   void
   add_diagnostic (const char *text,
-		  const diagnostic_info &diagnostic);
+		  const diagnostics::diagnostic_info &diagnostic);
 
   void
   set_tree_location (tree t, location *loc);
diff --git a/gcc/json.cc b/gcc/json.cc
index f3f3645..7153f08 100644
--- a/gcc/json.cc
+++ b/gcc/json.cc
@@ -289,6 +289,30 @@ object::print (pretty_printer *pp, bool formatted) const
   pp_character (pp, '}');
 }
 
+std::unique_ptr<value>
+object::clone () const
+{
+  return clone_as_object ();
+}
+
+std::unique_ptr<object>
+object::clone_as_object () const
+{
+  auto result = std::make_unique<object> ();
+
+  /* Iterate in the order that the keys were inserted.  */
+  unsigned i;
+  const char *key;
+  FOR_EACH_VEC_ELT (m_keys, i, key)
+    {
+      map_t &mut_map = const_cast<map_t &> (m_map);
+      value *value = *mut_map.get (key);
+      result->set (key, value->clone ());
+    }
+
+  return result;
+}
+
 /* Set the json::value * for KEY, taking ownership of V
    (and taking a copy of KEY if necessary).  */
 
@@ -443,6 +467,17 @@ array::print (pretty_printer *pp, bool formatted) const
   pp_character (pp, ']');
 }
 
+std::unique_ptr<value>
+array::clone () const
+{
+  auto result = std::make_unique<array> ();
+  unsigned i;
+  value *v;
+  FOR_EACH_VEC_ELT (m_elements, i, v)
+    result->append (v->clone ());
+  return result;
+}
+
 /* Append non-NULL value V to a json::array, taking ownership of V.  */
 
 void
@@ -473,6 +508,12 @@ float_number::print (pretty_printer *pp,
   pp_string (pp, tmp);
 }
 
+std::unique_ptr<value>
+float_number::clone () const
+{
+  return std::make_unique<float_number> (m_value);
+}
+
 /* class json::integer_number, a subclass of json::value, wrapping a long.  */
 
 /* Implementation of json::value::print for json::integer_number.  */
@@ -486,6 +527,11 @@ integer_number::print (pretty_printer *pp,
   pp_string (pp, tmp);
 }
 
+std::unique_ptr<value>
+integer_number::clone () const
+{
+  return std::make_unique<integer_number> (m_value);
+}
 
 /* class json::string, a subclass of json::value.  */
 
@@ -501,9 +547,10 @@ string::string (const char *utf8)
 string::string (const char *utf8, size_t len)
 {
   gcc_assert (utf8);
-  m_utf8 = XNEWVEC (char, len);
+  m_utf8 = XNEWVEC (char, len + 1);
   m_len = len;
   memcpy (m_utf8, utf8, len);
+  m_utf8[len] = '\0';
 }
 
 /* Implementation of json::value::print for json::string.  */
@@ -515,6 +562,12 @@ string::print (pretty_printer *pp,
   print_escaped_json_string (pp, m_utf8, m_len);
 }
 
+std::unique_ptr<value>
+string::clone () const
+{
+  return std::make_unique<string> (m_utf8, m_len);
+}
+
 /* class json::literal, a subclass of json::value.  */
 
 /* Implementation of json::value::print for json::literal.  */
@@ -539,6 +592,12 @@ literal::print (pretty_printer *pp,
     }
 }
 
+std::unique_ptr<value>
+literal::clone () const
+{
+  return std::make_unique<literal> (m_kind);
+}
+
 
 #if CHECKING_P
 
@@ -914,6 +973,65 @@ test_comparisons ()
   ASSERT_JSON_NE (arr_1, arr_2);
 }
 
+/* Ensure that json::string's get_string is usable as a C-style string.  */
+
+static void
+test_strcmp ()
+{
+  string str ("foobar", 3);
+  ASSERT_EQ (strcmp (str.get_string (), "foo"), 0);
+}
+
+static void
+test_cloning ()
+{
+  // Objects
+  {
+    object obj;
+    obj.set_string ("foo", "bar");
+
+    auto obj_clone = obj.clone ();
+    ASSERT_JSON_EQ (obj, *obj_clone);
+  }
+
+  // Arrays
+  {
+    array arr;
+    arr.append (std::make_unique<string> ("foo"));
+
+    auto arr_clone = arr.clone ();
+    ASSERT_JSON_EQ (arr, *arr_clone);
+  }
+
+  // float_number
+  {
+    float_number f_one (1.0);
+    auto f_clone = f_one.clone ();
+    ASSERT_JSON_EQ (f_one, *f_clone);
+  }
+
+  // integer_number
+  {
+    integer_number num (42);
+    auto num_clone = num.clone ();
+    ASSERT_JSON_EQ (num, *num_clone);
+  }
+
+  // string
+  {
+    string str ("foo");
+    auto str_clone = str.clone ();
+    ASSERT_JSON_EQ (str, *str_clone);
+  }
+
+  // literal
+  {
+    literal lit (JSON_TRUE);
+    auto lit_clone = lit.clone ();
+    ASSERT_JSON_EQ (lit, *lit_clone);
+  }
+}
+
 /* Run all of the selftests within this file.  */
 
 void
@@ -928,6 +1046,8 @@ json_cc_tests ()
   test_writing_literals ();
   test_formatting ();
   test_comparisons ();
+  test_strcmp ();
+  test_cloning ();
 }
 
 } // namespace selftest
diff --git a/gcc/json.h b/gcc/json.h
index da4da85..c706f2a 100644
--- a/gcc/json.h
+++ b/gcc/json.h
@@ -124,11 +124,13 @@ class value
   virtual ~value () {}
   virtual enum kind get_kind () const = 0;
   virtual void print (pretty_printer *pp, bool formatted) const = 0;
+  virtual std::unique_ptr<value> clone () const = 0;
 
   void dump (FILE *, bool formatted) const;
   void DEBUG_FUNCTION dump () const;
 
   virtual object *dyn_cast_object () { return nullptr; }
+  virtual string *dyn_cast_string () { return nullptr; }
 
   static int compare (const json::value &val_a, const json::value &val_b);
 
@@ -150,6 +152,7 @@ class object : public value
 
   enum kind get_kind () const final override { return JSON_OBJECT; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   object *dyn_cast_object () final override { return this; }
 
@@ -182,6 +185,11 @@ class object : public value
 
   static int compare (const json::object &obj_a, const json::object &obj_b);
 
+  size_t get_num_keys () const { return m_keys.length (); }
+  const char *get_key (size_t i) const { return m_keys[i]; }
+
+  std::unique_ptr<object> clone_as_object () const;
+
  private:
   typedef hash_map <char *, value *,
     simple_hashmap_traits<nofree_string_hash, value *> > map_t;
@@ -200,6 +208,7 @@ class array : public value
 
   enum kind get_kind () const final override { return JSON_ARRAY; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   void append (value *v);
   void append_string (const char *utf8_value);
@@ -241,6 +250,7 @@ class float_number : public value
 
   enum kind get_kind () const final override { return JSON_FLOAT; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   double get () const { return m_value; }
 
@@ -257,6 +267,7 @@ class integer_number : public value
 
   enum kind get_kind () const final override { return JSON_INTEGER; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   long get () const { return m_value; }
 
@@ -276,6 +287,8 @@ class string : public value
 
   enum kind get_kind () const final override { return JSON_STRING; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
+  string *dyn_cast_string () final override { return this; }
 
   const char *get_string () const { return m_utf8; }
   size_t get_length () const { return m_len; }
@@ -298,6 +311,7 @@ class literal : public value
 
   enum kind get_kind () const final override { return m_kind; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
  private:
   enum kind m_kind;
diff --git a/gcc/langhooks-def.h b/gcc/langhooks-def.h
index 010aaed..518509a 100644
--- a/gcc/langhooks-def.h
+++ b/gcc/langhooks-def.h
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "hooks.h"
 
-struct diagnostic_info;
+namespace diagnostics { struct diagnostic_info; }
 class substring_loc;
 
 /* Note to creators of new hooks:
@@ -49,9 +49,9 @@ extern void lhd_print_tree_nothing (FILE *, tree, int);
 extern const char *lhd_decl_printable_name (tree, int);
 extern const char *lhd_dwarf_name (tree, int);
 extern int lhd_types_compatible_p (tree, tree);
-extern void lhd_print_error_function (diagnostic_text_output_format &,
+extern void lhd_print_error_function (diagnostics::text_sink &,
 				      const char *,
-				      const struct diagnostic_info *);
+				      const diagnostics::diagnostic_info *);
 extern void lhd_set_decl_assembler_name (tree decl);
 extern void lhd_overwrite_decl_assembler_name (tree decl, tree name);
 extern bool lhd_warn_unused_global_decl (const_tree);
@@ -71,7 +71,7 @@ extern tree lhd_builtin_function (tree);
 extern tree lhd_enum_underlying_base_type (const_tree);
 
 /* Declarations of default tree inlining hooks.  */
-extern void lhd_initialize_diagnostics (diagnostic_context *);
+extern void lhd_initialize_diagnostics (diagnostics::context *);
 extern void lhd_init_options (unsigned int,
 			      struct cl_decoded_option *);
 extern bool lhd_complain_wrong_lang_p (const struct cl_option *);
diff --git a/gcc/langhooks.cc b/gcc/langhooks.cc
index 194701f..20d27a6 100644
--- a/gcc/langhooks.cc
+++ b/gcc/langhooks.cc
@@ -38,7 +38,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "stor-layout.h"
 #include "cgraph.h"
 #include "debug.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 /* Do nothing; in many cases the default hook.  */
 
@@ -330,7 +330,7 @@ global_decl_processing (void)
 
 /* Called to perform language-specific initialization of CTX.  */
 void
-lhd_initialize_diagnostics (diagnostic_context *ctx ATTRIBUTE_UNUSED)
+lhd_initialize_diagnostics (diagnostics::context *ctx ATTRIBUTE_UNUSED)
 {
 }
 
@@ -369,11 +369,11 @@ lhd_handle_option (size_t code ATTRIBUTE_UNUSED,
 /* The default function to print out name of current function that caused
    an error.  */
 void
-lhd_print_error_function (diagnostic_text_output_format &text_output,
+lhd_print_error_function (diagnostics::text_sink &text_output,
 			  const char *file,
-			  const diagnostic_info *diagnostic)
+			  const diagnostics::diagnostic_info *diagnostic)
 {
-  diagnostic_context *const context = &text_output.get_context ();
+  diagnostics::context *const context = &text_output.get_context ();
   if (diagnostic_last_function_changed (context, diagnostic))
     {
       pretty_printer *const pp = text_output.get_printer ();
diff --git a/gcc/langhooks.h b/gcc/langhooks.h
index cb03c83..3bf4152 100644
--- a/gcc/langhooks.h
+++ b/gcc/langhooks.h
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 /* FIXME: This file should be #include-d after tree.h (for enum tree_code).  */
 
-struct diagnostic_info;
+namespace diagnostics { struct diagnostic_info; }
 
 struct gimplify_omp_ctx;
 
@@ -421,7 +421,7 @@ struct lang_hooks
 
   /* Callback used to perform language-specific initialization for the
      global diagnostic context structure.  */
-  void (*initialize_diagnostics) (diagnostic_context *);
+  void (*initialize_diagnostics) (diagnostics::context *);
 
   /* Beginning the main source file.  */
   void (*preprocess_main_file) (cpp_reader *, line_maps *,
@@ -547,9 +547,9 @@ struct lang_hooks
 
   /* Called by diagnostic_report_current_function to print out function name
      for textual diagnostic output.  */
-  void (*print_error_function) (diagnostic_text_output_format &,
+  void (*print_error_function) (diagnostics::text_sink &,
 				const char *,
-				const struct diagnostic_info *);
+				const diagnostics::diagnostic_info *);
 
   /* Convert a character from the host's to the target's character
      set.  The character should be in what C calls the "basic source
diff --git a/gcc/lazily-created.h b/gcc/lazily-created.h
new file mode 100644
index 0000000..a6010ca
--- /dev/null
+++ b/gcc/lazily-created.h
@@ -0,0 +1,51 @@
+/* Template for deferring object creation until the object is needed.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_LAZILY_CREATED_H
+#define GCC_LAZILY_CREATED_H
+
+/* A template for deferring object creation for a type T until the object
+   is needed, to avoid potentially expensive creation of T unless it's
+   actually needed (e.g. for directed graphs associated with a diagnostic,
+   which are ignored by the default "text" sink).  */
+
+template <typename T>
+class lazily_created
+{
+ public:
+  virtual ~lazily_created () {}
+
+  const T &
+  get_or_create () const
+  {
+    if (!m_object)
+      m_object = create_object ();
+    gcc_assert (m_object);
+    return *m_object;
+  }
+
+private:
+  virtual std::unique_ptr<T>
+  create_object () const = 0;
+
+  mutable std::unique_ptr<T> m_object;
+};
+
+#endif /* ! GCC_LAZILY_CREATED_H */
diff --git a/gcc/libgdiagnostics++.h b/gcc/libgdiagnostics++.h
index 93b8f90..fc20398 100644
--- a/gcc/libgdiagnostics++.h
+++ b/gcc/libgdiagnostics++.h
@@ -34,6 +34,10 @@ class execution_path;
 class group;
 class manager;
 class diagnostic;
+class graph;
+class node;
+class edge;
+class message_buffer;
 
 /* Wrapper around a borrowed diagnostic_text_sink *.  */
 
@@ -131,6 +135,65 @@ public:
   const diagnostic_logical_location *m_inner;
 };
 
+/* Wrapper around a diagnostic_message_buffer *, with ownership.  */
+
+class message_buffer
+{
+public:
+  message_buffer () : m_inner (nullptr) {}
+  message_buffer (diagnostic_message_buffer *inner) : m_inner (inner) {}
+  ~message_buffer ()
+  {
+    if (m_inner)
+      diagnostic_message_buffer_release (m_inner);
+  }
+  message_buffer (const message_buffer &) = delete;
+  message_buffer (message_buffer &&other)
+  {
+    m_inner = other.m_inner;
+    other.m_inner = nullptr;
+  }
+  message_buffer& operator= (const message_buffer &) = delete;
+  message_buffer& operator= (message_buffer &&other)
+  {
+    if (m_inner)
+      diagnostic_message_buffer_release (m_inner);
+    m_inner = other.m_inner;
+    other.m_inner = nullptr;
+    return *this;
+  }
+
+  message_buffer&
+  operator+= (const char *str)
+  {
+    diagnostic_message_buffer_append_str (m_inner, str);
+    return *this;
+  }
+
+  message_buffer&
+  operator+= (char ch)
+  {
+    diagnostic_message_buffer_append_byte (m_inner, ch);
+    return *this;
+  }
+
+  message_buffer &
+  begin_url (const char *url)
+  {
+    diagnostic_message_buffer_begin_url (m_inner, url);
+    return *this;
+  }
+
+  message_buffer &
+  end_url ()
+  {
+    diagnostic_message_buffer_end_url (m_inner);
+    return *this;
+  }
+
+  diagnostic_message_buffer *m_inner;
+};
+
 /* RAII class around a diagnostic_execution_path *.  */
 
 class execution_path
@@ -188,6 +251,12 @@ public:
 		va_list *args)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (5, 0);
 
+  diagnostic_event_id
+  add_event_via_msg_buf (physical_location physical_loc,
+			 logical_location logical_loc,
+			 unsigned stack_depth,
+			 message_buffer &&msg_buf);
+
   diagnostic_execution_path *m_inner;
   bool m_owned;
 };
@@ -228,6 +297,10 @@ public:
 			   const char *text);
 
   void
+  add_location_with_label (physical_location loc,
+			   message_buffer &&text);
+
+  void
   set_logical_location (logical_location loc);
 
   void
@@ -246,6 +319,9 @@ public:
   take_execution_path (execution_path path);
 
   void
+  take_graph (graph g);
+
+  void
   finish (const char *fmt, ...)
     LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (2, 3);
@@ -255,6 +331,9 @@ public:
     LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (2, 0);
 
+  void
+  finish_via_msg_buf (message_buffer &&msg_buf);
+
   ::diagnostic * const m_inner;
 };
 
@@ -395,10 +474,108 @@ public:
   void
   set_analysis_target (file f);
 
+  void
+  take_global_graph (graph g);
+
+  void
+  set_debug_physical_locations (bool value);
+
   diagnostic_manager *m_inner;
   bool m_owned;
 };
 
+class graph
+{
+public:
+  graph () : m_inner (nullptr), m_owned (false) {}
+
+  graph (diagnostic_graph *graph)
+  : m_inner (graph), m_owned (true)
+  {}
+
+  graph (const diagnostic_graph *graph)
+  : m_inner (const_cast<diagnostic_graph *> (graph)),
+    m_owned (false)
+  {}
+
+  graph (const graph &other) = delete;
+  graph &operator= (const graph &other) = delete;
+
+  graph (graph &&other)
+  : m_inner (other.m_inner),
+    m_owned (other.m_owned)
+  {
+    other.m_inner = nullptr;
+    other.m_owned = false;
+  }
+
+  graph &operator= (graph &&other)
+  {
+    m_inner = other.m_inner;
+    m_owned = other.m_owned;
+    other.m_inner = nullptr;
+    other.m_owned = false;
+    return *this;
+  }
+
+  ~graph ()
+  {
+    if (m_owned)
+      diagnostic_graph_release (m_inner);
+  }
+
+  void
+  set_description (const char *);
+  void
+  set_description (message_buffer &&);
+
+  node
+  get_node_by_id (const char *id) const;
+
+  edge
+  get_edge_by_id (const char *id) const;
+
+  edge
+  add_edge (const char *id, node src_node, node dst_node, const char *label);
+  edge
+  add_edge (const char *id, node src_node, node dst_node, message_buffer &&label);
+
+  diagnostic_graph *m_inner;
+  bool m_owned;
+};
+
+// Borrowed pointer to a diagnostic_node.
+
+class node
+{
+public:
+  node () : m_inner (nullptr) {}
+  node (diagnostic_node *node_) : m_inner (node_) {}
+
+  void
+  set_label (const char *);
+  void
+  set_label (message_buffer &&);
+
+  void
+  set_location (physical_location loc);
+
+  void
+  set_logical_location (logical_location loc);
+
+  diagnostic_node *m_inner;
+};
+
+// Borrowed edge to a diagnostic_edge.
+
+class edge
+{
+public:
+  edge (diagnostic_edge *edge_) : m_inner (edge_) {}
+
+  diagnostic_edge *m_inner;
+};
+
 // Implementation
 
 // class file
@@ -496,6 +673,21 @@ execution_path::add_event_va (physical_location physical_loc,
 						 args);
 }
 
+inline diagnostic_event_id
+execution_path::add_event_via_msg_buf (physical_location physical_loc,
+				       logical_location logical_loc,
+				       unsigned stack_depth,
+				       message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  return diagnostic_execution_path_add_event_via_msg_buf (m_inner,
+							  physical_loc.m_inner,
+							  logical_loc.m_inner,
+							  stack_depth,
+							  inner_msg_buf);
+}
+
 // class group
 
 inline
@@ -539,6 +731,17 @@ diagnostic::add_location_with_label (physical_location loc,
 }
 
 inline void
+diagnostic::add_location_with_label (physical_location loc,
+				     message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  diagnostic_add_location_with_label_via_msg_buf (m_inner,
+						  loc.m_inner,
+						  inner_msg_buf);
+}
+
+inline void
 diagnostic::add_location (physical_location loc)
 {
   diagnostic_add_location (m_inner, loc.m_inner);
@@ -593,6 +796,14 @@ diagnostic::take_execution_path (execution_path path)
 }
 
 inline void
+diagnostic::take_graph (graph g)
+{
+  diagnostic_take_graph (m_inner,
+			 g.m_inner);
+  g.m_owned = false;
+}
+
+inline void
 diagnostic::finish (const char *fmt, ...)
 {
   va_list ap;
@@ -607,6 +818,14 @@ diagnostic::finish_va (const char *fmt, va_list *args)
   diagnostic_finish_va (m_inner, fmt, args);
 }
 
+inline void
+diagnostic::finish_via_msg_buf (message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  diagnostic_finish_via_msg_buf (m_inner, inner_msg_buf);
+}
+
 // class manager
 
 inline file
@@ -702,6 +921,103 @@ manager::set_analysis_target (file f)
   diagnostic_manager_set_analysis_target (m_inner, f.m_inner);
 }
 
+inline void
+manager::take_global_graph (graph g)
+{
+  diagnostic_manager_take_global_graph (m_inner,
+					g.m_inner);
+  g.m_owned = false;
+}
+
+inline void
+manager::set_debug_physical_locations (bool value)
+{
+  diagnostic_manager_set_debug_physical_locations (m_inner,
+						   value ? 1 : 0);
+}
+
+// class graph
+
+inline void
+graph::set_description (const char *desc)
+{
+  diagnostic_graph_set_description (m_inner, desc);
+}
+
+inline void
+graph::set_description (message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  diagnostic_graph_set_description_via_msg_buf (m_inner, inner_msg_buf);
+}
+
+inline node
+graph::get_node_by_id (const char *id) const
+{
+  return node (diagnostic_graph_get_node_by_id (m_inner, id));
+}
+
+inline edge
+graph::get_edge_by_id (const char *id) const
+{
+  return edge (diagnostic_graph_get_edge_by_id (m_inner, id));
+}
+
+inline edge
+graph::add_edge (const char *id,
+		 node src_node, node dst_node,
+		 const char *label)
+{
+  return edge (diagnostic_graph_add_edge (m_inner,
+					  id,
+					  src_node.m_inner,
+					  dst_node.m_inner,
+					  label));
+}
+
+inline edge
+graph::add_edge (const char *id,
+		 node src_node, node dst_node,
+		 message_buffer &&label)
+{
+  diagnostic_message_buffer *inner_label = label.m_inner;
+  label.m_inner = nullptr;
+  return edge (diagnostic_graph_add_edge_via_msg_buf (m_inner,
+						      id,
+						      src_node.m_inner,
+						      dst_node.m_inner,
+						      inner_label));
+}
+
+// class node
+
+inline void
+node::set_label (const char *label)
+{
+  diagnostic_node_set_label (m_inner, label);
+}
+
+inline void
+node::set_label (message_buffer &&label)
+{
+  diagnostic_message_buffer *inner_label = label.m_inner;
+  label.m_inner = nullptr;
+  diagnostic_node_set_label_via_msg_buf (m_inner, inner_label);
+}
+
+inline void
+node::set_location (physical_location loc)
+{
+  diagnostic_node_set_location (m_inner, loc.m_inner);
+}
+
+inline void
+node::set_logical_location (logical_location loc)
+{
+  diagnostic_node_set_logical_location (m_inner, loc.m_inner);
+}
+
 } // namespace libgdiagnostics
 
 #endif // #ifndef LIBGDIAGNOSTICSPP_H
diff --git a/gcc/libgdiagnostics-private.h b/gcc/libgdiagnostics-private.h
new file mode 100644
index 0000000..4186c67
--- /dev/null
+++ b/gcc/libgdiagnostics-private.h
@@ -0,0 +1,70 @@
+/* Private API entrypoints to libgdiagnostics purely for use by sarif-replay.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef LIBGDIAGNOSTICS_PRIVATE_H
+#define LIBGDIAGNOSTICS_PRIVATE_H
+
+#include "libgdiagnostics.h"
+
+namespace json { class object; }
+
+extern "C" {
+
+/* Private entrypoints, for use only by sarif-replay.
+   These are subject to removal without notice.  */
+
+/* Entrypoints added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+private_diagnostic_graph_set_property_bag (diagnostic_graph &graph,
+					   std::unique_ptr<json::object> properties);
+
+extern void
+private_diagnostic_node_set_property_bag (diagnostic_node &node,
+					  std::unique_ptr<json::object> properties);
+
+extern void
+private_diagnostic_edge_set_property_bag (diagnostic_edge &edge,
+					  std::unique_ptr<json::object> properties);
+
+/* Entrypoint added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_event_id
+private_diagnostic_execution_path_add_event_3 (diagnostic_execution_path *path,
+					       const diagnostic_physical_location *physical_loc,
+					       const diagnostic_logical_location *logical_loc,
+					       unsigned stack_depth,
+					       diagnostic_graph *state_graph,
+					       diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (5)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (6);
+
+/* Entrypoint added in LIBGDIAGNOSTICS_ABI_5.  */
+
+extern void
+private_diagnostic_set_nesting_level (diagnostic *diag,
+				      int nesting_level)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+} // extern "C"
+
+#endif  /* LIBGDIAGNOSTICS_PRIVATE_H  */
diff --git a/gcc/libgdiagnostics.cc b/gcc/libgdiagnostics.cc
index 74814c7..784e281 100644
--- a/gcc/libgdiagnostics.cc
+++ b/gcc/libgdiagnostics.cc
@@ -19,22 +19,30 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "config.h"
 #define INCLUDE_MAP
+#define INCLUDE_STRING
 #define INCLUDE_VECTOR
 #include "system.h"
 #include "coretypes.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-format-sarif.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-output-spec.h"
-#include "logical-location.h"
-#include "edit-context.h"
+#include "diagnostics/color.h"
+#include "diagnostics/file-cache.h"
+#include "diagnostics/url.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/output-spec.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/state-graphs.h"
+#include "diagnostics/logical-locations.h"
+#include "diagnostics/changes.h"
 #include "libgdiagnostics.h"
+#include "libgdiagnostics-private.h"
+#include "pretty-print-format-impl.h"
+#include "pretty-print-markup.h"
+#include "auto-obstack.h"
 
 class owned_nullable_string
 {
@@ -203,7 +211,7 @@ struct diagnostic_logical_location
 };
 
 static diagnostic_event_id
-as_diagnostic_event_id (diagnostic_event_id_t id)
+as_diagnostic_event_id (diagnostics::paths::event_id_t id)
 {
   return id.zero_based ();
 }
@@ -225,7 +233,7 @@ public:
 			FILE *dst_stream,
 			enum diagnostic_colorize colorize);
 
-  diagnostic_source_printing_options &get_source_printing_options ()
+  diagnostics::source_printing_options &get_source_printing_options ()
   {
     return m_source_printing;
   }
@@ -234,12 +242,103 @@ public:
   set_colorize (enum diagnostic_colorize colorize);
 
   static void
-  text_starter (diagnostic_text_output_format &text_output,
-		const diagnostic_info *diagnostic);
+  text_starter (diagnostics::text_sink &text_output,
+		const diagnostics::diagnostic_info *diagnostic);
 
 private:
-  diagnostic_text_output_format *m_inner_sink; // borrowed from dc
-  diagnostic_source_printing_options m_source_printing;
+  diagnostics::text_sink *m_inner_sink; // borrowed from dc
+  diagnostics::source_printing_options m_source_printing;
+};
+
+/* A token_printer that makes a deep copy of the pp_token_list
+   into another obstack.  */
+
+class copying_token_printer : public token_printer
+{
+public:
+  copying_token_printer (obstack &dst_obstack,
+			 pp_token_list &dst_token_list)
+  : m_dst_obstack (dst_obstack),
+    m_dst_token_list (dst_token_list)
+  {
+  }
+
+  void
+  print_tokens (pretty_printer *,
+		const pp_token_list &tokens) final override
+  {
+    for (auto iter = tokens.m_first; iter; iter = iter->m_next)
+      switch (iter->m_kind)
+	{
+	default:
+	  gcc_unreachable ();
+
+	case pp_token::kind::text:
+	  {
+	    const pp_token_text *sub = as_a <const pp_token_text *> (iter);
+	    /* Copy the text, with null terminator.  */
+	    obstack_grow (&m_dst_obstack, sub->m_value.get (),
+			  strlen (sub->m_value.get ()) + 1);
+	    m_dst_token_list.push_back_text
+	      (label_text::borrow (XOBFINISH (&m_dst_obstack,
+					      const char *)));
+	  }
+	  break;
+
+	case pp_token::kind::begin_color:
+	  {
+	    pp_token_begin_color *sub = as_a <pp_token_begin_color *> (iter);
+	    /* Copy the color, with null terminator.  */
+	    obstack_grow (&m_dst_obstack, sub->m_value.get (),
+			  strlen (sub->m_value.get ()) + 1);
+	    m_dst_token_list.push_back<pp_token_begin_color>
+	      (label_text::borrow (XOBFINISH (&m_dst_obstack,
+					      const char *)));
+	  }
+	  break;
+	case pp_token::kind::end_color:
+	  m_dst_token_list.push_back<pp_token_end_color> ();
+	  break;
+
+	case pp_token::kind::begin_quote:
+	  m_dst_token_list.push_back<pp_token_begin_quote> ();
+	  break;
+	case pp_token::kind::end_quote:
+	  m_dst_token_list.push_back<pp_token_end_quote> ();
+	  break;
+
+	case pp_token::kind::begin_url:
+	  {
+	    pp_token_begin_url *sub = as_a <pp_token_begin_url *> (iter);
+	    /* Copy the URL, with null terminator.  */
+	    obstack_grow (&m_dst_obstack, sub->m_value.get (),
+			  strlen (sub->m_value.get ()) + 1);
+	    m_dst_token_list.push_back<pp_token_begin_url>
+	      (label_text::borrow (XOBFINISH (&m_dst_obstack,
+					      const char *)));
+	  }
+	  break;
+	case pp_token::kind::end_url:
+	  m_dst_token_list.push_back<pp_token_end_url> ();
+	  break;
+
+	case pp_token::kind::event_id:
+	  {
+	    pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+	    m_dst_token_list.push_back<pp_token_event_id> (sub->m_event_id);
+	  }
+	  break;
+
+	case pp_token::kind::custom_data:
+	  /* These should have been eliminated by replace_custom_tokens.  */
+	  gcc_unreachable ();
+	  break;
+	}
+  }
+
+private:
+  obstack &m_dst_obstack;
+  pp_token_list &m_dst_token_list;
 };
 
 class sarif_sink : public sink
@@ -248,7 +347,108 @@ public:
   sarif_sink (diagnostic_manager &mgr,
 	      FILE *dst_stream,
 	      const diagnostic_file *main_input_file,
-	      const sarif_generation_options &sarif_gen_opts);
+	      const diagnostics::sarif_generation_options &sarif_gen_opts);
+};
+
+struct diagnostic_message_buffer
+{
+  diagnostic_message_buffer ()
+  : m_tokens (m_obstack)
+  {
+  }
+
+  diagnostic_message_buffer (const char *gmsgid,
+			     va_list *args)
+  : m_tokens (m_obstack)
+  {
+    text_info text (gmsgid, args, errno);
+    pretty_printer pp;
+    pp.set_output_stream (nullptr);
+    copying_token_printer tok_printer (m_obstack, m_tokens);
+    pp.set_token_printer (&tok_printer);
+    pp_format (&pp, &text);
+    pp_output_formatted_text (&pp, nullptr);
+  }
+
+
+  std::string to_string () const;
+
+  auto_obstack m_obstack;
+  pp_token_list m_tokens;
+};
+
+/* A pp_element subclass that replays the saved tokens in a
+   diagnostic_message_buffer.  */
+
+class pp_element_message_buffer : public pp_element
+{
+public:
+  pp_element_message_buffer (diagnostic_message_buffer &msg_buf)
+    : m_msg_buf (msg_buf)
+  {
+  }
+
+  void add_to_phase_2 (pp_markup::context &ctxt) final override
+  {
+    /* Convert to text, possibly with colorization, URLs, etc.  */
+    for (auto iter = m_msg_buf.m_tokens.m_first; iter; iter = iter->m_next)
+      switch (iter->m_kind)
+	{
+	default:
+	  gcc_unreachable ();
+
+	case pp_token::kind::text:
+	  {
+	    pp_token_text *sub = as_a <pp_token_text *> (iter);
+	    pp_string (&ctxt.m_pp, sub->m_value.get ());
+	    ctxt.push_back_any_text ();
+	  }
+	  break;
+
+	case pp_token::kind::begin_color:
+	  {
+	    pp_token_begin_color *sub = as_a <pp_token_begin_color *> (iter);
+	    ctxt.begin_highlight_color (sub->m_value.get ());
+	  }
+	  break;
+	case pp_token::kind::end_color:
+	  ctxt.end_highlight_color ();
+	  break;
+
+	case pp_token::kind::begin_quote:
+	  ctxt.begin_quote ();
+	  break;
+	case pp_token::kind::end_quote:
+	  ctxt.end_quote ();
+	  break;
+
+	case pp_token::kind::begin_url:
+	  {
+	    pp_token_begin_url *sub = as_a <pp_token_begin_url *> (iter);
+	    ctxt.begin_url (sub->m_value.get ());
+	  }
+	  break;
+	case pp_token::kind::end_url:
+	  ctxt.end_url ();
+	  break;
+
+	case pp_token::kind::event_id:
+	  {
+	    pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+	    gcc_assert (sub->m_event_id.known_p ());
+	    ctxt.add_event_id (sub->m_event_id);
+	  }
+	  break;
+
+	case pp_token::kind::custom_data:
+	  /* We don't have a way of handling custom_data tokens here.  */
+	  gcc_unreachable ();
+	  break;
+	}
+  }
+
+private:
+  diagnostic_message_buffer &m_msg_buf;
 };
 
 /* Helper for the linemap code.  */
@@ -259,19 +459,23 @@ round_alloc_size (size_t s)
   return s;
 }
 
-class impl_logical_location_manager : public logical_location_manager
+class impl_logical_location_manager
+  : public diagnostics::logical_locations::manager
 {
 public:
+  using key = diagnostics::logical_locations::key;
+  using kind = diagnostics::logical_locations::kind;
+
   static const diagnostic_logical_location *
-  ptr_from_key (logical_location k)
+  ptr_from_key (key k)
   {
     return k.cast_to<const diagnostic_logical_location *> ();
   }
 
-  static logical_location
+  static key
   key_from_ptr (const diagnostic_logical_location *ptr)
   {
-    return logical_location::from_ptr (ptr);
+    return key::from_ptr (ptr);
   }
 
   const char *get_short_name (key k) const final override
@@ -298,7 +502,7 @@ public:
       return nullptr;
   }
 
-  enum logical_location_kind get_kind (key k) const final override
+  kind get_kind (key k) const final override
   {
     auto loc = ptr_from_key (k);
     gcc_assert (loc);
@@ -308,45 +512,45 @@ public:
 	gcc_unreachable ();
 
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_FUNCTION:
-	return logical_location_kind::function;
+	return kind::function;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_MEMBER:
-	return logical_location_kind::member;
+	return kind::member;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_MODULE:
-	return logical_location_kind::module_;
+	return kind::module_;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_NAMESPACE:
-	return logical_location_kind::namespace_;
+	return kind::namespace_;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_TYPE:
-	return logical_location_kind::type;
+	return kind::type;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_RETURN_TYPE:
-	return logical_location_kind::return_type;
+	return kind::return_type;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_PARAMETER:
-	return logical_location_kind::parameter;
+	return kind::parameter;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_VARIABLE:
-	return logical_location_kind::variable;
+	return kind::variable;
 
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_ELEMENT:
-	return logical_location_kind::element;
+	return kind::element;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_ATTRIBUTE:
-	return logical_location_kind::attribute;
+	return kind::attribute;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_TEXT:
-	return logical_location_kind::text;
+	return kind::text;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_COMMENT:
-	return logical_location_kind::comment;
+	return kind::comment;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_PROCESSING_INSTRUCTION:
-	return logical_location_kind::processing_instruction;
+	return kind::processing_instruction;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_DTD:
-	return logical_location_kind::dtd;
+	return kind::dtd;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_DECLARATION:
-	return logical_location_kind::declaration;
+	return kind::declaration;
 
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_OBJECT:
-	  return logical_location_kind::object;
+	  return kind::object;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_ARRAY:
-	return logical_location_kind::array;
+	return kind::array;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_PROPERTY:
-	return logical_location_kind::property;
+	return kind::property;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_VALUE:
-	return logical_location_kind::value;
+	return kind::value;
       }
   }
 
@@ -365,25 +569,28 @@ public:
   }
 };
 
-class impl_diagnostic_client_data_hooks : public diagnostic_client_data_hooks
+class impl_diagnostic_client_data_hooks : public diagnostics::client_data_hooks
 {
 public:
   impl_diagnostic_client_data_hooks (diagnostic_manager &mgr)
   : m_mgr (mgr)
   {}
 
-  const client_version_info *get_any_version_info () const final override;
+  const diagnostics::client_version_info *
+  get_any_version_info () const final override;
 
-  const logical_location_manager *
+  const diagnostics::logical_locations::manager *
   get_logical_location_manager () const final override
   {
     return &m_logical_location_manager;
   }
-  logical_location get_current_logical_location () const final override;
+  diagnostics::logical_locations::key
+  get_current_logical_location () const final override;
 
   const char * maybe_get_sarif_source_language (const char *filename)
     const final override;
-  void add_sarif_invocation_properties (sarif_object &invocation_obj)
+  void
+  add_sarif_invocation_properties (diagnostics::sarif_object &invocation_obj)
     const final override;
 
 private:
@@ -391,7 +598,7 @@ private:
   impl_logical_location_manager m_logical_location_manager;
 };
 
-class impl_client_version_info : public client_version_info
+class impl_client_version_info : public diagnostics::client_version_info
 {
 public:
   const char *get_tool_name () const final override
@@ -432,7 +639,8 @@ struct diagnostic_manager
 public:
   diagnostic_manager ()
   : m_current_diag (nullptr),
-    m_prev_diag_logical_loc (nullptr)
+    m_prev_diag_logical_loc (nullptr),
+    m_debug_physical_locations (false)
   {
     linemap_init (&m_line_table, BUILTINS_LOCATION);
     m_line_table.m_reallocator = xrealloc;
@@ -450,23 +658,26 @@ public:
     m_dc.set_show_cwe (true);
     m_dc.set_show_rules (true);
     m_dc.m_show_column = true;
-    m_dc.m_source_printing.enabled = true;
-    m_dc.m_source_printing.colorize_source_p = true;
+    auto &source_printing_opts = m_dc.get_source_printing_options ();
+    source_printing_opts.enabled = true;
+    source_printing_opts.colorize_source_p = true;
 
     /* We don't currently expose a way for clients to manipulate the
        following.  */
-    m_dc.m_source_printing.show_labels_p = true;
-    m_dc.m_source_printing.show_line_numbers_p = true;
-    m_dc.m_source_printing.min_margin_width = 6;
+    source_printing_opts.show_labels_p = true;
+    source_printing_opts.show_line_numbers_p = true;
+    source_printing_opts.min_margin_width = 6;
     m_dc.set_path_format (DPF_INLINE_EVENTS);
 
     m_dc.m_client_aux_data = this;
     m_dc.set_client_data_hooks
       (std::make_unique<impl_diagnostic_client_data_hooks> (*this));
 
-    diagnostic_text_starter (&m_dc) = diagnostic_text_sink::text_starter;
+    diagnostics::text_starter (&m_dc) = diagnostic_text_sink::text_starter;
 
-    m_edit_context = std::make_unique <edit_context> (m_dc.get_file_cache ());
+    m_change_set
+      = std::make_unique <diagnostics::changes::change_set>
+	  (m_dc.get_file_cache ());
   }
 
   ~diagnostic_manager ()
@@ -487,9 +698,9 @@ public:
   }
 
   line_maps *get_line_table () { return &m_line_table; }
-  diagnostic_context &get_dc () { return m_dc; }
+  diagnostics::context &get_dc () { return m_dc; }
 
-  const logical_location_manager &
+  const diagnostics::logical_locations::manager &
   get_logical_location_manager () const
   {
     auto mgr = m_dc.get_logical_location_manager ();
@@ -504,9 +715,15 @@ public:
     m_sinks.push_back (std::move (sink));
   }
 
-  void emit (diagnostic &diag, const char *msgid, va_list *args)
+  void emit_va (diagnostic &diag, const char *msgid, va_list *args)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING(3, 0);
 
+  void emit (diagnostic &diag, const char *msgid, ...)
+    LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING(3, 4);
+
+  void emit_msg_buf (diagnostic &diag,
+		     diagnostic_message_buffer &msg_buf);
+
   diagnostic_file *
   new_file (const char *name,
 	    const char *sarif_source_language)
@@ -531,6 +748,9 @@ public:
   new_location_from_file_and_line (const diagnostic_file *file,
 				   diagnostic_line_num_t line_num)
   {
+    if (m_debug_physical_locations)
+      fprintf (stderr, "new_location_from_file_and_line (%s, %i)",
+	       file->get_name (), line_num);
     ensure_linemap_for_file_and_line (file, line_num);
     location_t loc = linemap_position_for_column (&m_line_table, 0);
     return new_location (loc);
@@ -541,6 +761,9 @@ public:
 				      diagnostic_line_num_t line_num,
 				      diagnostic_column_num_t column_num)
   {
+    if (m_debug_physical_locations)
+      fprintf (stderr, "new_location_from_file_line_column (%s, %i, %i)",
+	       file->get_name (), line_num, column_num);
     ensure_linemap_for_file_and_line (file, line_num);
     location_t loc = linemap_position_for_column (&m_line_table, column_num);
     return new_location (loc);
@@ -551,12 +774,23 @@ public:
 			   const diagnostic_physical_location *loc_start,
 			   const diagnostic_physical_location *loc_end)
   {
+    if (m_debug_physical_locations)
+      fprintf (stderr, "new_location_from_range (%p, %p, %p)",
+	       (const void *)loc_caret,
+	       (const void *)loc_start,
+	       (const void *)loc_end);
     return new_location
       (m_line_table.make_location (as_location_t (loc_caret),
 				   as_location_t (loc_start),
 				   as_location_t (loc_end)));
   }
 
+  void
+  set_debug_physical_locations (bool value)
+  {
+    m_debug_physical_locations = value;
+  }
+
   const diagnostic_logical_location *
   new_logical_location (enum diagnostic_logical_location_kind_t kind,
 			const diagnostic_logical_location *parent,
@@ -613,7 +847,8 @@ public:
 
   const diagnostic *get_current_diag () { return m_current_diag; }
 
-  const client_version_info *get_client_version_info () const
+  const diagnostics::client_version_info *
+  get_client_version_info () const
   {
     return &m_client_version_info;
   }
@@ -644,6 +879,9 @@ public:
     return m_prev_diag_logical_loc;
   }
 
+  void
+  take_global_graph (std::unique_ptr<diagnostic_graph> graph);
+
 private:
   void
   ensure_linemap_for_file_and_line (const diagnostic_file *file,
@@ -654,11 +892,17 @@ private:
       linemap_add (&m_line_table, LC_ENTER, false, file->get_name (), 0);
     else
       {
-	line_map *map
-	  = const_cast<line_map *>
-	    (linemap_add (&m_line_table, LC_RENAME_VERBATIM, false,
-			  file->get_name (), 0));
-	((line_map_ordinary *)map)->included_from = UNKNOWN_LOCATION;
+	line_map_ordinary *last_map
+	  = LINEMAPS_LAST_ORDINARY_MAP (&m_line_table);
+	if (last_map->to_file != file->get_name ()
+	    || linenum < last_map->to_line)
+	  {
+	    line_map *map
+	      = const_cast<line_map *>
+	      (linemap_add (&m_line_table, LC_RENAME_VERBATIM, false,
+			    file->get_name (), 0));
+	    ((line_map_ordinary *)map)->included_from = UNKNOWN_LOCATION;
+	  }
       }
     linemap_line_start (&m_line_table, linenum, 100);
   }
@@ -668,15 +912,23 @@ private:
   {
     if (loc == UNKNOWN_LOCATION)
       return nullptr;
+    if (m_debug_physical_locations)
+      fprintf (stderr, ": new_location (%lx)", loc);
     if (diagnostic_physical_location **slot = m_location_t_map.get (loc))
-      return *slot;
+      {
+	if (m_debug_physical_locations)
+	  fprintf (stderr, ": cache hit: %p\n", (const void *)*slot);
+	return *slot;
+      }
     diagnostic_physical_location *phys_loc
       = new diagnostic_physical_location (this, loc);
     m_location_t_map.put (loc, phys_loc);
+    if (m_debug_physical_locations)
+      fprintf (stderr, ": cache miss: %p\n", (const void *)phys_loc);
     return phys_loc;
   }
 
-  diagnostic_context m_dc;
+  diagnostics::context m_dc;
   line_maps m_line_table;
   impl_client_version_info m_client_version_info;
   std::vector<std::unique_ptr<sink>> m_sinks;
@@ -688,7 +940,8 @@ private:
   logical_locs_map_t m_logical_locs;
   const diagnostic *m_current_diag;
   const diagnostic_logical_location *m_prev_diag_logical_loc;
-  std::unique_ptr<edit_context> m_edit_context;
+  std::unique_ptr<diagnostics::changes::change_set> m_change_set;
+  bool m_debug_physical_locations;
 };
 
 class impl_rich_location : public rich_location
@@ -717,7 +970,7 @@ private:
   char *m_text;
 };
 
-class impl_rule : public diagnostic_metadata::rule
+class impl_rule : public diagnostics::metadata::rule
 {
 public:
   impl_rule (const char *title, const char *url)
@@ -743,23 +996,58 @@ private:
   owned_nullable_string m_url;
 };
 
-class libgdiagnostics_path_event : public diagnostic_event
+struct diagnostic_graph : public diagnostics::digraphs::digraph
+{
+  diagnostic_graph (diagnostic_manager &) {}
+
+  diagnostic_node *
+  add_node_with_id (std::string node_id,
+		    diagnostic_node *parent_node);
+  diagnostic_edge *
+  add_edge_with_label (const char *edge_id,
+		       diagnostic_node &src_node,
+		       diagnostic_node &dst_node,
+		       const char *label);
+};
+
+struct diagnostic_node : public diagnostics::digraphs::node
+{
+  diagnostic_node (diagnostic_graph &g,
+		   std::string id)
+  : node (g, std::move (id))
+  {
+  }
+};
+
+struct diagnostic_edge : public diagnostics::digraphs::edge
+{
+  diagnostic_edge (diagnostic_graph &g,
+		   const char *id,
+		   diagnostic_node &src_node,
+		   diagnostic_node &dst_node)
+  : edge (g, id, src_node, dst_node)
+  {
+  }
+};
+
+class libgdiagnostics_path_event : public diagnostics::paths::event
 {
 public:
   libgdiagnostics_path_event (const diagnostic_physical_location *physical_loc,
 			      const diagnostic_logical_location *logical_loc,
 			      unsigned stack_depth,
-			      const char *gmsgid,
-			      va_list *args)
+			      std::unique_ptr<diagnostic_graph> state_graph,
+			      std::unique_ptr<diagnostic_message_buffer> msg_buf)
   : m_physical_loc (physical_loc),
     m_logical_loc (logical_loc),
-    m_stack_depth (stack_depth)
+    m_stack_depth (stack_depth),
+    m_state_graph (std::move (state_graph)),
+    m_msg_buf (std::move (msg_buf))
   {
-    m_desc_uncolored = make_desc (gmsgid, args, false);
-    m_desc_colored = make_desc (gmsgid, args, true);
+    gcc_assert (m_msg_buf);
   }
 
-  /* diagnostic_event vfunc implementations.  */
+  /* diagnostics::paths::event vfunc implementations.  */
 
   location_t get_location () const final override
   {
@@ -773,13 +1061,15 @@ public:
 
   void print_desc (pretty_printer &pp) const final override
   {
-    if (pp_show_color (&pp))
-      pp_string (&pp, m_desc_colored.get ());
-    else
-      pp_string (&pp, m_desc_uncolored.get ());
+    if (m_msg_buf)
+      {
+	pp_element_message_buffer e_msg_buf (*m_msg_buf);
+	pp_printf (&pp, "%e", &e_msg_buf);
+      }
   }
 
-  logical_location get_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_logical_location () const final override
   {
     return impl_logical_location_manager::key_from_ptr (m_logical_loc);
   }
@@ -794,11 +1084,20 @@ public:
     return false; // TODO
   }
 
-  diagnostic_thread_id_t get_thread_id () const final override
+  diagnostics::paths::thread_id_t get_thread_id () const final override
   {
     return 0;
   }
 
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  maybe_make_diagnostic_state_graph (bool) const final override
+  {
+    if (!m_state_graph)
+      return nullptr;
+
+    return m_state_graph->clone ();
+  }
+
 private:
   static label_text make_desc (const char *gmsgid,
 			       va_list *args,
@@ -824,11 +1123,11 @@ private:
   const diagnostic_physical_location *m_physical_loc;
   const diagnostic_logical_location *m_logical_loc;
   unsigned m_stack_depth;
-  label_text m_desc_uncolored;
-  label_text m_desc_colored;
+  std::unique_ptr<diagnostic_graph> m_state_graph;
+  std::unique_ptr<diagnostic_message_buffer> m_msg_buf;
 };
 
-class libgdiagnostics_path_thread : public diagnostic_thread
+class libgdiagnostics_path_thread : public diagnostics::paths::thread
 {
 public:
   libgdiagnostics_path_thread (const char *name) : m_name (name) {}
@@ -843,43 +1142,62 @@ private:
 
 /* This has to be a "struct" as it is exposed in the C API.  */
 
-struct diagnostic_execution_path : public diagnostic_path
+struct diagnostic_execution_path : public diagnostics::paths::path
 {
-  diagnostic_execution_path (const logical_location_manager &logical_loc_mgr)
-  : diagnostic_path (logical_loc_mgr),
+  diagnostic_execution_path (const diagnostics::logical_locations::manager &logical_loc_mgr)
+  : diagnostics::paths::path (logical_loc_mgr),
     m_thread ("")
   {
   }
 
-  diagnostic_event_id_t
+  diagnostics::paths::event_id_t
   add_event_va (const diagnostic_physical_location *physical_loc,
 		const diagnostic_logical_location *logical_loc,
 		unsigned stack_depth,
+		std::unique_ptr<diagnostic_graph> state_graph,
 		const char *gmsgid,
 		va_list *args)
   {
+    auto msg_buf = std::make_unique<diagnostic_message_buffer> (gmsgid, args);
+
+    m_events.push_back
+      (std::make_unique<libgdiagnostics_path_event> (physical_loc,
+						     logical_loc,
+						     stack_depth,
+						     std::move (state_graph),
+						     std::move (msg_buf)));
+    return m_events.size () - 1;
+  }
+
+  diagnostic_event_id_t
+  add_event_via_msg_buf (const diagnostic_physical_location *physical_loc,
+			 const diagnostic_logical_location *logical_loc,
+			 unsigned stack_depth,
+			 std::unique_ptr<diagnostic_graph> state_graph,
+			 std::unique_ptr<diagnostic_message_buffer> msg_buf)
+  {
     m_events.push_back
       (std::make_unique<libgdiagnostics_path_event> (physical_loc,
 						     logical_loc,
 						     stack_depth,
-						     gmsgid,
-						     args));
+						     std::move (state_graph),
+						     std::move (msg_buf)));
     return m_events.size () - 1;
   }
 
-  /* diagnostic_path vfunc implementations.  */
+  /* diagnostics::paths::path vfunc implementations.  */
 
   unsigned num_events () const final override
   {
     return m_events.size ();
   }
-  const diagnostic_event & get_event (int idx) const final override
+  const diagnostics::paths::event & get_event (int idx) const final override
   {
     return *m_events[idx];
   }
   unsigned num_threads () const final override { return 1; }
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override
+  const diagnostics::paths::thread &
+  get_thread (diagnostics::paths::thread_id_t) const final override
   {
     return m_thread;
   }
@@ -888,9 +1206,10 @@ struct diagnostic_execution_path : public diagnostic_path
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override
   {
+    using logical_location = diagnostics::logical_locations::key;
     logical_location logical_loc_a
       = m_events[event_idx_a]->get_logical_location ();
-    logical_location logical_loc_b
+    logical_location  logical_loc_b
       = m_events[event_idx_b]->get_logical_location ();
 
     /* Pointer equality, as we uniqify logical location instances.  */
@@ -902,6 +1221,28 @@ private:
   std::vector<std::unique_ptr<libgdiagnostics_path_event>> m_events;
 };
 
+class prebuilt_digraphs
+  : public lazily_created<std::vector<std::unique_ptr<diagnostics::digraphs::digraph>>>
+{
+public:
+  using digraph = diagnostics::digraphs::digraph;
+
+  std::unique_ptr<std::vector<std::unique_ptr<digraph>>>
+  create_object () const final override
+  {
+    return std::make_unique<std::vector<std::unique_ptr<digraph>>> (std::move (m_digraphs));
+  }
+
+  void
+  take_graph (std::unique_ptr<diagnostic_graph> graph)
+  {
+    m_digraphs.push_back (std::move (graph));
+  }
+
+private:
+  mutable std::vector<std::unique_ptr<digraph>> m_digraphs;
+};
+
 /* This has to be a "struct" as it is exposed in the C API.  */
 
 struct diagnostic
@@ -913,8 +1254,11 @@ public:
     m_level (level),
     m_rich_loc (diag_mgr.get_line_table ()),
     m_logical_loc (nullptr),
-    m_path (nullptr)
-  {}
+    m_path (nullptr),
+    m_nesting_level (0)
+  {
+    m_metadata.set_lazy_digraphs (&m_graphs);
+  }
 
   diagnostic_manager &get_manager () const
   {
@@ -924,7 +1268,7 @@ public:
   enum diagnostic_level get_level () const { return m_level; }
 
   rich_location *get_rich_location () { return &m_rich_loc; }
-  const diagnostic_metadata *get_metadata () { return &m_metadata; }
+  const diagnostics::metadata *get_metadata () { return &m_metadata; }
 
   void set_cwe (unsigned cwe_id)
   {
@@ -964,6 +1308,19 @@ public:
   }
 
   void
+  add_location_with_label (const diagnostic_physical_location *loc,
+			   std::unique_ptr<diagnostic_message_buffer> msg_buf)
+  {
+    std::string str = msg_buf->to_string ();
+    std::unique_ptr<range_label> label
+      = std::make_unique <impl_range_label> (str.c_str ());
+    m_rich_loc.add_range (as_location_t (loc),
+			  SHOW_RANGE_WITHOUT_CARET,
+			  label.get ());
+    m_labels.push_back (std::move (label));
+  }
+
+  void
   set_logical_location (const diagnostic_logical_location *logical_loc)
   {
     m_logical_loc = logical_loc;
@@ -990,32 +1347,49 @@ public:
     m_rich_loc.set_path (path);
   }
 
+  void
+  take_graph (std::unique_ptr<diagnostic_graph> graph)
+  {
+    m_graphs.take_graph (std::move (graph));
+  }
+
+  const prebuilt_digraphs &
+  get_graphs () const
+  {
+    return m_graphs;
+  }
+
+  int get_nesting_level () const { return m_nesting_level; }
+  void set_nesting_level (int value) { m_nesting_level = value; }
+
 private:
   diagnostic_manager &m_diag_mgr;
   enum diagnostic_level m_level;
   impl_rich_location m_rich_loc;
   const diagnostic_logical_location *m_logical_loc;
-  diagnostic_metadata m_metadata;
+  diagnostics::metadata m_metadata;
+  prebuilt_digraphs m_graphs;
   std::vector<std::unique_ptr<range_label>> m_labels;
   std::vector<std::unique_ptr<impl_rule>> m_rules;
   std::unique_ptr<diagnostic_execution_path> m_path;
+  int m_nesting_level;
 };
 
-static diagnostic_t
-diagnostic_t_from_diagnostic_level (enum diagnostic_level level)
+static enum diagnostics::kind
+diagnostics_kind_from_diagnostic_level (enum diagnostic_level level)
 {
   switch (level)
     {
     default:
       gcc_unreachable ();
     case DIAGNOSTIC_LEVEL_ERROR:
-      return DK_ERROR;
+      return diagnostics::kind::error;
     case DIAGNOSTIC_LEVEL_WARNING:
-      return DK_WARNING;
+      return diagnostics::kind::warning;
     case DIAGNOSTIC_LEVEL_NOTE:
-      return DK_NOTE;
+      return diagnostics::kind::note;
     case DIAGNOSTIC_LEVEL_SORRY:
-      return DK_SORRY;
+      return diagnostics::kind::sorry;
     }
 }
 
@@ -1025,7 +1399,7 @@ diagnostic_file::set_buffered_content (const char *buf, size_t sz)
   m_content = std::make_unique<content_buffer> (buf, sz);
 
   // Populate file_cache:
-  file_cache &fc = m_mgr.get_dc ().get_file_cache ();
+  diagnostics::file_cache &fc = m_mgr.get_dc ().get_file_cache ();
   fc.add_buffered_content (m_name.get_str (), buf, sz);
 }
 
@@ -1043,13 +1417,13 @@ diagnostic_physical_location::get_file () const
 
 /* class impl_diagnostic_client_data_hooks.  */
 
-const client_version_info *
+const diagnostics::client_version_info *
 impl_diagnostic_client_data_hooks::get_any_version_info () const
 {
   return m_mgr.get_client_version_info ();
 }
 
-logical_location
+diagnostics::logical_locations::key
 impl_diagnostic_client_data_hooks::get_current_logical_location () const
 {
   gcc_assert (m_mgr.get_current_diag ());
@@ -1067,7 +1441,7 @@ maybe_get_sarif_source_language (const char *filename) const
 
 void
 impl_diagnostic_client_data_hooks::
-add_sarif_invocation_properties (sarif_object &) const
+add_sarif_invocation_properties (diagnostics::sarif_object &) const
 {
   // No-op.
 }
@@ -1078,10 +1452,10 @@ diagnostic_text_sink::diagnostic_text_sink (diagnostic_manager &mgr,
 					    FILE *dst_stream,
 					    enum diagnostic_colorize colorize)
 : sink (mgr),
-  m_source_printing (mgr.get_dc ().m_source_printing)
+  m_source_printing (mgr.get_dc ().get_source_printing_options ())
 {
   auto inner_sink
-    = std::make_unique<diagnostic_text_output_format> (mgr.get_dc (),
+    = std::make_unique<diagnostics::text_sink> (mgr.get_dc (),
 						       &m_source_printing);
   inner_sink->get_printer ()->set_output_stream (dst_stream);
   m_inner_sink = inner_sink.get ();
@@ -1111,11 +1485,11 @@ diagnostic_text_sink::set_colorize (enum diagnostic_colorize colorize)
 }
 
 void
-diagnostic_text_sink::text_starter (diagnostic_text_output_format &text_output,
-				    const diagnostic_info *info)
+diagnostic_text_sink::text_starter (diagnostics::text_sink &text_output,
+				    const diagnostics::diagnostic_info *info)
 {
-  gcc_assert (info->x_data);
-  const diagnostic &diag = *static_cast<const diagnostic *> (info->x_data);
+  gcc_assert (info->m_x_data);
+  const diagnostic &diag = *static_cast<const diagnostic *> (info->m_x_data);
   pretty_printer *pp = text_output.get_printer ();
   const diagnostic_logical_location *diag_logical_loc
     = diag.get_logical_location ();
@@ -1181,15 +1555,17 @@ diagnostic_text_sink::text_starter (diagnostic_text_output_format &text_output,
 
 /* class sarif_sink : public sink.  */
 
-sarif_sink::sarif_sink (diagnostic_manager &mgr,
-			FILE *dst_stream,
-			const diagnostic_file *main_input_file,
-			const sarif_generation_options &sarif_gen_opts)
+sarif_sink::
+sarif_sink (diagnostic_manager &mgr,
+	    FILE *dst_stream,
+	    const diagnostic_file *main_input_file,
+	    const diagnostics::sarif_generation_options &sarif_gen_opts)
 : sink (mgr)
 {
-  diagnostic_output_file output_file (dst_stream, false,
-				      label_text::borrow ("sarif_sink"));
-  auto serialization = std::make_unique<sarif_serialization_format_json> (true);
+  diagnostics::output_file output_file (dst_stream, false,
+					label_text::borrow ("sarif_sink"));
+  auto serialization
+    = std::make_unique<diagnostics::sarif_serialization_format_json> (true);
   auto inner_sink = make_sarif_sink (mgr.get_dc (),
 				     *mgr.get_line_table (),
 				     std::move (serialization),
@@ -1199,6 +1575,64 @@ sarif_sink::sarif_sink (diagnostic_manager &mgr,
   mgr.get_dc ().add_sink (std::move (inner_sink));
 }
 
+// struct diagnostic_message_buffer
+
+std::string
+diagnostic_message_buffer::to_string () const
+{
+  std::string result;
+
+  /* Convert to text, dropping colorization, URLs, etc.  */
+  for (auto iter = m_tokens.m_first; iter; iter = iter->m_next)
+    switch (iter->m_kind)
+      {
+      default:
+	gcc_unreachable ();
+
+      case pp_token::kind::text:
+	{
+	  pp_token_text *sub = as_a <pp_token_text *> (iter);
+	  result += sub->m_value.get ();
+	}
+	break;
+
+      case pp_token::kind::begin_color:
+      case pp_token::kind::end_color:
+	// Skip
+	break;
+
+      case pp_token::kind::begin_quote:
+	result += open_quote;
+	break;
+
+      case pp_token::kind::end_quote:
+	result += close_quote;
+	break;
+
+      case pp_token::kind::begin_url:
+      case pp_token::kind::end_url:
+	// Skip
+	break;
+
+      case pp_token::kind::event_id:
+	{
+	  pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+	  gcc_assert (sub->m_event_id.known_p ());
+	  result += '(';
+	  result += std::to_string (sub->m_event_id.one_based ());
+	  result += ')';
+	}
+	break;
+
+      case pp_token::kind::custom_data:
+	/* We don't have a way of handling custom_data tokens here.  */
+	gcc_unreachable ();
+	break;
+      }
+
+  return result;
+}
+
 /* struct diagnostic_manager.  */
 
 void
@@ -1206,12 +1640,12 @@ diagnostic_manager::write_patch (FILE *dst_stream)
 {
   pretty_printer pp;
   pp.set_output_stream (dst_stream);
-  m_edit_context->print_diff (&pp, true);
+  m_change_set->print_diff (&pp, true);
   pp_flush (&pp);
 }
 
 void
-diagnostic_manager::emit (diagnostic &diag, const char *msgid, va_list *args)
+diagnostic_manager::emit_va (diagnostic &diag, const char *msgid, va_list *args)
 {
   set_line_table_global ();
 
@@ -1220,26 +1654,46 @@ diagnostic_manager::emit (diagnostic &diag, const char *msgid, va_list *args)
   {
     m_dc.begin_group ();
 
-    diagnostic_info info;
+    diagnostics::diagnostic_info info;
 GCC_DIAGNOSTIC_PUSH_IGNORED(-Wsuggest-attribute=format)
     diagnostic_set_info (&info, msgid, args, diag.get_rich_location (),
-			 diagnostic_t_from_diagnostic_level (diag.get_level ()));
+			 diagnostics_kind_from_diagnostic_level
+			   (diag.get_level ()));
 GCC_DIAGNOSTIC_POP
-    info.metadata = diag.get_metadata ();
-    info.x_data = &diag;
+    info.m_metadata = diag.get_metadata ();
+    info.m_x_data = &diag;
+    m_dc.set_nesting_level (diag.get_nesting_level ());
     diagnostic_report_diagnostic (&m_dc, &info);
-
+    m_dc.set_nesting_level (0);
     m_dc.end_group ();
   }
 
   rich_location *rich_loc = diag.get_rich_location ();
   if (rich_loc->fixits_can_be_auto_applied_p ())
-    m_edit_context->add_fixits (rich_loc);
+    m_change_set->add_fixits (rich_loc);
 
   m_prev_diag_logical_loc = diag.get_logical_location ();
   m_current_diag = nullptr;
 }
 
+void
+diagnostic_manager::emit (diagnostic &diag, const char *msgid, ...)
+{
+  va_list args;
+  va_start (args, msgid);
+  emit_va (diag, msgid, &args);
+  va_end (args);
+}
+
+void
+diagnostic_manager::emit_msg_buf (diagnostic &diag,
+				  diagnostic_message_buffer &msg_buf)
+{
+
+  pp_element_message_buffer e_msg_buf (msg_buf);
+  emit (diag, "%e", &e_msg_buf);
+}
+
 diagnostic_execution_path *
 diagnostic_manager::new_execution_path ()
 {
@@ -1248,13 +1702,36 @@ diagnostic_manager::new_execution_path ()
   return new diagnostic_execution_path (*mgr);
 }
 
+void
+diagnostic_manager::take_global_graph (std::unique_ptr<diagnostic_graph> graph)
+{
+  class prebuilt_lazy_digraph : public lazily_created<diagnostics::digraphs::digraph>
+  {
+  public:
+    prebuilt_lazy_digraph (std::unique_ptr<diagnostic_graph> graph)
+      : m_graph (std::move (graph))
+    {
+    }
+
+    std::unique_ptr<diagnostics::digraphs::digraph>
+    create_object () const final override
+    {
+      return std::move (m_graph);
+    }
+
+  private:
+    mutable std::unique_ptr<diagnostic_graph> m_graph;
+  };
+
+  m_dc.report_global_digraph (prebuilt_lazy_digraph (std::move (graph)));
+}
 /* Error-checking at the API boundary.  */
 
 #define FAIL_IF_NULL(PTR_ARG) \
-  do {						    \
-    volatile const void *p = (PTR_ARG);		    \
-    if (!p) {					    \
-      fprintf (stderr, "%s: %s must be non-NULL\n",   \
+  do {							    \
+    volatile const void *ptr_arg = (PTR_ARG);		    \
+    if (!ptr_arg) {					    \
+      fprintf (stderr, "%s: %s must be non-NULL\n",	    \
 	       __func__, #PTR_ARG);		      \
       abort ();					      \
     }						    \
@@ -1388,7 +1865,7 @@ diagnostic_manager_add_sarif_sink (diagnostic_manager *diag_mgr,
   FAIL_IF_NULL (dst_stream);
   FAIL_IF_NULL (main_input_file);
 
-  sarif_generation_options sarif_gen_opts;
+  diagnostics::sarif_generation_options sarif_gen_opts;
   switch (version)
     {
     default:
@@ -1396,10 +1873,11 @@ diagnostic_manager_add_sarif_sink (diagnostic_manager *diag_mgr,
 	       __func__, (int)version);
       abort ();
     case DIAGNOSTIC_SARIF_VERSION_2_1_0:
-      sarif_gen_opts.m_version = sarif_version::v2_1_0;
+      sarif_gen_opts.m_version = diagnostics::sarif_version::v2_1_0;
       break;
     case DIAGNOSTIC_SARIF_VERSION_2_2_PRERELEASE:
-      sarif_gen_opts.m_version = sarif_version::v2_2_prerelease_2024_08_08;
+      sarif_gen_opts.m_version
+	= diagnostics::sarif_version::v2_2_prerelease_2024_08_08;
       break;
     }
 
@@ -1529,12 +2007,12 @@ diagnostic_manager_debug_dump_location (const diagnostic_manager *diag_mgr,
       diag_mgr->set_line_table_global ();
       const expanded_location exp_loc (expand_location (cpplib_loc));
 
-      diagnostic_context dc;
+      diagnostics::context dc;
       diagnostic_initialize (&dc, 0);
       dc.m_show_column = true;
 
-      diagnostic_text_output_format text_format (dc);
-      label_text loc_text = text_format.get_location_text (exp_loc);
+      diagnostics::text_sink text_output (dc);
+      label_text loc_text = text_output.get_location_text (exp_loc);
       fprintf (out, "%s", loc_text.get ());
 
       diagnostic_finish (&dc);
@@ -1873,10 +2351,12 @@ diagnostic_execution_path_add_event (diagnostic_execution_path *path,
 
   va_list args;
   va_start (args, gmsgid);
-  diagnostic_event_id_t result = path->add_event_va (physical_loc,
-						     logical_loc,
-						     stack_depth,
-						     gmsgid, &args);
+  diagnostics::paths::event_id_t result
+    = path->add_event_va (physical_loc,
+			  logical_loc,
+			  stack_depth,
+			  nullptr,
+			  gmsgid, &args);
   va_end (args);
 
   return as_diagnostic_event_id (result);
@@ -1895,10 +2375,12 @@ diagnostic_execution_path_add_event_va (diagnostic_execution_path *path,
   FAIL_IF_NULL (path);
   FAIL_IF_NULL (gmsgid);
 
-  diagnostic_event_id_t result = path->add_event_va (physical_loc,
-						     logical_loc,
-						     stack_depth,
-						     gmsgid, args);
+  diagnostics::paths::event_id_t result
+    = path->add_event_va (physical_loc,
+			  logical_loc,
+			  stack_depth,
+			  nullptr,
+			  gmsgid, args);
   return as_diagnostic_event_id (result);
 }
 
@@ -1928,7 +2410,7 @@ diagnostic_finish_va (diagnostic *diag, const char *gmsgid, va_list *args)
   else
     progname = "progname";
   auto_diagnostic_group d;
-  diag->get_manager ().emit (*diag, gmsgid, args);
+  diag->get_manager ().emit_va (*diag, gmsgid, args);
   delete diag;
 }
 
@@ -1987,7 +2469,7 @@ diagnostic_logical_location_get_decorated_name (const diagnostic_logical_locatio
 
 namespace {
 
-struct spec_context : public diagnostics_output_spec::context
+struct spec_context : public diagnostics::output_spec::context
 {
 public:
   spec_context (const char *option_name,
@@ -2048,3 +2530,504 @@ diagnostic_manager_set_analysis_target (diagnostic_manager *mgr,
 
   mgr->get_dc ().set_main_input_filename (file->get_name ());
 }
+
+/* Public entrypoint.  */
+
+diagnostic_node *
+diagnostic_graph::add_node_with_id (std::string node_id,
+				    diagnostic_node *parent_node)
+{
+  auto node_up
+    = std::make_unique<diagnostic_node> (*this, std::move (node_id));
+  diagnostic_node *new_node = node_up.get ();
+  if (parent_node)
+    parent_node->add_child (std::move (node_up));
+  else
+    add_node (std::move (node_up));
+  return new_node;
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph::add_edge_with_label (const char *edge_id,
+				       diagnostic_node &src_node,
+				       diagnostic_node &dst_node,
+				       const char *label)
+{
+  auto edge_up
+    = std::make_unique<diagnostic_edge> (*this, edge_id,
+					 src_node, dst_node);
+  diagnostic_edge *new_edge = edge_up.get ();
+  if (label)
+    new_edge->set_label (label);
+  add_edge (std::move (edge_up));
+  return new_edge;
+}
+
+/* Public entrypoint.  */
+
+diagnostic_graph *
+diagnostic_manager_new_graph (diagnostic_manager *manager)
+{
+  FAIL_IF_NULL (manager);
+
+  return new diagnostic_graph (*manager);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_manager_take_global_graph (diagnostic_manager *manager,
+				      diagnostic_graph *graph)
+{
+  FAIL_IF_NULL (manager);
+  FAIL_IF_NULL (graph);
+
+  manager->take_global_graph (std::unique_ptr<diagnostic_graph> (graph));
+}
+
+void
+diagnostic_take_graph (diagnostic *diag,
+		       diagnostic_graph *graph)
+{
+  FAIL_IF_NULL (diag);
+  FAIL_IF_NULL (graph);
+
+  diag->take_graph (std::unique_ptr<diagnostic_graph> (graph));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_graph_release (diagnostic_graph *graph)
+{
+  delete graph;
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_graph_set_description (diagnostic_graph *graph,
+				  const char *desc)
+{
+  FAIL_IF_NULL (graph);
+
+  graph->set_description (desc);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_node *
+diagnostic_graph_add_node (diagnostic_graph *graph,
+			   const char *node_id,
+			   diagnostic_node *parent_node)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (node_id);
+
+  return graph->add_node_with_id (node_id, parent_node);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph_add_edge (diagnostic_graph *graph,
+			   const char *edge_id,
+			   diagnostic_node *src_node,
+			   diagnostic_node *dst_node,
+			   const char *label)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (src_node);
+  FAIL_IF_NULL (dst_node);
+
+  return graph->add_edge_with_label (edge_id, *src_node, *dst_node, label);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_node *
+diagnostic_graph_get_node_by_id (diagnostic_graph *graph,
+				 const char *node_id)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (node_id);
+
+  return static_cast<diagnostic_node *> (graph->get_node_by_id (node_id));
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph_get_edge_by_id (diagnostic_graph *graph,
+				 const char *edge_id)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (edge_id);
+
+  return static_cast<diagnostic_edge *> (graph->get_edge_by_id (edge_id));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_location (diagnostic_node *node,
+			      const diagnostic_physical_location *loc)
+{
+  FAIL_IF_NULL (node);
+
+  node->set_physical_loc (loc ? loc->m_inner : UNKNOWN_LOCATION);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_label (diagnostic_node *node,
+			   const char *label)
+{
+  FAIL_IF_NULL (node);
+
+  node->set_label (label);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_logical_location (diagnostic_node *node,
+				      const diagnostic_logical_location *logical_loc)
+{
+  FAIL_IF_NULL (node);
+
+  node->set_logical_loc
+    (impl_logical_location_manager::key_from_ptr (logical_loc));
+}
+
+/* Private entrypoint.  */
+
+void
+private_diagnostic_graph_set_property_bag (diagnostic_graph &graph,
+					  std::unique_ptr<json::object> properties)
+{
+  graph.set_property_bag (std::move (properties));
+}
+
+/* Private entrypoint.  */
+
+void
+private_diagnostic_node_set_property_bag (diagnostic_node &node,
+					  std::unique_ptr<json::object> properties)
+{
+  node.set_property_bag (std::move (properties));
+}
+
+/* Private entrypoint.  */
+
+void
+private_diagnostic_edge_set_property_bag (diagnostic_edge &edge,
+					  std::unique_ptr<json::object> properties)
+{
+  edge.set_property_bag (std::move (properties));
+}
+
+/* Public entrypoint.  */
+
+diagnostic_message_buffer *
+diagnostic_message_buffer_new ()
+{
+  return new diagnostic_message_buffer ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_release (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  delete msg_buf;
+}
+
+void
+diagnostic_message_buffer_append_str (diagnostic_message_buffer *msg_buf,
+				      const char *p)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (p);
+  msg_buf->m_tokens.push_back_text (label_text::take (xstrdup (p)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_text (diagnostic_message_buffer *msg_buf,
+				       const char *p,
+				       size_t len)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (p);
+  msg_buf->m_tokens.push_back_text (label_text::take (xstrndup (p, len)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_byte (diagnostic_message_buffer *msg_buf,
+				       char ch)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back_byte (ch);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_printf (diagnostic_message_buffer *msg_buf,
+					 const char *fmt, ...)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (fmt);
+
+  va_list args;
+  va_start (args, fmt);
+
+  char *formatted_buf = xvasprintf (fmt, args);
+
+  va_end (args);
+
+  msg_buf->m_tokens.push_back_text (label_text::take (formatted_buf));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_event_id (diagnostic_message_buffer *msg_buf,
+					   diagnostic_event_id event_id)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_event_id> (event_id);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_begin_url (diagnostic_message_buffer *msg_buf,
+				     const char *url)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (url);
+  msg_buf->m_tokens.push_back<pp_token_begin_url>
+    (label_text::take (xstrdup (url)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_end_url> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_begin_quote (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_begin_quote> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_end_quote (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_end_quote> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_begin_color (diagnostic_message_buffer *msg_buf,
+				       const char *color)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (color);
+  msg_buf->m_tokens.push_back<pp_token_begin_color>
+    (label_text::take (xstrdup (color)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_end_color (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_end_color> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_dump (const diagnostic_message_buffer *msg_buf,
+				FILE *outf)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (outf);
+
+  msg_buf->m_tokens.dump (outf);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_finish_via_msg_buf (diagnostic *diag,
+			       diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (diag);
+  FAIL_IF_NULL (msg_buf);
+
+  if (const char *tool_name
+      = diag->get_manager ().get_client_version_info ()->m_name.get_str ())
+    progname = tool_name;
+  else
+    progname = "progname";
+  auto_diagnostic_group d;
+  diag->get_manager ().emit_msg_buf (*diag, *msg_buf);
+  delete diag;
+  delete msg_buf;
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_add_location_with_label_via_msg_buf (diagnostic *diag,
+						const diagnostic_physical_location *loc,
+						diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (diag);
+  diag->get_manager ().assert_valid_diagnostic_physical_location (loc);
+  FAIL_IF_NULL (msg_buf);
+
+  std::unique_ptr<diagnostic_message_buffer> msg_buf_up (msg_buf);
+  diag->add_location_with_label (loc, std::move (msg_buf_up));
+}
+
+/* Public entrypoint.  */
+
+diagnostic_event_id
+diagnostic_execution_path_add_event_via_msg_buf (diagnostic_execution_path *path,
+						 const diagnostic_physical_location *physical_loc,
+						 const diagnostic_logical_location *logical_loc,
+						 unsigned stack_depth,
+						 diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (path);
+  FAIL_IF_NULL (msg_buf);
+
+  std::unique_ptr<diagnostic_message_buffer> msg_buf_up (msg_buf);
+  diagnostic_event_id_t result
+    = path->add_event_via_msg_buf (physical_loc,
+				   logical_loc,
+				   stack_depth,
+				   nullptr,
+				   std::move (msg_buf_up));
+  return as_diagnostic_event_id (result);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_graph_set_description_via_msg_buf (diagnostic_graph *graph,
+					      diagnostic_message_buffer *desc)
+{
+  FAIL_IF_NULL (graph);
+
+  if (desc)
+    graph->set_description (desc->to_string ());
+  else
+    graph->set_description (nullptr);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph_add_edge_via_msg_buf (diagnostic_graph *graph,
+				       const char *edge_id,
+				       diagnostic_node *src_node,
+				       diagnostic_node *dst_node,
+				       diagnostic_message_buffer *label)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (src_node);
+  FAIL_IF_NULL (dst_node);
+
+  if (label)
+    {
+      std::string label_str (label->to_string ());
+      return graph->add_edge_with_label (edge_id, *src_node, *dst_node,
+					 label_str.c_str ());
+    }
+  else
+    return graph->add_edge_with_label (edge_id, *src_node, *dst_node,
+				       nullptr);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_label_via_msg_buf (diagnostic_node *node,
+				       diagnostic_message_buffer *label)
+{
+  FAIL_IF_NULL (node);
+
+  if (label)
+    node->set_label (label->to_string ());
+  else
+    node->set_label (nullptr);
+}
+
+/* Private entrypoint.  */
+
+diagnostic_event_id
+private_diagnostic_execution_path_add_event_3 (diagnostic_execution_path *path,
+					       const diagnostic_physical_location *physical_loc,
+					       const diagnostic_logical_location *logical_loc,
+					       unsigned stack_depth,
+					       diagnostic_graph *state_graph,
+					       diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (path);
+  FAIL_IF_NULL (msg_buf);
+
+  diagnostic_event_id_t result
+    = path->add_event_via_msg_buf
+	(physical_loc,
+	 logical_loc,
+	 stack_depth,
+	 std::unique_ptr <diagnostic_graph> (state_graph),
+	 std::unique_ptr <diagnostic_message_buffer> (msg_buf));
+
+  return as_diagnostic_event_id (result);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_manager_set_debug_physical_locations (diagnostic_manager *mgr,
+						 int value)
+{
+  FAIL_IF_NULL (mgr);
+  mgr->set_debug_physical_locations (value);
+}
+
+/* Private entrypoint.  */
+
+void
+private_diagnostic_set_nesting_level (diagnostic *diag,
+				      int nesting_level)
+{
+  FAIL_IF_NULL (diag);
+  diag->set_nesting_level (nesting_level);
+}
diff --git a/gcc/libgdiagnostics.h b/gcc/libgdiagnostics.h
index 9af2747..0ae56f0 100644
--- a/gcc/libgdiagnostics.h
+++ b/gcc/libgdiagnostics.h
@@ -50,6 +50,13 @@ extern "C" {
 #define LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL(ARG_NUM)
   /* empty; for the human reader */
 
+# if (LIBGDIAGNOSTICS_GCC_VERSION >= 4001)
+#  define LIBGDIAGNOSTICS_PARAM_FORMAT_STRING(FMT_KIND, FMT_ARG_NUM, ARGS_ARG_NUM) \
+     __attribute__ ((__format__ (FMT_KIND, FMT_ARG_NUM, ARGS_ARG_NUM)))
+# else
+#  define LIBGDIAGNOSTICS_PARAM_FORMAT_STRING(FMT_KIND, FMT_ARG_NUM, ARGS_ARG_NUM)
+# endif /* GNUC >= 4.1 */
+
 #define LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING(FMT_ARG_NUM, ARGS_ARG_NUM) \
   LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (FMT_ARG_NUM)
   /* In theory we'd also add
@@ -59,6 +66,10 @@ extern "C" {
      of -Wall but undocumented, and much fussier than I'd want to inflict
      on users of libgdiagnostics.  */
 
+#define LIBGDIAGNOSTICS_PARAM_PRINTF_FORMAT_STRING(FMT_ARG_NUM, ARGS_ARG_NUM) \
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (FMT_ARG_NUM) \
+  LIBGDIAGNOSTICS_PARAM_FORMAT_STRING(gnu_printf, FMT_ARG_NUM, ARGS_ARG_NUM)
+
 /**********************************************************************
  Data structures and types.
  All structs within the API are opaque.
@@ -230,6 +241,8 @@ enum diagnostic_level
 typedef struct diagnostic_execution_path diagnostic_execution_path;
 typedef int diagnostic_event_id;
 
+typedef struct diagnostic_message_buffer diagnostic_message_buffer;
+
 /**********************************************************************
  API entrypoints.
  **********************************************************************/
@@ -765,6 +778,353 @@ diagnostic_manager_set_analysis_target (diagnostic_manager *mgr,
   LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
   LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
 
+/* Directed graphs.  */
+
+typedef struct diagnostic_graph diagnostic_graph;
+typedef struct diagnostic_node diagnostic_node;
+typedef struct diagnostic_edge diagnostic_edge;
+
+/* Create a new graph.
+   This is owned by the caller and must have one of
+   diagnostic_manager_take_global_graph, diagnostic_take_graph,
+   or diagnostic_graph_release called on it.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_graph *
+diagnostic_manager_new_graph (diagnostic_manager *manager)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Report this graph "globally", taking ownership of it.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_manager_take_global_graph (diagnostic_manager *manager,
+				      diagnostic_graph *graph)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Add this graph to DIAG, transferring ownership to it.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_take_graph (diagnostic *diag,
+		      diagnostic_graph *graph)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Release this graph.  Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_graph_release (diagnostic_graph *graph)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (1);
+
+/* Set the description of GRAPH for use
+   in the value of the SARIF "description" property
+   (SARIF v2.1.0 section 3.39.2).
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_graph_set_description (diagnostic_graph *graph,
+				 const char *description)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Create and add a new node within GRAPH.
+   NODE_ID must be unique within nodes in GRAPH.
+   The new node is owned by GRAPH.
+   PARENT_NODE can be NULL (for a top-level node in the graph),
+   or non-null for a child node.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_node *
+diagnostic_graph_add_node (diagnostic_graph *graph,
+			   const char *node_id,
+			   diagnostic_node *parent_node)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (3);
+
+/* Create and add a new edge within GRAPH.
+
+   If non-null, then EDGE_ID must be unique within edges in GRAPH;
+   if EDGE_ID is null then a unique id of the form "edge0", "edge1", etc
+   will be used automatically.
+
+   The new edge is owned by GRAPH.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_edge *
+diagnostic_graph_add_edge (diagnostic_graph *graph,
+			   const char *edge_id,
+			   diagnostic_node *src_node,
+			   diagnostic_node *dst_node,
+			   const char *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (4)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (5);
+
+/* Get the node in GRAPH with the given id, or null.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_node *
+diagnostic_graph_get_node_by_id (diagnostic_graph *graph,
+				 const char *node_id)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Get the edge in GRAPH with the given id, or null.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_edge *
+diagnostic_graph_get_edge_by_id (diagnostic_graph *graph,
+				 const char *edge_id)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Set the label of NODE for use
+   in the value of the SARIF "label" property
+   (SARIF v2.1.0 section 3.40.3).
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_node_set_label (diagnostic_node *node,
+			   const char *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Set the physical location of NODE.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_node_set_location (diagnostic_node *node,
+			      const diagnostic_physical_location *loc)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Set the logical location of NODE.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_node_set_logical_location (diagnostic_node *node,
+				      const diagnostic_logical_location *logical_loc)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Message buffers.  */
+
+#define LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+/* Create a new diagnostic_message_buffer.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_message_buffer *
+diagnostic_message_buffer_new (void);
+
+/* Release a diagnostic_message_buffer that hasn't been used.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_release (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Append a UTF-8 encoded null-terminated string to the buffer.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_str (diagnostic_message_buffer *msg_buf,
+				      const char *p)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Append a UTF-8 encoded run of bytes to the buffer.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_text (diagnostic_message_buffer *msg_buf,
+				       const char *p,
+				       size_t len)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Append a byte to to the buffer.  This should be either
+   ASCII, or part of UTF-8 encoded text.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_byte (diagnostic_message_buffer *msg_buf,
+				       char ch)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Append a formatted string to the buffer, using the formatting rules
+   for "printf".
+   The string is assumed to be UTF-8 encoded.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_printf (diagnostic_message_buffer *msg_buf,
+					 const char *fmt, ...)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_PRINTF_FORMAT_STRING (2, 3);
+
+/* Append a diagnostic_event_id to the buffer in the form "(1)".
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_event_id (diagnostic_message_buffer *msg_buf,
+					   diagnostic_event_id event_id)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Begin a run of text associated with the given URL.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_begin_url (diagnostic_message_buffer *msg_buf,
+				     const char *url)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* End a run of text started with diagnostic_message_buffer_begin_url.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Begin a run of text to be printed in quotes.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_begin_quote (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* End a run of text started with diagnostic_message_buffer_begin_quote.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_end_quote (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Begin a run of text to be printed with color.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_begin_color (diagnostic_message_buffer *msg_buf,
+				       const char *color)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* End a run of text started with diagnostic_message_buffer_begin_color.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_end_color (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Write a debugging representation of MSG_BUG to OUTF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_dump (const diagnostic_message_buffer *msg_buf,
+				FILE *outf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* As diagnostic_finish, but takes ownership of MSG_BUF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_finish_via_msg_buf (diagnostic *diag,
+			       diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* As diagnostic_add_location_with_label but takes ownership of MSG_BUF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_add_location_with_label_via_msg_buf (diagnostic *diag,
+						const diagnostic_physical_location *loc,
+						diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (3);
+
+/* As diagnostic_execution_path_add_event but takes ownership of MSG_BUF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_event_id
+diagnostic_execution_path_add_event_via_msg_buf (diagnostic_execution_path *path,
+						 const diagnostic_physical_location *physical_loc,
+						 const diagnostic_logical_location *logical_loc,
+						 unsigned stack_depth,
+						 diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (5);
+
+/* Set the description of GRAPH for use
+   in the value of the SARIF "description" property
+   (SARIF v2.1.0 section 3.39.2).
+
+   Takes ownership of DESC, if non-null.
+
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_graph_set_description_via_msg_buf (diagnostic_graph *graph,
+					      diagnostic_message_buffer *desc)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Create and add a new edge within GRAPH.
+
+   If non-null, then EDGE_ID must be unique within edges in GRAPH;
+   if EDGE_ID is null then a unique id of the form "edge0", "edge1", etc
+   will be used automatically.
+
+   Takes ownership of LABEL, if non-null.
+
+   The new edge is owned by GRAPH.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_edge *
+diagnostic_graph_add_edge_via_msg_buf (diagnostic_graph *graph,
+				       const char *edge_id,
+				       diagnostic_node *src_node,
+				       diagnostic_node *dst_node,
+				       diagnostic_message_buffer *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (4)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (5);
+
+/* Set the label of NODE for use
+   in the value of the SARIF "label" property
+   (SARIF v2.1.0 section 3.40.3).
+
+   Takes ownership of LABEL, if non-null.
+
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_node_set_label_via_msg_buf (diagnostic_node *node,
+				       diagnostic_message_buffer *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* If non-zero, print debugging information to stderr when
+   creating diagnostic_physical_location instances.
+
+   Added in LIBGDIAGNOSTICS_ABI_5.  */
+#define LIBDIAGNOSTICS_HAVE_diagnostic_manager_set_debug_physical_locations
+
+extern void
+diagnostic_manager_set_debug_physical_locations (diagnostic_manager *mgr,
+						 int value);
+
 /* DEFERRED:
    - thread-safety
    - plural forms
@@ -772,6 +1132,7 @@ diagnostic_manager_set_analysis_target (diagnostic_manager *mgr,
    - locations within binary files
    - options and URLs for warnings
    - enable/disable of warnings by kind
+   - command-line arguments
    - plugin metadata.  */
 
 #ifdef __cplusplus
diff --git a/gcc/libgdiagnostics.map b/gcc/libgdiagnostics.map
index 49cabca..0400ca7 100644
--- a/gcc/libgdiagnostics.map
+++ b/gcc/libgdiagnostics.map
@@ -69,6 +69,7 @@ LIBGDIAGNOSTICS_ABI_0
     diagnostic_finish;
     diagnostic_finish_va;
 
+
     diagnostic_physical_location_get_file;
 
   local: *;
@@ -90,3 +91,61 @@ LIBGDIAGNOSTICS_ABI_2 {
     diagnostic_manager_add_sink_from_spec;
     diagnostic_manager_set_analysis_target;
 } LIBGDIAGNOSTICS_ABI_1;
+
+# Add hooks needed for diagnostic_graph support.
+LIBGDIAGNOSTICS_ABI_3 {
+  global:
+    diagnostic_manager_new_graph;
+    diagnostic_manager_take_global_graph;
+    diagnostic_take_graph;
+    diagnostic_graph_release;
+    diagnostic_graph_set_description;
+    diagnostic_graph_add_node;
+    diagnostic_graph_add_edge;
+    diagnostic_graph_get_node_by_id;
+    diagnostic_graph_get_edge_by_id;
+    diagnostic_node_set_label;
+    diagnostic_node_set_location;
+    diagnostic_node_set_logical_location;
+
+    # Private hooks used by sarif-replay
+    private_diagnostic_graph_set_property_bag;
+    private_diagnostic_node_set_property_bag;
+    private_diagnostic_edge_set_property_bag;
+} LIBGDIAGNOSTICS_ABI_2;
+
+# Add diagnostic_message_buffer
+LIBGDIAGNOSTICS_ABI_4 {
+  global:
+    diagnostic_message_buffer_new;
+    diagnostic_message_buffer_release;
+    diagnostic_message_buffer_append_str;
+    diagnostic_message_buffer_append_text;
+    diagnostic_message_buffer_append_byte;
+    diagnostic_message_buffer_append_printf;
+    diagnostic_message_buffer_append_event_id;
+    diagnostic_message_buffer_begin_url;
+    diagnostic_message_buffer_end_url;
+    diagnostic_message_buffer_begin_quote;
+    diagnostic_message_buffer_end_quote;
+    diagnostic_message_buffer_begin_color;
+    diagnostic_message_buffer_end_color;
+    diagnostic_message_buffer_dump;
+    diagnostic_finish_via_msg_buf;
+    diagnostic_add_location_with_label_via_msg_buf;
+    diagnostic_execution_path_add_event_via_msg_buf;
+    diagnostic_graph_set_description_via_msg_buf;
+    diagnostic_graph_add_edge_via_msg_buf;
+    diagnostic_node_set_label_via_msg_buf;
+
+    # Private hook used by sarif-replay
+    private_diagnostic_execution_path_add_event_3;
+} LIBGDIAGNOSTICS_ABI_3;
+
+LIBGDIAGNOSTICS_ABI_5 {
+  global:
+    diagnostic_manager_set_debug_physical_locations;
+
+    # Private hook used by sarif-replay
+    private_diagnostic_set_nesting_level;
+} LIBGDIAGNOSTICS_ABI_4;
diff --git a/gcc/libsarifreplay.cc b/gcc/libsarifreplay.cc
index 2d6c394..e8fc6d0 100644
--- a/gcc/libsarifreplay.cc
+++ b/gcc/libsarifreplay.cc
@@ -26,6 +26,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "libgdiagnostics++.h"
+#include "libgdiagnostics-private.h"
 #include "json-parsing.h"
 #include "intl.h"
 #include "sarif-spec-urls.def"
@@ -281,16 +282,18 @@ class annotation
 {
 public:
   annotation (libgdiagnostics::physical_location phys_loc,
-	      label_text label)
+	      libgdiagnostics::message_buffer label)
   : m_phys_loc (phys_loc),
     m_label (std::move (label))
   {
   }
 
   libgdiagnostics::physical_location m_phys_loc;
-  label_text m_label;
+  libgdiagnostics::message_buffer m_label;
 };
 
+using id_map = std::map<std::string, const json::string *>;
+
 class sarif_replayer
 {
 public:
@@ -330,7 +333,7 @@ private:
 
   enum status emit_sarif_as_diagnostics (const json::value &jv);
 
-  label_text
+  libgdiagnostics::message_buffer
   make_plain_text_within_result_message (const json::object *tool_component_obj,
 					 const json::object &message_obj,
 					 const json::object *rule_obj);
@@ -413,6 +416,30 @@ private:
 				      const json::object &run_obj,
 				      libgdiagnostics::execution_path &out);
 
+  // "graph" object (§3.39)
+  enum status
+  handle_graph_object (const json::object &graph_obj,
+		       const json::object &run_obj,
+		       libgdiagnostics::graph &out);
+  // "node" object (§3.40)
+  libgdiagnostics::node
+  handle_node_object (const json::object &node_obj,
+		      const json::object &run_obj,
+		      libgdiagnostics::graph &graph,
+		      libgdiagnostics::node parent_node,
+		      id_map &node_id_map);
+
+  // "edge" object (§3.41)
+  libgdiagnostics::edge
+  handle_edge_object (const json::object &edge_obj,
+		      libgdiagnostics::graph &graph,
+		      id_map &edge_id_map);
+
+  libgdiagnostics::node
+  get_graph_node_by_id_property (const json::object &edge_json_object,
+				 const property_spec_ref &id_prop,
+				 libgdiagnostics::graph &graph);
+
   // reportingDescriptor lookup (§3.52.3)
   const json::object *
   lookup_rule_by_id_in_tool (const char *rule_id,
@@ -446,7 +473,7 @@ private:
   {
     va_list ap;
     va_start (ap, gmsgid);
-    report_problem (jv, ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_ERROR);
+    report_problem (jv, &ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_ERROR);
     va_end (ap);
     return status::err_invalid_sarif;
   }
@@ -462,14 +489,25 @@ private:
   {
     va_list ap;
     va_start (ap, gmsgid);
-    report_problem (jv, ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_SORRY);
+    report_problem (jv, &ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_SORRY);
     va_end (ap);
     return status::err_unhandled_sarif;
   }
 
   void
+  report_note (const json::value &jv,
+	       const char *gmsgid, ...)
+    LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (3, 4)
+  {
+    va_list ap;
+    va_start (ap, gmsgid);
+    report_problem (jv, nullptr, gmsgid, &ap, DIAGNOSTIC_LEVEL_NOTE);
+    va_end (ap);
+  }
+
+  void
   report_problem (const json::value &jv,
-		  const spec_ref &ref,
+		  const spec_ref *ref,
 		  const char *gmsgid,
 		  va_list *args,
 		  enum diagnostic_level level)
@@ -481,11 +519,14 @@ private:
        There doesn't seem to be a systematic mapping from spec sections to
        HTML anchors, so we can't provide URLs
        (filed as https://github.com/oasis-tcs/sarif-spec/issues/533 ).  */
-    char *ref_desc = ref.make_description ();
-    char *ref_url = ref.make_url ();
-    diag.add_rule (ref_desc, ref_url);
-    free (ref_desc);
-    free (ref_url);
+    if (ref)
+      {
+	char *ref_desc = ref->make_description ();
+	char *ref_url = ref->make_url ();
+	diag.add_rule (ref_desc, ref_url);
+	free (ref_desc);
+	free (ref_url);
+      }
 
     auto loc_range
       = make_physical_location (m_control_mgr,
@@ -651,6 +692,38 @@ private:
 			      const string_property_value<ValueType> *value_arr,
 			      size_t num_values);
 
+  const json::object *
+  maybe_get_property_bag (const json::object &obj)
+  {
+    const property_spec_ref properties
+      ("object", "properties", "3.8.1");
+    return get_optional_property<json::object> (obj, properties);
+  }
+
+  /* Look for a property bag within OBJ.
+     If found, look for a property within it named PROPERTY_NAME
+     of the given type.
+     If successful, return the property's value.
+     Otherwise, return nullptr without complaining (unless the property bag
+     is itself not an object).  */
+  template <typename JsonType>
+  const JsonType *
+  maybe_get_property_bag_value (const json::object &obj,
+				const char *property_name)
+  {
+    auto property_bag_obj = maybe_get_property_bag (obj);
+    if (!property_bag_obj)
+      return nullptr;
+    const json::value *property_val = property_bag_obj->get (property_name);
+    if (!property_val)
+      return nullptr;
+    const JsonType *sub = dyn_cast<const JsonType *> (property_val);
+    if (!sub)
+      /* Property is wrong kind of value.  Don't treat this as an error.  */
+      return nullptr;
+    return sub;
+  }
+
   /* The manager to replay the SARIF files to.  */
   libgdiagnostics::manager m_output_mgr;
 
@@ -929,6 +1002,31 @@ sarif_replayer::handle_run_obj (const json::object &run_obj)
 	break;
       }
 
+  // §3.14.20 "graphs"
+  const property_spec_ref prop_graphs ("run", "graphs", "3.14.20");
+  if (const json::array *graphs
+      = get_optional_property<json::array> (run_obj,
+					    prop_graphs))
+    {
+      for (auto element : *graphs)
+	{
+	  if (const json::object *graph_json_obj
+	      = require_object_for_element (*element, prop_graphs))
+	    {
+	      libgdiagnostics::graph graph;
+	      enum status s = handle_graph_object (*graph_json_obj,
+						   run_obj,
+						   graph);
+	      if (s != status::ok)
+		return s;
+
+	      m_output_mgr.take_global_graph (std::move (graph));
+	    }
+	  else
+	    return status::err_invalid_sarif;
+      }
+    }
+
   return status::ok;
 }
 
@@ -1083,12 +1181,12 @@ sarif_replayer::get_level_from_level_str (const json::string &level_str)
 
 static void
 add_any_annotations (libgdiagnostics::diagnostic &diag,
-		     const std::vector<annotation> &annotations)
+		     std::vector<annotation> &annotations)
 {
   for (auto &annotation : annotations)
-    if (annotation.m_label.get ())
+    if (annotation.m_label.m_inner)
       diag.add_location_with_label (annotation.m_phys_loc,
-				    annotation.m_label.get ());
+				    std::move (annotation.m_label));
     else
       diag.add_location (annotation.m_phys_loc);
 }
@@ -1151,13 +1249,13 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
     }
 
   // §3.27.11 "message" property
-  label_text text;
+  libgdiagnostics::message_buffer msg_buf;
   if (auto message_obj
 	= get_optional_property<json::object> (result_obj, PROP_result_message))
-    text = make_plain_text_within_result_message (nullptr, // TODO: tool_component_obj,
-						  *message_obj,
-						  rule_obj);
-  if (!text.get ())
+    msg_buf = make_plain_text_within_result_message (nullptr, // TODO: tool_component_obj,
+						     *message_obj,
+						     rule_obj);
+  if (!msg_buf.m_inner)
     return status::err_invalid_sarif;
 
   // §3.27.12 "locations" property
@@ -1244,8 +1342,34 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
   if (path.m_inner)
     err.take_execution_path (std::move (path));
 
+  // §3.27.19 "graphs" property
+  const property_spec_ref prop_graphs ("result", "graphs", "3.27.19");
+  if (const json::array *graphs
+      = get_optional_property<json::array> (result_obj,
+					    prop_graphs))
+    {
+      for (auto element : *graphs)
+	{
+	  if (const json::object *graph_json_obj
+	      = require_object_for_element (*element, prop_graphs))
+	    {
+	      libgdiagnostics::graph graph;
+	      enum status s = handle_graph_object (*graph_json_obj,
+						   run_obj,
+						   graph);
+	      if (s != status::ok)
+		return s;
+
+	      err.take_graph (std::move (graph));
+	    }
+	  else
+	    return status::err_invalid_sarif;
+      }
+    }
+
   // §3.27.22 relatedLocations property
-  std::vector<std::pair<libgdiagnostics::diagnostic, label_text>> notes;
+  std::vector<std::pair<libgdiagnostics::diagnostic,
+			libgdiagnostics::message_buffer>> notes;
   const property_spec_ref prop_related_locations
     ("result", "relatedLocations", "3.27.22");
   if (auto related_locations_arr
@@ -1277,18 +1401,30 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
 						     prop_message))
 	    {
 	      /* Treat related locations with a message as a "note".  */
-	      label_text text
+	      libgdiagnostics::message_buffer msg_buf
 		(make_plain_text_within_result_message
 		 (tool_component_obj,
 		  *message_obj,
 		  rule_obj));
-	      if (!text.get ())
+	      if (!msg_buf.m_inner)
 		return status::err_invalid_sarif;
+
 	      auto note (m_output_mgr.begin_diagnostic (DIAGNOSTIC_LEVEL_NOTE));
 	      note.set_location (physical_loc);
 	      note.set_logical_location (logical_loc);
 	      add_any_annotations (note, annotations);
-	      notes.push_back ({std::move (note), std::move (text)});
+
+	      /* Look for "nestingLevel" property, as per
+		 "P3358R0 SARIF for Structured Diagnostics"
+		 https://wg21.link/P3358R0  */
+	      if (auto nesting_level
+		  = maybe_get_property_bag_value<json::integer_number>
+		  (*location_obj,
+		   "nestingLevel"))
+		private_diagnostic_set_nesting_level (note.m_inner,
+						      nesting_level->get ());
+
+	      notes.push_back ({std::move (note), std::move (msg_buf)});
 	    }
 	  else
 	    {
@@ -1311,14 +1447,14 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
 	  handle_fix_object (err, *fix_obj);
     }
 
-  err.finish ("%s", text.get ());
+  err.finish_via_msg_buf (std::move (msg_buf));
 
   // Flush any notes
   for (auto &iter : notes)
     {
       auto &note = iter.first;
-      auto &text = iter.second;
-      note.finish ("%s", text.get ());
+      auto &msg_buf = iter.second;
+      note.finish_via_msg_buf (std::move (msg_buf));
     }
 
   return status::ok;
@@ -1446,13 +1582,10 @@ maybe_consume_embedded_link (const char *&iter_src)
    and substitute for any placeholders (§3.11.5) and handle any
    embedded links (§3.11.6).
 
-   Limitations:
-   - we don't preserve destinations within embedded links
-
    MESSAGE_OBJ is "theMessage"
    RULE_OBJ is "theRule".  */
 
-label_text
+libgdiagnostics::message_buffer
 sarif_replayer::
 make_plain_text_within_result_message (const json::object *tool_component_obj,
 				       const json::object &message_obj,
@@ -1465,7 +1598,7 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 					       rule_obj,
 					       js_str);
   if (!original_text)
-    return label_text::borrow (nullptr);
+    return libgdiagnostics::message_buffer ();
 
   gcc_assert (js_str);
 
@@ -1475,7 +1608,7 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
     = get_optional_property<json::array> (message_obj, arguments_prop);
 
   /* Duplicate original_text, substituting any placeholders.  */
-  std::string accum;
+  libgdiagnostics::message_buffer result (diagnostic_message_buffer_new ());
 
   const char *iter_src = original_text;
   while (char ch = *iter_src)
@@ -1491,7 +1624,7 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 		 " but message object has no %qs property",
 		 (int)arg_idx,
 		 arguments_prop.get_property_name ());
-	      return label_text::borrow (nullptr);
+	      return libgdiagnostics::message_buffer ();
 	    }
 	  if (arg_idx >= arguments->length ())
 	    {
@@ -1502,20 +1635,20 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 		 arguments_prop.get_property_name (),
 		 (int)arg_idx);
 	      // TODO: might be nice to add a note showing the args
-	      return label_text::borrow (nullptr);
+	      return libgdiagnostics::message_buffer ();
 	    }
 	  auto replacement_jstr
 	    = require_string (*arguments->get (arg_idx), arguments_prop);
 	  if (!replacement_jstr)
-	    return label_text::borrow (nullptr);
-	  accum += replacement_jstr->get_string ();
+	    return libgdiagnostics::message_buffer ();
+	  result += replacement_jstr->get_string ();
 	}
       else if (ch == '{' || ch == '}')
 	{
 	  /* '{' and '}' are escaped by repeating them.  */
 	  if (iter_src[1] == ch)
 	    {
-	      accum += ch;
+	      result += ch;
 	      iter_src += 2;
 	    }
 	  else
@@ -1525,24 +1658,25 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 	      report_invalid_sarif (*js_str, msgs_with_placeholders,
 				    "unescaped '%c' within message string",
 				    ch);
-	      return label_text::borrow (nullptr);
+	      return libgdiagnostics::message_buffer ();
 	    }
 	}
       else if (auto link = maybe_consume_embedded_link (iter_src))
 	{
-	  accum += link->text;
-	  /* TODO: use the destination.  */
+	  result.begin_url (link->destination.c_str ());
+	  result += link->text.c_str ();
+	  result.end_url ();
 	  /* TODO: potentially could try to convert
 	     intra-sarif links into event ids.  */
 	}
       else
 	{
-	  accum += ch;
+	  result += ch;
 	  iter_src++;
 	}
     }
 
-  return label_text::take (xstrdup (accum.c_str ()));
+  return result;
 }
 
 /* Handle a value that should be a multiformatMessageString object (§3.12).
@@ -1676,7 +1810,7 @@ handle_thread_flow_location_object (const json::object &tflow_loc_obj,
 {
   libgdiagnostics::physical_location physical_loc;
   libgdiagnostics::logical_location logical_loc;
-  label_text message;
+  libgdiagnostics::message_buffer msg_buf;
   int stack_depth = 0;
 
   const property_spec_ref location_prop
@@ -1698,7 +1832,7 @@ handle_thread_flow_location_object (const json::object &tflow_loc_obj,
 	  = get_optional_property<json::object> (*location_obj,
 						 location_message))
 	{
-	  message = make_plain_text_within_result_message
+	  msg_buf = make_plain_text_within_result_message
 	    (nullptr,
 	     *message_obj,
 	     nullptr/* TODO.  */);
@@ -1740,16 +1874,29 @@ handle_thread_flow_location_object (const json::object &tflow_loc_obj,
 	}
     }
 
-  if (message.get ())
-    path.add_event (physical_loc,
-		    logical_loc,
-		    stack_depth,
-		    "%s", message.get ());
-  else
-    path.add_event (physical_loc,
-		    logical_loc,
-		    stack_depth,
-		    "");
+  libgdiagnostics::graph state_graph;
+  if (auto sarif_state_graph
+	= maybe_get_property_bag_value<json::object> (tflow_loc_obj,
+						      "gcc/diagnostics/paths/event/state_graph"))
+    {
+      enum status s
+	= handle_graph_object (*sarif_state_graph, run_obj, state_graph);
+      if (s != status::ok)
+	return s;
+    }
+
+  if (!msg_buf.m_inner)
+    msg_buf.m_inner = diagnostic_message_buffer_new ();
+
+  private_diagnostic_execution_path_add_event_3 (path.m_inner,
+						 physical_loc.m_inner,
+						 logical_loc.m_inner,
+						 stack_depth,
+						 state_graph.m_inner,
+						 msg_buf.m_inner);
+
+  state_graph.m_owned = false;
+  msg_buf.m_inner = nullptr;
 
   return status::ok;
 }
@@ -1828,7 +1975,7 @@ handle_location_object (const json::object &location_obj,
 	  if (s != status::ok)
 	    return s;
 
-	  label_text label;
+	  libgdiagnostics::message_buffer label;
 
 	  // §3.30.14 message property
 	  {
@@ -2142,6 +2289,270 @@ handle_logical_location_object (const json::object &logical_loc_obj,
   return status::ok;
 }
 
+// "graph" object (§3.39)
+
+enum status
+sarif_replayer::handle_graph_object (const json::object &graph_json_obj,
+				     const json::object &run_obj,
+				     libgdiagnostics::graph &out_graph)
+{
+  out_graph = libgdiagnostics::graph
+		(diagnostic_manager_new_graph (m_output_mgr.m_inner));
+
+  id_map node_id_map;
+  id_map edge_id_map;
+
+  if (auto properties = maybe_get_property_bag (graph_json_obj))
+    private_diagnostic_graph_set_property_bag (*out_graph.m_inner,
+					       properties->clone_as_object ());
+
+  // §3.39.2: MAY contain a "description" property
+  const property_spec_ref description_prop
+    ("graph", "description", "3.39.2");
+  if (auto description_obj
+      = get_optional_property<json::object> (graph_json_obj, description_prop))
+    {
+      auto msg_buf
+	= make_plain_text_within_result_message (&run_obj,
+						 *description_obj,
+						 nullptr);
+      if (!msg_buf.m_inner)
+	return status::err_invalid_sarif;
+      out_graph.set_description (std::move (msg_buf));
+    }
+
+  // §3.39.3: MAY contain a "nodes" property
+  const property_spec_ref nodes_prop
+    ("graph", "nodes", "3.39.3");
+  if (auto nodes_arr
+      = get_optional_property<json::array> (graph_json_obj, nodes_prop))
+    {
+      for (auto element : *nodes_arr)
+	{
+	  const json::object *node_json_obj
+	    = require_object_for_element (*element, nodes_prop);
+	  if (!node_json_obj)
+	    return status::err_invalid_sarif;
+	  libgdiagnostics::node node
+	    = handle_node_object (*node_json_obj, run_obj, out_graph,
+				  nullptr, node_id_map);
+	  if (node.m_inner == nullptr)
+	    return status::err_invalid_sarif;
+	}
+    }
+  else
+    // If we have no nodes, we can't handle the edges.
+    return status::ok;
+
+  // §3.39.4 edges property
+  const property_spec_ref edges_prop
+    ("graph", "edges", "3.39.4");
+  if (auto edges_arr
+      = get_optional_property<json::array> (graph_json_obj, edges_prop))
+    {
+      for (auto element : *edges_arr)
+	{
+	  const json::object *edge_json_obj
+	    = require_object_for_element (*element, edges_prop);
+	  if (!edge_json_obj)
+	    return status::err_invalid_sarif;
+	  libgdiagnostics::edge edge
+	    = handle_edge_object (*edge_json_obj, out_graph, edge_id_map);
+	  if (edge.m_inner == nullptr)
+	    return status::err_invalid_sarif;
+	}
+    }
+
+  return status::ok;
+}
+
+// "node" object (§3.40)
+
+libgdiagnostics::node
+sarif_replayer::handle_node_object (const json::object &node_json_obj,
+				    const json::object &run_obj,
+				    libgdiagnostics::graph &graph,
+				    libgdiagnostics::node parent_node,
+				    id_map &node_id_map)
+{
+  // §3.40.2 "id" property
+  const property_spec_ref id_prop ("node", "id", "3.40.2");
+  auto id_str = get_required_property<json::string> (node_json_obj, id_prop);
+  if (!id_str)
+    return nullptr;
+  const char *id = id_str->get_string ();
+  if (diagnostic_graph_get_node_by_id (graph.m_inner, id))
+    {
+      // Duplicate id; fail:
+      libgdiagnostics::group g (m_control_mgr);
+      report_invalid_sarif (*id_str,
+			    id_prop,
+			    "duplicate node id %qs within graph",
+			    id);
+      gcc_assert (node_id_map[id]);
+      report_note (*node_id_map[id],
+		   "%qs already used as node id within graph here",
+		   id);
+      return nullptr;
+    }
+  node_id_map[id] = id_str;
+
+  libgdiagnostics::node new_node
+    = libgdiagnostics::node (diagnostic_graph_add_node (graph.m_inner,
+							id,
+							parent_node.m_inner));
+  if (auto properties = maybe_get_property_bag (node_json_obj))
+    private_diagnostic_node_set_property_bag (*new_node.m_inner,
+					      properties->clone_as_object ());
+
+  // §3.40.3 "label" property
+  const property_spec_ref label_prop
+    ("node", "label", "3.40.3");
+  if (auto label_obj
+      = get_optional_property<json::object> (node_json_obj, label_prop))
+    {
+      auto msg_buf
+	= make_plain_text_within_result_message (&run_obj,
+						 *label_obj,
+						 nullptr);
+      if (!msg_buf.m_inner)
+	return nullptr;
+      new_node.set_label (std::move (msg_buf));
+    }
+
+  // §3.40.4 "location" property
+  const property_spec_ref location_prop ("node", "location", "3.40.4");
+  if (auto location_json_obj
+      = get_optional_property<json::object> (node_json_obj, location_prop))
+    {
+      libgdiagnostics::physical_location physical_loc;
+      libgdiagnostics::logical_location logical_loc;
+      enum status s = handle_location_object (*location_json_obj,
+					      run_obj,
+					      physical_loc,
+					      logical_loc,
+					      nullptr); // annotations
+      if (s != status::ok)
+	return nullptr;
+
+      new_node.set_location (physical_loc);
+      new_node.set_logical_location (logical_loc);
+    }
+
+  // §3.40.5: MAY contain a "children" property
+  const property_spec_ref children_prop
+    ("graph", "children", "3.40.5");
+  if (auto children_json_arr
+      = get_optional_property<json::array> (node_json_obj, children_prop))
+    {
+      for (auto json_element : *children_json_arr)
+	{
+	  const json::object *child_json_obj
+	    = require_object_for_element (*json_element, children_prop);
+	  if (!child_json_obj)
+	    return nullptr;
+	  libgdiagnostics::node child_node
+	    = handle_node_object (*child_json_obj, run_obj, graph,
+				  new_node, node_id_map);
+	  if (child_node.m_inner == nullptr)
+	    return nullptr;
+	}
+    }
+
+  return new_node;
+}
+
+// "edge" object (§3.41)
+
+libgdiagnostics::edge
+sarif_replayer::handle_edge_object (const json::object &edge_json_obj,
+				    libgdiagnostics::graph &graph,
+				    id_map &edge_id_map)
+{
+  // §3.41.2 "id" property
+  const property_spec_ref id_prop ("edge", "id", "3.41.2");
+  auto id_str = get_required_property<json::string> (edge_json_obj, id_prop);
+  if (!id_str)
+    return nullptr;
+  const char *id = id_str->get_string ();
+  if (diagnostic_graph_get_edge_by_id (graph.m_inner, id))
+    {
+      // Duplicate id; fail:
+      libgdiagnostics::group g (m_control_mgr);
+      report_invalid_sarif (*id_str,
+			    id_prop,
+			    "duplicate edge id %qs within graph",
+			    id);
+      gcc_assert (edge_id_map[id]);
+      report_note (*edge_id_map[id],
+		   "%qs already used as edge id within graph here",
+		   id);
+      return nullptr;
+    }
+  edge_id_map[id] = id_str;
+
+  // §3.41.3 "label" property
+  libgdiagnostics::message_buffer label;
+  const property_spec_ref label_prop
+    ("edge", "label", "3.41.3");
+  if (auto label_obj
+      = get_optional_property<json::object> (edge_json_obj, label_prop))
+    {
+      label = make_plain_text_within_result_message (nullptr,
+						     *label_obj,
+						     nullptr);
+      if (!label.m_inner)
+	return nullptr;
+    }
+
+  // §3.41.4 "sourceNodeId" property
+  const property_spec_ref src_id_prop ("edge", "sourceNodeId", "3.41.4");
+  auto src_node = get_graph_node_by_id_property (edge_json_obj,
+						 src_id_prop,
+						 graph);
+  if (!src_node.m_inner)
+    return nullptr;
+
+  // §3.41.5 "targetNodeId" property
+  const property_spec_ref dst_id_prop ("edge", "targetNodeId", "3.41.5");
+  auto dst_node = get_graph_node_by_id_property (edge_json_obj,
+						 dst_id_prop,
+						 graph);
+  if (!dst_node.m_inner)
+    return nullptr;
+
+  auto result = graph.add_edge (id, src_node, dst_node, std::move (label));
+
+  if (auto properties = maybe_get_property_bag (edge_json_obj))
+    private_diagnostic_edge_set_property_bag (*result.m_inner,
+					      properties->clone_as_object ());
+
+  return result;
+}
+
+libgdiagnostics::node
+sarif_replayer::
+get_graph_node_by_id_property (const json::object &edge_json_obj,
+			       const property_spec_ref &id_prop,
+			       libgdiagnostics::graph &graph)
+{
+  auto id_str = get_required_property<json::string> (edge_json_obj, id_prop);
+  if (!id_str)
+    return nullptr;
+  const char *id = id_str->get_string ();
+  auto node = graph.get_node_by_id (id);
+  if (!node.m_inner)
+    {
+      // id not found; complain:
+      report_invalid_sarif (*id_str,
+			    id_prop,
+			    "no node with id %qs in graph",
+			    id);
+      return nullptr;
+    }
+  return node;
+}
+
 // 3.52.3 reportingDescriptor lookup
 // "For an example of the interaction between ruleId and rule.id, see §3.52.4."
 
diff --git a/gcc/libsarifreplay.h b/gcc/libsarifreplay.h
index fb66014..313a905 100644
--- a/gcc/libsarifreplay.h
+++ b/gcc/libsarifreplay.h
@@ -31,6 +31,7 @@ struct replay_options
   bool m_echo_file;
   bool m_json_comments;
   bool m_verbose;
+  bool m_debug_physical_locations;
   enum diagnostic_colorize m_diagnostics_colorize;
 };
 
diff --git a/gcc/loop-unroll.cc b/gcc/loop-unroll.cc
index 6149cec..c80a6cb 100644
--- a/gcc/loop-unroll.cc
+++ b/gcc/loop-unroll.cc
@@ -1185,7 +1185,7 @@ decide_unroll_stupid (class loop *loop, int flags)
 
   /* Do not unroll loops with branches inside -- it increases number
      of mispredicts.
-     TODO: this heuristic needs tunning; call inside the loop body
+     TODO: this heuristic needs tuning; call inside the loop body
      is also relatively good reason to not unroll.  */
   if (num_loop_branches (loop) > 1)
     {
diff --git a/gcc/lra-constraints.cc b/gcc/lra-constraints.cc
index 68aaf86..83f8fda 100644
--- a/gcc/lra-constraints.cc
+++ b/gcc/lra-constraints.cc
@@ -114,6 +114,7 @@
 #include "target.h"
 #include "rtl.h"
 #include "tree.h"
+#include "stmt.h"
 #include "predict.h"
 #include "df.h"
 #include "memmodel.h"
@@ -2168,6 +2169,7 @@ process_alt_operands (int only_alternative)
   bool costly_p;
   enum reg_class cl;
   const HARD_REG_SET *cl_filter;
+  HARD_REG_SET hard_reg_constraint;
 
   /* Calculate some data common for all alternatives to speed up the
      function.	*/
@@ -2416,14 +2418,15 @@ process_alt_operands (int only_alternative)
 			if (curr_static_id->operand[nop].type == OP_INOUT
 			    || curr_static_id->operand[m].type == OP_INOUT)
 			  break;
-			/* Operands don't match.  If the operands are
-			   different user defined explicit hard
+			/* Operands don't match.  For asm if the operands
+			   are different user defined explicit hard
 			   registers, then we cannot make them match
 			   when one is early clobber operand.  */
 			if ((REG_P (*curr_id->operand_loc[nop])
 			     || SUBREG_P (*curr_id->operand_loc[nop]))
 			    && (REG_P (*curr_id->operand_loc[m])
-				|| SUBREG_P (*curr_id->operand_loc[m])))
+				|| SUBREG_P (*curr_id->operand_loc[m]))
+			    && INSN_CODE (curr_insn) < 0)
 			  {
 			    rtx nop_reg = *curr_id->operand_loc[nop];
 			    if (SUBREG_P (nop_reg))
@@ -2544,6 +2547,17 @@ process_alt_operands (int only_alternative)
 		  cl_filter = nullptr;
 		  goto reg;
 
+		case '{':
+		    {
+		      int regno = decode_hard_reg_constraint (p);
+		      gcc_assert (regno >= 0);
+		      cl = REGNO_REG_CLASS (regno);
+		      CLEAR_HARD_REG_SET (hard_reg_constraint);
+		      SET_HARD_REG_BIT (hard_reg_constraint, regno);
+		      cl_filter = &hard_reg_constraint;
+		      goto reg;
+		    }
+
 		default:
 		  cn = lookup_constraint (p);
 		  switch (get_constraint_type (cn))
@@ -3328,19 +3342,15 @@ process_alt_operands (int only_alternative)
 		  first_conflict_j = j;
 		last_conflict_j = j;
 		/* Both the earlyclobber operand and conflicting operand
-		   cannot both be user defined hard registers.  */
+		   cannot both be user defined hard registers for asm.
+		   Let curr_insn_transform diagnose it.  */
 		if (HARD_REGISTER_P (operand_reg[i])
 		    && REG_USERVAR_P (operand_reg[i])
 		    && operand_reg[j] != NULL_RTX
 		    && HARD_REGISTER_P (operand_reg[j])
-		    && REG_USERVAR_P (operand_reg[j]))
-		  {
-		    /* For asm, let curr_insn_transform diagnose it.  */
-		    if (INSN_CODE (curr_insn) < 0)
+		    && REG_USERVAR_P (operand_reg[j])
+		    && INSN_CODE (curr_insn) < 0)
 		      return false;
-		    fatal_insn ("unable to generate reloads for "
-				"impossible constraints:", curr_insn);
-		  }
 	      }
 	  if (last_conflict_j < 0)
 	    continue;
@@ -3930,6 +3940,16 @@ process_address_1 (int nop, bool check_only_p,
       enum reg_class cl;
       rtx set;
       rtx_insn *insns, *last_insn;
+
+      cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
+			   get_index_code (&ad), curr_insn);
+
+      if (REG_P (*ad.base_term)
+	  && ira_class_subset_p[get_reg_class (REGNO (*ad.base_term))][cl])
+	/* It seems base reg is already in the base reg class and changing it
+	   does not make a progress.  So reload the whole inner address.  */
+	goto reload_inner_addr;
+
       /* Try to reload base into register only if the base is invalid
          for the address but with valid offset, case (4) above.  */
       start_sequence ();
@@ -3975,8 +3995,6 @@ process_address_1 (int nop, bool check_only_p,
 	    {
 	      *ad.base_term = XEXP (SET_SRC (set), 0);
 	      *ad.disp_term = XEXP (SET_SRC (set), 1);
-	      cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
-				   get_index_code (&ad), curr_insn);
 	      regno = REGNO (*ad.base_term);
 	      if (regno >= FIRST_PSEUDO_REGISTER
 		  && cl != lra_get_allocno_class (regno))
@@ -4019,11 +4037,11 @@ process_address_1 (int nop, bool check_only_p,
     }
   else
     {
-      enum reg_class cl = base_reg_class (ad.mode, ad.as,
-					  SCRATCH, SCRATCH,
-					  curr_insn);
-      rtx addr = *ad.inner;
-
+      enum reg_class cl;
+      rtx addr;
+    reload_inner_addr:
+      cl = base_reg_class (ad.mode, ad.as, SCRATCH, SCRATCH, curr_insn);
+      addr = *ad.inner;
       new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
       /* addr => new_base.  */
       lra_emit_move (new_reg, addr);
@@ -4044,14 +4062,21 @@ process_address (int nop, bool check_only_p,
 		 rtx_insn **before, rtx_insn **after)
 {
   bool res = false;
-
-  while (process_address_1 (nop, check_only_p, before, after))
+  /* Use enough iterations to process all address parts:  */
+  for (int i = 0; i < 10; i++)
     {
-      if (check_only_p)
-	return true;
-      res = true;
+      if (!process_address_1 (nop, check_only_p, before, after))
+	{
+	  return res;
+	}
+      else
+	{
+	  if (check_only_p)
+	    return true;
+	  res = true;
+	}
     }
-  return res;
+  fatal_insn ("unable to reload address in ", curr_insn);
 }
 
 /* Override the generic address_reload_context in order to
diff --git a/gcc/lto-cgraph.cc b/gcc/lto-cgraph.cc
index ec34f65..0af2e88 100644
--- a/gcc/lto-cgraph.cc
+++ b/gcc/lto-cgraph.cc
@@ -718,11 +718,12 @@ output_profile_summary (struct lto_simple_output_block *ob)
 {
   if (profile_info)
     {
-      /* We do not output num and run_max, they are not used by
-         GCC profile feedback and they are difficult to merge from multiple
-         units.  */
       unsigned runs = (profile_info->runs);
       streamer_write_uhwi_stream (ob->main_stream, runs);
+      streamer_write_gcov_count_stream (ob->main_stream,
+					profile_info->sum_max);
+      streamer_write_gcov_count_stream (ob->main_stream,
+					profile_info->cutoff);
 
       /* IPA-profile computes hot bb threshold based on cumulated
 	 whole program profile.  We need to stream it down to ltrans.  */
@@ -1678,6 +1679,8 @@ input_profile_summary (class lto_input_block *ib,
   if (runs)
     {
       file_data->profile_info.runs = runs;
+      file_data->profile_info.sum_max = streamer_read_gcov_count (ib);
+      file_data->profile_info.cutoff = streamer_read_gcov_count (ib);
 
       /* IPA-profile computes hot bb threshold based on cumulated
 	 whole program profile.  We need to stream it down to ltrans.  */
@@ -1719,6 +1722,8 @@ merge_profile_summaries (struct lto_file_decl_data **file_data_vec)
 
   profile_info = XCNEW (gcov_summary);
   profile_info->runs = max_runs;
+  profile_info->sum_max = 0;
+  profile_info->cutoff = 0;
 
   /* If merging already happent at WPA time, we are done.  */
   if (flag_ltrans)
@@ -1735,6 +1740,14 @@ merge_profile_summaries (struct lto_file_decl_data **file_data_vec)
 
 	scale = RDIV (node->count_materialization_scale * max_runs,
                       node->lto_file_data->profile_info.runs);
+	gcov_type sum_max = RDIV (node->lto_file_data->profile_info.sum_max * max_runs,
+				  node->lto_file_data->profile_info.runs);
+	gcov_type cutoff = RDIV (node->lto_file_data->profile_info.cutoff * max_runs,
+				 node->lto_file_data->profile_info.runs);
+	if (sum_max > profile_info->sum_max)
+	  profile_info->sum_max = sum_max;
+	if (cutoff > profile_info->cutoff)
+	  profile_info->cutoff = cutoff;
 	node->count_materialization_scale = scale;
 	if (scale < 0)
 	  fatal_error (input_location, "Profile information in %s corrupted",
diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc
index 27405e8..03fca97 100644
--- a/gcc/lto-wrapper.cc
+++ b/gcc/lto-wrapper.cc
@@ -2265,22 +2265,23 @@ cont:
   obstack_free (&argv_obstack, NULL);
 }
 
-/* Concrete implementation of diagnostic_option_manager for LTO.  */
+/* Concrete implementation of diagnostics::option_id_manager for LTO.  */
 
-class lto_diagnostic_option_manager : public gcc_diagnostic_option_manager
+class lto_diagnostic_option_id_manager
+  : public gcc_diagnostic_option_id_manager
 {
 public:
-  lto_diagnostic_option_manager ()
-  : gcc_diagnostic_option_manager (0 /* lang_mask */)
+  lto_diagnostic_option_id_manager ()
+  : gcc_diagnostic_option_id_manager (0 /* lang_mask */)
   {
   }
-  int option_enabled_p (diagnostic_option_id) const final override
+  int option_enabled_p (diagnostics::option_id) const final override
   {
     return true;
   }
-  char *make_option_name (diagnostic_option_id,
-			  diagnostic_t,
-			  diagnostic_t) const final override
+  char *make_option_name (diagnostics::option_id,
+			  enum diagnostics::kind,
+			  enum diagnostics::kind) const final override
   {
     return nullptr;
   }
@@ -2307,8 +2308,8 @@ main (int argc, char *argv[])
   diagnostic_initialize (global_dc, 0);
   diagnostic_color_init (global_dc);
   diagnostic_urls_init (global_dc);
-  global_dc->set_option_manager
-    (::make_unique<lto_diagnostic_option_manager> (), 0);
+  global_dc->set_option_id_manager
+    (::make_unique<lto_diagnostic_option_id_manager> (), 0);
 
   if (atexit (lto_wrapper_cleanup) != 0)
     fatal_error (input_location, "%<atexit%> failed");
diff --git a/gcc/m2/ChangeLog b/gcc/m2/ChangeLog
index f7254f9..ad10605 100644
--- a/gcc/m2/ChangeLog
+++ b/gcc/m2/ChangeLog
@@ -1,3 +1,135 @@
+2025-08-01  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121354
+	* gm2-compiler/M2GenGCC.mod (FoldHigh): Rewrite.
+	(IsUnboundedArray): New procedure function.
+
+2025-07-31  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121314
+	* mc-boot/GFormatStrings.cc (PerformFormatString): Rebuilt.
+	* mc-boot/GM2EXCEPTION.cc (M2EXCEPTION_M2Exception): Rebuilt.
+	* mc-boot/GSFIO.cc (SFIO_GetFileName): Rebuilt.
+	* mc-boot/GSFIO.h (SFIO_GetFileName): Rebuilt.
+	* mc-boot/Gdecl.cc: Rebuilt.
+	* mc-boot/GmcFileName.h: Rebuilt.
+	* mc/decl.mod (getStringChar): New procedure function.
+	(getStringContents): Call getStringChar.
+	(addQuotes): New procedure function.
+	(foldBinary): Call addQuotes to add delimiting quotes
+	to the new string.
+
+2025-07-29  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	* gm2-compiler/M2GenGCC.mod (FoldBecomes): Remove all
+	local variables.
+	(CodeIndrX): Remove length.
+	Remove newstr.
+	* gm2-compiler/M2Range.mod (FoldTypeIndrX): Remove desType.
+
+2025-07-29  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121289
+	* gm2-compiler/M2Students.def (CheckVariableAgainstKeyword): New
+	parameter tok.
+	* gm2-compiler/M2Students.mod (CheckVariableAgainstKeyword): New
+	parameter tok.
+	Pass tok to PerformVariableKeywordCheck.
+	(PerformVariableKeywordCheck): New parameter tok.
+	Pass tok to MetaErrorStringT0.
+	* gm2-compiler/P2SymBuild.mod (BuildVariable): Pass tok to
+	CheckVariableAgainstKeyword.
+	* gm2-libs-iso/LowLong.mod (except): Replace with ...
+	(exceptSrc): ... this.
+	* gm2-libs-iso/LowReal.mod (except): Replace with ...
+	(exceptSrc): ... this.
+	* gm2-libs-iso/LowShort.mod (except): Replace with ...
+	(exceptSrc): ... this.
+	* gm2-libs-iso/Processes.mod (Wait): Replace from with fromCor.
+	* gm2-libs-iso/RndFile.mod (EndPos): Replace end with endP.
+	* gm2-libs/SCmdArgs.mod (GetArg): Replace start with startPos.
+	Replace end with endPos.
+	(NArg): Replace start with startPos.
+	Replace end with endPos.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gm2-gcc/m2linemap.cc: Update usage of "diagnostic_info" to
+	explicitly refer to "diagnostics::diagnostic_info".
+	* gm2-gcc/rtegraph.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gm2-gcc/m2linemap.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* gm2-gcc/rtegraph.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gm2-gcc/m2color.cc: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+
+2025-07-23  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	* gm2-libs/M2EXCEPTION.mod (M2Exception): Add return
+	exException in case Raise completes.
+
+2025-07-23  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121164
+	* gm2-compiler/P1SymBuild.mod: Remove PutProcTypeParam.
+	Remove PutProcTypeParam.
+	(CheckFileName): Remove.
+	(P1EndBuildDefinitionModule): Correct spelling.
+	(P1EndBuildImplementationModule): Ditto.
+	(P1EndBuildProgramModule): Ditto.
+	(EndBuildInnerModule): Ditto.
+	* gm2-compiler/P2SymBuild.mod (P2EndBuildDefModule): Correct
+	spelling.
+	(P2EndBuildImplementationModule): Ditto.
+	(P2EndBuildProgramModule): Ditto.
+	(EndBuildInnerModule): Ditto.
+	(CheckFormalParameterSection): Ditto.
+	* gm2-compiler/P3SymBuild.mod (P3EndBuildDefModule): Ditto.
+	* gm2-compiler/PCSymBuild.mod (PCEndBuildDefModule): Ditto.
+	(fixupProcedureType): Pass tok to PutProcTypeVarParam.
+	Pass tok to PutProcTypeParam.
+	* gm2-compiler/SymbolTable.def (PutProcTypeParam): Add tok
+	parameter.
+	(PutProcTypeVarParam): Ditto.
+	* gm2-compiler/SymbolTable.mod (SymParam): At change type to
+	CARDINAL.
+	New field FullTok.
+	New field Scope.
+	(SymVarParam): At change type to CARDINAL.
+	New field FullTok.
+	New field Scope.
+	(GetVarDeclTok): Check ShadowVar for NulSym and return At.
+	(PutParam): Initialize FullTok.
+	Initialize At.
+	Initialize Scope.
+	(PutVarParam): Initialize FullTok.
+	Assign At.
+	Initialize Scope.
+	(AddProcedureProcTypeParam): Add tok parameter.
+	(GetScope): Add ParamSym and VarParamSym clause.
+	(PutProcTypeVarParam): Add tok parameter.
+	Initialize At.
+	Initialize FullTok.
+	(GetDeclaredDefinition): Clause ParamSym return At.
+	Clause VarParamSym return At.
+	(GetDeclaredModule): Ditto.
+	(PutDeclaredDefinition): Remove clause ParamSym.
+	Remove clause VarParamSym.
+	(PutDeclaredModule): Remove clause ParamSym.
+	Remove clause VarParamSym.
+
+2025-07-18  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121164
+	* gm2-libs/ARRAYOFCHAR.mod (Write): Rename 2nd parameter
+	name a to str.
+
 2025-07-01  Gaius Mulley  <gaiusmod2@gmail.com>
 
 	PR modula2/120912
diff --git a/gcc/m2/gm2-compiler/M2GenGCC.mod b/gcc/m2/gm2-compiler/M2GenGCC.mod
index 4a9ced3..2440b2a 100644
--- a/gcc/m2/gm2-compiler/M2GenGCC.mod
+++ b/gcc/m2/gm2-compiler/M2GenGCC.mod
@@ -2903,9 +2903,6 @@ END CheckStop ;
 *)
 
 PROCEDURE FoldBecomes (p: WalkAction; bb: BasicBlock; quad: CARDINAL) ;
-VAR
-   op            : QuadOperator ;
-   des, op2, expr: CARDINAL ;
 BEGIN
    IF DeclaredOperandsBecomes (p, quad)
    THEN
@@ -6442,37 +6439,52 @@ END ResolveHigh ;
 
 
 (*
+   IsUnboundedArray - return TRUE if symbol is an unbounded array.
+*)
+
+PROCEDURE IsUnboundedArray (sym: CARDINAL) : BOOLEAN ;
+BEGIN
+   IF IsParameter (sym) OR IsVar (sym)
+   THEN
+      RETURN IsUnbounded (GetType (sym))
+   END ;
+   RETURN FALSE
+END IsUnboundedArray ;
+
+
+(*
    FoldHigh - if the array is not dynamic then we should be able to
               remove the HighOp quadruple and assign op1 with
-              the known compile time HIGH(op3).
+              the known compile time HIGH(array).
 *)
 
 PROCEDURE FoldHigh (tokenno: CARDINAL; p: WalkAction;
-                    quad: CARDINAL; op1, dim, op3: CARDINAL) ;
+                    quad: CARDINAL; op1, dim, array: CARDINAL) ;
 VAR
    t       : tree ;
    location: location_t ;
 BEGIN
-   (* firstly ensure that any constant literal is declared *)
-   TryDeclareConstant(tokenno, op3) ;
-   location := TokenToLocation(tokenno) ;
-   IF GccKnowsAbout(op3) AND CompletelyResolved(op3)
+   (* Firstly ensure that any constant literal is declared.  *)
+   TryDeclareConstant (tokenno, array) ;
+   location := TokenToLocation (tokenno) ;
+   IF (NOT IsUnboundedArray (array)) AND
+      GccKnowsAbout (array) AND CompletelyResolved (array)
    THEN
-      t := ResolveHigh(tokenno, dim, op3) ;
-      (* fine, we can take advantage of this and fold constants *)
-      IF IsConst(op1) AND (t#tree(NIL))
+      t := ResolveHigh (tokenno, dim, array) ;
+      (* We can take advantage of this and fold constants.  *)
+      IF IsConst (op1) AND (t # tree (NIL))
       THEN
-         PutConst(op1, Cardinal) ;
-         AddModGcc(op1,
-                   DeclareKnownConstant(location, GetCardinalType(),
-                                        ToCardinal(location, t))) ;
-         p(op1) ;
+         PutConst (op1, Cardinal) ;
+         AddModGcc (op1,
+                    DeclareKnownConstant (location, GetCardinalType (),
+                                          ToCardinal (location, t))) ;
+         p (op1) ;
          NoChange := FALSE ;
-         SubQuad(quad)
+         SubQuad (quad)
       ELSE
-         (* we can still fold the expression, but not the assignment, however, we will
-            not do this here but in CodeHigh
-         *)
+         (* We can still fold the expression but not the assignment,
+            we will not do this here but in CodeHigh when the result
+            can be stored.  *)
       END
    END
 END FoldHigh ;
@@ -8154,8 +8166,6 @@ VAR
    rightpos,
    typepos,
    indrxpos        : CARDINAL ;
-   length,
-   newstr          : tree ;
    location        : location_t ;
 BEGIN
    GetQuadOtok (quad, indrxpos, op, left, type, right,
diff --git a/gcc/m2/gm2-compiler/M2Range.mod b/gcc/m2/gm2-compiler/M2Range.mod
index dcac2ba..f1516d3 100644
--- a/gcc/m2/gm2-compiler/M2Range.mod
+++ b/gcc/m2/gm2-compiler/M2Range.mod
@@ -1869,14 +1869,12 @@ END FoldTypeAssign ;
 
 PROCEDURE FoldTypeIndrX (q: CARDINAL; tokenNo: CARDINAL; des, expr: CARDINAL; r: CARDINAL) ;
 VAR
-   desType,
    exprType: CARDINAL ;
 BEGIN
    (* Need to skip over a variable or temporary in des and expr so
       long as expr is not a procedure.  In the case of des = *expr,
       both expr and des will be variables due to the property of
       indirection.  *)
-   desType := GetType (des) ;
    IF IsProcedure (expr)
    THEN
       (* Must not GetType for a procedure as it gives the return type.  *)
diff --git a/gcc/m2/gm2-compiler/M2Students.def b/gcc/m2/gm2-compiler/M2Students.def
index 7d67a0a..a3ecdcd 100644
--- a/gcc/m2/gm2-compiler/M2Students.def
+++ b/gcc/m2/gm2-compiler/M2Students.def
@@ -39,7 +39,7 @@ EXPORT QUALIFIED StudentVariableCheck, CheckVariableAgainstKeyword ;
                                  as a keyword except for its case.
 *)
 
-PROCEDURE CheckVariableAgainstKeyword (name: Name) ;
+PROCEDURE CheckVariableAgainstKeyword (tok: CARDINAL; name: Name) ;
 
 
 (*
diff --git a/gcc/m2/gm2-compiler/M2Students.mod b/gcc/m2/gm2-compiler/M2Students.mod
index e539eb0..3df160a 100644
--- a/gcc/m2/gm2-compiler/M2Students.mod
+++ b/gcc/m2/gm2-compiler/M2Students.mod
@@ -25,7 +25,7 @@ IMPLEMENTATION MODULE M2Students ;
 FROM SymbolTable IMPORT FinalSymbol, IsVar, IsProcedure, IsModule,
                         GetMainModule, IsType, NulSym, IsRecord, GetSymName, GetNth, GetNthProcedure, GetDeclaredMod, NoOfParam ;
 FROM NameKey IMPORT GetKey, WriteKey, MakeKey, IsSameExcludingCase, NulName, makekey, KeyToCharStar ;
-FROM M2MetaError IMPORT MetaErrorString0, MetaError2 ;
+FROM M2MetaError IMPORT MetaErrorStringT0, MetaError2 ;
 FROM Lists IMPORT List, InitList, IsItemInList, IncludeItemIntoList ;
 FROM M2Reserved IMPORT IsReserved, toktype ;
 FROM DynamicStrings IMPORT String, InitString, KillString, ToUpper, InitStringCharStar, string, Mark, ToUpper, Dup ;
@@ -78,11 +78,11 @@ END IsNotADuplicateName ;
                                  as a keyword except for its case.
 *)
 
-PROCEDURE CheckVariableAgainstKeyword (name: Name) ;
+PROCEDURE CheckVariableAgainstKeyword (tok: CARDINAL; name: Name) ;
 BEGIN
    IF StyleChecking
    THEN
-      PerformVariableKeywordCheck (name)
+      PerformVariableKeywordCheck (tok, name)
    END
 END CheckVariableAgainstKeyword ;
 
@@ -91,7 +91,7 @@ END CheckVariableAgainstKeyword ;
    PerformVariableKeywordCheck - performs the check and constructs the metaerror notes if appropriate.
 *)
 
-PROCEDURE PerformVariableKeywordCheck (name: Name) ;
+PROCEDURE PerformVariableKeywordCheck (tok: CARDINAL; name: Name) ;
 VAR
    upper : Name ;
    token : toktype ;
@@ -105,9 +105,11 @@ BEGIN
    THEN
       IF IsNotADuplicateName (name)
       THEN
-         MetaErrorString0 (Sprintf2 (Mark (InitString ('either the identifier has the same name as a keyword or alternatively a keyword has the wrong case ({%%K%s} and {!%%O:{%%K%s}})')),
-                                     upperS, orig)) ;
-         MetaErrorString0 (Sprintf1 (Mark (InitString ('the symbol name {!%%O:{%%K%s}} is legal as an identifier, however as such it might cause confusion and is considered bad programming practice')), orig))
+         MetaErrorStringT0 (tok,
+                            Sprintf2 (Mark (InitString ('either the identifier has the same name as a keyword or alternatively a keyword has the wrong case ({%%K%s} and {!%%O:{%%K%s}})')),
+                                      upperS, orig)) ;
+         MetaErrorStringT0 (tok,
+                            Sprintf1 (Mark (InitString ('the symbol name {!%%O:{%%K%s}} is legal as an identifier, however as such it might cause confusion and is considered bad programming practice')), orig))
       END
    END ;
    upperS := KillString (upperS) ;
diff --git a/gcc/m2/gm2-compiler/P1SymBuild.mod b/gcc/m2/gm2-compiler/P1SymBuild.mod
index 02d4ac7..08a0fc3 100644
--- a/gcc/m2/gm2-compiler/P1SymBuild.mod
+++ b/gcc/m2/gm2-compiler/P1SymBuild.mod
@@ -79,7 +79,6 @@ FROM SymbolTable IMPORT NulSym,
                         MakeSubscript, PutSubscript,
                         PutArray, GetType, IsArray,
                         IsProcType, MakeProcType,
-                        PutProcTypeVarParam, PutProcTypeParam,
                         PutProcedureBuiltin, PutProcedureInline,
                         GetSymName,
                         ResolveImports, PutDeclared,
@@ -109,42 +108,6 @@ VAR
 
 
 (*
-   CheckFileName - checks to see that the module name matches the file name.
-*)
-
-(*
-PROCEDURE CheckFileName (tok: CARDINAL; name: Name; ModuleType: ARRAY OF CHAR) ;
-VAR
-   ext,
-   basename: INTEGER ;
-   s,
-   FileName: String ;
-BEGIN
-   FileName := GetFileName() ;
-   basename := RIndex(FileName, '/', 0) ;
-   IF basename=-1
-   THEN
-      basename := 0
-   END ;
-   ext := RIndex(FileName, '.', 0) ;
-   IF ext=-1
-   THEN
-      ext := 0
-   END ;
-   FileName := Slice(FileName, basename, ext) ;
-   IF EqualCharStar(FileName, KeyToCharStar(name))
-   THEN
-      FileName := KillString(FileName)
-   ELSE
-      s := ConCat (InitString (ModuleType),
-                   Mark (InitString (" module name {%1Ea} is inconsistant with the filename {%F{%2a}}"))) ;
-      MetaErrorString2 (s, MakeError (tok, name), MakeErrorS (tok, FileName))
-   END
-END CheckFileName ;
-*)
-
-
-(*
    StartBuildDefinitionModule - Creates a definition module and starts
                                 a new scope.
 
@@ -227,7 +190,7 @@ BEGIN
    END ;
    IF NameStart#NameEnd
    THEN
-      MetaError1 ('inconsistant definition module name {%1Wa}', MakeError (start, NameStart))
+      MetaError1 ('inconsistent definition module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END P1EndBuildDefinitionModule ;
@@ -301,7 +264,7 @@ BEGIN
    IF NameStart#NameEnd
    THEN
       MetaErrorT1 (end,
-                   'inconsistant implementation module name {%1Wa}', MakeError (start, NameStart))
+                   'inconsistent implementation module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END P1EndBuildImplementationModule ;
@@ -381,7 +344,7 @@ BEGIN
    IF NameStart#NameEnd
    THEN
       MetaErrorT1 (end,
-                   'inconsistant program module name {%1Wa}', MakeError (start, NameStart))
+                   'inconsistent program module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END P1EndBuildProgramModule ;
@@ -446,7 +409,7 @@ BEGIN
    IF NameStart#NameEnd
    THEN
       MetaErrorT1 (end,
-                   'inconsistant inner module name {%1Wa}', MakeError (start, NameStart))
+                   'inconsistent inner module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END EndBuildInnerModule ;
diff --git a/gcc/m2/gm2-compiler/P2SymBuild.mod b/gcc/m2/gm2-compiler/P2SymBuild.mod
index 5c82ec8..54e624f 100644
--- a/gcc/m2/gm2-compiler/P2SymBuild.mod
+++ b/gcc/m2/gm2-compiler/P2SymBuild.mod
@@ -356,7 +356,7 @@ BEGIN
    END ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant definition module name, module began as (%a) and ended with (%a)', NameStart, NameEnd)
+      WriteFormat2('inconsistent definition module name, module began as (%a) and ended with (%a)', NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
 END P2EndBuildDefModule ;
@@ -425,7 +425,7 @@ BEGIN
    PopT (NameEnd) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat1('inconsistant implementation module name %a', NameStart)
+      WriteFormat1('inconsistent implementation module name %a', NameStart)
    END ;
    M2Error.LeaveErrorScope
 END P2EndBuildImplementationModule ;
@@ -499,7 +499,7 @@ BEGIN
    END ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant program module name %a does not match %a', NameStart, NameEnd)
+      WriteFormat2('inconsistent program module name %a does not match %a', NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
 END P2EndBuildProgramModule ;
@@ -564,7 +564,7 @@ BEGIN
    PopT(NameEnd) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant inner module name %a does not match %a',
+      WriteFormat2('inconsistent inner module name %a does not match %a',
                    NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
@@ -1179,8 +1179,8 @@ BEGIN
    PopT (n) ;
    i := 1 ;
    WHILE i <= n DO
-      CheckVariableAgainstKeyword (OperandT (n+1-i)) ;
       tok := OperandTok (n+1-i) ;
+      CheckVariableAgainstKeyword (tok, OperandT (n+1-i)) ;
       Var := MakeVar (tok, OperandT (n+1-i)) ;
       AtAddress := OperandA (n+1-i) ;
       IF AtAddress # NulSym
@@ -1835,13 +1835,13 @@ BEGIN
          (* WarnStringAt (InitString ('parampos?'), OperandTok (pi)) ;  *)
          IF Unbounded AND (NOT IsUnboundedParam (ProcSym, prevkind, ParamTotal+i))
          THEN
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             'the parameter {%3EHa} was not declared as an ARRAY OF type',
                             'the parameter {%3EVa} was declared as an ARRAY OF type',
                             ParamTotal+i, ProcSym, curkind, prevkind)
          ELSIF (NOT Unbounded) AND IsUnboundedParam (ProcSym, prevkind, ParamTotal+i)
          THEN
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             'the parameter {%3EHa} was declared as an ARRAY OF type',
                             'the parameter {%3EVa} was not declared as an ARRAY OF type',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1850,7 +1850,7 @@ BEGIN
          THEN
             IF GetDimension (GetNthParam (ProcSym, prevkind, ParamTotal+1)) # ndim
             THEN
-               ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+               ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                                'the dynamic array parameter {%3EHa} was declared with a different of dimensions',
                                'the dynamic array parameter {%3EVa} was declared with a different of dimensions',
                                ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1859,14 +1859,14 @@ BEGIN
          IF isVarParam AND (NOT IsVarParam (ProcSym, prevkind, ParamTotal+i))
          THEN
             (* Expecting non VAR parameter.  *)
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             '{%3EHa} was not declared as a {%kVAR} parameter',
                             '{%3EVa} was declared as a {%kVAR} parameter',
                             ParamTotal+i, ProcSym, curkind, prevkind)
          ELSIF (NOT isVarParam) AND IsVarParam (ProcSym, prevkind, ParamTotal+i)
          THEN
             (* Expecting VAR pamarater.  *)
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             '{%3EHa} was declared as a {%kVAR} parameter',
                             '{%3EVa} was not declared as a {%kVAR} parameter',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1877,7 +1877,7 @@ BEGIN
             IF GetSymName (ParamI) # OperandT (pi)
             THEN
                (* Different parameter names.  *)
-               ParameterError ('procedure {%%1a} in the %s differs from the %s, {%%2N} parameter name is inconsistant, %s',
+               ParameterError ('procedure {%%1a} in the %s differs from the %s, {%%2N} parameter name is inconsistent, %s',
                             'named as {%3EVa}',
                             'named as {%3EVa}',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1897,7 +1897,7 @@ BEGIN
             (NOT IsUnknown(SkipType(ParamIType)))
          THEN
             (* Different parameter types.  *)
-            ParameterError ('declaration in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             'the parameter {%3EHa} was declared with a different type',
                             'the parameter {%3EVa} was declared with a different type',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -3072,10 +3072,10 @@ BEGIN
    IF Var=VarTok
    THEN
       (* VAR parameter *)
-      PutProcTypeVarParam(ProcTypeSym, TypeSym, IsUnbounded(TypeSym))
+      PutProcTypeVarParam (tok, ProcTypeSym, TypeSym, IsUnbounded (TypeSym))
    ELSE
       (* Non VAR parameter *)
-      PutProcTypeParam(ProcTypeSym, TypeSym, IsUnbounded(TypeSym))
+      PutProcTypeParam (tok, ProcTypeSym, TypeSym, IsUnbounded (TypeSym))
    END ;
    PushT(ProcTypeSym) ;
    Annotate("%1s(%1d)||proc type")
diff --git a/gcc/m2/gm2-compiler/P3SymBuild.mod b/gcc/m2/gm2-compiler/P3SymBuild.mod
index c4877f7..096057e 100644
--- a/gcc/m2/gm2-compiler/P3SymBuild.mod
+++ b/gcc/m2/gm2-compiler/P3SymBuild.mod
@@ -126,7 +126,7 @@ BEGIN
    PopT(NameStart) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant definition module was named (%a) and concluded as (%a)',
+      WriteFormat2('inconsistent definition module was named (%a) and concluded as (%a)',
                    NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
diff --git a/gcc/m2/gm2-compiler/PCSymBuild.mod b/gcc/m2/gm2-compiler/PCSymBuild.mod
index 3bffe86..4db2730 100644
--- a/gcc/m2/gm2-compiler/PCSymBuild.mod
+++ b/gcc/m2/gm2-compiler/PCSymBuild.mod
@@ -269,7 +269,7 @@ BEGIN
    PopT(NameStart) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant definition module was named (%a) and concluded as (%a)',
+      WriteFormat2('inconsistent definition module was named (%a) and concluded as (%a)',
                    NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
@@ -1168,9 +1168,9 @@ BEGIN
          par := GetParam (p, i) ;
          IF IsParameterVar (par)
          THEN
-            PutProcTypeVarParam (t, GetType (par), IsParameterUnbounded (par))
+            PutProcTypeVarParam (tok, t, GetType (par), IsParameterUnbounded (par))
          ELSE
-            PutProcTypeParam (t, GetType (par), IsParameterUnbounded (par))
+            PutProcTypeParam (tok, t, GetType (par), IsParameterUnbounded (par))
          END ;
          INC(i)
       END ;
diff --git a/gcc/m2/gm2-compiler/SymbolTable.def b/gcc/m2/gm2-compiler/SymbolTable.def
index 2a9865a..5b93f29 100644
--- a/gcc/m2/gm2-compiler/SymbolTable.def
+++ b/gcc/m2/gm2-compiler/SymbolTable.def
@@ -2650,7 +2650,8 @@ PROCEDURE MakeProcType (tok: CARDINAL; ProcTypeName: Name) : CARDINAL ;
                       ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeParam (tok: CARDINAL;
+                            Sym: CARDINAL;
                             ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 
 
@@ -2659,7 +2660,8 @@ PROCEDURE PutProcTypeParam (Sym: CARDINAL;
                          ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeVarParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeVarParam (tok: CARDINAL;
+                               Sym: CARDINAL;
                                ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 
 
diff --git a/gcc/m2/gm2-compiler/SymbolTable.mod b/gcc/m2/gm2-compiler/SymbolTable.mod
index ff661dc..00946e5 100644
--- a/gcc/m2/gm2-compiler/SymbolTable.mod
+++ b/gcc/m2/gm2-compiler/SymbolTable.mod
@@ -463,9 +463,11 @@ TYPE
                                               (* of param.                   *)
                  Type          : CARDINAL ;   (* Index to the type of param. *)
                  IsUnbounded   : BOOLEAN ;    (* Is it an ARRAY OF Type?     *)
+                 Scope         : CARDINAL ;   (* Procedure declaration.      *)
                  ShadowVar     : CARDINAL ;   (* The local variable used to  *)
                                               (* shadow this parameter.      *)
-                 At            : Where ;      (* Where was sym declared/used *)
+                 FullTok,                     (* name: type virtual token.   *)
+                 At            : CARDINAL ;   (* Where was sym declared.     *)
               END ;
 
    SymVarParam = RECORD
@@ -476,9 +478,11 @@ TYPE
                     HeapVar       : CARDINAL ;(* The pointer value on heap.  *)
                                               (* Only used by static         *)
                                               (* analysis.                   *)
+                    Scope         : CARDINAL ;(* Procedure declaration.      *)
                     ShadowVar     : CARDINAL ;(* The local variable used to  *)
                                               (* shadow this parameter.      *)
-                    At            : Where ;   (* Where was sym declared/used *)
+                    FullTok,                  (* name: type virtual token.   *)
+                    At            : CARDINAL ;(* Where was sym declared.     *)
                  END ;
 
    ConstStringVariant = (m2str, cstr, m2nulstr, cnulstr) ;
@@ -4456,9 +4460,19 @@ BEGIN
       pSym := GetPsym (sym) ;
       IF IsParameterVar (sym)
       THEN
-         RETURN GetVarDeclTok (pSym^.VarParam.ShadowVar)
+         IF pSym^.VarParam.ShadowVar = NulSym
+         THEN
+            RETURN pSym^.VarParam.At
+         ELSE
+            RETURN GetVarDeclTok (pSym^.VarParam.ShadowVar)
+         END
       ELSE
-         RETURN GetVarDeclTok (pSym^.Param.ShadowVar)
+         IF pSym^.Param.ShadowVar = NulSym
+         THEN
+            RETURN pSym^.Param.At
+         ELSE
+            RETURN GetVarDeclTok (pSym^.Param.ShadowVar)
+         END
       END
    ELSIF IsVar (sym)
    THEN
@@ -4549,9 +4563,9 @@ BEGIN
    THEN
       IF IsParameterVar (sym)
       THEN
-         RETURN GetVarDeclFullTok (pSym^.VarParam.ShadowVar)
+         RETURN pSym^.VarParam.FullTok
       ELSE
-         RETURN GetVarDeclFullTok (pSym^.Param.ShadowVar)
+         RETURN pSym^.Param.FullTok
       END
    ELSIF IsVar (sym)
    THEN
@@ -10651,8 +10665,10 @@ BEGIN
             name := ParamName ;
             Type := ParamType ;
             IsUnbounded := isUnbounded ;
+            Scope := Sym ;
             ShadowVar := NulSym ;
-            InitWhereDeclaredTok(tok, At)
+            FullTok := MakeVirtual2Tok (tok, typetok) ;
+            At := tok
          END
       END ;
       AddParameter (Sym, kind, ParSym) ;
@@ -10671,7 +10687,7 @@ BEGIN
             pSym^.Param.ShadowVar := VariableSym
          END
       END ;
-      AddProcedureProcTypeParam (Sym, ParamType, isUnbounded, FALSE)
+      AddProcedureProcTypeParam (tok, Sym, ParamType, isUnbounded, FALSE)
    END ;
    RETURN( TRUE )
 END PutParam ;
@@ -10708,9 +10724,11 @@ BEGIN
             name := ParamName ;
             Type := ParamType ;
             IsUnbounded := isUnbounded ;
+            Scope := Sym ;
             ShadowVar := NulSym ;
             HeapVar := NulSym ;  (* Will contain a pointer value.  *)
-            InitWhereDeclaredTok(tok, At)
+            FullTok := MakeVirtual2Tok (tok, typetok) ;
+            At := tok
          END
       END ;
       AddParameter (Sym, kind, ParSym) ;
@@ -10729,7 +10747,7 @@ BEGIN
             pSym^.VarParam.ShadowVar := VariableSym
          END
       END ;
-      AddProcedureProcTypeParam (Sym, ParamType, isUnbounded, TRUE)
+      AddProcedureProcTypeParam (tok, Sym, ParamType, isUnbounded, TRUE)
    END ;
    RETURN( TRUE )
 END PutVarParam ;
@@ -10816,7 +10834,8 @@ END AddParameter ;
                                associated with procedure Sym.
 *)
 
-PROCEDURE AddProcedureProcTypeParam (Sym, ParamType: CARDINAL;
+PROCEDURE AddProcedureProcTypeParam (tok: CARDINAL;
+                                     Sym, ParamType: CARDINAL;
                                      isUnbounded, isVarParam: BOOLEAN) ;
 VAR
    pSym: PtrToSymbol ;
@@ -10829,10 +10848,12 @@ BEGIN
                     THEN
                        IF isVarParam
                        THEN
-                          PutProcTypeVarParam (Procedure.ProcedureType,
+                          PutProcTypeVarParam (tok,
+                                               Procedure.ProcedureType,
                                                ParamType, isUnbounded)
                        ELSE
-                          PutProcTypeParam (Procedure.ProcedureType,
+                          PutProcTypeParam (tok,
+                                            Procedure.ProcedureType,
                                             ParamType, isUnbounded)
                        END
                     END
@@ -13027,18 +13048,8 @@ BEGIN
       ConstLitSym        : RETURN( ConstLit.Scope ) |
       ConstStringSym     : RETURN( ConstString.Scope ) |
       ConstVarSym        : RETURN( ConstVar.Scope ) |
-      ParamSym           : IF Param.ShadowVar = NulSym
-                           THEN
-                              RETURN NulSym
-                           ELSE
-                              RETURN( GetScope (Param.ShadowVar) )
-                           END |
-      VarParamSym        : IF VarParam.ShadowVar = NulSym
-                           THEN
-                              RETURN NulSym
-                           ELSE
-                              RETURN( GetScope (VarParam.ShadowVar) )
-                           END |
+      ParamSym           : RETURN( Param.Scope ) |
+      VarParamSym        : RETURN( VarParam.Scope ) |
       UndefinedSym       : RETURN( NulSym ) |
       PartialUnboundedSym: InternalError ('should not be requesting the scope of a PartialUnbounded symbol')
 
@@ -13186,7 +13197,8 @@ END MakeProcType ;
                       ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeParam (tok: CARDINAL;
+                            Sym: CARDINAL;
                             ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 VAR
    pSym  : PtrToSymbol ;
@@ -13201,7 +13213,8 @@ BEGIN
          Type := ParamType ;
          IsUnbounded := isUnbounded ;
          ShadowVar := NulSym ;
-         InitWhereDeclared(At)
+         FullTok := tok ;
+         At := tok
       END
    END ;
    AddParameter (Sym, ProperProcedure, ParSym)
@@ -13213,7 +13226,8 @@ END PutProcTypeParam ;
                          ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeVarParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeVarParam (tok: CARDINAL;
+                               Sym: CARDINAL;
                                ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 VAR
    pSym  : PtrToSymbol ;
@@ -13228,7 +13242,8 @@ BEGIN
          Type := ParamType ;
          IsUnbounded := isUnbounded ;
          ShadowVar := NulSym ;
-         InitWhereDeclared(At)
+         FullTok := tok ;
+         At := tok
       END
    END ;
    AddParameter (Sym, ProperProcedure, ParSym)
@@ -13918,8 +13933,8 @@ BEGIN
       UnboundedSym       : RETURN( Unbounded.At.DefDeclared ) |
       ProcedureSym       : RETURN( Procedure.At.DefDeclared ) |
       ProcTypeSym        : RETURN( ProcType.At.DefDeclared ) |
-      ParamSym           : RETURN( Param.At.DefDeclared ) |
-      VarParamSym        : RETURN( VarParam.At.DefDeclared ) |
+      ParamSym           : RETURN( Param.At ) |
+      VarParamSym        : RETURN( VarParam.At ) |
       ConstStringSym     : RETURN( ConstString.At.DefDeclared ) |
       ConstLitSym        : RETURN( ConstLit.At.DefDeclared ) |
       ConstVarSym        : RETURN( ConstVar.At.DefDeclared ) |
@@ -13968,8 +13983,8 @@ BEGIN
       UnboundedSym       : RETURN( Unbounded.At.ModDeclared ) |
       ProcedureSym       : RETURN( Procedure.At.ModDeclared ) |
       ProcTypeSym        : RETURN( ProcType.At.ModDeclared ) |
-      ParamSym           : RETURN( Param.At.ModDeclared ) |
-      VarParamSym        : RETURN( VarParam.At.ModDeclared ) |
+      ParamSym           : RETURN( Param.At ) |
+      VarParamSym        : RETURN( VarParam.At ) |
       ConstStringSym     : RETURN( ConstString.At.ModDeclared ) |
       ConstLitSym        : RETURN( ConstLit.At.ModDeclared ) |
       ConstVarSym        : RETURN( ConstVar.At.ModDeclared ) |
@@ -14019,8 +14034,6 @@ BEGIN
       UnboundedSym       : Unbounded.At.DefDeclared := tok |
       ProcedureSym       : Procedure.At.DefDeclared := tok |
       ProcTypeSym        : ProcType.At.DefDeclared := tok |
-      ParamSym           : Param.At.DefDeclared := tok |
-      VarParamSym        : VarParam.At.DefDeclared := tok |
       ConstStringSym     : ConstString.At.DefDeclared := tok |
       ConstLitSym        : ConstLit.At.DefDeclared := tok |
       ConstVarSym        : ConstVar.At.DefDeclared := tok |
@@ -14067,8 +14080,6 @@ BEGIN
       UnboundedSym       : Unbounded.At.ModDeclared := tok |
       ProcedureSym       : Procedure.At.ModDeclared := tok |
       ProcTypeSym        : ProcType.At.ModDeclared := tok |
-      ParamSym           : Param.At.ModDeclared := tok |
-      VarParamSym        : VarParam.At.ModDeclared := tok |
       ConstStringSym     : ConstString.At.ModDeclared := tok |
       ConstLitSym        : ConstLit.At.ModDeclared := tok |
       ConstVarSym        : ConstVar.At.ModDeclared := tok |
@@ -14323,8 +14334,10 @@ BEGIN
       UnboundedSym       : RETURN( Unbounded.At.FirstUsed ) |
       ProcedureSym       : RETURN( Procedure.At.FirstUsed ) |
       ProcTypeSym        : RETURN( ProcType.At.FirstUsed ) |
+      (*
       ParamSym           : RETURN( Param.At.FirstUsed ) |
       VarParamSym        : RETURN( VarParam.At.FirstUsed ) |
+      *)
       ConstStringSym     : RETURN( ConstString.At.FirstUsed ) |
       ConstLitSym        : RETURN( ConstLit.At.FirstUsed ) |
       ConstVarSym        : RETURN( ConstVar.At.FirstUsed ) |
diff --git a/gcc/m2/gm2-gcc/m2color.cc b/gcc/m2/gm2-gcc/m2color.cc
index fd30f46..a638d7c 100644
--- a/gcc/m2/gm2-gcc/m2color.cc
+++ b/gcc/m2/gm2-gcc/m2color.cc
@@ -23,7 +23,7 @@ along with GNU Modula-2; see the file COPYING3.  If not see
 #include "m2color.h"
 
 #include "gcc-consolidation.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 
 
 char *
diff --git a/gcc/m2/gm2-gcc/m2linemap.cc b/gcc/m2/gm2-gcc/m2linemap.cc
index feb62b3..89a1080 100644
--- a/gcc/m2/gm2-gcc/m2linemap.cc
+++ b/gcc/m2/gm2-gcc/m2linemap.cc
@@ -188,12 +188,13 @@ m2linemap_ErrorAt (location_t location, char *message)
 static void
 m2linemap_ErrorAtf_1 (location_t location, const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, message);
-  diagnostic_set_info (&diagnostic, message, &ap, &richloc, DK_ERROR);
+  diagnostic_set_info (&diagnostic, message, &ap, &richloc,
+		       diagnostics::kind::error);
   diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
 }
@@ -209,12 +210,13 @@ m2linemap_ErrorAtf (location_t location, const char *message)
 static void
 m2linemap_WarningAtf_1 (location_t location, const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, message);
-  diagnostic_set_info (&diagnostic, message, &ap, &richloc, DK_WARNING);
+  diagnostic_set_info (&diagnostic, message, &ap, &richloc,
+		       diagnostics::kind::warning);
   diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
 }
@@ -230,12 +232,13 @@ m2linemap_WarningAtf (location_t location, const char *message)
 static void
 m2linemap_NoteAtf_1 (location_t location, const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, message);
-  diagnostic_set_info (&diagnostic, message, &ap, &richloc, DK_NOTE);
+  diagnostic_set_info (&diagnostic, message, &ap, &richloc,
+		       diagnostics::kind::note);
   diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
 }
diff --git a/gcc/m2/gm2-gcc/rtegraph.cc b/gcc/m2/gm2-gcc/rtegraph.cc
index cd12582..f7b3065 100644
--- a/gcc/m2/gm2-gcc/rtegraph.cc
+++ b/gcc/m2/gm2-gcc/rtegraph.cc
@@ -283,9 +283,10 @@ rtegraph_lookup (gimple *g, tree fndecl, bool is_call)
 /* rte_error_at - wraps up an error message.  */
 
 static void
-rte_error_at (location_t location, diagnostic_t kind, const char *message, ...)
+rte_error_at (location_t location, diagnostics::kind kind,
+	      const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
@@ -338,7 +339,7 @@ access_string (tree t, const char **value)
    give the function context which might be misleading if this is inlined.  */
 
 static void
-generate_report (gimple *stmt, const char *report, diagnostic_t kind)
+generate_report (gimple *stmt, const char *report, diagnostics::kind kind)
 {
   if (gimple_call_num_args (stmt) == 5)
     {
@@ -393,15 +394,16 @@ const char *rtenode::create_message (const char *with_name, const char *without_
   return message;
 }
 
-/* error_message issue an DK_ERROR from grtenode.  */
+/* error_message issue an diagnostics::kind::error from grtenode.  */
 
 void rtenode::error_message (void)
 {
   if (grtenode != NULL)
-    generate_report (grtenode, "runtime error will occur", DK_ERROR);
+    generate_report (grtenode, "runtime error will occur",
+		     diagnostics::kind::error);
 }
 
-/* warning_message issue an DK_WARNING from grtenode.  */
+/* warning_message issue an diagnostics::kind::warning from grtenode.  */
 
 void rtenode::warning_message (void)
 {
@@ -409,15 +411,16 @@ void rtenode::warning_message (void)
     ("runtime error will occur if an exported procedure is called from %s",
      "runtime error will occur if an exported procedure is called");
   if (grtenode != NULL)
-    generate_report (grtenode, message, DK_WARNING);
+    generate_report (grtenode, message, diagnostics::kind::warning);
 }
 
-/* note_message issue an DK_NOTE from grtenode.  */
+/* note_message issue an diagnostics::kind::note from grtenode.  */
 
 void rtenode::note_message (void)
 {
   if (grtenode != NULL)
-    generate_report (grtenode, "runtime will occur if this procedure is called", DK_NOTE);
+    generate_report (grtenode, "runtime will occur if this procedure is called",
+		     diagnostics::kind::note);
 }
 
 /* dump_vec display contents of vector array list.  */
diff --git a/gcc/m2/gm2-libs-iso/LowLong.mod b/gcc/m2/gm2-libs-iso/LowLong.mod
index 92c7d91..f611923 100644
--- a/gcc/m2/gm2-libs-iso/LowLong.mod
+++ b/gcc/m2/gm2-libs-iso/LowLong.mod
@@ -182,7 +182,7 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
+      RAISE(exceptSrc, ORD(badparam),
             'LowLong.trunc: cannot truncate to a negative number of digits') ;
       RETURN x
    ELSE
@@ -230,7 +230,7 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
+      RAISE(exceptSrc, ORD(badparam),
             'LowLong.round: cannot round to a negative number of digits') ;
       RETURN x
    ELSE
@@ -287,12 +287,12 @@ END currentMode ;
 
 PROCEDURE IsLowException () : BOOLEAN ;
 BEGIN
-   RETURN( IsExceptionalExecution() AND IsCurrentSource(except) )
+   RETURN( IsExceptionalExecution () AND IsCurrentSource (exceptSrc) )
 END IsLowException ;
 
 
 VAR
-   except: ExceptionSource ;
+   exceptSrc: ExceptionSource ;
 BEGIN
-   AllocateSource(except)
+   AllocateSource (exceptSrc)
 END LowLong.
diff --git a/gcc/m2/gm2-libs-iso/LowReal.mod b/gcc/m2/gm2-libs-iso/LowReal.mod
index 580f36b..6d9ea00 100644
--- a/gcc/m2/gm2-libs-iso/LowReal.mod
+++ b/gcc/m2/gm2-libs-iso/LowReal.mod
@@ -183,8 +183,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowReal.trunc: cannot truncate to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowReal.trunc: cannot truncate to a negative number of digits') ;
       RETURN x
    ELSE
       r := dtoa(x, maxsignificant, 100, point, sign) ;
@@ -231,8 +231,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowReal.round: cannot round to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowReal.round: cannot round to a negative number of digits') ;
       RETURN x
    ELSE
       s := RealToFloatString(x, n) ;
@@ -288,12 +288,12 @@ END currentMode ;
 
 PROCEDURE IsLowException () : BOOLEAN ;
 BEGIN
-   RETURN( IsExceptionalExecution() AND IsCurrentSource(except) )
+   RETURN( IsExceptionalExecution () AND IsCurrentSource (exceptSrc) )
 END IsLowException ;
 
 
 VAR
-   except: ExceptionSource ;
+   exceptSrc: ExceptionSource ;
 BEGIN
-   AllocateSource(except)
+   AllocateSource (exceptSrc)
 END LowReal.
diff --git a/gcc/m2/gm2-libs-iso/LowShort.mod b/gcc/m2/gm2-libs-iso/LowShort.mod
index 8531a88..62e4887 100644
--- a/gcc/m2/gm2-libs-iso/LowShort.mod
+++ b/gcc/m2/gm2-libs-iso/LowShort.mod
@@ -183,8 +183,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowLong.trunc: cannot truncate to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowLong.trunc: cannot truncate to a negative number of digits') ;
       RETURN x
    ELSE
       r := dtoa(x, maxsignificant, 100, point, sign) ;
@@ -231,8 +231,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowLong.round: cannot round to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowLong.round: cannot round to a negative number of digits') ;
       RETURN x
    ELSE
       s := RealToFloatString(x, n) ;
@@ -288,12 +288,12 @@ END currentMode ;
 
 PROCEDURE IsLowException () : BOOLEAN ;
 BEGIN
-   RETURN( IsExceptionalExecution() AND IsCurrentSource(except) )
+   RETURN( IsExceptionalExecution () AND IsCurrentSource (exceptSrc) )
 END IsLowException ;
 
 
 VAR
-   except: ExceptionSource ;
+   exceptSrc: ExceptionSource ;
 BEGIN
-   AllocateSource(except)
+   AllocateSource (exceptSrc)
 END LowShort.
diff --git a/gcc/m2/gm2-libs-iso/Processes.mod b/gcc/m2/gm2-libs-iso/Processes.mod
index 8ef22c0..b0c1b69 100644
--- a/gcc/m2/gm2-libs-iso/Processes.mod
+++ b/gcc/m2/gm2-libs-iso/Processes.mod
@@ -441,7 +441,7 @@ PROCEDURE Wait ;
 VAR
    calling,
    best   : ProcessId ;
-   from   : COROUTINE ;
+   fromCor: COROUTINE ;
 BEGIN
    IF debugging
    THEN
@@ -451,17 +451,17 @@ BEGIN
    OnWaitingQueue (calling) ;
    best := chooseProcess () ;
    currentId := best ;
-   from := calling^.context ;
+   fromCor := calling^.context ;
    IF debugging
    THEN
       displayProcesses ("Wait about to perform IOTRANSFER")
    END ;
-   IOTRANSFER (from, currentId^.context) ;
+   IOTRANSFER (fromCor, currentId^.context) ;
    IF debugging
    THEN
       displayProcesses ("Wait after IOTRANSFER")
    END ;
-   currentId^.context := from ;
+   currentId^.context := fromCor ;
    currentId := calling ;
    OnReadyQueue (calling) ;
    IF debugging
diff --git a/gcc/m2/gm2-libs-iso/RndFile.mod b/gcc/m2/gm2-libs-iso/RndFile.mod
index e04cd8f..0a2264a 100644
--- a/gcc/m2/gm2-libs-iso/RndFile.mod
+++ b/gcc/m2/gm2-libs-iso/RndFile.mod
@@ -398,9 +398,9 @@ PROCEDURE EndPos (cid: ChanId): FilePos;
      position after which there have been no writes.
   *)
 VAR
-   d  : DeviceTablePtr ;
-   end,
-   old: FilePos ;
+   d   : DeviceTablePtr ;
+   endP,
+   old : FilePos ;
 BEGIN
    IF IsRndFile(cid)
    THEN
@@ -410,9 +410,9 @@ BEGIN
          old := CurrentPos(cid) ;
          FIO.SetPositionFromEnd(RTio.GetFile(cid), 0) ;
          checkErrno(dev, d) ;
-         end := CurrentPos(cid) ;
+         endP := CurrentPos(cid) ;
          FIO.SetPositionFromBeginning(RTio.GetFile(cid), old) ;
-         RETURN( end )
+         RETURN( endP )
       END
    ELSE
       RAISEdevException(cid, did, IOChan.wrongDevice,
diff --git a/gcc/m2/gm2-libs/ARRAYOFCHAR.mod b/gcc/m2/gm2-libs/ARRAYOFCHAR.mod
index f27378a..41aaf68 100644
--- a/gcc/m2/gm2-libs/ARRAYOFCHAR.mod
+++ b/gcc/m2/gm2-libs/ARRAYOFCHAR.mod
@@ -34,14 +34,14 @@ IMPORT StrLib ;
    Write - writes a string to file f.
 *)
 
-PROCEDURE Write (f: File; a: ARRAY OF CHAR) ;
+PROCEDURE Write (f: File; str: ARRAY OF CHAR) ;
 VAR
    len, i: CARDINAL ;
 BEGIN
-   len := StrLib.StrLen (a) ;
+   len := StrLib.StrLen (str) ;
    i := 0 ;
    WHILE i < len DO
-      WriteChar (f, a[i]) ;
+      WriteChar (f, str[i]) ;
       INC (i)
    END
 END Write ;
diff --git a/gcc/m2/gm2-libs/M2EXCEPTION.mod b/gcc/m2/gm2-libs/M2EXCEPTION.mod
index a17310f..e92b16d 100644
--- a/gcc/m2/gm2-libs/M2EXCEPTION.mod
+++ b/gcc/m2/gm2-libs/M2EXCEPTION.mod
@@ -33,7 +33,7 @@ FROM RTExceptions IMPORT EHBlock, GetExceptionBlock, GetNumber, Raise,
 
 (* If the program or coroutine is in the exception state then return the enumeration
    value representing the exception cause.  If it is not in the exception state then
-   raises and exception (exException).  *)
+   raises an exException exception.  *)
 
 PROCEDURE M2Exception () : M2Exceptions;
 VAR
@@ -45,7 +45,8 @@ BEGIN
    IF n = MAX (CARDINAL)
    THEN
       Raise (ORD (exException), ADR (__FILE__), __LINE__, __COLUMN__, ADR (__FUNCTION__),
-             ADR ('current coroutine is not in the exceptional execution state'))
+             ADR ('current coroutine is not in the exceptional execution state')) ;
+      RETURN exException
    ELSE
       RETURN VAL (M2Exceptions, n)
    END
diff --git a/gcc/m2/gm2-libs/SCmdArgs.mod b/gcc/m2/gm2-libs/SCmdArgs.mod
index ed76fc4..8443d5f 100644
--- a/gcc/m2/gm2-libs/SCmdArgs.mod
+++ b/gcc/m2/gm2-libs/SCmdArgs.mod
@@ -132,26 +132,27 @@ PROCEDURE GetArg (CmdLine: String;
 VAR
    i         : CARDINAL ;
    sn,
-   start, end: INTEGER ;
+   startPos,
+   endPos    : INTEGER ;
    ch        : CHAR ;
 BEGIN
    i := 0 ;
-   start := 0 ;
-   end := Length(CmdLine) ;
+   startPos := 0 ;
+   endPos := Length(CmdLine) ;
    WHILE i<n DO
-      start := skipWhite(CmdLine, start, end) ;
-      sn := skipNextArg(CmdLine, start, end) ;
-      IF sn<end
+      startPos := skipWhite(CmdLine, startPos, endPos) ;
+      sn := skipNextArg(CmdLine, startPos, endPos) ;
+      IF sn<endPos
       THEN
-         start := sn ;
+         startPos := sn ;
          INC(i)
       ELSE
          RETURN( FALSE )
       END
    END ;
-   start := skipWhite(CmdLine, start, end) ;
-   sn := skipNextArg(CmdLine, start, end) ;
-   Argi := Slice(CmdLine, start, sn) ;
+   startPos := skipWhite(CmdLine, startPos, endPos) ;
+   sn := skipNextArg(CmdLine, startPos, endPos) ;
+   Argi := Slice(CmdLine, startPos, sn) ;
    RETURN( TRUE )
 END GetArg ;
 
@@ -165,17 +166,18 @@ PROCEDURE Narg (CmdLine: String) : CARDINAL ;
 VAR
    n         : CARDINAL ;
    s,
-   start, end: INTEGER ;
+   startPos,
+   endPos    : INTEGER ;
 BEGIN
    n := 0 ;
-   start := 0 ;
-   end := Length(CmdLine) ;
+   startPos := 0 ;
+   endPos := Length(CmdLine) ;
    LOOP
-      start := skipWhite(CmdLine, start, end) ;
-      s := skipNextArg(CmdLine, start, end) ;
-      IF s<end
+      startPos := skipWhite(CmdLine, startPos, endPos) ;
+      s := skipNextArg(CmdLine, startPos, endPos) ;
+      IF s<endPos
       THEN
-         start := s ;
+         startPos := s ;
          INC(n)
       ELSE
          RETURN( n )
diff --git a/gcc/m2/mc-boot/GFormatStrings.cc b/gcc/m2/mc-boot/GFormatStrings.cc
index f4c4fd6..ad7e7d8 100644
--- a/gcc/m2/mc-boot/GFormatStrings.cc
+++ b/gcc/m2/mc-boot/GFormatStrings.cc
@@ -464,7 +464,7 @@ static DynamicStrings_String PerformFormatString (DynamicStrings_String fmt, int
               /* avoid dangling else.  */
               afterperc += 1;
               Cast ((unsigned char *) &u, (sizeof (u)-1), (const unsigned char *) w, _w_high);
-              in = DynamicStrings_ConCat (in, DynamicStrings_Slice (fmt, (*startpos), nextperc));
+              in = Copy (fmt, in, (*startpos), nextperc);
               in = DynamicStrings_ConCat (in, StringConvert_CardinalToString (u, static_cast<unsigned int> (width), leader, 16, true));
               (*startpos) = afterperc;
               DSdbExit (static_cast<DynamicStrings_String> (NULL));
@@ -475,7 +475,7 @@ static DynamicStrings_String PerformFormatString (DynamicStrings_String fmt, int
               /* avoid dangling else.  */
               afterperc += 1;
               Cast ((unsigned char *) &u, (sizeof (u)-1), (const unsigned char *) w, _w_high);
-              in = DynamicStrings_ConCat (in, DynamicStrings_Slice (fmt, (*startpos), nextperc));
+              in = Copy (fmt, in, (*startpos), nextperc);
               in = DynamicStrings_ConCat (in, StringConvert_CardinalToString (u, static_cast<unsigned int> (width), leader, 10, false));
               (*startpos) = afterperc;
               DSdbExit (static_cast<DynamicStrings_String> (NULL));
diff --git a/gcc/m2/mc-boot/GM2EXCEPTION.cc b/gcc/m2/mc-boot/GM2EXCEPTION.cc
index 62d47f0..6baff3c 100644
--- a/gcc/m2/mc-boot/GM2EXCEPTION.cc
+++ b/gcc/m2/mc-boot/GM2EXCEPTION.cc
@@ -34,7 +34,6 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
        typedef struct { PROC_t proc; } PROC;
 #   endif
 
-#   include "Gmcrts.h"
 #define _M2EXCEPTION_C
 
 #include "GM2EXCEPTION.h"
@@ -51,18 +50,19 @@ extern "C" M2EXCEPTION_M2Exceptions M2EXCEPTION_M2Exception (void)
 
   /* If the program or coroutine is in the exception state then return the enumeration
    value representing the exception cause.  If it is not in the exception state then
-   raises and exception (exException).  */
+   raises an exException exception.  */
   e = RTExceptions_GetExceptionBlock ();
   n = RTExceptions_GetNumber (e);
   if (n == (UINT_MAX))
     {
       RTExceptions_Raise ( ((unsigned int) (M2EXCEPTION_exException)), const_cast<void*> (static_cast<const void*>("../../gcc/m2/gm2-libs/M2EXCEPTION.mod")), 47, 6, const_cast<void*> (static_cast<const void*>("M2Exception")), const_cast<void*> (static_cast<const void*>("current coroutine is not in the exceptional execution state")));
+      return M2EXCEPTION_exException;
     }
   else
     {
       return (M2EXCEPTION_M2Exceptions) (n);
     }
-  ReturnException ("../../gcc/m2/gm2-libs/M2EXCEPTION.def", 25, 1);
+  /* static analysis guarentees a RETURN statement will be used before here.  */
   __builtin_unreachable ();
 }
 
diff --git a/gcc/m2/mc-boot/GSFIO.cc b/gcc/m2/mc-boot/GSFIO.cc
index 6ae0d5e..f8c13d3 100644
--- a/gcc/m2/mc-boot/GSFIO.cc
+++ b/gcc/m2/mc-boot/GSFIO.cc
@@ -99,6 +99,13 @@ extern "C" DynamicStrings_String SFIO_WriteS (FIO_File file, DynamicStrings_Stri
 
 extern "C" DynamicStrings_String SFIO_ReadS (FIO_File file);
 
+/*
+   GetFileName - return a new string containing the name of the file.
+                 The string should be killed by the caller.
+*/
+
+extern "C" DynamicStrings_String SFIO_GetFileName (FIO_File file);
+
 
 /*
    Exists - returns TRUE if a file named, fname exists for reading.
@@ -207,6 +214,19 @@ extern "C" DynamicStrings_String SFIO_ReadS (FIO_File file)
   __builtin_unreachable ();
 }
 
+
+/*
+   GetFileName - return a new string containing the name of the file.
+                 The string should be killed by the caller.
+*/
+
+extern "C" DynamicStrings_String SFIO_GetFileName (FIO_File file)
+{
+  return DynamicStrings_InitStringCharStar (FIO_getFileName (file));
+  /* static analysis guarentees a RETURN statement will be used before here.  */
+  __builtin_unreachable ();
+}
+
 extern "C" void _M2_SFIO_init (__attribute__((unused)) int argc, __attribute__((unused)) char *argv[], __attribute__((unused)) char *envp[])
 {
 }
diff --git a/gcc/m2/mc-boot/GSFIO.h b/gcc/m2/mc-boot/GSFIO.h
index 42ffc48..93c8099 100644
--- a/gcc/m2/mc-boot/GSFIO.h
+++ b/gcc/m2/mc-boot/GSFIO.h
@@ -103,6 +103,13 @@ EXTERN DynamicStrings_String SFIO_WriteS (FIO_File file, DynamicStrings_String s
 */
 
 EXTERN DynamicStrings_String SFIO_ReadS (FIO_File file);
+
+/*
+   GetFileName - return a new string containing the name of the file.
+                 The string should be killed by the caller.
+*/
+
+EXTERN DynamicStrings_String SFIO_GetFileName (FIO_File file);
 #   ifdef __cplusplus
 }
 #   endif
diff --git a/gcc/m2/mc-boot/Gdecl.cc b/gcc/m2/mc-boot/Gdecl.cc
index ae03483..94ea098 100644
--- a/gcc/m2/mc-boot/Gdecl.cc
+++ b/gcc/m2/mc-boot/Gdecl.cc
@@ -2550,6 +2550,14 @@ static bool isLeafString (decl_node__opaque n);
 static DynamicStrings_String getLiteralStringContents (decl_node__opaque n);
 
 /*
+   getStringChar - if the string is delimited by single
+                   or double quotes then strip both
+                   quotes from the string.
+*/
+
+static DynamicStrings_String getStringChar (decl_node__opaque n);
+
+/*
    getStringContents - return the string contents of a constant, literal,
                        string or a constexp node.
 */
@@ -2569,7 +2577,13 @@ static nameKey_Name addNames (decl_node__opaque a, decl_node__opaque b);
 static decl_node__opaque resolveString (decl_node__opaque n);
 
 /*
-   foldBinary -
+   addQuotes - adds delimiter quote char to string.
+*/
+
+static DynamicStrings_String addQuotes (DynamicStrings_String s, char quote);
+
+/*
+   foldBinary - attempt to fold binary + for string constants.
 */
 
 static decl_node__opaque foldBinary (decl_nodeT k, decl_node__opaque l, decl_node__opaque r, decl_node__opaque res);
@@ -7590,6 +7604,32 @@ static DynamicStrings_String getLiteralStringContents (decl_node__opaque n)
 
 
 /*
+   getStringChar - if the string is delimited by single
+                   or double quotes then strip both
+                   quotes from the string.
+*/
+
+static DynamicStrings_String getStringChar (decl_node__opaque n)
+{
+  DynamicStrings_String s;
+
+  s = getString (n);
+  if (((DynamicStrings_char (s, 0)) == '\'') && ((DynamicStrings_char (s, -1)) == '\''))
+    {
+      s = DynamicStrings_Slice (s, 1, -1);
+    }
+  else if (((DynamicStrings_char (s, 0)) == '"') && ((DynamicStrings_char (s, -1)) == '"'))
+    {
+      /* avoid dangling else.  */
+      s = DynamicStrings_Slice (s, 1, -1);
+    }
+  return s;
+  /* static analysis guarentees a RETURN statement will be used before here.  */
+  __builtin_unreachable ();
+}
+
+
+/*
    getStringContents - return the string contents of a constant, literal,
                        string or a constexp node.
 */
@@ -7608,7 +7648,7 @@ static DynamicStrings_String getStringContents (decl_node__opaque n)
   else if (isString (n))
     {
       /* avoid dangling else.  */
-      return getString (n);
+      return getStringChar (n);
     }
   else if (isConstExp (n))
     {
@@ -7672,11 +7712,29 @@ static decl_node__opaque resolveString (decl_node__opaque n)
 
 
 /*
-   foldBinary -
+   addQuotes - adds delimiter quote char to string.
+*/
+
+static DynamicStrings_String addQuotes (DynamicStrings_String s, char quote)
+{
+  DynamicStrings_String qs;
+
+  s = DynamicStrings_ConCatChar (s, quote);
+  qs = DynamicStrings_InitStringChar (quote);
+  qs = DynamicStrings_ConCat (qs, DynamicStrings_Mark (s));
+  return qs;
+  /* static analysis guarentees a RETURN statement will be used before here.  */
+  __builtin_unreachable ();
+}
+
+
+/*
+   foldBinary - attempt to fold binary + for string constants.
 */
 
 static decl_node__opaque foldBinary (decl_nodeT k, decl_node__opaque l, decl_node__opaque r, decl_node__opaque res)
 {
+  char qc;
   decl_node__opaque n;
   DynamicStrings_String ls;
   DynamicStrings_String rs;
@@ -7686,7 +7744,12 @@ static decl_node__opaque foldBinary (decl_nodeT k, decl_node__opaque l, decl_nod
     {
       ls = getStringContents (l);
       rs = getStringContents (r);
+      qc = '\'';
+      /* Add unquoted contents.  */
       ls = DynamicStrings_Add (ls, rs);
+      /* Add quote.  */
+      ls = addQuotes (ls, qc);
+      /* Build new string.  */
       n = static_cast<decl_node__opaque> (decl_makeString (nameKey_makekey (DynamicStrings_string (ls))));
       ls = DynamicStrings_KillString (ls);
       rs = DynamicStrings_KillString (rs);
@@ -22789,7 +22852,7 @@ static decl_node__opaque doDupExpr (decl_node__opaque n)
         break;
 
       case decl_length:
-        M2RTS_HALT (-1);
+        M2RTS_HALT (-1);  /* length should have been converted into unary.  */
         __builtin_unreachable ();
         break;
 
diff --git a/gcc/m2/mc-boot/GmcFileName.h b/gcc/m2/mc-boot/GmcFileName.h
index 11f1512..6c7ec75 100644
--- a/gcc/m2/mc-boot/GmcFileName.h
+++ b/gcc/m2/mc-boot/GmcFileName.h
@@ -50,7 +50,7 @@ extern "C" {
                        given a module and an extension. This file name
                        length will be operating system specific.
                        String, Extension, is concatenated onto
-                       Module and thus it is safe to `Mark' the extension
+                       Module and thus it is safe to Mark the extension
                        for garbage collection.
 */
 
diff --git a/gcc/m2/mc/decl.mod b/gcc/m2/mc/decl.mod
index 342487e..197ca5e 100644
--- a/gcc/m2/mc/decl.mod
+++ b/gcc/m2/mc/decl.mod
@@ -4643,6 +4643,28 @@ END getLiteralStringContents ;
 
 
 (*
+   getStringChar - if the string is delimited by single
+                   or double quotes then strip both
+                   quotes from the string.
+*)
+
+PROCEDURE getStringChar (n: node) : String ;
+VAR
+   s: String ;
+BEGIN
+   s := getString (n) ;
+   IF (DynamicStrings.char (s, 0) = "'") AND (DynamicStrings.char (s, -1) = "'")
+   THEN
+      s := DynamicStrings.Slice (s, 1, -1)
+   ELSIF (DynamicStrings.char (s, 0) = '"') AND (DynamicStrings.char (s, -1) = '"')
+   THEN
+      s := DynamicStrings.Slice (s, 1, -1)
+   END ;
+   RETURN s
+END getStringChar ;
+
+
+(*
    getStringContents - return the string contents of a constant, literal,
                        string or a constexp node.
 *)
@@ -4657,7 +4679,7 @@ BEGIN
       RETURN getLiteralStringContents (n)
    ELSIF isString (n)
    THEN
-      RETURN getString (n)
+      RETURN getStringChar (n)
    ELSIF isConstExp (n)
    THEN
       RETURN getStringContents (n^.unaryF.arg)
@@ -4709,11 +4731,27 @@ END resolveString ;
 
 
 (*
-   foldBinary -
+   addQuotes - adds delimiter quote char to string.
+*)
+
+PROCEDURE addQuotes (s: String; quote: CHAR) : String ;
+VAR
+   qs: String ;
+BEGIN
+   s := DynamicStrings.ConCatChar (s, quote) ;
+   qs := DynamicStrings.InitStringChar (quote) ;
+   qs := DynamicStrings.ConCat (qs, DynamicStrings.Mark (s)) ;
+   RETURN qs
+END addQuotes ;
+
+
+(*
+   foldBinary - attempt to fold binary + for string constants.
 *)
 
 PROCEDURE foldBinary (k: nodeT; l, r: node; res: node) : node ;
 VAR
+   qc: CHAR ;
    n : node ;
    ls,
    rs: String ;
@@ -4723,7 +4761,12 @@ BEGIN
    THEN
       ls := getStringContents (l) ;
       rs := getStringContents (r) ;
+      qc := "'" ;
+      (* Add unquoted contents.  *)
       ls := DynamicStrings.Add (ls, rs) ;
+      (* Add quote.  *)
+      ls := addQuotes (ls, qc) ;
+      (* Build new string.  *)
       n := makeString (makekey (DynamicStrings.string (ls))) ;
       ls := DynamicStrings.KillString (ls) ;
       rs := DynamicStrings.KillString (rs)
diff --git a/gcc/machmode.h b/gcc/machmode.h
index 467681d9..2f2d349 100644
--- a/gcc/machmode.h
+++ b/gcc/machmode.h
@@ -958,7 +958,8 @@ private:
 
 /* Find the best mode to use to access a bit field.  */
 
-extern bool get_best_mode (int, int, poly_uint64, poly_uint64, unsigned int,
+extern bool get_best_mode (HOST_WIDE_INT, HOST_WIDE_INT,
+			   poly_uint64, poly_uint64, unsigned int,
 			   unsigned HOST_WIDE_INT, bool, scalar_int_mode *);
 
 /* Determine alignment, 1<=result<=BIGGEST_ALIGNMENT.  */
diff --git a/gcc/match.pd b/gcc/match.pd
index f4416d9..82e6e29 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -3500,7 +3500,18 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
      wide_int c2 = wi::to_wide (@2);
      wide_int sum = wi::add (c1, c2);
     }
-    (if (wi::eq_p (sum, wi::max_value (precision, SIGNED)))))))
+    (if (wi::eq_p (sum, wi::max_value (precision, SIGNED))))))
+
+(match (signed_integer_sat_add @0 @1)
+  /* T SUM = (T)((UT)X + (UT)IMM)
+     SAT_S_ADD = (X ^ SUM) < 0 && (X ^ IMM) >= 0 ? (-(T)(X < 0) ^ MAX) : SUM  */
+   (cond^ (ge (bit_ior:c (bit_xor:c @0 INTEGER_CST@1)
+                       (bit_not (bit_xor:c @0 (nop_convert@2 (plus (nop_convert @0)
+                       INTEGER_CST@3)))))
+                   integer_zerop)
+           (signed_integer_sat_val @0)
+           @2)
+   (if (wi::eq_p (wi::to_wide (@1), wi::to_wide (@3))))))
 
 /* Saturation sub for signed integer.  */
 (if (INTEGRAL_TYPE_P (type) && !TYPE_UNSIGNED (type))
@@ -3572,6 +3583,48 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 	 || (wi::eq_p (int_cst_1, itype_max) && wi::eq_p (int_cst_2, limit_1)))
 	 && wi::eq_p (int_cst_3, otype_max)))))))
 
+/* Saturation mult for unsigned integer.  */
+(if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type))
+  (match (unsigned_integer_sat_mul @0 @1)
+   /* SAT_U_MUL (X, Y) = {
+	WT x = (WT)a * (WT)b;
+	T max = -1;
+	if (x > (WT)(max))
+	  return max;
+	else
+	  return (T)x;
+      }
+      while WT is uint128_t, T is uint8_t, uint16_t, uint32_t or uint64_t.  */
+   (convert (min (widen_mult:c@3 (convert@4 (convert @0))
+				 (convert@5 (convert @1)))
+		   INTEGER_CST@2))
+   (if (types_match (type, @0, @1))
+    (with
+     {
+      unsigned prec = TYPE_PRECISION (type);
+      unsigned widen_prec = TYPE_PRECISION (TREE_TYPE (@3));
+      unsigned cvt4_prec = TYPE_PRECISION (TREE_TYPE (@4));
+      unsigned cvt5_prec = TYPE_PRECISION (TREE_TYPE (@5));
+      wide_int c2 = wi::to_wide (@2);
+      wide_int max = wi::mask (prec, false, widen_prec);
+      bool c2_is_max_p = wi::eq_p (c2, max);
+      bool widen_mult_p = cvt4_prec == cvt5_prec && widen_prec == cvt5_prec * 2;
+     }
+     (if (widen_prec > prec && c2_is_max_p && widen_mult_p)))))
+  (match (unsigned_integer_sat_mul @0 @1)
+   (convert (min (mult:c@3 (convert @0) (convert @1)) INTEGER_CST@2))
+   (if (types_match (type, @0, @1))
+    (with
+     {
+      unsigned prec = TYPE_PRECISION (type);
+      unsigned widen_prec = TYPE_PRECISION (TREE_TYPE (@3));
+      wide_int c2 = wi::to_wide (@2);
+      wide_int max = wi::mask (prec, false, widen_prec);
+      bool c2_is_max_p = wi::eq_p (c2, max);
+     }
+     (if (widen_prec > prec && c2_is_max_p)))))
+)
+
 /* The boundary condition for case 10: IMM = 1:
    SAT_U_SUB = X >= IMM ? (X - IMM) : 0.
    simplify (X != 0 ? X + ~0 : 0) to X - (X != 0).  */
@@ -3674,6 +3727,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
    (if ((TREE_CODE (@1) == INTEGER_CST
 	 && TREE_CODE (@2) == INTEGER_CST)
 	|| ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
+   || (VECTOR_TYPE_P (TREE_TYPE (@1))
+   && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, code2))
 	     || POINTER_TYPE_P (TREE_TYPE (@1)))
 	    && bitwise_equal_p (@1, @2)))
     (with
@@ -3712,27 +3767,39 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
       (if (code1 == EQ_EXPR && val) @3)
       (if (code1 == EQ_EXPR && !val) { constant_boolean_node (false, type); })
       (if (code1 == NE_EXPR && !val && allbits) @4)
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == GE_EXPR
 	   && cmp == 0
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (gt @c0 (convert @1)))
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == LE_EXPR
 	   && cmp == 0
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (lt @c0 (convert @1)))
       /* (a != (b+1)) & (a > b) -> a > (b+1) */
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == GT_EXPR
 	   && one_after
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (gt @c0 (convert @1)))
       /* (a != (b-1)) & (a < b) -> a < (b-1) */
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == LT_EXPR
 	   && one_before
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (lt @c0 (convert @1)))
      )
     )
@@ -3751,6 +3818,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
   (if ((TREE_CODE (@1) == INTEGER_CST
 	&& TREE_CODE (@2) == INTEGER_CST)
        || ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
+       || (VECTOR_TYPE_P (TREE_TYPE (@1))
+       && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, code2))
 	    || POINTER_TYPE_P (TREE_TYPE (@1)))
 	   && operand_equal_p (@1, @2)))
    (with
@@ -3801,6 +3870,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
    (if ((TREE_CODE (@1) == INTEGER_CST
 	 && TREE_CODE (@2) == INTEGER_CST)
 	|| ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
+   || (VECTOR_TYPE_P (TREE_TYPE (@1)))
 	    || POINTER_TYPE_P (TREE_TYPE (@1)))
 	    && bitwise_equal_p (@1, @2)))
     (with
@@ -3842,24 +3912,36 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
       (if (code1 == EQ_EXPR
            && code2 == GT_EXPR
 	   && cmp == 0
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (ge @c0 @2))
       (if (code1 == EQ_EXPR
            && code2 == LT_EXPR
 	   && cmp == 0
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (le @c0 @2))
       /* (a == (b-1)) | (a >= b) -> a >= (b-1) */
       (if (code1 == EQ_EXPR
            && code2 == GE_EXPR
 	   && one_before
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (ge @c0 (convert @1)))
       /* (a == (b+1)) | (a <= b) -> a <= (b-1) */
       (if (code1 == EQ_EXPR
            && code2 == LE_EXPR
 	   && one_after
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (le @c0 (convert @1)))
      )
     )
@@ -3924,7 +4006,10 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
       rcmp (ne le gt ne lt ge)
   (simplify
    (op:c (cmp1:c @0 @1) (cmp2 @0 @1))
-   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) || POINTER_TYPE_P (TREE_TYPE (@0)))
+   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
+      || POINTER_TYPE_P (TREE_TYPE (@0))
+      || (VECTOR_TYPE_P (TREE_TYPE (@1))
+      && expand_vec_cmp_expr_p (TREE_TYPE (@0), type, rcmp)))
     (rcmp @0 @1)))))
 
 /* Optimize (a CMP b) == (a CMP b)  */
@@ -3933,7 +4018,10 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
      rcmp (eq gt le eq ge lt)
  (simplify
   (eq:c (cmp1:c @0 @1) (cmp2 @0 @1))
-  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) || POINTER_TYPE_P (TREE_TYPE (@0)))
+  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
+		|| POINTER_TYPE_P (TREE_TYPE (@0))
+      || (VECTOR_TYPE_P (TREE_TYPE (@0))
+      && expand_vec_cmp_expr_p (TREE_TYPE (@0), type,  rcmp)))
     (rcmp @0 @1))))
 
 /* (type)([0,1]@a != 0) -> (type)a
@@ -5939,6 +6027,15 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 	       && !expand_vec_cond_expr_p (TREE_TYPE (@1), TREE_TYPE (@0)))))
    (vec_cond @0 (op! @1 @3) (op! @2 @4))))
 
+/* (@0 ? @2 : @3) lop (@1 ? @2 : @3)  -->  (@0 lop @1) ? @2 : @3.  */
+(for lop (bit_and bit_ior bit_xor)
+   (simplify
+   (lop
+      (vec_cond @0 integer_minus_onep@2 integer_zerop@3)
+      (vec_cond @1 @2 @3))
+      (if (target_supports_op_p (TREE_TYPE (@0), lop, optab_vector))
+   (vec_cond (lop @0 @1) @2 @3))))
+
 /* (c ? a : b) op d  -->  c ? (a op d) : (b op d) */
  (simplify
   (op (vec_cond:s @0 @1 @2) @3)
@@ -6950,6 +7047,22 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
     (icmp @0 @1)
     (if (ic == ncmp)
      (ncmp @0 @1)))))
+ /* ((cast)cmp) - 1 -> -(cast)icmp . */
+ (simplify
+  (plus (convert? (cmp@2 @0 @1)) integer_minus_onep)
+  (if (TYPE_PRECISION (type) > 1
+       && INTEGRAL_TYPE_P (TREE_TYPE (@2)) && TYPE_PRECISION (TREE_TYPE (@2)) == 1)
+   /* Comparison inversion may be impossible for trapping math,
+      invert_tree_comparison will tell us.  But we can't use
+      a computed operator in the replacement tree thus we have
+      to play the trick below.  */
+   (with { enum tree_code ic = invert_tree_comparison
+             (cmp, HONOR_NANS (@0));
+           tree cmptype = TREE_TYPE (@2); }
+    (if (ic == icmp)
+     (negate (convert (icmp:cmptype @0 @1)))
+     (if (ic == ncmp)
+      (negate (convert (ncmp:cmptype @0 @1))))))))
  /* The following bits are handled by fold_binary_op_with_conditional_arg.  */
  (simplify
   (ne (cmp@2 @0 @1) integer_zerop)
diff --git a/gcc/opt-problem.cc b/gcc/opt-problem.cc
index 8324feb..e0c7601 100644
--- a/gcc/opt-problem.cc
+++ b/gcc/opt-problem.cc
@@ -42,7 +42,7 @@ along with GCC; see the file COPYING3.  If not see
 
 opt_problem::opt_problem (const dump_location_t &loc,
 			  const char *fmt, va_list *ap)
-: m_optinfo (loc, OPTINFO_KIND_FAILURE, current_pass)
+: m_optinfo (loc, optinfo::kind::failure, current_pass)
 {
   /* We shouldn't be bothering to construct these objects if
      dumping isn't enabled.  */
diff --git a/gcc/optabs-query.cc b/gcc/optabs-query.cc
index f5ca98d..5335d0d 100644
--- a/gcc/optabs-query.cc
+++ b/gcc/optabs-query.cc
@@ -719,13 +719,9 @@ supports_vec_gather_load_p (machine_mode mode, vec<int> *elsvals)
 	= (icode != CODE_FOR_nothing) ? 1 : -1;
     }
 
-  /* For gather the optab's operand indices do not match the IFN's because
-     the latter does not have the extension operand (operand 3).  It is
-     implicitly added during expansion so we use the IFN's else index + 1.
-     */
   if (elsvals && icode != CODE_FOR_nothing)
     get_supported_else_vals
-      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD) + 1, *elsvals);
+      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD), *elsvals);
 
   return this_fn_optabs->supports_vec_gather_load[mode] > 0;
 }
diff --git a/gcc/optabs-tree.cc b/gcc/optabs-tree.cc
index 6dfe8ee..0de74c7 100644
--- a/gcc/optabs-tree.cc
+++ b/gcc/optabs-tree.cc
@@ -82,6 +82,9 @@ optab_for_tree_code (enum tree_code code, const_tree type,
 	return unknown_optab;
       /* FALLTHRU */
     case RDIV_EXPR:
+      gcc_assert (FLOAT_TYPE_P (type)
+		  || ALL_FIXED_POINT_MODE_P (TYPE_MODE (type)));
+      /* FALLTHRU */
     case TRUNC_DIV_EXPR:
     case EXACT_DIV_EXPR:
       if (TYPE_SATURATING (type))
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 0c1435d..87a8b85 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -134,8 +134,8 @@ OPTAB_NX(smul_optab, "mul$P$a3")
 OPTAB_NX(smul_optab, "mul$F$a3")
 OPTAB_VL(smulv_optab, "mulv$I$a3", MULT, "mul", '3', gen_intv_fp_libfunc)
 OPTAB_VX(smulv_optab, "mul$F$a3")
-OPTAB_NL(ssmul_optab, "ssmul$Q$a3", SS_MULT, "ssmul", '3', gen_signed_fixed_libfunc)
-OPTAB_NL(usmul_optab, "usmul$Q$a3", US_MULT, "usmul", '3', gen_unsigned_fixed_libfunc)
+OPTAB_NL(ssmul_optab, "ssmul$a3", SS_MULT, "ssmul", '3', gen_signed_fixed_libfunc)
+OPTAB_NL(usmul_optab, "usmul$a3", US_MULT, "usmul", '3', gen_unsigned_fixed_libfunc)
 OPTAB_NL(sdiv_optab, "div$a3", DIV, "div", '3', gen_int_fp_signed_fixed_libfunc)
 OPTAB_VL(sdivv_optab, "divv$I$a3", DIV, "divv", '3', gen_int_libfunc)
 OPTAB_VX(sdivv_optab, "div$F$a3")
diff --git a/gcc/optc-gen.awk b/gcc/optc-gen.awk
index 3d074e4..4e9b538 100644
--- a/gcc/optc-gen.awk
+++ b/gcc/optc-gen.awk
@@ -497,7 +497,7 @@ print "                           const struct cl_decoded_option *decoded,   "
 print "                           unsigned int lang_mask, int kind,          "
 print "                           location_t loc,                            "
 print "                           const struct cl_option_handlers *handlers, "
-print "                           diagnostic_context *dc)                    "
+print "                           diagnostics::context *dc)                  "
 print "{                                                                     "
 print "  size_t scode = decoded->opt_index;                                  "
 print "  HOST_WIDE_INT value = decoded->value;                               "
@@ -554,7 +554,7 @@ for (i = 0; i < n_langs; i++) {
     print "                           unsigned int lang_mask" mark_unused ", int kind" mark_unused ",          "
     print "                           location_t loc" mark_unused ",                            "
     print "                           const struct cl_option_handlers *handlers" mark_unused ", "
-    print "                           diagnostic_context *dc" mark_unused ")                    "
+    print "                           diagnostics::context *dc" mark_unused ")                  "
     print "{                                                                     "
     print "  enum opt_code code = (enum opt_code) scode;                         "
     print "                                                                      "
diff --git a/gcc/optc-save-gen.awk b/gcc/optc-save-gen.awk
index a3d7e5a..31756ec 100644
--- a/gcc/optc-save-gen.awk
+++ b/gcc/optc-save-gen.awk
@@ -1313,6 +1313,12 @@ for (i = 0; i < n_opts; i++) {
 		# offloading is enabled.
 		if (flag_set_p("Target", flags[i]))
 			var_target_opt[n_opt_val] = 1;
+
+		# These options should not be passed from host to target, but
+		# are not actually target specific.
+		if (flag_set_p("NoOffload", flags[i]))
+			var_target_opt[n_opt_val] = 2;
+
 		n_opt_val++;
 	}
 }
@@ -1393,7 +1399,7 @@ for (i = 0; i < n_opt_val; i++) {
 		# Do not stream out target-specific opts if offloading is
 		# enabled.
 		if (var_target_opt[i])
-			print "  if (!lto_stream_offload_p)"
+			print "  if (!lto_stream_offload_p) {"
 		# If applicable, encode the streamed value.
 		if (var_opt_optimize_init[i]) {
 			print "  if (" var_opt_optimize_init[i] " > (" var_opt_val_type[i] ") 10)";
@@ -1403,6 +1409,8 @@ for (i = 0; i < n_opt_val; i++) {
 		} else {
 			print "  bp_pack_var_len_" sgn " (bp, ptr->" name");";
 		}
+		if (var_target_opt[i])
+			print "}"
 	}
 }
 print "  for (size_t i = 0; i < ARRAY_SIZE (ptr->explicit_mask); i++)";
@@ -1418,10 +1426,14 @@ print "                           struct cl_optimization *ptr ATTRIBUTE_UNUSED)"
 print "{";
 for (i = 0; i < n_opt_val; i++) {
 	name = var_opt_val[i]
-        if (var_target_opt[i]) {
+        if (var_target_opt[i] == 1) {
 		print "#ifdef ACCEL_COMPILER"
 		print "#error accel compiler cannot define Optimization attribute for target-specific option " name;
 		print "#else"
+	} else if (var_target_opt[i] == 2) {
+		print "#ifdef ACCEL_COMPILER"
+		print "  ptr->" name " = global_options." name ";"
+		print "#else"
 	}
 	otype = var_opt_val_type[i];
 	if (otype ~ "^const char \\**$") {
@@ -1489,6 +1501,9 @@ for (i = 0; i < n_opts; i++) {
 	if (flag_set_p("Warning", flags[i]))
 		continue;
 
+	if (flag_set_p("NoOffload", flags[i]))
+		continue;
+
 	if (name in checked_options)
 		continue;
 	checked_options[name]++
diff --git a/gcc/opth-gen.awk b/gcc/opth-gen.awk
index b6613d3..91fcf56 100644
--- a/gcc/opth-gen.awk
+++ b/gcc/opth-gen.awk
@@ -357,7 +357,7 @@ print "                           const struct cl_decoded_option *decoded,   "
 print "                           unsigned int lang_mask, int kind,          "
 print "                           location_t loc,                            "
 print "                           const struct cl_option_handlers *handlers, "
-print "                           diagnostic_context *dc);                   "
+print "                           diagnostics::context *dc);                   "
 for (i = 0; i < n_langs; i++) {
     lang_name = lang_sanitized_name(langs[i]);
     print "bool"
@@ -368,7 +368,7 @@ for (i = 0; i < n_langs; i++) {
     print "                           unsigned int lang_mask, int kind,"
     print "                           location_t loc,"
     print "                           const struct cl_option_handlers *handlers,"
-    print "                           diagnostic_context *dc);"
+    print "                           diagnostics::context *dc);"
 }
 print "void cpp_handle_option_auto (const struct gcc_options * opts, size_t scode,"
 print "                             struct cpp_options * cpp_opts);"
diff --git a/gcc/optinfo-emit-json.cc b/gcc/optinfo-emit-json.cc
index 28c8a98..ae686a9 100644
--- a/gcc/optinfo-emit-json.cc
+++ b/gcc/optinfo-emit-json.cc
@@ -143,7 +143,7 @@ optrecord_json_writer::add_record (const optinfo *optinfo)
   add_record (obj);
 
   /* Potentially push the scope.  */
-  if (optinfo->get_kind () == OPTINFO_KIND_SCOPE)
+  if (optinfo->get_kind () == optinfo::kind::scope)
     {
       json::array *children = new json::array ();
       obj->set ("children", children);
@@ -334,7 +334,7 @@ optrecord_json_writer::optinfo_to_json (const optinfo *optinfo)
   obj->set ("impl_location",
 	    impl_location_to_json (optinfo->get_impl_location ()));
 
-  const char *kind_str = optinfo_kind_to_string (optinfo->get_kind ());
+  const char *kind_str = optinfo::kind_to_string (optinfo->get_kind ());
   obj->set_string ("kind", kind_str);
   json::array *message = new json::array ();
   obj->set ("message", message);
@@ -345,12 +345,12 @@ optrecord_json_writer::optinfo_to_json (const optinfo *optinfo)
 	{
 	default:
 	  gcc_unreachable ();
-	case OPTINFO_ITEM_KIND_TEXT:
+	case optinfo_item::kind::text:
 	  {
 	    message->append_string (item->get_text ());
 	  }
 	  break;
-	case OPTINFO_ITEM_KIND_TREE:
+	case optinfo_item::kind::tree:
 	  {
 	    json::object *json_item = new json::object ();
 	    json_item->set_string ("expr", item->get_text ());
@@ -363,7 +363,7 @@ optrecord_json_writer::optinfo_to_json (const optinfo *optinfo)
 	    message->append (json_item);
 	  }
 	  break;
-	case OPTINFO_ITEM_KIND_GIMPLE:
+	case optinfo_item::kind::gimple:
 	  {
 	    json::object *json_item = new json::object ();
 	    json_item->set_string ("stmt", item->get_text ());
@@ -376,7 +376,7 @@ optrecord_json_writer::optinfo_to_json (const optinfo *optinfo)
 	    message->append (json_item);
 	  }
 	  break;
-	case OPTINFO_ITEM_KIND_SYMTAB_NODE:
+	case optinfo_item::kind::symtab_node:
 	  {
 	    json::object *json_item = new json::object ();
 	    json_item->set_string ("symtab_node", item->get_text ());
diff --git a/gcc/optinfo.cc b/gcc/optinfo.cc
index 51e9fd6..dd5a551 100644
--- a/gcc/optinfo.cc
+++ b/gcc/optinfo.cc
@@ -36,9 +36,9 @@ along with GCC; see the file COPYING3.  If not see
 
 /* optinfo_item's ctor.  Takes ownership of TEXT.  */
 
-optinfo_item::optinfo_item (enum optinfo_item_kind kind, location_t location,
+optinfo_item::optinfo_item (enum kind kind_, location_t location,
 			    char *text)
-: m_kind (kind), m_location (location), m_text (text)
+: m_kind (kind_), m_location (location), m_text (text)
 {
 }
 
@@ -52,19 +52,19 @@ optinfo_item::~optinfo_item ()
 /* Get a string from KIND.  */
 
 const char *
-optinfo_kind_to_string (enum optinfo_kind kind)
+optinfo::kind_to_string (enum kind kind_)
 {
-  switch (kind)
+  switch (kind_)
     {
     default:
       gcc_unreachable ();
-    case OPTINFO_KIND_SUCCESS:
+    case kind::success:
       return "success";
-    case OPTINFO_KIND_FAILURE:
+    case kind::failure:
       return "failure";
-    case OPTINFO_KIND_NOTE:
+    case kind::note:
       return "note";
-    case OPTINFO_KIND_SCOPE:
+    case kind::scope:
       return "scope";
     }
 }
@@ -91,19 +91,19 @@ optinfo::add_item (std::unique_ptr<optinfo_item> item)
 
 /* Get MSG_* flags corresponding to KIND.  */
 
-static dump_flags_t
-optinfo_kind_to_dump_flag (enum optinfo_kind kind)
+dump_flags_t
+optinfo::kind_to_dump_flag (enum kind kind_)
 {
-  switch (kind)
+  switch (kind_)
     {
     default:
       gcc_unreachable ();
-    case OPTINFO_KIND_SUCCESS:
+    case kind::success:
       return MSG_OPTIMIZED_LOCATIONS;
-    case OPTINFO_KIND_FAILURE:
+    case kind::failure:
       return MSG_MISSED_OPTIMIZATION;
-    case OPTINFO_KIND_NOTE:
-    case OPTINFO_KIND_SCOPE:
+    case kind::note:
+    case kind::scope:
       return MSG_NOTE;
     }
 }
@@ -114,7 +114,7 @@ optinfo_kind_to_dump_flag (enum optinfo_kind kind)
 void
 optinfo::emit_for_opt_problem () const
 {
-  dump_flags_t dump_kind = optinfo_kind_to_dump_flag (get_kind ());
+  dump_flags_t dump_kind = kind_to_dump_flag (get_kind ());
   dump_kind |= MSG_PRIORITY_REEMITTED;
 
   /* Re-emit to "immediate" destinations, without creating a new optinfo.  */
@@ -134,14 +134,14 @@ void
 optinfo::handle_dump_file_kind (dump_flags_t dump_kind)
 {
   /* Any optinfo for a "scope" should have been emitted separately.  */
-  gcc_assert (m_kind != OPTINFO_KIND_SCOPE);
+  gcc_assert (m_kind != kind::scope);
 
   if (dump_kind & MSG_OPTIMIZED_LOCATIONS)
-    m_kind = OPTINFO_KIND_SUCCESS;
+    m_kind = kind::success;
   else if (dump_kind & MSG_MISSED_OPTIMIZATION)
-    m_kind = OPTINFO_KIND_FAILURE;
+    m_kind = kind::failure;
   else if (dump_kind & MSG_NOTE)
-    m_kind = OPTINFO_KIND_NOTE;
+    m_kind = kind::note;
 }
 
 /* Return true if any of the active optinfo destinations make use
diff --git a/gcc/optinfo.h b/gcc/optinfo.h
index 1e0fb22..ca9457f 100644
--- a/gcc/optinfo.h
+++ b/gcc/optinfo.h
@@ -74,18 +74,6 @@ class optinfo_item;
 
 extern bool optinfo_wants_inlining_info_p ();
 
-/* The various kinds of optinfo.  */
-
-enum optinfo_kind
-{
-  OPTINFO_KIND_SUCCESS,
-  OPTINFO_KIND_FAILURE,
-  OPTINFO_KIND_NOTE,
-  OPTINFO_KIND_SCOPE
-};
-
-extern const char *optinfo_kind_to_string (enum optinfo_kind kind);
-
 class dump_context;
 
 /* A bundle of information describing part of an optimization.  */
@@ -95,10 +83,19 @@ class optinfo
   friend class dump_context;
 
  public:
+  /* The various kinds of optinfo.  */
+  enum class kind
+  {
+    success,
+    failure,
+    note,
+    scope
+  };
+
   optinfo (const dump_location_t &loc,
-	   enum optinfo_kind kind,
+	   enum kind kind_,
 	   opt_pass *pass)
-  : m_loc (loc), m_kind (kind), m_pass (pass), m_items ()
+  : m_loc (loc), m_kind (kind_), m_pass (pass), m_items ()
   {}
   ~optinfo ();
 
@@ -111,7 +108,7 @@ class optinfo
   const dump_impl_location_t &
   get_impl_location () const { return m_loc.get_impl_location (); }
 
-  enum optinfo_kind get_kind () const { return m_kind; }
+  enum kind get_kind () const { return m_kind; }
   opt_pass *get_pass () const { return m_pass; }
   unsigned int num_items () const { return m_items.length (); }
   const optinfo_item *get_item (unsigned int i) const { return m_items[i]; }
@@ -123,6 +120,9 @@ class optinfo
 
   void emit_for_opt_problem () const;
 
+  static const char *kind_to_string (enum kind k);
+  static dump_flags_t kind_to_dump_flag (enum kind k);
+
  private:
   /* Pre-canned ways of manipulating the optinfo, for use by friend class
      dump_context.  */
@@ -130,37 +130,36 @@ class optinfo
 
  private:
   dump_location_t m_loc;
-  enum optinfo_kind m_kind;
+  enum kind m_kind;
   opt_pass *m_pass;
   auto_vec <optinfo_item *> m_items;
 };
 
-/* An enum for discriminating between different kinds of optinfo_item.  */
-
-enum optinfo_item_kind
-{
-  OPTINFO_ITEM_KIND_TEXT,
-  OPTINFO_ITEM_KIND_TREE,
-  OPTINFO_ITEM_KIND_GIMPLE,
-  OPTINFO_ITEM_KIND_SYMTAB_NODE
-};
-
 /* An item within an optinfo.  */
 
 class optinfo_item
 {
  public:
-  optinfo_item (enum optinfo_item_kind kind, location_t location,
+  /* An enum for discriminating between different kinds of optinfo_item.  */
+  enum class kind
+  {
+    text,
+    tree,
+    gimple,
+    symtab_node
+  };
+
+  optinfo_item (enum kind kind_, location_t location,
 		char *text);
   ~optinfo_item ();
 
-  enum optinfo_item_kind get_kind () const { return m_kind; }
+  enum kind get_kind () const { return m_kind; }
   location_t get_location () const { return m_location; }
   const char *get_text () const { return m_text; }
 
  private:
   /* Metadata (e.g. for optimization records).  */
-  enum optinfo_item_kind m_kind;
+  enum kind m_kind;
   location_t m_location;
 
   /* The textual form of the item, owned by the item.  */
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 03b1a79..e6d7f4d 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -1282,9 +1282,9 @@ keep:
 
 /* Handle option DECODED for the language indicated by LANG_MASK,
    using the handlers in HANDLERS and setting fields in OPTS and
-   OPTS_SET.  KIND is the diagnostic_t if this is a diagnostics
-   option, DK_UNSPECIFIED otherwise, and LOC is the location of the
-   option for options from the source file, UNKNOWN_LOCATION
+   OPTS_SET.  KIND is the enum diagnostics::kind if this is a diagnostics
+   option, diagnostics::kind::unspecified otherwise, and LOC is the location
+   of the option for options from the source file, UNKNOWN_LOCATION
    otherwise.  GENERATED_P is true for an option generated as part of
    processing another option or otherwise generated internally, false
    for one explicitly passed by the user.  control_warning_option
@@ -1298,7 +1298,7 @@ handle_option (struct gcc_options *opts,
 	       const struct cl_decoded_option *decoded,
 	       unsigned int lang_mask, int kind, location_t loc,
 	       const struct cl_option_handlers *handlers,
-	       bool generated_p, diagnostic_context *dc)
+	       bool generated_p, diagnostics::context *dc)
 {
   size_t opt_index = decoded->opt_index;
   const char *arg = decoded->arg;
@@ -1336,7 +1336,7 @@ handle_generated_option (struct gcc_options *opts,
 			 size_t opt_index, const char *arg, HOST_WIDE_INT value,
 			 unsigned int lang_mask, int kind, location_t loc,
 			 const struct cl_option_handlers *handlers,
-			 bool generated_p, diagnostic_context *dc)
+			 bool generated_p, diagnostics::context *dc)
 {
   struct cl_decoded_option decoded;
 
@@ -1604,7 +1604,7 @@ read_cmdline_option (struct gcc_options *opts,
 		     location_t loc,
 		     unsigned int lang_mask,
 		     const struct cl_option_handlers *handlers,
-		     diagnostic_context *dc)
+		     diagnostics::context *dc)
 {
   const struct cl_option *option;
   const char *opt = decoded->orig_option_with_args_text;
@@ -1645,7 +1645,8 @@ read_cmdline_option (struct gcc_options *opts,
 
   gcc_assert (!decoded->errors);
 
-  if (!handle_option (opts, opts_set, decoded, lang_mask, DK_UNSPECIFIED,
+  if (!handle_option (opts, opts_set, decoded, lang_mask,
+		      static_cast<int> (diagnostics::kind::unspecified),
 		      loc, handlers, false, dc))
     error_at (loc, "unrecognized command-line option %qs", opt);
 }
@@ -1658,7 +1659,7 @@ read_cmdline_option (struct gcc_options *opts,
 void
 set_option (struct gcc_options *opts, struct gcc_options *opts_set,
 	    size_t opt_index, HOST_WIDE_INT value, const char *arg,
-	    int kind, location_t loc, diagnostic_context *dc,
+	    int kind, location_t loc, diagnostics::context *dc,
 	    HOST_WIDE_INT mask /* = 0 */)
 {
   const struct cl_option *option = &cl_options[opt_index];
@@ -1668,8 +1669,10 @@ set_option (struct gcc_options *opts, struct gcc_options *opts_set,
   if (!flag_var)
     return;
 
-  if ((diagnostic_t) kind != DK_UNSPECIFIED && dc != NULL)
-    diagnostic_classify_diagnostic (dc, opt_index, (diagnostic_t) kind, loc);
+  if ((enum diagnostics::kind) kind != diagnostics::kind::unspecified
+      && dc != nullptr)
+    diagnostic_classify_diagnostic (dc, opt_index,
+				    (enum diagnostics::kind) kind, loc);
 
   if (opts_set != NULL)
     set_flag_var = option_flag_var (opt_index, opts_set);
@@ -1873,8 +1876,8 @@ option_enabled (int opt_idx, unsigned lang_mask, void *opts)
 }
 
 int
-compiler_diagnostic_option_manager::
-option_enabled_p (diagnostic_option_id opt_id) const
+compiler_diagnostic_option_id_manager::
+option_enabled_p (diagnostics::option_id opt_id) const
 {
   return option_enabled (opt_id.m_idx, m_lang_mask, m_opts);
 }
@@ -1941,7 +1944,7 @@ control_warning_option (unsigned int opt_index, int kind, const char *arg,
 			const struct cl_option_handlers *handlers,
 			struct gcc_options *opts,
 			struct gcc_options *opts_set,
-			diagnostic_context *dc)
+			diagnostics::context *dc)
 {
   if (cl_options[opt_index].alias_target != N_OPTS)
     {
@@ -1954,7 +1957,9 @@ control_warning_option (unsigned int opt_index, int kind, const char *arg,
   if (opt_index == OPT_SPECIAL_ignore || opt_index == OPT_SPECIAL_warn_removed)
     return;
   if (dc)
-    diagnostic_classify_diagnostic (dc, opt_index, (diagnostic_t) kind, loc);
+    diagnostic_classify_diagnostic (dc, opt_index,
+				    static_cast<enum diagnostics::kind> (kind),
+				    loc);
   if (imply)
     {
       /* -Werror=foo implies -Wfoo.  */
diff --git a/gcc/opts-diagnostic.cc b/gcc/opts-diagnostic.cc
index d2c3a93..70d7d74 100644
--- a/gcc/opts-diagnostic.cc
+++ b/gcc/opts-diagnostic.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 /* This file implements the options -fdiagnostics-add-output=,
    -fdiagnostics-set-output=.  Most of the work is done
-   by diagnostic-output-spec.cc so it can be shared by libgdiagnostics.  */
+   by diagnostics/output-spec.cc so it can be shared by libgdiagnostics.  */
 
 #include "config.h"
 #define INCLUDE_ARRAY
@@ -31,7 +31,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "version.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-output-spec.h"
+#include "diagnostics/output-spec.h"
 #include "opts.h"
 #include "options.h"
 
@@ -39,19 +39,19 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace {
 
-struct opt_spec_context : public diagnostics_output_spec::gcc_spec_context
+struct opt_spec_context : public diagnostics::output_spec::dc_spec_context
 {
 public:
   opt_spec_context (const gcc_options &opts,
-		    diagnostic_context &dc,
+		    diagnostics::context &dc,
 		    line_maps *location_mgr,
 		    location_t loc,
 		    const char *option_name)
-  : gcc_spec_context (dc,
-		      location_mgr,
-		      location_mgr,
-		      loc,
-		      option_name),
+  : dc_spec_context (dc,
+		     location_mgr,
+		     location_mgr,
+		     loc,
+		     option_name),
     m_opts (opts)
   {}
 
@@ -70,7 +70,7 @@ public:
 
 void
 handle_OPT_fdiagnostics_add_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc)
 {
@@ -89,7 +89,7 @@ handle_OPT_fdiagnostics_add_output_ (const gcc_options &opts,
 
 void
 handle_OPT_fdiagnostics_set_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc)
 {
@@ -103,5 +103,5 @@ handle_OPT_fdiagnostics_set_output_ (const gcc_options &opts,
     return;
 
   sink->set_main_input_filename (opts.x_main_input_filename);
-  dc.set_output_format (std::move (sink));
+  dc.set_sink (std::move (sink));
 }
diff --git a/gcc/opts-diagnostic.h b/gcc/opts-diagnostic.h
index 384fbb3..25ade86 100644
--- a/gcc/opts-diagnostic.h
+++ b/gcc/opts-diagnostic.h
@@ -20,54 +20,56 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_OPTS_DIAGNOSTIC_H
 #define GCC_OPTS_DIAGNOSTIC_H
 
-/* Abstract subclass of diagnostic_option_manager for gcc options.  */
+/* Abstract subclass of diagnostics::option_id_manager for gcc options.  */
 
-class gcc_diagnostic_option_manager : public diagnostic_option_manager
+class gcc_diagnostic_option_id_manager : public diagnostics::option_id_manager
 {
 public:
-  char *make_option_url (diagnostic_option_id option_id) const final override;
+  char *make_option_url (diagnostics::option_id option_id) const final override;
 
 protected:
-  gcc_diagnostic_option_manager (unsigned lang_mask)
+  gcc_diagnostic_option_id_manager (unsigned lang_mask)
   : m_lang_mask (lang_mask)
   {}
 
   unsigned m_lang_mask;
 };
 
-/* Concrete implementation of diagnostic_option_manager for compiler.  */
+/* Concrete implementation of diagnostics::option_id_manager for compiler.  */
 
-class compiler_diagnostic_option_manager : public gcc_diagnostic_option_manager
+class compiler_diagnostic_option_id_manager
+  : public gcc_diagnostic_option_id_manager
 {
 public:
-  compiler_diagnostic_option_manager (const diagnostic_context &context,
-				      unsigned lang_mask,
-				      void *opts)
-  : gcc_diagnostic_option_manager (lang_mask),
+  compiler_diagnostic_option_id_manager (const diagnostics::context &context,
+					 unsigned lang_mask,
+					 void *opts)
+  : gcc_diagnostic_option_id_manager (lang_mask),
     m_context (context),
     m_opts (opts)
   {
   }
 
-  int option_enabled_p (diagnostic_option_id option_id) const final override;
-  char *make_option_name (diagnostic_option_id option_id,
-			  diagnostic_t orig_diag_kind,
-			  diagnostic_t diag_kind) const final override;
+  int option_enabled_p (diagnostics::option_id option_id) const final override;
+  char *
+  make_option_name (diagnostics::option_id option_id,
+		    enum diagnostics::kind orig_diag_kind,
+		    enum diagnostics::kind diag_kind) const final override;
 
 private:
-  const diagnostic_context &m_context;
+  const diagnostics::context &m_context;
   void *m_opts;
 };
 
 extern void
 handle_OPT_fdiagnostics_add_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc);
 
 extern void
 handle_OPT_fdiagnostics_set_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc);
 #endif
diff --git a/gcc/opts-global.cc b/gcc/opts-global.cc
index b9b42d3..701a6bb 100644
--- a/gcc/opts-global.cc
+++ b/gcc/opts-global.cc
@@ -181,7 +181,7 @@ lang_handle_option (struct gcc_options *opts,
 		    unsigned int lang_mask ATTRIBUTE_UNUSED, int kind,
 		    location_t loc,
 		    const struct cl_option_handlers *handlers,
-		    diagnostic_context *dc,
+		    diagnostics::context *dc,
 		    void (*) (void))
 {
   gcc_assert (opts == &global_options);
@@ -215,7 +215,7 @@ read_cmdline_options (struct gcc_options *opts, struct gcc_options *opts_set,
 		      location_t loc,
 		      unsigned int lang_mask,
 		      const struct cl_option_handlers *handlers,
-		      diagnostic_context *dc)
+		      diagnostics::context *dc)
 {
   unsigned int i;
 
@@ -312,7 +312,7 @@ void
 decode_options (struct gcc_options *opts, struct gcc_options *opts_set,
 		struct cl_decoded_option *decoded_options,
 		unsigned int decoded_options_count,
-		location_t loc, diagnostic_context *dc,
+		location_t loc, diagnostics::context *dc,
 		void (*target_option_override_hook) (void))
 {
   struct cl_option_handlers handlers;
diff --git a/gcc/opts.cc b/gcc/opts.cc
index 6ca1ec7..b6d25bf 100644
--- a/gcc/opts.cc
+++ b/gcc/opts.cc
@@ -31,8 +31,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "common/common-target.h"
 #include "spellcheck.h"
 #include "opt-suggestions.h"
-#include "diagnostic-color.h"
-#include "diagnostic-format.h"
+#include "diagnostics/color.h"
+#include "diagnostics/sink.h"
 #include "version.h"
 #include "selftest.h"
 #include "file-prefix-map.h"
@@ -347,7 +347,7 @@ static void set_debug_level (uint32_t dinfo, int extended,
 			     location_t loc);
 static void set_fast_math_flags (struct gcc_options *opts, int set);
 static void decode_d_option (const char *arg, struct gcc_options *opts,
-			     location_t loc, diagnostic_context *dc);
+			     location_t loc, diagnostics::context *dc);
 static void set_unsafe_math_optimizations_flags (struct gcc_options *opts,
 						 int set);
 static void enable_warning_as_error (const char *arg, int value,
@@ -356,7 +356,7 @@ static void enable_warning_as_error (const char *arg, int value,
 				     struct gcc_options *opts,
 				     struct gcc_options *opts_set,
 				     location_t loc,
-				     diagnostic_context *dc);
+				     diagnostics::context *dc);
 
 /* Handle a back-end option; arguments and return value as for
    handle_option.  */
@@ -368,10 +368,11 @@ target_handle_option (struct gcc_options *opts,
 		      unsigned int lang_mask ATTRIBUTE_UNUSED, int kind,
 		      location_t loc,
 		      const struct cl_option_handlers *handlers ATTRIBUTE_UNUSED,
-		      diagnostic_context *dc, void (*) (void))
+		      diagnostics::context *dc, void (*) (void))
 {
   gcc_assert (dc == global_dc);
-  gcc_assert (kind == DK_UNSPECIFIED);
+  gcc_assert (static_cast<diagnostics::kind> (kind)
+	      == diagnostics::kind::unspecified);
   return targetm_common.handle_option (opts, opts_set, decoded, loc);
 }
 
@@ -469,7 +470,7 @@ maybe_default_option (struct gcc_options *opts,
 		      unsigned int lang_mask,
 		      const struct cl_option_handlers *handlers,
 		      location_t loc,
-		      diagnostic_context *dc)
+		      diagnostics::context *dc)
 {
   const struct cl_option *option = &cl_options[default_opt->opt_index];
   bool enabled;
@@ -535,14 +536,18 @@ maybe_default_option (struct gcc_options *opts,
   if (enabled)
     handle_generated_option (opts, opts_set, default_opt->opt_index,
 			     default_opt->arg, default_opt->value,
-			     lang_mask, DK_UNSPECIFIED, loc,
+			     lang_mask,
+			     static_cast<int> (diagnostics::kind::unspecified),
+			     loc,
 			     handlers, true, dc);
   else if (default_opt->arg == NULL
 	   && !option->cl_reject_negative
 	   && !(option->flags & CL_PARAMS))
     handle_generated_option (opts, opts_set, default_opt->opt_index,
 			     default_opt->arg, !default_opt->value,
-			     lang_mask, DK_UNSPECIFIED, loc,
+			     lang_mask,
+			     static_cast<int> (diagnostics::kind::unspecified),
+			     loc,
 			     handlers, true, dc);
 }
 
@@ -559,7 +564,7 @@ maybe_default_options (struct gcc_options *opts,
 		       unsigned int lang_mask,
 		       const struct cl_option_handlers *handlers,
 		       location_t loc,
-		       diagnostic_context *dc)
+		       diagnostics::context *dc)
 {
   size_t i;
 
@@ -727,7 +732,7 @@ default_options_optimization (struct gcc_options *opts,
 			      location_t loc,
 			      unsigned int lang_mask,
 			      const struct cl_option_handlers *handlers,
-			      diagnostic_context *dc)
+			      diagnostics::context *dc)
 {
   unsigned int i;
   int opt2;
@@ -1411,11 +1416,14 @@ finish_options (struct gcc_options *opts, struct gcc_options *opts_set,
 	opts->x_debug_info_level = DINFO_LEVEL_NONE;
     }
 
+  /* Also enable markers with -fauto-profile even when debug info is disabled,
+     so we assign same discriminators and can read back the profile info.  */
   if (!opts_set->x_debug_nonbind_markers_p)
     opts->x_debug_nonbind_markers_p
       = (opts->x_optimize
-	 && opts->x_debug_info_level >= DINFO_LEVEL_NORMAL
-	 && (dwarf_debuginfo_p (opts) || codeview_debuginfo_p ())
+	 && ((opts->x_debug_info_level >= DINFO_LEVEL_NORMAL
+	      && (dwarf_debuginfo_p (opts) || codeview_debuginfo_p ()))
+	     || opts->x_flag_auto_profile)
 	 && !(opts->x_flag_selective_scheduling
 	      || opts->x_flag_selective_scheduling2));
 
@@ -2700,7 +2708,7 @@ common_handle_option (struct gcc_options *opts,
 		      unsigned int lang_mask, int kind ATTRIBUTE_UNUSED,
 		      location_t loc,
 		      const struct cl_option_handlers *handlers,
-		      diagnostic_context *dc,
+		      diagnostics::context *dc,
 		      void (*target_option_override_hook) (void))
 {
   size_t scode = decoded->opt_index;
@@ -2954,19 +2962,19 @@ common_handle_option (struct gcc_options *opts,
       break;
 
     case OPT_fdiagnostics_show_caret:
-      dc->m_source_printing.enabled = value;
+      dc->get_source_printing_options ().enabled = value;
       break;
 
     case OPT_fdiagnostics_show_event_links:
-      dc->m_source_printing.show_event_links_p = value;
+      dc->get_source_printing_options ().show_event_links_p = value;
       break;
 
     case OPT_fdiagnostics_show_labels:
-      dc->m_source_printing.show_labels_p = value;
+      dc->get_source_printing_options ().show_labels_p = value;
       break;
 
     case OPT_fdiagnostics_show_line_numbers:
-      dc->m_source_printing.show_line_numbers_p = value;
+      dc->get_source_printing_options ().show_line_numbers_p = value;
       break;
 
     case OPT_fdiagnostics_color_:
@@ -2982,10 +2990,10 @@ common_handle_option (struct gcc_options *opts,
 	  const char *basename = (opts->x_dump_base_name ? opts->x_dump_base_name
 				  : opts->x_main_input_basename);
 	  gcc_assert (dc);
-	  diagnostic_output_format_init (*dc,
-					 opts->x_main_input_filename, basename,
-					 (enum diagnostics_output_format)value,
-					 opts->x_flag_diagnostics_json_formatting);
+	  diagnostics::output_format_init (*dc,
+					   opts->x_main_input_filename, basename,
+					   (enum diagnostics_output_format)value,
+					   opts->x_flag_diagnostics_json_formatting);
 	  break;
 	}
 
@@ -3044,7 +3052,7 @@ common_handle_option (struct gcc_options *opts,
       break;
 
     case OPT_fdiagnostics_minimum_margin_width_:
-      dc->m_source_printing.min_margin_width = value;
+      dc->get_source_printing_options ().min_margin_width = value;
       break;
 
     case OPT_fdump_:
@@ -3082,7 +3090,7 @@ common_handle_option (struct gcc_options *opts,
 
     case OPT_fmessage_length_:
       pp_set_line_maximum_length (dc->get_reference_printer (), value);
-      diagnostic_set_caret_max_width (dc, value);
+      dc->set_caret_max_width (value);
       break;
 
     case OPT_fopt_info:
@@ -3314,7 +3322,9 @@ common_handle_option (struct gcc_options *opts,
 
     case OPT_pedantic_errors:
       dc->m_pedantic_errors = 1;
-      control_warning_option (OPT_Wpedantic, DK_ERROR, NULL, value,
+      control_warning_option (OPT_Wpedantic,
+			      static_cast<int> (diagnostics::kind::error),
+			      NULL, value,
 			      loc, lang_mask,
 			      handlers, opts, opts_set,
                               dc);
@@ -3615,7 +3625,7 @@ set_debug_level (uint32_t dinfo, int extended, const char *arg,
    abort ().)  */
 
 static void
-setup_core_dumping (diagnostic_context *dc)
+setup_core_dumping (diagnostics::context *dc)
 {
 #ifdef SIGABRT
   signal (SIGABRT, SIG_DFL);
@@ -3639,7 +3649,7 @@ setup_core_dumping (diagnostic_context *dc)
 
 static void
 decode_d_option (const char *arg, struct gcc_options *opts,
-		 location_t loc, diagnostic_context *dc)
+		 location_t loc, diagnostics::context *dc)
 {
   int c;
 
@@ -3688,7 +3698,7 @@ enable_warning_as_error (const char *arg, int value, unsigned int lang_mask,
 			 const struct cl_option_handlers *handlers,
 			 struct gcc_options *opts,
 			 struct gcc_options *opts_set,
-			 location_t loc, diagnostic_context *dc)
+			 location_t loc, diagnostics::context *dc)
 {
   char *new_option;
   int option_index;
@@ -3714,7 +3724,9 @@ enable_warning_as_error (const char *arg, int value, unsigned int lang_mask,
 	      "controls warnings", arg, new_option);
   else
     {
-      const diagnostic_t kind = value ? DK_ERROR : DK_WARNING;
+      const enum diagnostics::kind kind = (value
+					   ? diagnostics::kind::error
+					   : diagnostics::kind::warning);
       const char *arg = NULL;
 
       if (cl_options[option_index].flags & CL_JOINED)
@@ -3732,16 +3744,17 @@ enable_warning_as_error (const char *arg, int value, unsigned int lang_mask,
    as -Werror.  */
 
 char *
-compiler_diagnostic_option_manager::
-make_option_name (diagnostic_option_id option_id,
-		  diagnostic_t orig_diag_kind,
-		  diagnostic_t diag_kind) const
+compiler_diagnostic_option_id_manager::
+make_option_name (diagnostics::option_id option_id,
+		  enum diagnostics::kind orig_diag_kind,
+		  enum diagnostics::kind diag_kind) const
 {
   if (option_id.m_idx)
     {
       /* A warning classified as an error.  */
-      if ((orig_diag_kind == DK_WARNING || orig_diag_kind == DK_PEDWARN)
-	  && diag_kind == DK_ERROR)
+      if ((orig_diag_kind == diagnostics::kind::warning
+	   || orig_diag_kind == diagnostics::kind::pedwarn)
+	  && diag_kind == diagnostics::kind::error)
 	return concat (cl_options[OPT_Werror_].opt_text,
 		       /* Skip over "-W".  */
 		       cl_options[option_id.m_idx].opt_text + 2,
@@ -3751,8 +3764,9 @@ make_option_name (diagnostic_option_id option_id,
 	return xstrdup (cl_options[option_id.m_idx].opt_text);
     }
   /* A warning without option classified as an error.  */
-  else if ((orig_diag_kind == DK_WARNING || orig_diag_kind == DK_PEDWARN
-	    || diag_kind == DK_WARNING)
+  else if ((orig_diag_kind == diagnostics::kind::warning
+	    || orig_diag_kind == diagnostics::kind::pedwarn
+	    || diag_kind == diagnostics::kind::warning)
 	   && m_context.warning_as_error_requested_p ())
     return xstrdup (cl_options[OPT_Werror].opt_text);
   else
@@ -3809,8 +3823,8 @@ get_option_url_suffix (int option_index, unsigned lang_mask)
    which enabled a diagnostic.  */
 
 char *
-gcc_diagnostic_option_manager::
-make_option_url (diagnostic_option_id option_id) const
+gcc_diagnostic_option_id_manager::
+make_option_url (diagnostics::option_id option_id) const
 {
   if (option_id.m_idx)
     {
diff --git a/gcc/opts.h b/gcc/opts.h
index 382faab..ea92c49 100644
--- a/gcc/opts.h
+++ b/gcc/opts.h
@@ -313,7 +313,7 @@ struct cl_option_handler_func
 		   const struct cl_decoded_option *decoded,
 		   unsigned int lang_mask, int kind, location_t loc,
 		   const struct cl_option_handlers *handlers,
-		   diagnostic_context *dc,
+		   diagnostics::context *dc,
 		   void (*target_option_override_hook) (void));
 
   /* The mask that must have some bit in common with the flags for the
@@ -390,7 +390,7 @@ extern void decode_options (struct gcc_options *opts,
 			    struct cl_decoded_option *decoded_options,
 			    unsigned int decoded_options_count,
 			    location_t loc,
-			    diagnostic_context *dc,
+			    diagnostics::context *dc,
 			    void (*target_option_override_hook) (void));
 extern int option_enabled (int opt_idx, unsigned lang_mask, void *opts);
 
@@ -399,7 +399,7 @@ extern bool get_option_state (struct gcc_options *, int,
 extern void set_option (struct gcc_options *opts,
 			struct gcc_options *opts_set,
 			size_t opt_index, HOST_WIDE_INT value, const char *arg,
-			int kind, location_t loc, diagnostic_context *dc,
+			int kind, location_t loc, diagnostics::context *dc,
 			HOST_WIDE_INT = 0);
 extern void *option_flag_var (int opt_index, struct gcc_options *opts);
 bool handle_generated_option (struct gcc_options *opts,
@@ -408,7 +408,7 @@ bool handle_generated_option (struct gcc_options *opts,
 			      HOST_WIDE_INT value,
 			      unsigned int lang_mask, int kind, location_t loc,
 			      const struct cl_option_handlers *handlers,
-			      bool generated_p, diagnostic_context *dc);
+			      bool generated_p, diagnostics::context *dc);
 void generate_option (size_t opt_index, const char *arg, HOST_WIDE_INT value,
 		      unsigned int lang_mask,
 		      struct cl_decoded_option *decoded);
@@ -420,14 +420,14 @@ extern void read_cmdline_option (struct gcc_options *opts,
 				 location_t loc,
 				 unsigned int lang_mask,
 				 const struct cl_option_handlers *handlers,
-				 diagnostic_context *dc);
+				 diagnostics::context *dc);
 extern void control_warning_option (unsigned int opt_index, int kind,
 				    const char *arg, bool imply, location_t loc,
 				    unsigned int lang_mask,
 				    const struct cl_option_handlers *handlers,
 				    struct gcc_options *opts,
 				    struct gcc_options *opts_set,
-				    diagnostic_context *dc);
+				    diagnostics::context *dc);
 extern char *write_langs (unsigned int mask);
 extern void print_ignored_options (void);
 extern void handle_common_deferred_options (void);
@@ -442,7 +442,7 @@ extern bool common_handle_option (struct gcc_options *opts,
 				  unsigned int lang_mask, int kind,
 				  location_t loc,
 				  const struct cl_option_handlers *handlers,
-				  diagnostic_context *dc,
+				  diagnostics::context *dc,
 				  void (*target_option_override_hook) (void));
 extern bool target_handle_option (struct gcc_options *opts,
 				  struct gcc_options *opts_set,
@@ -450,7 +450,7 @@ extern bool target_handle_option (struct gcc_options *opts,
 				  unsigned int lang_mask, int kind,
 				  location_t loc,
 				  const struct cl_option_handlers *handlers,
-				  diagnostic_context *dc,
+				  diagnostics::context *dc,
 				  void (*target_option_override_hook) (void));
 extern void finish_options (struct gcc_options *opts,
 			    struct gcc_options *opts_set,
@@ -466,7 +466,7 @@ extern void default_options_optimization (struct gcc_options *opts,
 					  location_t loc,
 					  unsigned int lang_mask,
 					  const struct cl_option_handlers *handlers,
-					  diagnostic_context *dc);
+					  diagnostics::context *dc);
 extern void set_struct_debug_option (struct gcc_options *opts,
 				     location_t loc,
 				     const char *value);
diff --git a/gcc/output.h b/gcc/output.h
index 372d63c..51c2d36 100644
--- a/gcc/output.h
+++ b/gcc/output.h
@@ -545,6 +545,9 @@ extern GTY(()) section *bss_noswitch_section;
 extern GTY(()) section *in_section;
 extern GTY(()) bool in_cold_section_p;
 
+/* MAX bit alignment for mergable sections. */
+#define MAX_ALIGN_MERGABLE 256
+
 extern section *get_unnamed_section (unsigned int, void (*) (const char *),
 				     const char *);
 extern section *get_section (const char *, unsigned int, tree,
@@ -557,6 +560,9 @@ extern rtx get_section_anchor (struct object_block *, HOST_WIDE_INT,
 extern section *mergeable_constant_section (machine_mode,
 					    unsigned HOST_WIDE_INT,
 					    unsigned int);
+extern section *mergeable_constant_section (unsigned HOST_WIDE_INT,
+					    unsigned HOST_WIDE_INT,
+					    unsigned int);
 extern section *function_section (tree);
 extern section *unlikely_text_section (void);
 extern section *current_function_section (void);
@@ -639,4 +645,6 @@ extern int default_address_cost (rtx, machine_mode, addr_space_t, bool);
 /* Stack usage.  */
 extern void output_stack_usage (void);
 
+extern void verify_reg_names_in_constraints ();
+
 #endif /* ! GCC_OUTPUT_H */
diff --git a/gcc/params.opt b/gcc/params.opt
index 31aa0bd..ac1b2c7 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -1177,7 +1177,7 @@ Common Joined UInteger Var(param_uninit_control_dep_attempts) Init(1000) Integer
 Maximum number of nested calls to search for control dependencies during uninitialized variable analysis.
 
 -param=uninit-max-chain-len=
-Common Joined UInteger Var(param_uninit_max_chain_len) Init(8) IntegerRange(1, 128) Param Optimization
+Common Joined UInteger Var(param_uninit_max_chain_len) Init(12) IntegerRange(1, 128) Param Optimization
 Maximum number of predicates anded for each predicate ored in the normalized
 predicate chain.
 
@@ -1226,7 +1226,7 @@ Common Joined UInteger Var(param_use_canonical_types) Init(1) IntegerRange(0, 1)
 Whether to use canonical types.
 
 -param=vect-epilogues-nomask=
-Common Joined UInteger Var(param_vect_epilogues_nomask) Init(1) IntegerRange(0, 1) Param Optimization
+Common Joined UInteger Var(param_vect_epilogues_nomask) Init(1) IntegerRange(0, 1) Param Optimization NoOffload
 Enable loop epilogue vectorization using smaller vector size.
 
 -param=vect-max-layout-candidates=
@@ -1246,11 +1246,11 @@ Common Joined UInteger Var(param_vect_max_version_for_alignment_checks) Init(6)
 Bound on number of runtime checks inserted by the vectorizer's loop versioning for alignment check.
 
 -param=vect-partial-vector-usage=
-Common Joined UInteger Var(param_vect_partial_vector_usage) Init(2) IntegerRange(0, 2) Param Optimization
+Common Joined UInteger Var(param_vect_partial_vector_usage) Init(2) IntegerRange(0, 2) Param Optimization NoOffload
 Controls how loop vectorizer uses partial vectors.  0 means never, 1 means only for loops whose need to iterate can be removed, 2 means for all loops.  The default value is 2.
 
 -param=vect-inner-loop-cost-factor=
-Common Joined UInteger Var(param_vect_inner_loop_cost_factor) Init(50) IntegerRange(1, 10000) Param Optimization
+Common Joined UInteger Var(param_vect_inner_loop_cost_factor) Init(50) IntegerRange(1, 10000) Param Optimization NoOffload
 The maximum factor which the loop vectorizer applies to the cost of statements in an inner loop relative to the loop being vectorized.
 
 -param=vect-induction-float=
diff --git a/gcc/predict.cc b/gcc/predict.cc
index 872f54d..5639d81 100644
--- a/gcc/predict.cc
+++ b/gcc/predict.cc
@@ -245,7 +245,10 @@ unlikely_executed_edge_p (edge e)
 {
   return (e->src->count == profile_count::zero ()
 	  || e->probability == profile_probability::never ())
-	 || (e->flags & (EDGE_EH | EDGE_FAKE));
+	 || (e->flags & EDGE_FAKE)
+	 /* If we read profile and know EH edge is executed, trust it.
+	    Otherwise we consider EH edges never executed.  */
+	 || ((e->flags & EDGE_EH) && !e->probability.reliable_p ());
 }
 
 /* Return true if edge E of function FUN is probably never executed.  */
@@ -830,6 +833,26 @@ unlikely_executed_stmt_p (gimple *stmt)
 {
   if (!is_gimple_call (stmt))
     return false;
+
+  /* Those calls are inserted by optimizers when code is known to be
+     unreachable or undefined.  */
+  if (gimple_call_builtin_p (stmt, BUILT_IN_UNREACHABLE)
+      || gimple_call_builtin_p (stmt, BUILT_IN_UNREACHABLE_TRAP)
+      || gimple_call_builtin_p (stmt, BUILT_IN_TRAP))
+    return false;
+
+  /* Checks below do not need to be fully reliable.  Cold attribute may be
+     misplaced by user and in the presence of comdat we may result in call to
+     function with 0 profile having non-zero profile.
+
+     We later detect that profile is lost and will drop the profile of the
+     comdat.
+
+     So if we think profile count is reliable, do not try to apply these
+     heuristics.  */
+  if (gimple_bb (stmt)->count.reliable_p ()
+      && gimple_bb (stmt)->count.nonzero_p ())
+    return gimple_bb (stmt)->count == profile_count::zero ();
   /* NORETURN attribute alone is not strong enough: exit() may be quite
      likely executed once during program run.  */
   if (gimple_call_fntype (stmt)
@@ -3269,7 +3292,8 @@ tree_estimate_probability (bool dry_run)
   calculate_dominance_info (CDI_POST_DOMINATORS);
   /* Decide which edges are known to be unlikely.  This improves later
      branch prediction. */
-  determine_unlikely_bbs ();
+  if (!dry_run)
+    determine_unlikely_bbs ();
 
   bb_predictions = new hash_map<const_basic_block, edge_prediction *>;
   ssa_expected_value = new hash_map<int_hash<unsigned, 0>, expected_value>;
diff --git a/gcc/pretty-print-format-impl.h b/gcc/pretty-print-format-impl.h
index cbbd21f..2aded64 100644
--- a/gcc/pretty-print-format-impl.h
+++ b/gcc/pretty-print-format-impl.h
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #define GCC_PRETTY_PRINT_FORMAT_IMPL_H
 
 #include "pretty-print.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 /* A struct representing a pending item to be printed within
    pp_format.
@@ -224,14 +224,14 @@ struct pp_token_end_url : public pp_token
 
 struct pp_token_event_id : public pp_token
 {
-  pp_token_event_id (diagnostic_event_id_t event_id)
+  pp_token_event_id (diagnostics::paths::event_id_t event_id)
   : pp_token (kind::event_id),
     m_event_id (event_id)
   {
     gcc_assert (event_id.known_p ());
   }
 
-  diagnostic_event_id_t m_event_id;
+  diagnostics::paths::event_id_t m_event_id;
 };
 
 template <>
@@ -334,6 +334,7 @@ public:
     push_back (std::move (tok));
   }
   void push_back_text (label_text &&text);
+  void push_back_byte (char ch);
   void push_back (std::unique_ptr<pp_token> tok);
   void push_back_list (pp_token_list &&list);
 
diff --git a/gcc/pretty-print-markup.h b/gcc/pretty-print-markup.h
index 18e298c..e4346f1 100644
--- a/gcc/pretty-print-markup.h
+++ b/gcc/pretty-print-markup.h
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_PRETTY_PRINT_MARKUP_H
 #define GCC_PRETTY_PRINT_MARKUP_H
 
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 
 class pp_token_list;
 
@@ -45,6 +45,11 @@ public:
   void begin_highlight_color (const char *color_name);
   void end_highlight_color ();
 
+  void begin_url (const char *url);
+  void end_url ();
+
+  void add_event_id (diagnostic_event_id_t event_id);
+
   void push_back_any_text ();
 
   pretty_printer &m_pp;
diff --git a/gcc/pretty-print.cc b/gcc/pretty-print.cc
index 6ecfcb2..77d40ec 100644
--- a/gcc/pretty-print.cc
+++ b/gcc/pretty-print.cc
@@ -27,9 +27,10 @@ along with GCC; see the file COPYING3.  If not see
 #include "pretty-print-format-impl.h"
 #include "pretty-print-markup.h"
 #include "pretty-print-urlifier.h"
-#include "diagnostic-color.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/color.h"
+#include "diagnostics/event-id.h"
 #include "diagnostic-highlight-colors.h"
+#include "auto-obstack.h"
 #include "selftest.h"
 
 #if HAVE_ICONV
@@ -714,7 +715,7 @@ static int
 decode_utf8_char (const unsigned char *, size_t len, unsigned int *);
 static void pp_quoted_string (pretty_printer *, const char *, size_t = -1);
 
-static void
+extern void
 default_token_printer (pretty_printer *pp,
 		       const pp_token_list &tokens);
 
@@ -1327,6 +1328,15 @@ pp_token_list::push_back_text (label_text &&text)
 }
 
 void
+pp_token_list::push_back_byte (char ch)
+{
+  char buf[2];
+  buf[0] = ch;
+  buf[1] = '\0';
+  push_back_text (label_text::take (xstrdup (buf)));
+}
+
+void
 pp_token_list::push_back (std::unique_ptr<pp_token> tok)
 {
   if (!m_first)
@@ -2034,6 +2044,16 @@ format_phase_2 (pretty_printer *pp,
 	    pp_string (pp, va_arg (*text.m_args_ptr, const char *));
 	  break;
 
+	case 'B':
+	  {
+	    string_slice s = *va_arg (*text.m_args_ptr, string_slice *);
+	    if (quote)
+	      pp_quoted_string (pp, s.begin (), s.size ());
+	    else
+	      pp_string_n (pp, s.begin (), s.size ());
+	    break;
+	  }
+
 	case 'p':
 	  pp_pointer (pp, va_arg (*text.m_args_ptr, void *));
 	  break;
@@ -2177,38 +2197,6 @@ format_phase_2 (pretty_printer *pp,
       gcc_assert (!formatters[argno]);
 }
 
-struct auto_obstack
-{
-  auto_obstack ()
-  {
-    obstack_init (&m_obstack);
-  }
-
-  ~auto_obstack ()
-  {
-    obstack_free (&m_obstack, NULL);
-  }
-
-  operator obstack & () { return m_obstack; }
-
-  void grow (const void *src, size_t length)
-  {
-    obstack_grow (&m_obstack, src, length);
-  }
-
-  void *object_base () const
-  {
-    return m_obstack.object_base;
-  }
-
-  size_t object_size () const
-  {
-    return obstack_object_size (&m_obstack);
-  }
-
-  obstack m_obstack;
-};
-
 /* Phase 3 of formatting a message (phases 1 and 2 done by pp_format).
 
    Pop a pp_formatted_chunks from chunk_obstack, collecting all the tokens from
@@ -2261,7 +2249,7 @@ pp_output_formatted_text (pretty_printer *pp,
 
 /* Default implementation of token printing.  */
 
-static void
+void
 default_token_printer (pretty_printer *pp,
 		       const pp_token_list &tokens)
 {
@@ -3189,6 +3177,29 @@ pp_markup::context::end_highlight_color ()
 }
 
 void
+pp_markup::context::begin_url (const char *url)
+{
+  push_back_any_text ();
+  m_formatted_token_list->push_back<pp_token_begin_url>
+    (label_text::take (xstrdup (url)));
+}
+
+void
+pp_markup::context::end_url ()
+{
+  push_back_any_text ();
+  m_formatted_token_list->push_back<pp_token_end_url> ();
+}
+
+void
+pp_markup::context::add_event_id (diagnostic_event_id_t event_id)
+{
+  gcc_assert (event_id.known_p ());
+  push_back_any_text ();
+  m_formatted_token_list->push_back<pp_token_event_id> (event_id);
+}
+
+void
 pp_markup::context::push_back_any_text ()
 {
   obstack *cur_obstack = m_buf.m_obstack;
@@ -3402,8 +3413,8 @@ test_pp_format ()
 			    "foo", 0x12345678);
   /* Verify "%@".  */
   {
-    diagnostic_event_id_t first (2);
-    diagnostic_event_id_t second (7);
+    diagnostics::paths::event_id_t first (2);
+    diagnostics::paths::event_id_t second (7);
 
     ASSERT_PP_FORMAT_2 ("first `free' at (3); second `free' at (8)",
 			"first %<free%> at %@; second %<free%> at %@",
diff --git a/gcc/pretty-print.h b/gcc/pretty-print.h
index 6cd9150..ced2e3c 100644
--- a/gcc/pretty-print.h
+++ b/gcc/pretty-print.h
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "obstack.h"
 #include "rich-location.h"
-#include "diagnostic-url.h"
+#include "diagnostics/url.h"
 
 /* Maximum number of format string arguments.  */
 #define PP_NL_ARGMAX   30
diff --git a/gcc/profile-count.cc b/gcc/profile-count.cc
index 190bbeb..8f05a79 100644
--- a/gcc/profile-count.cc
+++ b/gcc/profile-count.cc
@@ -32,6 +32,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cgraph.h"
 #include "wide-int.h"
 #include "sreal.h"
+#include "profile.h"
 
 /* Names from profile_quality enum values.  */
 
@@ -557,7 +558,7 @@ profile_count::operator* (const sreal &num) const
   sreal scaled = num * m_val;
   gcc_checking_assert (scaled >= 0);
   profile_count ret;
-  if (m_val > max_count)
+  if (scaled > max_count)
     ret.m_val = max_count;
   else
     ret.m_val = scaled.to_nearest_int ();
@@ -570,3 +571,27 @@ profile_count::operator*= (const sreal &num)
 {
   return *this * num;
 }
+
+/* Make counter forcibly nonzero.  */
+profile_count
+profile_count::force_nonzero () const
+{
+  if (!initialized_p ())
+    return *this;
+  profile_count ret = *this;
+  /* Generally values are forced non-zero to handle inconsistent profile 
+     where count 0 needs to be scaled up to non-zero.
+
+     Use cutoff value here to avoid situation where profile has large
+     cutoff and we perform count = count * num / den where num is non-zero
+     and den is 0.   If profile was scaled by large factor, forcing value
+     to 1 would lead to large scale factor.  */
+  gcov_unsigned_t small = profile_info ? profile_info->cutoff / 2 + 1
+			  : 1;
+  if (ret.m_val < small)
+    {
+      ret.m_val = small;
+      ret.m_quality = MIN (m_quality, ADJUSTED);
+    }
+  return ret;
+}
diff --git a/gcc/profile-count.h b/gcc/profile-count.h
index 2160540..c893aec 100644
--- a/gcc/profile-count.h
+++ b/gcc/profile-count.h
@@ -597,7 +597,7 @@ public:
      There are two exceptions - edges leading to noreturn edges and edges
      predicted by number of iterations heuristics are predicted well.  This macro
      should be able to distinguish those, but at the moment it simply check for
-     noreturn heuristic that is only one giving probability over 99% or bellow
+     noreturn heuristic that is only one giving probability over 99% or below
      1%.  In future we might want to propagate reliability information across the
      CFG if we find this information useful on multiple places.   */
   bool probably_reliable_p () const
@@ -1112,18 +1112,7 @@ public:
     }
 
   /* Make counter forcibly nonzero.  */
-  profile_count force_nonzero () const
-    {
-      if (!initialized_p ())
-	return *this;
-      profile_count ret = *this;
-      if (ret.m_val == 0)
-	{
-	  ret.m_val = 1;
-	  ret.m_quality = MIN (m_quality, ADJUSTED);
-	}
-      return ret;
-    }
+  profile_count force_nonzero () const;
 
   profile_count max (profile_count other) const
     {
diff --git a/gcc/pta-andersen.cc b/gcc/pta-andersen.cc
new file mode 100644
index 0000000..0253f05
--- /dev/null
+++ b/gcc/pta-andersen.cc
@@ -0,0 +1,2565 @@
+/* Andersen-style solver for tree based points-to analysis
+   Copyright (C) 2005-2025 Free Software Foundation, Inc.
+   Contributed by Daniel Berlin <dberlin@dberlin.org>
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "backend.h"
+
+#include "tree-ssa-structalias.h"
+#include "pta-andersen.h"
+
+/* During variable substitution and the offline version of indirect
+   cycle finding, we create nodes to represent dereferences and
+   address taken constraints.  These represent where these start and
+   end.  */
+#define FIRST_REF_NODE (varmap).length ()
+#define LAST_REF_NODE (FIRST_REF_NODE + (FIRST_REF_NODE - 1))
+
+#define EXECUTE_IF_IN_NONNULL_BITMAP(a, b, c, d)	\
+  if (a)						\
+    EXECUTE_IF_SET_IN_BITMAP (a, b, c, d)
+
+using namespace pointer_analysis;
+
+/* Used for predecessor bitmaps.  */
+static bitmap_obstack predbitmap_obstack;
+
+/* Used for per-solver-iteration bitmaps.  */
+static bitmap_obstack iteration_obstack;
+
+typedef struct constraint_graph *constraint_graph_t;
+
+/* The constraint graph is represented as an array of bitmaps
+   containing successor nodes.  */
+
+struct constraint_graph
+{
+  /* Size of this graph, which may be different than the number of
+     nodes in the variable map.  */
+  unsigned int size;
+
+  /* Explicit successors of each node.  */
+  bitmap *succs;
+
+  /* Implicit predecessors of each node (Used for variable
+     substitution). */
+  bitmap *implicit_preds;
+
+  /* Explicit predecessors of each node (Used for variable substitution).  */
+  bitmap *preds;
+
+  /* Indirect cycle representatives, or -1 if the node has no indirect
+     cycles.  */
+  int *indirect_cycles;
+
+  /* Representative node for a node.  rep[a] == a unless the node has
+     been unified.  */
+  unsigned int *rep;
+
+  /* Equivalence class representative for a label.  This is used for
+     variable substitution.  */
+  int *eq_rep;
+
+  /* Pointer equivalence label for a node.  All nodes with the same
+     pointer equivalence label can be unified together at some point
+     (either during constraint optimization or after the constraint
+     graph is built).  */
+  unsigned int *pe;
+
+  /* Pointer equivalence representative for a label.  This is used to
+     handle nodes that are pointer equivalent but not location
+     equivalent.  We can unite these once the addressof constraints
+     are transformed into initial points-to sets.  */
+  int *pe_rep;
+
+  /* Pointer equivalence label for each node, used during variable
+     substitution.  */
+  unsigned int *pointer_label;
+
+  /* Location equivalence label for each node, used during location
+     equivalence finding.  */
+  unsigned int *loc_label;
+
+  /* Pointed-by set for each node, used during location equivalence
+     finding.  This is pointed-by rather than pointed-to, because it
+     is constructed using the predecessor graph.  */
+  bitmap *pointed_by;
+
+  /* Points to sets for pointer equivalence.  This is *not* the actual
+     points-to sets for nodes.  */
+  bitmap *points_to;
+
+  /* Bitmap of nodes where the bit is set if the node is a direct
+     node.  Used for variable substitution.  */
+  sbitmap direct_nodes;
+
+  /* Bitmap of nodes where the bit is set if the node is address
+     taken.  Used for variable substitution.  */
+  bitmap address_taken;
+
+  /* Vector of complex constraints for each graph node.  Complex
+     constraints are those involving dereferences or offsets that are
+     not 0.  */
+  vec<constraint_t> *complex;
+};
+
+static constraint_graph_t graph;
+
+static void unify_nodes (constraint_graph_t, unsigned int, unsigned int, bool);
+
+
+/* Return the representative node for NODE, if NODE has been unioned
+   with another NODE.
+   This function performs path compression along the way to finding
+   the representative.  */
+
+static unsigned int
+find (unsigned int node)
+{
+  gcc_checking_assert (node < graph->size);
+  if (graph->rep[node] != node)
+    return graph->rep[node] = find (graph->rep[node]);
+  return node;
+}
+
+/* Union the TO and FROM nodes to the TO nodes.
+   Note that at some point in the future, we may want to do
+   union-by-rank, in which case we are going to have to return the
+   node we unified to.  */
+
+static bool
+unite (unsigned int to, unsigned int from)
+{
+  gcc_checking_assert (to < graph->size && from < graph->size);
+  if (to != from && graph->rep[from] != to)
+    {
+      graph->rep[from] = to;
+      return true;
+    }
+  return false;
+}
+
+/* Perform path compression for all nodes in the node representatives
+   union-find structure.  */
+
+static void
+union_find_compress_all (void)
+{
+  unsigned int i;
+  for (i = 0; i < graph->size; i++)
+    find (i);
+}
+
+/* Print the constraint graph in dot format.  */
+
+static void
+dump_constraint_graph (FILE *file)
+{
+  unsigned int i;
+
+  /* Only print the graph if it has already been initialized:  */
+  if (!graph)
+    return;
+
+  /* Prints the header of the dot file:  */
+  fprintf (file, "strict digraph {\n");
+  fprintf (file, "  node [\n    shape = box\n  ]\n");
+  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
+  fprintf (file, "\n  // List of nodes and complex constraints in "
+	   "the constraint graph:\n");
+
+  /* The next lines print the nodes in the graph together with the
+     complex constraints attached to them.  */
+  for (i = 1; i < graph->size; i++)
+    {
+      if (i == FIRST_REF_NODE)
+	continue;
+      if (find (i) != i)
+	continue;
+      if (i < FIRST_REF_NODE)
+	fprintf (file, "\"%s\"", get_varinfo (i)->name);
+      else
+	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+      if (graph->complex[i].exists ())
+	{
+	  unsigned j;
+	  constraint_t c;
+	  fprintf (file, " [label=\"\\N\\n");
+	  for (j = 0; graph->complex[i].iterate (j, &c); ++j)
+	    {
+	      dump_constraint (file, c);
+	      fprintf (file, "\\l");
+	    }
+	  fprintf (file, "\"]");
+	}
+      fprintf (file, ";\n");
+    }
+
+  /* Go over the edges.  */
+  fprintf (file, "\n  // Edges in the constraint graph:\n");
+  for (i = 1; i < graph->size; i++)
+    {
+      unsigned j;
+      bitmap_iterator bi;
+      if (find (i) != i)
+	continue;
+      EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i], 0, j, bi)
+	{
+	  unsigned to = find (j);
+	  if (i == to)
+	    continue;
+	  if (i < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
+	  else
+	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+	  fprintf (file, " -> ");
+	  if (to < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (to)->name);
+	  else
+	    fprintf (file, "\"*%s\"", get_varinfo (to - FIRST_REF_NODE)->name);
+	  fprintf (file, ";\n");
+	}
+    }
+
+  /* Prints the tail of the dot file.  */
+  fprintf (file, "}\n");
+}
+
+/* Print out the constraint graph to stderr.  */
+
+DEBUG_FUNCTION void
+debug_constraint_graph (void)
+{
+  dump_constraint_graph (stderr);
+}
+
+
+/* SOLVER FUNCTIONS
+
+   The solver is a simple worklist solver, that works on the following
+   algorithm:
+
+   sbitmap changed_nodes = all zeroes;
+   changed_count = 0;
+   For each node that is not already collapsed:
+       changed_count++;
+       set bit in changed nodes
+
+   while (changed_count > 0)
+   {
+     compute topological ordering for constraint graph
+
+     find and collapse cycles in the constraint graph (updating
+     changed if necessary)
+
+     for each node (n) in the graph in topological order:
+       changed_count--;
+
+       Process each complex constraint associated with the node,
+       updating changed if necessary.
+
+       For each outgoing edge from n, propagate the solution from n to
+       the destination of the edge, updating changed as necessary.
+
+   }  */
+
+/* Return true if two constraint expressions A and B are equal.  */
+
+static bool
+constraint_expr_equal (struct constraint_expr a, struct constraint_expr b)
+{
+  return a.type == b.type && a.var == b.var && a.offset == b.offset;
+}
+
+/* Return true if constraint expression A is less than constraint expression
+   B.  This is just arbitrary, but consistent, in order to give them an
+   ordering.  */
+
+static bool
+constraint_expr_less (struct constraint_expr a, struct constraint_expr b)
+{
+  if (a.type == b.type)
+    {
+      if (a.var == b.var)
+	return a.offset < b.offset;
+      else
+	return a.var < b.var;
+    }
+  else
+    return a.type < b.type;
+}
+
+/* Return true if constraint A is less than constraint B.  This is just
+   arbitrary, but consistent, in order to give them an ordering.  */
+
+static bool
+constraint_less (const constraint_t &a, const constraint_t &b)
+{
+  if (constraint_expr_less (a->lhs, b->lhs))
+    return true;
+  else if (constraint_expr_less (b->lhs, a->lhs))
+    return false;
+  else
+    return constraint_expr_less (a->rhs, b->rhs);
+}
+
+/* Return true if two constraints A and B are equal.  */
+
+static bool
+constraint_equal (const constraint &a, const constraint &b)
+{
+  return constraint_expr_equal (a.lhs, b.lhs)
+    && constraint_expr_equal (a.rhs, b.rhs);
+}
+
+/* Find a constraint LOOKFOR in the sorted constraint vector VEC.  */
+
+static constraint_t
+constraint_vec_find (vec<constraint_t> vec,
+		     constraint &lookfor)
+{
+  unsigned int place;
+  constraint_t found;
+
+  if (!vec.exists ())
+    return NULL;
+
+  place = vec.lower_bound (&lookfor, constraint_less);
+  if (place >= vec.length ())
+    return NULL;
+  found = vec[place];
+  if (!constraint_equal (*found, lookfor))
+    return NULL;
+  return found;
+}
+
+/* Union two constraint vectors, TO and FROM.  Put the result in TO.
+   Returns true of TO set is changed.  */
+
+static bool
+constraint_set_union (vec<constraint_t> *to,
+		      vec<constraint_t> *from)
+{
+  int i;
+  constraint_t c;
+  bool any_change = false;
+
+  FOR_EACH_VEC_ELT (*from, i, c)
+    {
+      if (constraint_vec_find (*to, *c) == NULL)
+	{
+	  unsigned int place = to->lower_bound (c, constraint_less);
+	  to->safe_insert (place, c);
+	  any_change = true;
+	}
+    }
+  return any_change;
+}
+
+/* Expands the solution in SET to all sub-fields of variables included.  */
+
+static bitmap
+solution_set_expand (bitmap set, bitmap *expanded)
+{
+  bitmap_iterator bi;
+  unsigned j;
+
+  if (*expanded)
+    return *expanded;
+
+  *expanded = BITMAP_ALLOC (&iteration_obstack);
+
+  /* In a first pass expand variables, once for each head to avoid
+     quadratic behavior, to include all sub-fields.  */
+  unsigned prev_head = 0;
+  EXECUTE_IF_SET_IN_BITMAP (set, 0, j, bi)
+    {
+      varinfo_t v = get_varinfo (j);
+      if (v->is_artificial_var
+	  || v->is_full_var)
+	continue;
+      if (v->head != prev_head)
+	{
+	  varinfo_t head = get_varinfo (v->head);
+	  unsigned num = 1;
+	  for (varinfo_t n = vi_next (head); n != NULL; n = vi_next (n))
+	    {
+	      if (n->id != head->id + num)
+		{
+		  /* Usually sub variables are adjacent but since we
+		     create pointed-to restrict representatives there
+		     can be gaps as well.  */
+		  bitmap_set_range (*expanded, head->id, num);
+		  head = n;
+		  num = 1;
+		}
+	      else
+		num++;
+	    }
+
+	  bitmap_set_range (*expanded, head->id, num);
+	  prev_head = v->head;
+	}
+    }
+
+  /* And finally set the rest of the bits from SET in an efficient way.  */
+  bitmap_ior_into (*expanded, set);
+
+  return *expanded;
+}
+
+/* Union solution sets TO and DELTA, and add INC to each member of DELTA in the
+   process.  */
+
+static bool
+set_union_with_increment (bitmap to, bitmap delta, HOST_WIDE_INT inc,
+			  bitmap *expanded_delta)
+{
+  bool changed = false;
+  bitmap_iterator bi;
+  unsigned int i;
+
+  /* If the solution of DELTA contains anything it is good enough to transfer
+     this to TO.  */
+  if (bitmap_bit_p (delta, anything_id))
+    return bitmap_set_bit (to, anything_id);
+
+  /* If the offset is unknown we have to expand the solution to
+     all subfields.  */
+  if (inc == UNKNOWN_OFFSET)
+    {
+      delta = solution_set_expand (delta, expanded_delta);
+      changed |= bitmap_ior_into (to, delta);
+      return changed;
+    }
+
+  /* For non-zero offset union the offsetted solution into the destination.  */
+  EXECUTE_IF_SET_IN_BITMAP (delta, 0, i, bi)
+    {
+      varinfo_t vi = get_varinfo (i);
+
+      /* If this is a variable with just one field just set its bit
+	 in the result.  */
+      if (vi->is_artificial_var
+	  || vi->is_unknown_size_var
+	  || vi->is_full_var)
+	changed |= bitmap_set_bit (to, i);
+      else
+	{
+	  HOST_WIDE_INT fieldoffset = vi->offset + inc;
+	  unsigned HOST_WIDE_INT size = vi->size;
+
+	  /* If the offset makes the pointer point to before the
+	     variable use offset zero for the field lookup.  */
+	  if (fieldoffset < 0)
+	    vi = get_varinfo (vi->head);
+	  else
+	    vi = first_or_preceding_vi_for_offset (vi, fieldoffset);
+
+	  do
+	    {
+	      changed |= bitmap_set_bit (to, vi->id);
+	      if (vi->is_full_var
+		  || vi->next == 0)
+		break;
+
+	      /* We have to include all fields that overlap the current field
+		 shifted by inc.  */
+	      vi = vi_next (vi);
+	    }
+	  while (vi->offset < fieldoffset + size);
+	}
+    }
+
+  return changed;
+}
+
+/* Insert constraint C into the list of complex constraints for graph
+   node VAR.  */
+
+static void
+insert_into_complex (constraint_graph_t graph,
+		     unsigned int var, constraint_t c)
+{
+  vec<constraint_t> complex = graph->complex[var];
+  unsigned int place = complex.lower_bound (c, constraint_less);
+
+  /* Only insert constraints that do not already exist.  */
+  if (place >= complex.length ()
+      || !constraint_equal (*c, *complex[place]))
+    graph->complex[var].safe_insert (place, c);
+}
+
+
+/* Condense two variable nodes into a single variable node, by moving
+   all associated info from FROM to TO.  Returns true if TO node's
+   constraint set changes after the merge.  */
+
+static bool
+merge_node_constraints (constraint_graph_t graph, unsigned int to,
+			unsigned int from)
+{
+  unsigned int i;
+  constraint_t c;
+  bool any_change = false;
+
+  gcc_checking_assert (find (from) == to);
+
+  /* Move all complex constraints from src node into to node.  */
+  FOR_EACH_VEC_ELT (graph->complex[from], i, c)
+    {
+      /* In complex constraints for node FROM, we may have either
+	 a = *FROM, and *FROM = a, or an offseted constraint which are
+	 always added to the rhs node's constraints.  */
+
+      if (c->rhs.type == DEREF)
+	c->rhs.var = to;
+      else if (c->lhs.type == DEREF)
+	c->lhs.var = to;
+      else
+	c->rhs.var = to;
+
+    }
+  any_change = constraint_set_union (&graph->complex[to],
+				     &graph->complex[from]);
+  graph->complex[from].release ();
+  return any_change;
+}
+
+/* Remove edges involving NODE from GRAPH.  */
+
+static void
+clear_edges_for_node (constraint_graph_t graph, unsigned int node)
+{
+  if (graph->succs[node])
+    BITMAP_FREE (graph->succs[node]);
+}
+
+/* Merge GRAPH nodes FROM and TO into node TO.  */
+
+static void
+merge_graph_nodes (constraint_graph_t graph, unsigned int to,
+		   unsigned int from)
+{
+  if (graph->indirect_cycles[from] != -1)
+    {
+      /* If we have indirect cycles with the from node, and we have
+	 none on the to node, the to node has indirect cycles from the
+	 from node now that they are unified.
+	 If indirect cycles exist on both, unify the nodes that they
+	 are in a cycle with, since we know they are in a cycle with
+	 each other.  */
+      if (graph->indirect_cycles[to] == -1)
+	graph->indirect_cycles[to] = graph->indirect_cycles[from];
+    }
+
+  /* Merge all the successor edges.  */
+  if (graph->succs[from])
+    {
+      if (!graph->succs[to])
+	graph->succs[to] = BITMAP_ALLOC (&pta_obstack);
+      bitmap_ior_into (graph->succs[to],
+		       graph->succs[from]);
+    }
+
+  clear_edges_for_node (graph, from);
+}
+
+
+/* Add an indirect graph edge to GRAPH, going from TO to FROM if
+   it doesn't exist in the graph already.  */
+
+static void
+add_implicit_graph_edge (constraint_graph_t graph, unsigned int to,
+			 unsigned int from)
+{
+  if (to == from)
+    return;
+
+  if (!graph->implicit_preds[to])
+    graph->implicit_preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
+
+  if (bitmap_set_bit (graph->implicit_preds[to], from))
+    stats.num_implicit_edges++;
+}
+
+/* Add a predecessor graph edge to GRAPH, going from TO to FROM if
+   it doesn't exist in the graph already.
+   Return false if the edge already existed, true otherwise.  */
+
+static void
+add_pred_graph_edge (constraint_graph_t graph, unsigned int to,
+		     unsigned int from)
+{
+  if (!graph->preds[to])
+    graph->preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
+  bitmap_set_bit (graph->preds[to], from);
+}
+
+/* Add a graph edge to GRAPH, going from FROM to TO if
+   it doesn't exist in the graph already.
+   Return false if the edge already existed, true otherwise.  */
+
+static bool
+add_graph_edge (constraint_graph_t graph, unsigned int to,
+		unsigned int from)
+{
+  if (to == from)
+    {
+      return false;
+    }
+  else
+    {
+      bool r = false;
+
+      if (!graph->succs[from])
+	graph->succs[from] = BITMAP_ALLOC (&pta_obstack);
+
+      /* The graph solving process does not avoid "triangles", thus
+	 there can be multiple paths from a node to another involving
+	 intermediate other nodes.  That causes extra copying which is
+	 most difficult to avoid when the intermediate node is ESCAPED
+	 because there are no edges added from ESCAPED.  Avoid
+	 adding the direct edge FROM -> TO when we have FROM -> ESCAPED
+	 and TO contains ESCAPED.
+	 ???  Note this is only a heuristic, it does not prevent the
+	 situation from occuring.  The heuristic helps PR38474 and
+	 PR99912 significantly.  */
+      if (to < FIRST_REF_NODE
+	  && bitmap_bit_p (graph->succs[from], find (escaped_id))
+	  && bitmap_bit_p (get_varinfo (find (to))->solution, escaped_id))
+	{
+	  stats.num_avoided_edges++;
+	  return false;
+	}
+
+      if (bitmap_set_bit (graph->succs[from], to))
+	{
+	  r = true;
+	  if (to < FIRST_REF_NODE && from < FIRST_REF_NODE)
+	    stats.num_edges++;
+	}
+      return r;
+    }
+}
+
+/* Initialize the constraint graph structure to contain SIZE nodes.  */
+
+static void
+init_graph (unsigned int size)
+{
+  unsigned int j;
+
+  bitmap_obstack_initialize (&predbitmap_obstack);
+
+  graph = XCNEW (struct constraint_graph);
+  graph->size = size;
+  graph->succs = XCNEWVEC (bitmap, graph->size);
+  graph->indirect_cycles = XNEWVEC (int, graph->size);
+  graph->rep = XNEWVEC (unsigned int, graph->size);
+  /* ??? Macros do not support template types with multiple arguments,
+     so we use a typedef to work around it.  */
+  typedef vec<constraint_t> vec_constraint_t_heap;
+  graph->complex = XCNEWVEC (vec_constraint_t_heap, size);
+  graph->pe = XCNEWVEC (unsigned int, graph->size);
+  graph->pe_rep = XNEWVEC (int, graph->size);
+
+  for (j = 0; j < graph->size; j++)
+    {
+      graph->rep[j] = j;
+      graph->pe_rep[j] = -1;
+      graph->indirect_cycles[j] = -1;
+    }
+}
+
+/* Build the constraint graph, adding only predecessor edges right now.  */
+
+static void
+build_pred_graph (void)
+{
+  int i;
+  constraint_t c;
+  unsigned int j;
+
+  graph->implicit_preds = XCNEWVEC (bitmap, graph->size);
+  graph->preds = XCNEWVEC (bitmap, graph->size);
+  graph->pointer_label = XCNEWVEC (unsigned int, graph->size);
+  graph->loc_label = XCNEWVEC (unsigned int, graph->size);
+  graph->pointed_by = XCNEWVEC (bitmap, graph->size);
+  graph->points_to = XCNEWVEC (bitmap, graph->size);
+  graph->eq_rep = XNEWVEC (int, graph->size);
+  graph->direct_nodes = sbitmap_alloc (graph->size);
+  graph->address_taken = BITMAP_ALLOC (&predbitmap_obstack);
+  bitmap_clear (graph->direct_nodes);
+
+  for (j = 1; j < FIRST_REF_NODE; j++)
+    {
+      if (!get_varinfo (j)->is_special_var)
+	bitmap_set_bit (graph->direct_nodes, j);
+    }
+
+  for (j = 0; j < graph->size; j++)
+    graph->eq_rep[j] = -1;
+
+  for (j = 0; j < varmap.length (); j++)
+    graph->indirect_cycles[j] = -1;
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      struct constraint_expr lhs = c->lhs;
+      struct constraint_expr rhs = c->rhs;
+      unsigned int lhsvar = lhs.var;
+      unsigned int rhsvar = rhs.var;
+
+      if (lhs.type == DEREF)
+	{
+	  /* *x = y.  */
+	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
+	    {
+	      if (lhs.var == anything_id)
+		add_pred_graph_edge (graph, storedanything_id, rhsvar);
+	      else
+		add_pred_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
+	    }
+	}
+      else if (rhs.type == DEREF)
+	{
+	  /* x = *y */
+	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
+	    add_pred_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
+	  else
+	    bitmap_clear_bit (graph->direct_nodes, lhsvar);
+	}
+      else if (rhs.type == ADDRESSOF)
+	{
+	  varinfo_t v;
+
+	  /* x = &y */
+	  if (graph->points_to[lhsvar] == NULL)
+	    graph->points_to[lhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
+	  bitmap_set_bit (graph->points_to[lhsvar], rhsvar);
+
+	  if (graph->pointed_by[rhsvar] == NULL)
+	    graph->pointed_by[rhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
+	  bitmap_set_bit (graph->pointed_by[rhsvar], lhsvar);
+
+	  /* Implicitly, *x = y */
+	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
+
+	  /* All related variables are no longer direct nodes.  */
+	  bitmap_clear_bit (graph->direct_nodes, rhsvar);
+	  v = get_varinfo (rhsvar);
+	  if (!v->is_full_var)
+	    {
+	      v = get_varinfo (v->head);
+	      do
+		{
+		  bitmap_clear_bit (graph->direct_nodes, v->id);
+		  v = vi_next (v);
+		}
+	      while (v != NULL);
+	    }
+	  bitmap_set_bit (graph->address_taken, rhsvar);
+	}
+      else if (lhsvar > anything_id
+	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
+	{
+	  /* x = y */
+	  add_pred_graph_edge (graph, lhsvar, rhsvar);
+	  /* Implicitly, *x = *y */
+	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar,
+				   FIRST_REF_NODE + rhsvar);
+	}
+      else if (lhs.offset != 0 || rhs.offset != 0)
+	{
+	  if (rhs.offset != 0)
+	    bitmap_clear_bit (graph->direct_nodes, lhs.var);
+	  else if (lhs.offset != 0)
+	    bitmap_clear_bit (graph->direct_nodes, rhs.var);
+	}
+    }
+}
+
+/* Build the constraint graph, adding successor edges.  */
+
+static void
+build_succ_graph (void)
+{
+  unsigned i, t;
+  constraint_t c;
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      struct constraint_expr lhs;
+      struct constraint_expr rhs;
+      unsigned int lhsvar;
+      unsigned int rhsvar;
+
+      if (!c)
+	continue;
+
+      lhs = c->lhs;
+      rhs = c->rhs;
+      lhsvar = find (lhs.var);
+      rhsvar = find (rhs.var);
+
+      if (lhs.type == DEREF)
+	{
+	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
+	    {
+	      if (lhs.var == anything_id)
+		add_graph_edge (graph, storedanything_id, rhsvar);
+	      else
+		add_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
+	    }
+	}
+      else if (rhs.type == DEREF)
+	{
+	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
+	    add_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
+	}
+      else if (rhs.type == ADDRESSOF)
+	{
+	  /* x = &y */
+	  gcc_checking_assert (find (rhs.var) == rhs.var);
+	  bitmap_set_bit (get_varinfo (lhsvar)->solution, rhsvar);
+	}
+      else if (lhsvar > anything_id
+	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
+	{
+	  add_graph_edge (graph, lhsvar, rhsvar);
+	}
+    }
+
+  /* Add edges from STOREDANYTHING to all nodes that can receive pointers.  */
+  t = find (storedanything_id);
+  for (i = integer_id + 1; i < FIRST_REF_NODE; ++i)
+    {
+      if (get_varinfo (i)->may_have_pointers)
+	add_graph_edge (graph, find (i), t);
+    }
+
+  /* Everything stored to ANYTHING also potentially escapes.  */
+  add_graph_edge (graph, find (escaped_id), t);
+}
+
+
+/* Changed variables on the last iteration.  */
+static bitmap changed;
+
+/* Strongly Connected Component visitation info.  */
+
+class scc_info
+{
+public:
+  scc_info (size_t size);
+  ~scc_info ();
+
+  auto_sbitmap visited;
+  auto_sbitmap deleted;
+  unsigned int *dfs;
+  unsigned int *node_mapping;
+  int current_index;
+  auto_vec<unsigned> scc_stack;
+};
+
+
+/* Recursive routine to find strongly connected components in GRAPH.
+   SI is the SCC info to store the information in, and N is the id of current
+   graph node we are processing.
+
+   This is Tarjan's strongly connected component finding algorithm, as
+   modified by Nuutila to keep only non-root nodes on the stack.
+   The algorithm can be found in "On finding the strongly connected
+   connected components in a directed graph" by Esko Nuutila and Eljas
+   Soisalon-Soininen, in Information Processing Letters volume 49,
+   number 1, pages 9-14.  */
+
+static void
+scc_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
+{
+  unsigned int i;
+  bitmap_iterator bi;
+  unsigned int my_dfs;
+
+  bitmap_set_bit (si->visited, n);
+  si->dfs[n] = si->current_index ++;
+  my_dfs = si->dfs[n];
+
+  /* Visit all the successors.  */
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[n], 0, i, bi)
+    {
+      unsigned int w;
+
+      if (i > LAST_REF_NODE)
+	break;
+
+      w = find (i);
+      if (bitmap_bit_p (si->deleted, w))
+	continue;
+
+      if (!bitmap_bit_p (si->visited, w))
+	scc_visit (graph, si, w);
+
+      unsigned int t = find (w);
+      gcc_checking_assert (find (n) == n);
+      if (si->dfs[t] < si->dfs[n])
+	si->dfs[n] = si->dfs[t];
+    }
+
+  /* See if any components have been identified.  */
+  if (si->dfs[n] == my_dfs)
+    {
+      if (si->scc_stack.length () > 0
+	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
+	{
+	  bitmap scc = BITMAP_ALLOC (NULL);
+	  unsigned int lowest_node;
+	  bitmap_iterator bi;
+
+	  bitmap_set_bit (scc, n);
+
+	  while (si->scc_stack.length () != 0
+		 && si->dfs[si->scc_stack.last ()] >= my_dfs)
+	    {
+	      unsigned int w = si->scc_stack.pop ();
+
+	      bitmap_set_bit (scc, w);
+	    }
+
+	  lowest_node = bitmap_first_set_bit (scc);
+	  gcc_assert (lowest_node < FIRST_REF_NODE);
+
+	  /* Collapse the SCC nodes into a single node, and mark the
+	     indirect cycles.  */
+	  EXECUTE_IF_SET_IN_BITMAP (scc, 0, i, bi)
+	    {
+	      if (i < FIRST_REF_NODE)
+		{
+		  if (unite (lowest_node, i))
+		    unify_nodes (graph, lowest_node, i, false);
+		}
+	      else
+		{
+		  unite (lowest_node, i);
+		  graph->indirect_cycles[i - FIRST_REF_NODE] = lowest_node;
+		}
+	    }
+	  bitmap_set_bit (si->deleted, lowest_node);
+	}
+      else
+	bitmap_set_bit (si->deleted, n);
+    }
+  else
+    si->scc_stack.safe_push (n);
+}
+
+/* Unify node FROM into node TO, updating the changed count if
+   necessary when UPDATE_CHANGED is true.  */
+
+static void
+unify_nodes (constraint_graph_t graph, unsigned int to, unsigned int from,
+	     bool update_changed)
+{
+  gcc_checking_assert (to != from && find (to) == to);
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    fprintf (dump_file, "Unifying %s to %s\n",
+	     get_varinfo (from)->name,
+	     get_varinfo (to)->name);
+
+  if (update_changed)
+    stats.unified_vars_dynamic++;
+  else
+    stats.unified_vars_static++;
+
+  merge_graph_nodes (graph, to, from);
+  if (merge_node_constraints (graph, to, from))
+    {
+      if (update_changed)
+	bitmap_set_bit (changed, to);
+    }
+
+  /* Mark TO as changed if FROM was changed.  If TO was already marked
+     as changed, decrease the changed count.  */
+
+  if (update_changed
+      && bitmap_clear_bit (changed, from))
+    bitmap_set_bit (changed, to);
+  varinfo_t fromvi = get_varinfo (from);
+  if (fromvi->solution)
+    {
+      /* If the solution changes because of the merging, we need to mark
+	 the variable as changed.  */
+      varinfo_t tovi = get_varinfo (to);
+      if (bitmap_ior_into (tovi->solution, fromvi->solution))
+	{
+	  if (update_changed)
+	    bitmap_set_bit (changed, to);
+	}
+
+      BITMAP_FREE (fromvi->solution);
+      if (fromvi->oldsolution)
+	BITMAP_FREE (fromvi->oldsolution);
+
+      if (stats.iterations > 0
+	  && tovi->oldsolution)
+	BITMAP_FREE (tovi->oldsolution);
+    }
+  if (graph->succs[to])
+    bitmap_clear_bit (graph->succs[to], to);
+}
+
+/* Add a copy edge FROM -> TO, optimizing special cases.  Returns TRUE
+   if the solution of TO changed.  */
+
+static bool
+solve_add_graph_edge (constraint_graph_t graph, unsigned int to,
+		      unsigned int from)
+{
+  /* Adding edges from the special vars is pointless.
+     They don't have sets that can change.  */
+  if (get_varinfo (from)->is_special_var)
+    return bitmap_ior_into (get_varinfo (to)->solution,
+			    get_varinfo (from)->solution);
+  /* Merging the solution from ESCAPED needlessly increases
+     the set.  Use ESCAPED as representative instead.  */
+  else if (from == find (escaped_id))
+    return bitmap_set_bit (get_varinfo (to)->solution, escaped_id);
+  else if (get_varinfo (from)->may_have_pointers
+	   && add_graph_edge (graph, to, from))
+    return bitmap_ior_into (get_varinfo (to)->solution,
+			    get_varinfo (from)->solution);
+  return false;
+}
+
+/* Process a constraint C that represents x = *(y + off), using DELTA as the
+   starting solution for y.  */
+
+static void
+do_sd_constraint (constraint_graph_t graph, constraint_t c,
+		  bitmap delta, bitmap *expanded_delta)
+{
+  unsigned int lhs = c->lhs.var;
+  bool flag = false;
+  bitmap sol = get_varinfo (lhs)->solution;
+  unsigned int j;
+  bitmap_iterator bi;
+  HOST_WIDE_INT roffset = c->rhs.offset;
+
+  /* Our IL does not allow this.  */
+  gcc_checking_assert (c->lhs.offset == 0);
+
+  /* If the solution of Y contains anything it is good enough to transfer
+     this to the LHS.  */
+  if (bitmap_bit_p (delta, anything_id))
+    {
+      flag |= bitmap_set_bit (sol, anything_id);
+      goto done;
+    }
+
+  /* If we do not know at with offset the rhs is dereferenced compute
+     the reachability set of DELTA, conservatively assuming it is
+     dereferenced at all valid offsets.  */
+  if (roffset == UNKNOWN_OFFSET)
+    {
+      delta = solution_set_expand (delta, expanded_delta);
+      /* No further offset processing is necessary.  */
+      roffset = 0;
+    }
+
+  /* For each variable j in delta (Sol(y)), add
+     an edge in the graph from j to x, and union Sol(j) into Sol(x).  */
+  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
+    {
+      varinfo_t v = get_varinfo (j);
+      HOST_WIDE_INT fieldoffset = v->offset + roffset;
+      unsigned HOST_WIDE_INT size = v->size;
+      unsigned int t;
+
+      if (v->is_full_var)
+	;
+      else if (roffset != 0)
+	{
+	  if (fieldoffset < 0)
+	    v = get_varinfo (v->head);
+	  else
+	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
+	}
+
+      /* We have to include all fields that overlap the current field
+	 shifted by roffset.  */
+      do
+	{
+	  t = find (v->id);
+
+	  flag |= solve_add_graph_edge (graph, lhs, t);
+
+	  if (v->is_full_var
+	      || v->next == 0)
+	    break;
+
+	  v = vi_next (v);
+	}
+      while (v->offset < fieldoffset + size);
+    }
+
+done:
+  /* If the LHS solution changed, mark the var as changed.  */
+  if (flag)
+    bitmap_set_bit (changed, lhs);
+}
+
+/* Process a constraint C that represents *(x + off) = y using DELTA
+   as the starting solution for x.  */
+
+static void
+do_ds_constraint (constraint_t c, bitmap delta, bitmap *expanded_delta)
+{
+  unsigned int rhs = c->rhs.var;
+  bitmap sol = get_varinfo (rhs)->solution;
+  unsigned int j;
+  bitmap_iterator bi;
+  HOST_WIDE_INT loff = c->lhs.offset;
+  bool escaped_p = false;
+
+  /* Our IL does not allow this.  */
+  gcc_checking_assert (c->rhs.offset == 0);
+
+  /* If the solution of y contains ANYTHING simply use the ANYTHING
+     solution.  This avoids needlessly increasing the points-to sets.  */
+  if (bitmap_bit_p (sol, anything_id))
+    sol = get_varinfo (find (anything_id))->solution;
+
+  /* If the solution for x contains ANYTHING we have to merge the
+     solution of y into all pointer variables which we do via
+     STOREDANYTHING.  */
+  if (bitmap_bit_p (delta, anything_id))
+    {
+      unsigned t = find (storedanything_id);
+      if (solve_add_graph_edge (graph, t, rhs))
+	bitmap_set_bit (changed, t);
+      return;
+    }
+
+  /* If we do not know at with offset the rhs is dereferenced compute
+     the reachability set of DELTA, conservatively assuming it is
+     dereferenced at all valid offsets.  */
+  if (loff == UNKNOWN_OFFSET)
+    {
+      delta = solution_set_expand (delta, expanded_delta);
+      loff = 0;
+    }
+
+  /* For each member j of delta (Sol(x)), add an edge from y to j and
+     union Sol(y) into Sol(j) */
+  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
+    {
+      varinfo_t v = get_varinfo (j);
+      unsigned int t;
+      HOST_WIDE_INT fieldoffset = v->offset + loff;
+      unsigned HOST_WIDE_INT size = v->size;
+
+      if (v->is_full_var)
+	;
+      else if (loff != 0)
+	{
+	  if (fieldoffset < 0)
+	    v = get_varinfo (v->head);
+	  else
+	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
+	}
+
+      /* We have to include all fields that overlap the current field
+	 shifted by loff.  */
+      do
+	{
+	  if (v->may_have_pointers)
+	    {
+	      /* If v is a global variable then this is an escape point.  */
+	      if (v->is_global_var
+		  && !escaped_p)
+		{
+		  t = find (escaped_id);
+		  if (add_graph_edge (graph, t, rhs)
+		      && bitmap_ior_into (get_varinfo (t)->solution, sol))
+		    bitmap_set_bit (changed, t);
+		  /* Enough to let rhs escape once.  */
+		  escaped_p = true;
+		}
+
+	      if (v->is_special_var)
+		break;
+
+	      t = find (v->id);
+
+	      if (solve_add_graph_edge (graph, t, rhs))
+		bitmap_set_bit (changed, t);
+	    }
+
+	  if (v->is_full_var
+	      || v->next == 0)
+	    break;
+
+	  v = vi_next (v);
+	}
+      while (v->offset < fieldoffset + size);
+    }
+}
+
+/* Handle a non-simple (simple meaning requires no iteration),
+   constraint (IE *x = &y, x = *y, *x = y, and x = y with offsets involved).  */
+
+static void
+do_complex_constraint (constraint_graph_t graph, constraint_t c, bitmap delta,
+		       bitmap *expanded_delta)
+{
+  if (c->lhs.type == DEREF)
+    {
+      if (c->rhs.type == ADDRESSOF)
+	{
+	  gcc_unreachable ();
+	}
+      else
+	{
+	  /* *x = y */
+	  do_ds_constraint (c, delta, expanded_delta);
+	}
+    }
+  else if (c->rhs.type == DEREF)
+    {
+      /* x = *y */
+      if (!(get_varinfo (c->lhs.var)->is_special_var))
+	do_sd_constraint (graph, c, delta, expanded_delta);
+    }
+  else
+    {
+      bitmap tmp;
+      bool flag = false;
+
+      gcc_checking_assert (c->rhs.type == SCALAR && c->lhs.type == SCALAR
+			   && c->rhs.offset != 0 && c->lhs.offset == 0);
+      tmp = get_varinfo (c->lhs.var)->solution;
+
+      flag = set_union_with_increment (tmp, delta, c->rhs.offset,
+				       expanded_delta);
+
+      if (flag)
+	bitmap_set_bit (changed, c->lhs.var);
+    }
+}
+
+/* Initialize and return a new SCC info structure.  */
+
+scc_info::scc_info (size_t size) :
+  visited (size), deleted (size), current_index (0), scc_stack (1)
+{
+  bitmap_clear (visited);
+  bitmap_clear (deleted);
+  node_mapping = XNEWVEC (unsigned int, size);
+  dfs = XCNEWVEC (unsigned int, size);
+
+  for (size_t i = 0; i < size; i++)
+    node_mapping[i] = i;
+}
+
+/* Free an SCC info structure pointed to by SI.  */
+
+scc_info::~scc_info ()
+{
+  free (node_mapping);
+  free (dfs);
+}
+
+
+/* Find indirect cycles in GRAPH that occur, using strongly connected
+   components, and note them in the indirect cycles map.
+
+   This technique comes from Ben Hardekopf and Calvin Lin,
+   "It Pays to be Lazy: Fast and Accurate Pointer Analysis for Millions of
+   Lines of Code", submitted to PLDI 2007.  */
+
+static void
+find_indirect_cycles (constraint_graph_t graph)
+{
+  unsigned int i;
+  unsigned int size = graph->size;
+  scc_info si (size);
+
+  for (i = 0; i < MIN (LAST_REF_NODE, size); i++)
+    if (!bitmap_bit_p (si.visited, i) && find (i) == i)
+      scc_visit (graph, &si, i);
+}
+
+/* Visit the graph in topological order starting at node N, and store the
+   order in TOPO_ORDER using VISITED to indicate visited nodes.  */
+
+static void
+topo_visit (constraint_graph_t graph, vec<unsigned> &topo_order,
+	    sbitmap visited, unsigned int n)
+{
+  bitmap_iterator bi;
+  unsigned int j;
+
+  bitmap_set_bit (visited, n);
+
+  if (graph->succs[n])
+    EXECUTE_IF_SET_IN_BITMAP (graph->succs[n], 0, j, bi)
+      {
+	unsigned k = find (j);
+	if (!bitmap_bit_p (visited, k))
+	  topo_visit (graph, topo_order, visited, k);
+      }
+
+  /* Also consider copy with offset complex constraints as implicit edges.  */
+  for (auto c : graph->complex[n])
+    {
+      /* Constraints are ordered so that SCALAR = SCALAR appear first.  */
+      if (c->lhs.type != SCALAR || c->rhs.type != SCALAR)
+	break;
+      gcc_checking_assert (c->rhs.var == n);
+      unsigned k = find (c->lhs.var);
+      if (!bitmap_bit_p (visited, k))
+	topo_visit (graph, topo_order, visited, k);
+    }
+
+  topo_order.quick_push (n);
+}
+
+/* Compute a topological ordering for GRAPH, and return the result.  */
+
+static auto_vec<unsigned>
+compute_topo_order (constraint_graph_t graph)
+{
+  unsigned int i;
+  unsigned int size = graph->size;
+
+  auto_sbitmap visited (size);
+  bitmap_clear (visited);
+
+  /* For the heuristic in add_graph_edge to work optimally make sure to
+     first visit the connected component of the graph containing
+     ESCAPED.  Do this by extracting the connected component
+     with ESCAPED and append that to all other components as solve_graph
+     pops from the order.  */
+  auto_vec<unsigned> tail (size);
+  topo_visit (graph, tail, visited, find (escaped_id));
+
+  auto_vec<unsigned> topo_order (size);
+
+  for (i = 0; i != size; ++i)
+    if (!bitmap_bit_p (visited, i) && find (i) == i)
+      topo_visit (graph, topo_order, visited, i);
+
+  topo_order.splice (tail);
+  return topo_order;
+}
+
+/* Structure used to for hash value numbering of pointer equivalence
+   classes.  */
+
+typedef struct equiv_class_label
+{
+  hashval_t hashcode;
+  unsigned int equivalence_class;
+  bitmap labels;
+} *equiv_class_label_t;
+typedef const struct equiv_class_label *const_equiv_class_label_t;
+
+/* Equiv_class_label hashtable helpers.  */
+
+struct equiv_class_hasher : nofree_ptr_hash <equiv_class_label>
+{
+  static inline hashval_t hash (const equiv_class_label *);
+  static inline bool equal (const equiv_class_label *,
+			    const equiv_class_label *);
+};
+
+/* A hashtable for mapping a bitmap of labels->pointer equivalence
+   classes.  */
+static hash_table<equiv_class_hasher> *pointer_equiv_class_table;
+
+/* A hashtable for mapping a bitmap of labels->location equivalence
+   classes.  */
+static hash_table<equiv_class_hasher> *location_equiv_class_table;
+
+/* Hash function for a equiv_class_label_t.  */
+
+inline hashval_t
+equiv_class_hasher::hash (const equiv_class_label *ecl)
+{
+  return ecl->hashcode;
+}
+
+/* Equality function for two equiv_class_label_t's.  */
+
+inline bool
+equiv_class_hasher::equal (const equiv_class_label *eql1,
+			   const equiv_class_label *eql2)
+{
+  return (eql1->hashcode == eql2->hashcode
+	  && bitmap_equal_p (eql1->labels, eql2->labels));
+}
+
+struct obstack equiv_class_obstack;
+
+/* Lookup a equivalence class in TABLE by the bitmap of LABELS with
+   hash HAS it contains.  Sets *REF_LABELS to the bitmap LABELS
+   is equivalent to.  */
+
+static equiv_class_label *
+equiv_class_lookup_or_add (hash_table<equiv_class_hasher> *table,
+			   bitmap labels)
+{
+  equiv_class_label **slot;
+  equiv_class_label ecl;
+
+  ecl.labels = labels;
+  ecl.hashcode = bitmap_hash (labels);
+  slot = table->find_slot (&ecl, INSERT);
+  if (!*slot)
+    {
+      *slot = XOBNEW (&equiv_class_obstack, struct equiv_class_label);
+      (*slot)->labels = labels;
+      (*slot)->hashcode = ecl.hashcode;
+      (*slot)->equivalence_class = 0;
+    }
+
+  return *slot;
+}
+
+
+/* Perform offline variable substitution.
+
+   This is a worst case quadratic time way of identifying variables
+   that must have equivalent points-to sets, including those caused by
+   static cycles, and single entry subgraphs, in the constraint graph.
+
+   The technique is described in "Exploiting Pointer and Location
+   Equivalence to Optimize Pointer Analysis.  In the 14th International
+   Static Analysis Symposium (SAS), August 2007."  It is known as the
+   "HU" algorithm, and is equivalent to value numbering the collapsed
+   constraint graph including evaluating unions.
+
+   The general method of finding equivalence classes is as follows:
+   Add fake nodes (REF nodes) and edges for *a = b and a = *b constraints.
+   Initialize all non-REF nodes to be direct nodes.
+   For each constraint a = a U {b}, we set pts(a) = pts(a) u {fresh
+   variable}
+   For each constraint containing the dereference, we also do the same
+   thing.
+
+   We then compute SCC's in the graph and unify nodes in the same SCC,
+   including pts sets.
+
+   For each non-collapsed node x:
+    Visit all unvisited explicit incoming edges.
+    Ignoring all non-pointers, set pts(x) = Union of pts(a) for y
+    where y->x.
+    Lookup the equivalence class for pts(x).
+     If we found one, equivalence_class(x) = found class.
+     Otherwise, equivalence_class(x) = new class, and new_class is
+    added to the lookup table.
+
+   All direct nodes with the same equivalence class can be replaced
+   with a single representative node.
+   All unlabeled nodes (label == 0) are not pointers and all edges
+   involving them can be eliminated.
+   We perform these optimizations during rewrite_constraints
+
+   In addition to pointer equivalence class finding, we also perform
+   location equivalence class finding.  This is the set of variables
+   that always appear together in points-to sets.  We use this to
+   compress the size of the points-to sets.  */
+
+/* Current maximum pointer equivalence class id.  */
+static int pointer_equiv_class;
+
+/* Current maximum location equivalence class id.  */
+static int location_equiv_class;
+
+/* Recursive routine to find strongly connected components in GRAPH,
+   and label it's nodes with DFS numbers.  */
+
+static void
+condense_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
+{
+  unsigned int i;
+  bitmap_iterator bi;
+  unsigned int my_dfs;
+
+  gcc_checking_assert (si->node_mapping[n] == n);
+  bitmap_set_bit (si->visited, n);
+  si->dfs[n] = si->current_index ++;
+  my_dfs = si->dfs[n];
+
+  /* Visit all the successors.  */
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
+    {
+      unsigned int w = si->node_mapping[i];
+
+      if (bitmap_bit_p (si->deleted, w))
+	continue;
+
+      if (!bitmap_bit_p (si->visited, w))
+	condense_visit (graph, si, w);
+
+      unsigned int t = si->node_mapping[w];
+      gcc_checking_assert (si->node_mapping[n] == n);
+      if (si->dfs[t] < si->dfs[n])
+	si->dfs[n] = si->dfs[t];
+    }
+
+  /* Visit all the implicit predecessors.  */
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->implicit_preds[n], 0, i, bi)
+    {
+      unsigned int w = si->node_mapping[i];
+
+      if (bitmap_bit_p (si->deleted, w))
+	continue;
+
+      if (!bitmap_bit_p (si->visited, w))
+	condense_visit (graph, si, w);
+
+      unsigned int t = si->node_mapping[w];
+      gcc_assert (si->node_mapping[n] == n);
+      if (si->dfs[t] < si->dfs[n])
+	si->dfs[n] = si->dfs[t];
+    }
+
+  /* See if any components have been identified.  */
+  if (si->dfs[n] == my_dfs)
+    {
+      if (si->scc_stack.length () != 0
+	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
+	{
+	  /* Find the first node of the SCC and do non-bitmap work.  */
+	  bool direct_p = true;
+	  unsigned first = si->scc_stack.length ();
+	  do
+	    {
+	      --first;
+	      unsigned int w = si->scc_stack[first];
+	      si->node_mapping[w] = n;
+	      if (!bitmap_bit_p (graph->direct_nodes, w))
+		direct_p = false;
+	    }
+	  while (first > 0
+		 && si->dfs[si->scc_stack[first - 1]] >= my_dfs);
+	  if (!direct_p)
+	    bitmap_clear_bit (graph->direct_nodes, n);
+
+	  /* Want to reduce to node n, push that first.  */
+	  si->scc_stack.reserve (1);
+	  si->scc_stack.quick_push (si->scc_stack[first]);
+	  si->scc_stack[first] = n;
+
+	  unsigned scc_size = si->scc_stack.length () - first;
+	  unsigned split = scc_size / 2;
+	  unsigned carry = scc_size - split * 2;
+	  while (split > 0)
+	    {
+	      for (unsigned i = 0; i < split; ++i)
+		{
+		  unsigned a = si->scc_stack[first + i];
+		  unsigned b = si->scc_stack[first + split + carry + i];
+
+		  /* Unify our nodes.  */
+		  if (graph->preds[b])
+		    {
+		      if (!graph->preds[a])
+			std::swap (graph->preds[a], graph->preds[b]);
+		      else
+			bitmap_ior_into_and_free (graph->preds[a],
+						  &graph->preds[b]);
+		    }
+		  if (graph->implicit_preds[b])
+		    {
+		      if (!graph->implicit_preds[a])
+			std::swap (graph->implicit_preds[a],
+				   graph->implicit_preds[b]);
+		      else
+			bitmap_ior_into_and_free (graph->implicit_preds[a],
+						  &graph->implicit_preds[b]);
+		    }
+		  if (graph->points_to[b])
+		    {
+		      if (!graph->points_to[a])
+			std::swap (graph->points_to[a], graph->points_to[b]);
+		      else
+			bitmap_ior_into_and_free (graph->points_to[a],
+						  &graph->points_to[b]);
+		    }
+		}
+	      unsigned remain = split + carry;
+	      split = remain / 2;
+	      carry = remain - split * 2;
+	    }
+	  /* Actually pop the SCC.  */
+	  si->scc_stack.truncate (first);
+	}
+      bitmap_set_bit (si->deleted, n);
+    }
+  else
+    si->scc_stack.safe_push (n);
+}
+
+/* Label pointer equivalences.
+
+   This performs a value numbering of the constraint graph to
+   discover which variables will always have the same points-to sets
+   under the current set of constraints.
+
+   The way it value numbers is to store the set of points-to bits
+   generated by the constraints and graph edges.  This is just used as a
+   hash and equality comparison.  The *actual set of points-to bits* is
+   completely irrelevant, in that we don't care about being able to
+   extract them later.
+
+   The equality values (currently bitmaps) just have to satisfy a few
+   constraints, the main ones being:
+   1. The combining operation must be order independent.
+   2. The end result of a given set of operations must be unique iff the
+      combination of input values is unique
+   3. Hashable.  */
+
+static void
+label_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
+{
+  unsigned int i, first_pred;
+  bitmap_iterator bi;
+
+  bitmap_set_bit (si->visited, n);
+
+  /* Label and union our incoming edges's points to sets.  */
+  first_pred = -1U;
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
+    {
+      unsigned int w = si->node_mapping[i];
+      if (!bitmap_bit_p (si->visited, w))
+	label_visit (graph, si, w);
+
+      /* Skip unused edges.  */
+      if (w == n || graph->pointer_label[w] == 0)
+	continue;
+
+      if (graph->points_to[w])
+	{
+	  if (!graph->points_to[n])
+	    {
+	      if (first_pred == -1U)
+		first_pred = w;
+	      else
+		{
+		  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
+		  bitmap_ior (graph->points_to[n],
+			      graph->points_to[first_pred],
+			      graph->points_to[w]);
+		}
+	    }
+	  else
+	    bitmap_ior_into (graph->points_to[n], graph->points_to[w]);
+	}
+    }
+
+  /* Indirect nodes get fresh variables and a new pointer equiv class.  */
+  if (!bitmap_bit_p (graph->direct_nodes, n))
+    {
+      if (!graph->points_to[n])
+	{
+	  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
+	  if (first_pred != -1U)
+	    bitmap_copy (graph->points_to[n], graph->points_to[first_pred]);
+	}
+      bitmap_set_bit (graph->points_to[n], FIRST_REF_NODE + n);
+      graph->pointer_label[n] = pointer_equiv_class++;
+      equiv_class_label_t ecl;
+      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
+				       graph->points_to[n]);
+      ecl->equivalence_class = graph->pointer_label[n];
+      return;
+    }
+
+  /* If there was only a single non-empty predecessor the pointer equiv
+     class is the same.  */
+  if (!graph->points_to[n])
+    {
+      if (first_pred != -1U)
+	{
+	  graph->pointer_label[n] = graph->pointer_label[first_pred];
+	  graph->points_to[n] = graph->points_to[first_pred];
+	}
+      return;
+    }
+
+  if (!bitmap_empty_p (graph->points_to[n]))
+    {
+      equiv_class_label_t ecl;
+      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
+				       graph->points_to[n]);
+      if (ecl->equivalence_class == 0)
+	ecl->equivalence_class = pointer_equiv_class++;
+      else
+	{
+	  BITMAP_FREE (graph->points_to[n]);
+	  graph->points_to[n] = ecl->labels;
+	}
+      graph->pointer_label[n] = ecl->equivalence_class;
+    }
+}
+
+/* Print the pred graph in dot format.  */
+
+static void
+dump_pred_graph (class scc_info *si, FILE *file)
+{
+  unsigned int i;
+
+  /* Only print the graph if it has already been initialized:  */
+  if (!graph)
+    return;
+
+  /* Prints the header of the dot file:  */
+  fprintf (file, "strict digraph {\n");
+  fprintf (file, "  node [\n    shape = box\n  ]\n");
+  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
+  fprintf (file, "\n  // List of nodes and complex constraints in "
+	   "the constraint graph:\n");
+
+  /* The next lines print the nodes in the graph together with the
+     complex constraints attached to them.  */
+  for (i = 1; i < graph->size; i++)
+    {
+      if (i == FIRST_REF_NODE)
+	continue;
+      if (si->node_mapping[i] != i)
+	continue;
+      if (i < FIRST_REF_NODE)
+	fprintf (file, "\"%s\"", get_varinfo (i)->name);
+      else
+	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+      if (graph->points_to[i]
+	  && !bitmap_empty_p (graph->points_to[i]))
+	{
+	  if (i < FIRST_REF_NODE)
+	    fprintf (file, "[label=\"%s = {", get_varinfo (i)->name);
+	  else
+	    fprintf (file, "[label=\"*%s = {",
+		     get_varinfo (i - FIRST_REF_NODE)->name);
+	  unsigned j;
+	  bitmap_iterator bi;
+	  EXECUTE_IF_SET_IN_BITMAP (graph->points_to[i], 0, j, bi)
+	    fprintf (file, " %d", j);
+	  fprintf (file, " }\"]");
+	}
+      fprintf (file, ";\n");
+    }
+
+  /* Go over the edges.  */
+  fprintf (file, "\n  // Edges in the constraint graph:\n");
+  for (i = 1; i < graph->size; i++)
+    {
+      unsigned j;
+      bitmap_iterator bi;
+      if (si->node_mapping[i] != i)
+	continue;
+      EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[i], 0, j, bi)
+	{
+	  unsigned from = si->node_mapping[j];
+	  if (from < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (from)->name);
+	  else
+	    fprintf (file, "\"*%s\"",
+		     get_varinfo (from - FIRST_REF_NODE)->name);
+	  fprintf (file, " -> ");
+	  if (i < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
+	  else
+	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+	  fprintf (file, ";\n");
+	}
+    }
+
+  /* Prints the tail of the dot file.  */
+  fprintf (file, "}\n");
+}
+
+/* Perform offline variable substitution, discovering equivalence
+   classes, and eliminating non-pointer variables.  */
+
+static class scc_info *
+perform_var_substitution (constraint_graph_t graph)
+{
+  unsigned int i;
+  unsigned int size = graph->size;
+  scc_info *si = new scc_info (size);
+
+  bitmap_obstack_initialize (&iteration_obstack);
+  gcc_obstack_init (&equiv_class_obstack);
+  pointer_equiv_class_table = new hash_table<equiv_class_hasher> (511);
+  location_equiv_class_table
+    = new hash_table<equiv_class_hasher> (511);
+  pointer_equiv_class = 1;
+  location_equiv_class = 1;
+
+  /* Condense the nodes, which means to find SCC's, count incoming
+     predecessors, and unite nodes in SCC's.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
+      condense_visit (graph, si, si->node_mapping[i]);
+
+  if (dump_file && (dump_flags & TDF_GRAPH))
+    {
+      fprintf (dump_file, "\n\n// The constraint graph before var-substitution "
+	       "in dot format:\n");
+      dump_pred_graph (si, dump_file);
+      fprintf (dump_file, "\n\n");
+    }
+
+  bitmap_clear (si->visited);
+  /* Actually the label the nodes for pointer equivalences.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
+      label_visit (graph, si, si->node_mapping[i]);
+
+  /* Calculate location equivalence labels.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      bitmap pointed_by;
+      bitmap_iterator bi;
+      unsigned int j;
+
+      if (!graph->pointed_by[i])
+	continue;
+      pointed_by = BITMAP_ALLOC (&iteration_obstack);
+
+      /* Translate the pointed-by mapping for pointer equivalence
+	 labels.  */
+      EXECUTE_IF_SET_IN_BITMAP (graph->pointed_by[i], 0, j, bi)
+	{
+	  bitmap_set_bit (pointed_by,
+			  graph->pointer_label[si->node_mapping[j]]);
+	}
+      /* The original pointed_by is now dead.  */
+      BITMAP_FREE (graph->pointed_by[i]);
+
+      /* Look up the location equivalence label if one exists, or make
+	 one otherwise.  */
+      equiv_class_label_t ecl;
+      ecl = equiv_class_lookup_or_add (location_equiv_class_table, pointed_by);
+      if (ecl->equivalence_class == 0)
+	ecl->equivalence_class = location_equiv_class++;
+      else
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file, "Found location equivalence for node %s\n",
+		     get_varinfo (i)->name);
+	  BITMAP_FREE (pointed_by);
+	}
+      graph->loc_label[i] = ecl->equivalence_class;
+
+    }
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    for (i = 1; i < FIRST_REF_NODE; i++)
+      {
+	unsigned j = si->node_mapping[i];
+	if (j != i)
+	  {
+	    fprintf (dump_file, "%s node id %d ",
+		     bitmap_bit_p (graph->direct_nodes, i)
+		     ? "Direct" : "Indirect", i);
+	    if (i < FIRST_REF_NODE)
+	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
+	    else
+	      fprintf (dump_file, "\"*%s\"",
+		       get_varinfo (i - FIRST_REF_NODE)->name);
+	    fprintf (dump_file, " mapped to SCC leader node id %d ", j);
+	    if (j < FIRST_REF_NODE)
+	      fprintf (dump_file, "\"%s\"\n", get_varinfo (j)->name);
+	    else
+	      fprintf (dump_file, "\"*%s\"\n",
+		       get_varinfo (j - FIRST_REF_NODE)->name);
+	  }
+	else
+	  {
+	    fprintf (dump_file,
+		     "Equivalence classes for %s node id %d ",
+		     bitmap_bit_p (graph->direct_nodes, i)
+		     ? "direct" : "indirect", i);
+	    if (i < FIRST_REF_NODE)
+	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
+	    else
+	      fprintf (dump_file, "\"*%s\"",
+		       get_varinfo (i - FIRST_REF_NODE)->name);
+	    fprintf (dump_file,
+		     ": pointer %d, location %d\n",
+		     graph->pointer_label[i], graph->loc_label[i]);
+	  }
+      }
+
+  /* Quickly eliminate our non-pointer variables.  */
+
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      unsigned int node = si->node_mapping[i];
+
+      if (graph->pointer_label[node] == 0)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file,
+		     "%s is a non-pointer variable, eliminating edges.\n",
+		     get_varinfo (node)->name);
+	  stats.nonpointer_vars++;
+	  clear_edges_for_node (graph, node);
+	}
+    }
+
+  return si;
+}
+
+/* Free information that was only necessary for variable
+   substitution.  */
+
+static void
+free_var_substitution_info (class scc_info *si)
+{
+  delete si;
+  free (graph->pointer_label);
+  free (graph->loc_label);
+  free (graph->pointed_by);
+  free (graph->points_to);
+  free (graph->eq_rep);
+  sbitmap_free (graph->direct_nodes);
+  delete pointer_equiv_class_table;
+  pointer_equiv_class_table = NULL;
+  delete location_equiv_class_table;
+  location_equiv_class_table = NULL;
+  obstack_free (&equiv_class_obstack, NULL);
+  bitmap_obstack_release (&iteration_obstack);
+}
+
+/* Return an existing node that is equivalent to NODE, which has
+   equivalence class LABEL, if one exists.  Return NODE otherwise.  */
+
+static unsigned int
+find_equivalent_node (constraint_graph_t graph,
+		      unsigned int node, unsigned int label)
+{
+  /* If the address version of this variable is unused, we can
+     substitute it for anything else with the same label.
+     Otherwise, we know the pointers are equivalent, but not the
+     locations, and we can unite them later.  */
+
+  if (!bitmap_bit_p (graph->address_taken, node))
+    {
+      gcc_checking_assert (label < graph->size);
+
+      if (graph->eq_rep[label] != -1)
+	{
+	  /* Unify the two variables since we know they are equivalent.  */
+	  if (unite (graph->eq_rep[label], node))
+	    unify_nodes (graph, graph->eq_rep[label], node, false);
+	  return graph->eq_rep[label];
+	}
+      else
+	{
+	  graph->eq_rep[label] = node;
+	  graph->pe_rep[label] = node;
+	}
+    }
+  else
+    {
+      gcc_checking_assert (label < graph->size);
+      graph->pe[node] = label;
+      if (graph->pe_rep[label] == -1)
+	graph->pe_rep[label] = node;
+    }
+
+  return node;
+}
+
+/* Unite pointer equivalent but not location equivalent nodes in
+   GRAPH.  This may only be performed once variable substitution is
+   finished.  */
+
+static void
+unite_pointer_equivalences (constraint_graph_t graph)
+{
+  unsigned int i;
+
+  /* Go through the pointer equivalences and unite them to their
+     representative, if they aren't already.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      unsigned int label = graph->pe[i];
+      if (label)
+	{
+	  int label_rep = graph->pe_rep[label];
+
+	  if (label_rep == -1)
+	    continue;
+
+	  label_rep = find (label_rep);
+	  if (label_rep >= 0 && unite (label_rep, find (i)))
+	    unify_nodes (graph, label_rep, i, false);
+	}
+    }
+}
+
+/* Move complex constraints to the GRAPH nodes they belong to.  */
+
+static void
+move_complex_constraints (constraint_graph_t graph)
+{
+  int i;
+  constraint_t c;
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      if (c)
+	{
+	  struct constraint_expr lhs = c->lhs;
+	  struct constraint_expr rhs = c->rhs;
+
+	  if (lhs.type == DEREF)
+	    {
+	      insert_into_complex (graph, lhs.var, c);
+	    }
+	  else if (rhs.type == DEREF)
+	    {
+	      if (!(get_varinfo (lhs.var)->is_special_var))
+		insert_into_complex (graph, rhs.var, c);
+	    }
+	  else if (rhs.type != ADDRESSOF && lhs.var > anything_id
+		   && (lhs.offset != 0 || rhs.offset != 0))
+	    {
+	      insert_into_complex (graph, rhs.var, c);
+	    }
+	}
+    }
+}
+
+/* Optimize and rewrite complex constraints while performing
+   collapsing of equivalent nodes.  SI is the SCC_INFO that is the
+   result of perform_variable_substitution.  */
+
+static void
+rewrite_constraints (constraint_graph_t graph,
+		     class scc_info *si)
+{
+  int i;
+  constraint_t c;
+
+  if (flag_checking)
+    {
+      for (unsigned int j = 0; j < graph->size; j++)
+	gcc_assert (find (j) == j);
+    }
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      struct constraint_expr lhs = c->lhs;
+      struct constraint_expr rhs = c->rhs;
+      unsigned int lhsvar = find (lhs.var);
+      unsigned int rhsvar = find (rhs.var);
+      unsigned int lhsnode, rhsnode;
+      unsigned int lhslabel, rhslabel;
+
+      lhsnode = si->node_mapping[lhsvar];
+      rhsnode = si->node_mapping[rhsvar];
+      lhslabel = graph->pointer_label[lhsnode];
+      rhslabel = graph->pointer_label[rhsnode];
+
+      /* See if it is really a non-pointer variable, and if so, ignore
+	 the constraint.  */
+      if (lhslabel == 0)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    {
+
+	      fprintf (dump_file, "%s is a non-pointer variable, "
+		       "ignoring constraint:",
+		       get_varinfo (lhs.var)->name);
+	      dump_constraint (dump_file, c);
+	      fprintf (dump_file, "\n");
+	    }
+	  constraints[i] = NULL;
+	  continue;
+	}
+
+      if (rhslabel == 0)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    {
+
+	      fprintf (dump_file, "%s is a non-pointer variable, "
+		       "ignoring constraint:",
+		       get_varinfo (rhs.var)->name);
+	      dump_constraint (dump_file, c);
+	      fprintf (dump_file, "\n");
+	    }
+	  constraints[i] = NULL;
+	  continue;
+	}
+
+      lhsvar = find_equivalent_node (graph, lhsvar, lhslabel);
+      rhsvar = find_equivalent_node (graph, rhsvar, rhslabel);
+      c->lhs.var = lhsvar;
+      c->rhs.var = rhsvar;
+    }
+}
+
+/* Eliminate indirect cycles involving NODE.  Return true if NODE was
+   part of an SCC, false otherwise.  */
+
+static bool
+eliminate_indirect_cycles (unsigned int node)
+{
+  if (graph->indirect_cycles[node] != -1
+      && !bitmap_empty_p (get_varinfo (node)->solution))
+    {
+      unsigned int i;
+      auto_vec<unsigned> queue;
+      int queuepos;
+      unsigned int to = find (graph->indirect_cycles[node]);
+      bitmap_iterator bi;
+
+      /* We can't touch the solution set and call unify_nodes
+	 at the same time, because unify_nodes is going to do
+	 bitmap unions into it.  */
+
+      EXECUTE_IF_SET_IN_BITMAP (get_varinfo (node)->solution, 0, i, bi)
+	{
+	  if (find (i) == i && i != to)
+	    {
+	      if (unite (to, i))
+		queue.safe_push (i);
+	    }
+	}
+
+      for (queuepos = 0;
+	   queue.iterate (queuepos, &i);
+	   queuepos++)
+	{
+	  unify_nodes (graph, to, i, true);
+	}
+      return true;
+    }
+  return false;
+}
+
+/* Solve the constraint graph GRAPH using our worklist solver.
+   This is based on the PW* family of solvers from the "Efficient Field
+   Sensitive Pointer Analysis for C" paper.
+   It works by iterating over all the graph nodes, processing the complex
+   constraints and propagating the copy constraints, until everything stops
+   changed.  This corresponds to steps 6-8 in the solving list given above.  */
+
+static void
+solve_graph (constraint_graph_t graph)
+{
+  unsigned int size = graph->size;
+  unsigned int i;
+  bitmap pts;
+
+  changed = BITMAP_ALLOC (NULL);
+
+  /* Mark all initial non-collapsed nodes as changed.  */
+  for (i = 1; i < size; i++)
+    {
+      varinfo_t ivi = get_varinfo (i);
+      if (find (i) == i && !bitmap_empty_p (ivi->solution)
+	  && ((graph->succs[i] && !bitmap_empty_p (graph->succs[i]))
+	      || graph->complex[i].length () > 0))
+	bitmap_set_bit (changed, i);
+    }
+
+  /* Allocate a bitmap to be used to store the changed bits.  */
+  pts = BITMAP_ALLOC (&pta_obstack);
+
+  while (!bitmap_empty_p (changed))
+    {
+      unsigned int i;
+      stats.iterations++;
+
+      bitmap_obstack_initialize (&iteration_obstack);
+
+      auto_vec<unsigned> topo_order = compute_topo_order (graph);
+      while (topo_order.length () != 0)
+	{
+	  i = topo_order.pop ();
+
+	  /* If this variable is not a representative, skip it.  */
+	  if (find (i) != i)
+	    continue;
+
+	  /* In certain indirect cycle cases, we may merge this
+	     variable to another.  */
+	  if (eliminate_indirect_cycles (i) && find (i) != i)
+	    continue;
+
+	  /* If the node has changed, we need to process the
+	     complex constraints and outgoing edges again.  For complex
+	     constraints that modify i itself, like the common group of
+	       callarg = callarg + UNKNOWN;
+	       callarg = *callarg + UNKNOWN;
+	       *callarg = callescape;
+	     make sure to iterate immediately because that maximizes
+	     cache reuse and expands the graph quickest, leading to
+	     better visitation order in the next iteration.  */
+	  while (bitmap_clear_bit (changed, i))
+	    {
+	      bitmap solution;
+	      vec<constraint_t> &complex = graph->complex[i];
+	      varinfo_t vi = get_varinfo (i);
+	      bool solution_empty;
+
+	      /* Compute the changed set of solution bits.  If anything
+		 is in the solution just propagate that.  */
+	      if (bitmap_bit_p (vi->solution, anything_id))
+		{
+		  /* If anything is also in the old solution there is
+		     nothing to do.
+		     ???  But we shouldn't ended up with "changed" set ...  */
+		  if (vi->oldsolution
+		      && bitmap_bit_p (vi->oldsolution, anything_id))
+		    break;
+		  bitmap_copy (pts, get_varinfo (find (anything_id))->solution);
+		}
+	      else if (vi->oldsolution)
+		bitmap_and_compl (pts, vi->solution, vi->oldsolution);
+	      else
+		bitmap_copy (pts, vi->solution);
+
+	      if (bitmap_empty_p (pts))
+		break;
+
+	      if (vi->oldsolution)
+		bitmap_ior_into (vi->oldsolution, pts);
+	      else
+		{
+		  vi->oldsolution = BITMAP_ALLOC (&oldpta_obstack);
+		  bitmap_copy (vi->oldsolution, pts);
+		}
+
+	      solution = vi->solution;
+	      solution_empty = bitmap_empty_p (solution);
+
+	      /* Process the complex constraints.  */
+	      hash_set<constraint_t> *cvisited = nullptr;
+	      if (flag_checking)
+		cvisited = new hash_set<constraint_t>;
+	      bitmap expanded_pts = NULL;
+	      for (unsigned j = 0; j < complex.length (); ++j)
+		{
+		  constraint_t c = complex[j];
+		  /* At unification time only the directly involved nodes
+		     will get their complex constraints updated.  Update
+		     our complex constraints now but keep the constraint
+		     vector sorted and clear of duplicates.  Also make
+		     sure to evaluate each prevailing constraint only once.  */
+		  unsigned int new_lhs = find (c->lhs.var);
+		  unsigned int new_rhs = find (c->rhs.var);
+		  if (c->lhs.var != new_lhs || c->rhs.var != new_rhs)
+		    {
+		      constraint tem = *c;
+		      tem.lhs.var = new_lhs;
+		      tem.rhs.var = new_rhs;
+		      unsigned int place
+			= complex.lower_bound (&tem, constraint_less);
+		      c->lhs.var = new_lhs;
+		      c->rhs.var = new_rhs;
+		      if (place != j)
+			{
+			  complex.ordered_remove (j);
+			  if (j < place)
+			    --place;
+			  if (place < complex.length ())
+			    {
+			      if (constraint_equal (*complex[place], *c))
+				{
+				  j--;
+				  continue;
+				}
+			      else
+				complex.safe_insert (place, c);
+			    }
+			  else
+			    complex.quick_push (c);
+			  if (place > j)
+			    {
+			      j--;
+			      continue;
+			    }
+			}
+		    }
+
+		  /* The only complex constraint that can change our
+		     solution to non-empty, given an empty solution,
+		     is a constraint where the lhs side is receiving
+		     some set from elsewhere.  */
+		  if (cvisited && cvisited->add (c))
+		    gcc_unreachable ();
+		  if (!solution_empty || c->lhs.type != DEREF)
+		    do_complex_constraint (graph, c, pts, &expanded_pts);
+		}
+	      if (cvisited)
+		{
+		  /* When checking, verify the order of constraints is
+		     maintained and each constraint is evaluated exactly
+		     once.  */
+		  for (unsigned j = 1; j < complex.length (); ++j)
+		    gcc_assert (constraint_less (complex[j-1], complex[j]));
+		  gcc_assert (cvisited->elements () == complex.length ());
+		  delete cvisited;
+		}
+	      BITMAP_FREE (expanded_pts);
+
+	      solution_empty = bitmap_empty_p (solution);
+
+	      if (!solution_empty)
+		{
+		  bitmap_iterator bi;
+		  unsigned eff_escaped_id = find (escaped_id);
+		  unsigned j;
+
+		  /* Propagate solution to all successors.  */
+		  unsigned to_remove = ~0U;
+		  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i],
+						0, j, bi)
+		    {
+		      if (to_remove != ~0U)
+			{
+			  bitmap_clear_bit (graph->succs[i], to_remove);
+			  to_remove = ~0U;
+			}
+		      unsigned int to = find (j);
+		      if (to != j)
+			{
+			  /* Update the succ graph, avoiding duplicate
+			     work.  */
+			  to_remove = j;
+			  if (! bitmap_set_bit (graph->succs[i], to))
+			    continue;
+			  /* We eventually end up processing 'to' twice
+			     as it is undefined whether bitmap iteration
+			     iterates over bits set during iteration.
+			     Play safe instead of doing tricks.  */
+			}
+		      /* Don't try to propagate to ourselves.  */
+		      if (to == i)
+			{
+			  to_remove = j;
+			  continue;
+			}
+		      /* Early node unification can lead to edges from
+			 escaped - remove them.  */
+		      if (i == eff_escaped_id)
+			{
+			  to_remove = j;
+			  if (bitmap_set_bit (get_varinfo (to)->solution,
+					      escaped_id))
+			    bitmap_set_bit (changed, to);
+			  continue;
+			}
+
+		      if (bitmap_ior_into (get_varinfo (to)->solution, pts))
+			bitmap_set_bit (changed, to);
+		    }
+		  if (to_remove != ~0U)
+		    bitmap_clear_bit (graph->succs[i], to_remove);
+		}
+	    }
+	}
+      bitmap_obstack_release (&iteration_obstack);
+    }
+
+  BITMAP_FREE (pts);
+  BITMAP_FREE (changed);
+  bitmap_obstack_release (&oldpta_obstack);
+}
+
+void
+delete_graph (void)
+{
+  unsigned int i;
+  for (i = 0; i < graph->size; i++)
+    graph->complex[i].release ();
+  free (graph->complex);
+
+  free (graph->succs);
+  free (graph->pe);
+  free (graph->pe_rep);
+  free (graph->indirect_cycles);
+  /* We are not doing free (graph->rep) since the representatives mapping is
+     needed outside of the solver too.  */
+  free (graph);
+}
+
+/* Remove the REF and ADDRESS edges from GRAPH, as well as all the
+   predecessor edges.  */
+
+static void
+remove_preds_and_fake_succs (constraint_graph_t graph)
+{
+  unsigned int i;
+
+  /* Clear the implicit ref and address nodes from the successor
+     lists.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      if (graph->succs[i])
+	bitmap_clear_range (graph->succs[i], FIRST_REF_NODE,
+			    FIRST_REF_NODE * 2);
+    }
+
+  /* Free the successor list for the non-ref nodes.  */
+  for (i = FIRST_REF_NODE + 1; i < graph->size; i++)
+    {
+      if (graph->succs[i])
+	BITMAP_FREE (graph->succs[i]);
+    }
+
+  /* Now reallocate the size of the successor list as, and blow away
+     the predecessor bitmaps.  */
+  graph->size = varmap.length ();
+  graph->succs = XRESIZEVEC (bitmap, graph->succs, graph->size);
+
+  free (graph->implicit_preds);
+  graph->implicit_preds = NULL;
+  free (graph->preds);
+  graph->preds = NULL;
+  bitmap_obstack_release (&predbitmap_obstack);
+}
+
+namespace pointer_analysis {
+
+/* Solve the constraint set.  The entry function of the solver.  */
+
+void
+solve_constraints (void)
+{
+  class scc_info *si;
+
+  /* Sort varinfos so that ones that cannot be pointed to are last.
+     This makes bitmaps more efficient.  */
+  unsigned int *map = XNEWVEC (unsigned int, varmap.length ());
+  for (unsigned i = 0; i < integer_id + 1; ++i)
+    map[i] = i;
+  /* Start with address-taken vars, followed by not address-taken vars
+     to move vars never appearing in the points-to solution bitmaps last.  */
+  unsigned j = integer_id + 1;
+  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
+    if (varmap[varmap[i]->head]->address_taken)
+      map[i] = j++;
+  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
+    if (! varmap[varmap[i]->head]->address_taken)
+      map[i] = j++;
+  /* Shuffle varmap according to map.  */
+  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
+    {
+      while (map[varmap[i]->id] != i)
+	std::swap (varmap[i], varmap[map[varmap[i]->id]]);
+      gcc_assert (bitmap_empty_p (varmap[i]->solution));
+      varmap[i]->id = i;
+      varmap[i]->next = map[varmap[i]->next];
+      varmap[i]->head = map[varmap[i]->head];
+    }
+  /* Finally rewrite constraints.  */
+  for (unsigned i = 0; i < constraints.length (); ++i)
+    {
+      constraints[i]->lhs.var = map[constraints[i]->lhs.var];
+      constraints[i]->rhs.var = map[constraints[i]->rhs.var];
+    }
+  free (map);
+
+  if (dump_file)
+    fprintf (dump_file,
+	     "\nCollapsing static cycles and doing variable "
+	     "substitution\n");
+
+  init_graph (varmap.length () * 2);
+
+  if (dump_file)
+    fprintf (dump_file, "Building predecessor graph\n");
+  build_pred_graph ();
+
+  if (dump_file)
+    fprintf (dump_file, "Detecting pointer and location "
+	     "equivalences\n");
+  si = perform_var_substitution (graph);
+
+  if (dump_file)
+    fprintf (dump_file, "Rewriting constraints and unifying "
+	     "variables\n");
+  rewrite_constraints (graph, si);
+
+  build_succ_graph ();
+
+  free_var_substitution_info (si);
+
+  /* Attach complex constraints to graph nodes.  */
+  move_complex_constraints (graph);
+
+  if (dump_file)
+    fprintf (dump_file, "Uniting pointer but not location equivalent "
+	     "variables\n");
+  unite_pointer_equivalences (graph);
+
+  if (dump_file)
+    fprintf (dump_file, "Finding indirect cycles\n");
+  find_indirect_cycles (graph);
+
+  /* Implicit nodes and predecessors are no longer necessary at this
+     point.  */
+  remove_preds_and_fake_succs (graph);
+
+  if (dump_file && (dump_flags & TDF_GRAPH))
+    {
+      fprintf (dump_file, "\n\n// The constraint graph before solve-graph "
+	       "in dot format:\n");
+      dump_constraint_graph (dump_file);
+      fprintf (dump_file, "\n\n");
+    }
+
+  if (dump_file)
+    fprintf (dump_file, "Solving graph\n");
+
+  solve_graph (graph);
+
+  if (dump_file && (dump_flags & TDF_GRAPH))
+    {
+      fprintf (dump_file, "\n\n// The constraint graph after solve-graph "
+	       "in dot format:\n");
+      dump_constraint_graph (dump_file);
+      fprintf (dump_file, "\n\n");
+    }
+
+  /* The mapping node -> representative is one of the outputs of the
+     solver.  */
+  union_find_compress_all ();
+  var_rep = graph->rep;
+
+  delete_graph ();
+}
+
+} // namespace pointer_analysis
diff --git a/gcc/pta-andersen.h b/gcc/pta-andersen.h
new file mode 100644
index 0000000..190a273
--- /dev/null
+++ b/gcc/pta-andersen.h
@@ -0,0 +1,31 @@
+/* Andersen-style solver for tree based points-to analysis
+   Copyright (C) 2005-2025 Free Software Foundation, Inc.
+   Contributed by Daniel Berlin <dberlin@dberlin.org>
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef PTA_ANDERSEN_H
+#define PTA_ANDERSEN_H
+
+namespace pointer_analysis {
+
+/* Solve the constraint set.  */
+void solve_constraints (void);
+
+} // namespace pointer_analysis
+
+#endif /* PTA_ANDERSEN_H  */
diff --git a/gcc/read-rtl-function.cc b/gcc/read-rtl-function.cc
index fb9c955..1f08c50 100644
--- a/gcc/read-rtl-function.cc
+++ b/gcc/read-rtl-function.cc
@@ -1065,7 +1065,10 @@ function_reader::read_rtx_operand_r (rtx x)
   if (regno == -1)
     fatal_at (loc, "unrecognized register: '%s'", name.string);
 
-  set_regno_raw (x, regno, 1);
+  int nregs = 1;
+  if (HARD_REGISTER_NUM_P (regno))
+    nregs = hard_regno_nregs (regno, GET_MODE (x));
+  set_regno_raw (x, regno, nregs);
 
   /* Consolidate singletons.  */
   x = consolidate_singletons (x);
diff --git a/gcc/recog.cc b/gcc/recog.cc
index edf22cb..67d7fa6 100644
--- a/gcc/recog.cc
+++ b/gcc/recog.cc
@@ -25,6 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "target.h"
 #include "rtl.h"
 #include "tree.h"
+#include "stmt.h"
 #include "cfghooks.h"
 #include "df.h"
 #include "memmodel.h"
@@ -2363,7 +2364,8 @@ asm_operand_ok (rtx op, const char *constraint, const char **constraints)
 	    {
 	    case CT_REGISTER:
 	      if (!result
-		  && reg_class_for_constraint (cn) != NO_REGS
+		  && (reg_class_for_constraint (cn) != NO_REGS
+		      || constraint[0] == '{')
 		  && GET_MODE (op) != BLKmode
 		  && register_operand (op, VOIDmode))
 		result = 1;
@@ -3304,6 +3306,13 @@ constrain_operands (int strict, alternative_mask alternatives)
 		  win = true;
 		break;
 
+	      case '{':
+		if ((REG_P (op) && HARD_REGISTER_P (op)
+		     && (int) REGNO (op) == decode_hard_reg_constraint (p))
+		    || !reload_completed)
+		  win = true;
+		break;
+
 	      default:
 		{
 		  enum constraint_num cn = lookup_constraint (p);
diff --git a/gcc/rtl-error.cc b/gcc/rtl-error.cc
index 06d060c..7391a2f 100644
--- a/gcc/rtl-error.cc
+++ b/gcc/rtl-error.cc
@@ -28,7 +28,7 @@ along with GCC; see the file COPYING3.  If not see
 
 static location_t location_for_asm (const rtx_insn *);
 static void diagnostic_for_asm (const rtx_insn *, const char *, va_list *,
-				diagnostic_t) ATTRIBUTE_GCC_DIAG(2,0);
+				enum diagnostics::kind) ATTRIBUTE_GCC_DIAG(2,0);
 
 /* Figure the location of the given INSN.  */
 static location_t
@@ -64,9 +64,9 @@ location_for_asm (const rtx_insn *insn)
    and each ASM_OPERANDS records its own source file and line.  */
 static void
 diagnostic_for_asm (const rtx_insn *insn, const char *msg, va_list *args_ptr,
-		    diagnostic_t kind)
+		    enum diagnostics::kind kind)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location richloc (line_table, location_for_asm (insn));
 
   diagnostic_set_info (&diagnostic, msg, args_ptr,
@@ -80,7 +80,7 @@ error_for_asm (const rtx_insn *insn, const char *gmsgid, ...)
   va_list ap;
 
   va_start (ap, gmsgid);
-  diagnostic_for_asm (insn, gmsgid, &ap, DK_ERROR);
+  diagnostic_for_asm (insn, gmsgid, &ap, diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -90,7 +90,7 @@ warning_for_asm (const rtx_insn *insn, const char *gmsgid, ...)
   va_list ap;
 
   va_start (ap, gmsgid);
-  diagnostic_for_asm (insn, gmsgid, &ap, DK_WARNING);
+  diagnostic_for_asm (insn, gmsgid, &ap, diagnostics::kind::warning);
   va_end (ap);
 }
 
diff --git a/gcc/rust/ChangeLog b/gcc/rust/ChangeLog
index 2a5219d..f103c7e 100644
--- a/gcc/rust/ChangeLog
+++ b/gcc/rust/ChangeLog
@@ -1,3 +1,37 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* resolve/rust-ice-finalizer.cc: Update usage of "diagnostic_info"
+	to explicitly refer to "diagnostics::diagnostic_info".
+	* resolve/rust-ice-finalizer.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* backend/rust-tree.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* backend/rust-tree.h: Likewise.
+	* resolve/rust-ast-resolve-expr.cc: Likewise.
+	* resolve/rust-ice-finalizer.cc: Likewise.
+	* resolve/rust-ice-finalizer.h: Likewise.
+	* resolve/rust-late-name-resolver-2.0.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* resolve/rust-ast-resolve-expr.cc: Update for diagnostic_text_finalizer
+	becoming diagnostics::text_finalizer.
+	* resolve/rust-late-name-resolver-2.0.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* resolve/rust-ice-finalizer.cc: Update for move of diagnostics
+	output formats into namespace "diagnostics" as "sinks".
+	* resolve/rust-ice-finalizer.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* rust-diagnostics.cc: Update #include for move of
+	"diagnostic-metadata.h" to "diagnostics/metadata.h", and update
+	for move of diagnostic_metadata to diagnostics::metadata.
+
 2025-06-26  Martin Jambor  <mjambor@suse.cz>
 
 	* checks/errors/borrowck/rust-borrow-checker-diagnostics.cc
diff --git a/gcc/rust/Make-lang.in b/gcc/rust/Make-lang.in
index 835e113..8cdf6c9 100644
--- a/gcc/rust/Make-lang.in
+++ b/gcc/rust/Make-lang.in
@@ -73,6 +73,9 @@ GRS_OBJS = \
     rust/rust-lex.o \
     rust/rust-cfg-parser.o \
     rust/rust-parse.o \
+    rust/rust-parse-impl-proc-macro.o \
+    rust/rust-parse-impl-macro.o \
+    rust/rust-parse-impl-lexer.o \
     rust/rust-ast.o \
     rust/rust-ast-formatting.o \
     rust/rust-path.o \
@@ -81,6 +84,7 @@ GRS_OBJS = \
     rust/rust-ast-dump.o \
     rust/rust-ast-collector.o \
     rust/rust-ast-visitor.o \
+    rust/rust-hir-visitor.o \
     rust/rust-hir-dump.o \
     rust/rust-session-manager.o \
     rust/rust-compile.o \
@@ -92,14 +96,15 @@ GRS_OBJS = \
     rust/rust-cfg-strip.o \
     rust/rust-expand-visitor.o \
     rust/rust-ast-builder.o \
-    rust/rust-ast-builder-type.o \
     rust/rust-derive.o \
+    rust/rust-derive-cmp-common.o \
     rust/rust-derive-clone.o \
     rust/rust-derive-copy.o \
     rust/rust-derive-debug.o \
     rust/rust-derive-default.o \
     rust/rust-derive-partial-eq.o \
     rust/rust-derive-eq.o \
+    rust/rust-derive-ord.o \
     rust/rust-derive-hash.o \
     rust/rust-proc-macro.o \
     rust/rust-macro-invoc-lexer.o \
@@ -113,6 +118,7 @@ GRS_OBJS = \
     rust/rust-macro-builtins-log-debug.o \
     rust/rust-macro-builtins-test-bench.o \
     rust/rust-macro-builtins-format-args.o \
+    rust/rust-macro-builtins-offset-of.o \
     rust/rust-macro-builtins-location.o \
     rust/rust-macro-builtins-include.o \
     rust/rust-token-tree-desugar.o \
@@ -123,7 +129,6 @@ GRS_OBJS = \
     rust/rust-keyword-values.o \
     rust/rust-abi.o \
     rust/rust-token-converter.o \
-    rust/rust-macro.o \
     rust/rust-ast-lower.o \
     rust/rust-ast-lower-base.o \
     rust/rust-ast-lower-pattern.o \
@@ -204,6 +209,7 @@ GRS_OBJS = \
     rust/rust-lint-marklive.o \
     rust/rust-lint-unused-var.o \
     rust/rust-readonly-check.o \
+    rust/rust-readonly-check2.o \
     rust/rust-hir-type-check-path.o \
     rust/rust-unsafe-checker.o \
     rust/rust-hir-pattern-analysis.o \
@@ -237,11 +243,16 @@ GRS_OBJS = \
     rust/rust-punycode.o \
     rust/rust-unwrap-segment.o \
     rust/rust-edition.o \
+    rust/rust-ggc.o \
 	rust/rust-expand-format-args.o \
 	rust/rust-lang-item.o \
 	rust/rust-collect-lang-items.o \
+	rust/rust-expression-yeast.o \
 	rust/rust-desugar-for-loops.o \
+	rust/rust-desugar-while-let.o \
 	rust/rust-desugar-question-mark.o \
+	rust/rust-desugar-apit.o \
+	rust/rust-desugar-try-block.o \
     $(END)
 # removed object files from here
 
diff --git a/gcc/rust/ast/rust-ast-builder-type.cc b/gcc/rust/ast/rust-ast-builder-type.cc
deleted file mode 100644
index 13126b4..0000000
--- a/gcc/rust/ast/rust-ast-builder-type.cc
+++ /dev/null
@@ -1,163 +0,0 @@
-// Copyright (C) 2020-2024 Free Software Foundation, Inc.
-
-// This file is part of GCC.
-
-// GCC is free software; you can redistribute it and/or modify it under
-// the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3, or (at your option) any later
-// version.
-
-// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-// WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-// for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with GCC; see the file COPYING3.  If not see
-// <http://www.gnu.org/licenses/>.
-
-#include "rust-ast-builder-type.h"
-#include "rust-ast-builder.h"
-#include "rust-ast-full.h"
-#include "rust-common.h"
-
-namespace Rust {
-namespace AST {
-
-ASTTypeBuilder::ASTTypeBuilder () : translated (nullptr) {}
-
-Type *
-ASTTypeBuilder::build (Type &type)
-{
-  ASTTypeBuilder builder;
-  type.accept_vis (builder);
-  rust_assert (builder.translated != nullptr);
-  return builder.translated;
-}
-
-void
-ASTTypeBuilder::visit (BareFunctionType &fntype)
-{
-  /* TODO */
-}
-
-void
-ASTTypeBuilder::visit (TupleType &tuple)
-{
-  std::vector<std::unique_ptr<Type> > elems;
-  for (auto &elem : tuple.get_elems ())
-    {
-      Type *t = ASTTypeBuilder::build (*elem.get ());
-      std::unique_ptr<Type> ty (t);
-      elems.push_back (std::move (ty));
-    }
-  translated = new TupleType (std::move (elems), tuple.get_locus ());
-}
-
-void
-ASTTypeBuilder::visit (TypePath &path)
-{
-  std::vector<std::unique_ptr<TypePathSegment> > segments;
-  for (auto &seg : path.get_segments ())
-    {
-      switch (seg->get_type ())
-	{
-	  case TypePathSegment::REG: {
-	    const TypePathSegment &segment
-	      = (const TypePathSegment &) (*seg.get ());
-	    TypePathSegment *s
-	      = new TypePathSegment (segment.get_ident_segment (),
-				     segment.get_separating_scope_resolution (),
-				     segment.get_locus ());
-	    std::unique_ptr<TypePathSegment> sg (s);
-	    segments.push_back (std::move (sg));
-	  }
-	  break;
-
-	  case TypePathSegment::GENERIC: {
-	    TypePathSegmentGeneric &generic
-	      = (TypePathSegmentGeneric &) (*seg.get ());
-
-	    GenericArgs args
-	      = Builder::new_generic_args (generic.get_generic_args ());
-	    TypePathSegmentGeneric *s
-	      = new TypePathSegmentGeneric (generic.get_ident_segment (), false,
-					    std::move (args),
-					    generic.get_locus ());
-	    std::unique_ptr<TypePathSegment> sg (s);
-	    segments.push_back (std::move (sg));
-	  }
-	  break;
-
-	  case TypePathSegment::FUNCTION: {
-	    rust_unreachable ();
-	    // TODO
-	    // const TypePathSegmentFunction &fn
-	    //   = (const TypePathSegmentFunction &) (*seg.get ());
-	  }
-	  break;
-	}
-    }
-
-  translated = new TypePath (std::move (segments), path.get_locus (),
-			     path.has_opening_scope_resolution_op ());
-}
-
-void
-ASTTypeBuilder::visit (QualifiedPathInType &path)
-{
-  /* TODO */
-}
-
-void
-ASTTypeBuilder::visit (ArrayType &type)
-{
-  /* TODO */
-}
-
-void
-ASTTypeBuilder::visit (ReferenceType &type)
-{
-  /* TODO */
-}
-
-void
-ASTTypeBuilder::visit (RawPointerType &type)
-{
-  /* TODO */
-}
-
-void
-ASTTypeBuilder::visit (SliceType &type)
-{
-  Type *t = ASTTypeBuilder::build (type.get_elem_type ());
-  std::unique_ptr<Type> ty (t);
-  translated = new SliceType (std::move (ty), type.get_locus ());
-}
-
-void
-ASTTypeBuilder::visit (InferredType &type)
-{
-  translated = new InferredType (type.get_locus ());
-}
-
-void
-ASTTypeBuilder::visit (NeverType &type)
-{
-  translated = new NeverType (type.get_locus ());
-}
-
-void
-ASTTypeBuilder::visit (TraitObjectTypeOneBound &type)
-{
-  /* TODO */
-}
-
-void
-ASTTypeBuilder::visit (TraitObjectType &type)
-{
-  /* TODO */
-}
-
-} // namespace AST
-} // namespace Rust
diff --git a/gcc/rust/ast/rust-ast-builder-type.h b/gcc/rust/ast/rust-ast-builder-type.h
deleted file mode 100644
index b67ae3b..0000000
--- a/gcc/rust/ast/rust-ast-builder-type.h
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright (C) 2020-2024 Free Software Foundation, Inc.
-
-// This file is part of GCC.
-
-// GCC is free software; you can redistribute it and/or modify it under
-// the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3, or (at your option) any later
-// version.
-
-// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-// WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-// for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with GCC; see the file COPYING3.  If not see
-// <http://www.gnu.org/licenses/>.
-
-#ifndef RUST_AST_BUILDER_TYPE
-#define RUST_AST_BUILDER_TYPE
-
-#include "rust-ast-visitor.h"
-
-namespace Rust {
-namespace AST {
-
-class ASTTypeBuilder : public DefaultASTVisitor
-{
-protected:
-  using DefaultASTVisitor::visit;
-
-public:
-  static Type *build (Type &type);
-
-  void visit (BareFunctionType &fntype) override;
-  void visit (TupleType &tuple) override;
-  void visit (TypePath &path) override;
-  void visit (QualifiedPathInType &path) override;
-  void visit (ArrayType &type) override;
-  void visit (ReferenceType &type) override;
-  void visit (RawPointerType &type) override;
-  void visit (SliceType &type) override;
-  void visit (InferredType &type) override;
-  void visit (NeverType &type) override;
-  void visit (TraitObjectTypeOneBound &type) override;
-  void visit (TraitObjectType &type) override;
-
-private:
-  ASTTypeBuilder ();
-
-  Type *translated;
-};
-
-} // namespace AST
-} // namespace Rust
-
-#endif // RUST_AST_BUILDER_TYPE
diff --git a/gcc/rust/ast/rust-ast-builder.cc b/gcc/rust/ast/rust-ast-builder.cc
index 08c52b1..ed10ce7 100644
--- a/gcc/rust/ast/rust-ast-builder.cc
+++ b/gcc/rust/ast/rust-ast-builder.cc
@@ -18,7 +18,6 @@
 
 #include "rust-ast-builder.h"
 #include "optional.h"
-#include "rust-ast-builder-type.h"
 #include "rust-ast.h"
 #include "rust-common.h"
 #include "rust-expr.h"
@@ -29,7 +28,6 @@
 #include "rust-pattern.h"
 #include "rust-system.h"
 #include "rust-token.h"
-#include <memory>
 
 namespace Rust {
 namespace AST {
@@ -332,6 +330,12 @@ Builder::block () const
 }
 
 std::unique_ptr<BlockExpr>
+Builder::block (std::unique_ptr<Expr> &&tail_expr) const
+{
+  return block (tl::nullopt, std::move (tail_expr));
+}
+
+std::unique_ptr<BlockExpr>
 Builder::block (std::vector<std::unique_ptr<Stmt>> &&stmts,
 		std::unique_ptr<Expr> &&tail_expr) const
 {
@@ -442,6 +446,14 @@ Builder::field_access (std::unique_ptr<Expr> &&instance,
     new FieldAccessExpr (std::move (instance), field, {}, loc));
 }
 
+std::unique_ptr<StructPatternField>
+Builder::struct_pattern_ident_pattern (std::string field_name,
+				       std::unique_ptr<Pattern> &&pattern)
+{
+  return std::make_unique<StructPatternFieldIdentPat> (
+    field_name, std::move (pattern), std::vector<Attribute> (), loc);
+}
+
 std::unique_ptr<Pattern>
 Builder::wildcard () const
 {
@@ -482,9 +494,14 @@ MatchCase
 Builder::match_case (std::unique_ptr<Pattern> &&pattern,
 		     std::unique_ptr<Expr> &&expr)
 {
-  return MatchCase (match_arm (std::move (pattern)), std::move (expr));
+  return match_case (match_arm (std::move (pattern)), std::move (expr));
 }
 
+MatchCase
+Builder::match_case (MatchArm &&arm, std::unique_ptr<Expr> &&expr)
+{
+  return MatchCase (std::move (arm), std::move (expr));
+}
 std::unique_ptr<Expr>
 Builder::loop (std::vector<std::unique_ptr<Stmt>> &&stmts)
 {
@@ -523,11 +540,14 @@ Builder::generic_type_param (
 				      std::vector<Attribute> ());
 }
 
-std::unique_ptr<Type>
-Builder::new_type (Type &type)
+std::unique_ptr<Stmt>
+Builder::discriminant_value (std::string binding_name, std::string instance)
 {
-  Type *t = ASTTypeBuilder::build (type);
-  return std::unique_ptr<Type> (t);
+  auto intrinsic = ptrify (
+    path_in_expression ({"core", "intrinsics", "discriminant_value"}, true));
+
+  return let (identifier_pattern (binding_name), nullptr,
+	      call (std::move (intrinsic), identifier (instance)));
 }
 
 std::unique_ptr<GenericParam>
@@ -547,6 +567,16 @@ Builder::new_lifetime_param (LifetimeParam &param)
 }
 
 std::unique_ptr<GenericParam>
+Builder::new_const_param (ConstGenericParam &param) const
+{
+  return std::make_unique<ConstGenericParam> (param.get_name (),
+					      param.get_type ().clone_type (),
+					      param.get_default_value (),
+					      param.get_outer_attrs (),
+					      param.get_locus ());
+}
+
+std::unique_ptr<GenericParam>
 Builder::new_type_param (
   TypeParam &param, std::vector<std::unique_ptr<TypeParamBound>> extra_bounds)
 {
@@ -557,7 +587,7 @@ Builder::new_type_param (
   std::unique_ptr<Type> type = nullptr;
 
   if (param.has_type ())
-    type = new_type (param.get_type ());
+    type = param.get_type ().reconstruct ();
 
   for (auto &&extra_bound : extra_bounds)
     type_param_bounds.emplace_back (std::move (extra_bound));
@@ -566,7 +596,8 @@ Builder::new_type_param (
     {
       switch (b->get_bound_type ())
 	{
-	  case TypeParamBound::TypeParamBoundType::TRAIT: {
+	case TypeParamBound::TypeParamBoundType::TRAIT:
+	  {
 	    const TraitBound &tb = (const TraitBound &) *b.get ();
 	    const TypePath &path = tb.get_type_path ();
 
@@ -591,7 +622,8 @@ Builder::new_type_param (
 	      {
 		switch (seg->get_type ())
 		  {
-		    case TypePathSegment::REG: {
+		  case TypePathSegment::REG:
+		    {
 		      const TypePathSegment &segment
 			= (const TypePathSegment &) (*seg.get ());
 		      TypePathSegment *s = new TypePathSegment (
@@ -603,7 +635,8 @@ Builder::new_type_param (
 		    }
 		    break;
 
-		    case TypePathSegment::GENERIC: {
+		  case TypePathSegment::GENERIC:
+		    {
 		      TypePathSegmentGeneric &generic
 			= (TypePathSegmentGeneric &) (*seg.get ());
 
@@ -617,7 +650,8 @@ Builder::new_type_param (
 		    }
 		    break;
 
-		    case TypePathSegment::FUNCTION: {
+		  case TypePathSegment::FUNCTION:
+		    {
 		      rust_unreachable ();
 		      // TODO
 		      // const TypePathSegmentFunction &fn
@@ -639,7 +673,8 @@ Builder::new_type_param (
 	  }
 	  break;
 
-	  case TypeParamBound::TypeParamBoundType::LIFETIME: {
+	case TypeParamBound::TypeParamBoundType::LIFETIME:
+	  {
 	    const Lifetime &l = (const Lifetime &) *b.get ();
 
 	    auto bl = new Lifetime (l.get_lifetime_type (),
@@ -682,7 +717,7 @@ Builder::new_generic_args (GenericArgs &args)
   for (auto &binding : args.get_binding_args ())
     {
       Type &t = *binding.get_type_ptr ().get ();
-      std::unique_ptr<Type> ty = new_type (t);
+      std::unique_ptr<Type> ty = t.reconstruct ();
       GenericArgsBinding b (binding.get_identifier (), std::move (ty),
 			    binding.get_locus ());
       binding_args.push_back (std::move (b));
@@ -690,19 +725,25 @@ Builder::new_generic_args (GenericArgs &args)
 
   for (auto &arg : args.get_generic_args ())
     {
+      tl::optional<GenericArg> new_arg = tl::nullopt;
+
       switch (arg.get_kind ())
 	{
-	  case GenericArg::Kind::Type: {
-	    std::unique_ptr<Type> ty = new_type (arg.get_type ());
-	    GenericArg arg = GenericArg::create_type (std::move (ty));
-	  }
+	case GenericArg::Kind::Type:
+	  new_arg = GenericArg::create_type (arg.get_type ().reconstruct ());
 	  break;
-
-	default:
-	  // FIXME
-	  rust_unreachable ();
+	case GenericArg::Kind::Either:
+	  new_arg
+	    = GenericArg::create_ambiguous (arg.get_path (), arg.get_locus ());
+	  break;
+	case GenericArg::Kind::Const:
+	  new_arg
+	    = GenericArg::create_const (arg.get_expression ().clone_expr ());
+	  // FIXME: Use `reconstruct()` here, not `clone_expr()`
 	  break;
 	}
+
+      generic_args.emplace_back (*new_arg);
     }
 
   return GenericArgs (std::move (lifetime_args), std::move (generic_args),
diff --git a/gcc/rust/ast/rust-ast-builder.h b/gcc/rust/ast/rust-ast-builder.h
index 41ce118..843bab8 100644
--- a/gcc/rust/ast/rust-ast-builder.h
+++ b/gcc/rust/ast/rust-ast-builder.h
@@ -24,6 +24,7 @@
 #include "rust-ast.h"
 #include "rust-item.h"
 #include "rust-operators.h"
+#include <initializer_list>
 
 namespace Rust {
 namespace AST {
@@ -51,6 +52,19 @@ vec (std::unique_ptr<T> &&t1, std::unique_ptr<T> &&t2)
   return v;
 }
 
+template <typename T>
+std::vector<std::unique_ptr<T>>
+vec (std::unique_ptr<T> &&t1, std::unique_ptr<T> &&t2, std::unique_ptr<T> &&t3)
+{
+  auto v = std::vector<std::unique_ptr<T>> ();
+
+  v.emplace_back (std::move (t1));
+  v.emplace_back (std::move (t2));
+  v.emplace_back (std::move (t3));
+
+  return v;
+}
+
 /* Pointer-ify something */
 template <typename T>
 static std::unique_ptr<T>
@@ -117,6 +131,9 @@ public:
   /* Create an empty block */
   std::unique_ptr<BlockExpr> block () const;
 
+  /* Create a block with just a tail expression */
+  std::unique_ptr<BlockExpr> block (std::unique_ptr<Expr> &&tail_expr) const;
+
   /* Create an early return expression with an optional expression */
   std::unique_ptr<Expr> return_expr (std::unique_ptr<Expr> &&to_return
 				     = nullptr);
@@ -254,6 +271,10 @@ public:
   std::unique_ptr<Expr> field_access (std::unique_ptr<Expr> &&instance,
 				      std::string field) const;
 
+  std::unique_ptr<StructPatternField>
+  struct_pattern_ident_pattern (std::string field_name,
+				std::unique_ptr<Pattern> &&pattern);
+
   /* Create a wildcard pattern (`_`) */
   std::unique_ptr<Pattern> wildcard () const;
   /* Create a reference pattern (`&pattern`) */
@@ -268,6 +289,7 @@ public:
   MatchArm match_arm (std::unique_ptr<Pattern> &&pattern);
   MatchCase match_case (std::unique_ptr<Pattern> &&pattern,
 			std::unique_ptr<Expr> &&expr);
+  MatchCase match_case (MatchArm &&arm, std::unique_ptr<Expr> &&expr);
 
   /* Create a loop expression */
   std::unique_ptr<Expr> loop (std::vector<std::unique_ptr<Stmt>> &&stmts);
@@ -285,11 +307,20 @@ public:
 		      std::vector<std::unique_ptr<TypeParamBound>> &&bounds,
 		      std::unique_ptr<Type> &&type = nullptr);
 
-  static std::unique_ptr<Type> new_type (Type &type);
+  /**
+   * Create a let statement with the discriminant value of a given enum
+   * instance. This helper exists since it is a common operation in a lot of the
+   * derive implementations, and it sucks to repeat all the steps every time.
+   */
+  std::unique_ptr<Stmt> discriminant_value (std::string binding_name,
+					    std::string instance = "self");
 
   static std::unique_ptr<GenericParam>
   new_lifetime_param (LifetimeParam &param);
 
+  std::unique_ptr<GenericParam>
+  new_const_param (ConstGenericParam &param) const;
+
   static std::unique_ptr<GenericParam> new_type_param (
     TypeParam &param,
     std::vector<std::unique_ptr<TypeParamBound>> extra_trait_bounds = {});
@@ -298,11 +329,13 @@ public:
 
   static GenericArgs new_generic_args (GenericArgs &args);
 
-private:
-  /**
-   * Location of the generated AST nodes
-   */
+  /* Location of the generated AST nodes */
   location_t loc;
+
+private:
+  /* Some constexpr helpers for some of the builders */
+  static constexpr std::initializer_list<const char *> discriminant_value_path
+    = {"core", "intrinsics", "discriminant_value"};
 };
 
 } // namespace AST
diff --git a/gcc/rust/ast/rust-ast-collector.cc b/gcc/rust/ast/rust-ast-collector.cc
index c850e96..3d9ea78 100644
--- a/gcc/rust/ast/rust-ast-collector.cc
+++ b/gcc/rust/ast/rust-ast-collector.cc
@@ -18,6 +18,7 @@
 
 #include "rust-ast-collector.h"
 #include "rust-ast.h"
+#include "rust-builtin-ast-nodes.h"
 #include "rust-diagnostics.h"
 #include "rust-expr.h"
 #include "rust-item.h"
@@ -154,20 +155,24 @@ TokenCollector::visit (Attribute &attrib)
     {
       switch (attrib.get_attr_input ().get_attr_input_type ())
 	{
-	  case AST::AttrInput::AttrInputType::LITERAL: {
+	case AST::AttrInput::AttrInputType::LITERAL:
+	  {
 	    visit (static_cast<AttrInputLiteral &> (attrib.get_attr_input ()));
 	    break;
 	  }
-	  case AST::AttrInput::AttrInputType::MACRO: {
+	case AST::AttrInput::AttrInputType::MACRO:
+	  {
 	    visit (static_cast<AttrInputMacro &> (attrib.get_attr_input ()));
 	    break;
 	  }
-	  case AST::AttrInput::AttrInputType::META_ITEM: {
+	case AST::AttrInput::AttrInputType::META_ITEM:
+	  {
 	    visit (static_cast<AttrInputMetaItemContainer &> (
 	      attrib.get_attr_input ()));
 	    break;
 	  }
-	  case AST::AttrInput::AttrInputType::TOKEN_TREE: {
+	case AST::AttrInput::AttrInputType::TOKEN_TREE:
+	  {
 	    visit (static_cast<DelimTokenTree &> (attrib.get_attr_input ()));
 	    break;
 	  }
@@ -634,7 +639,8 @@ TokenCollector::visit (GenericArg &arg)
     case GenericArg::Kind::Type:
       visit (arg.get_type ());
       break;
-      case GenericArg::Kind::Either: {
+    case GenericArg::Kind::Either:
+      {
 	auto path = arg.get_path ();
 	push (Rust::Token::make_identifier (UNDEF_LOCATION, std::move (path)));
       }
@@ -789,7 +795,8 @@ TokenCollector::visit (Literal &lit, location_t locus)
       push (Rust::Token::make_float (locus, std::move (value),
 				     lit.get_type_hint ()));
       break;
-      case Literal::LitType::BOOL: {
+    case Literal::LitType::BOOL:
+      {
 	if (value == Values::Keywords::FALSE_LITERAL)
 	  push (Rust::Token::make (FALSE_LITERAL, locus));
 	else if (value == Values::Keywords::TRUE_LITERAL)
@@ -833,13 +840,13 @@ TokenCollector::visit (MetaItemLitExpr &item)
 }
 
 void
-TokenCollector::visit (MetaItemPathLit &item)
+TokenCollector::visit (MetaItemPathExpr &item)
 {
-  auto path = item.get_path ();
-  auto lit = item.get_literal ();
+  auto &path = item.get_path ();
+  auto &expr = item.get_expr ();
   visit (path);
-  push (Rust::Token::make (COLON, item.get_locus ()));
-  visit (lit);
+  push (Rust::Token::make (EQUAL, item.get_locus ()));
+  visit (expr);
 }
 
 void
@@ -864,7 +871,8 @@ TokenCollector::visit (BorrowExpr &expr)
 	push (Rust::Token::make (MUT, UNDEF_LOCATION));
     }
 
-  visit (expr.get_borrowed_expr ());
+  if (expr.has_borrow_expr ())
+    visit (expr.get_borrowed_expr ());
 }
 
 void
@@ -1264,12 +1272,34 @@ TokenCollector::visit (BlockExpr &expr)
 }
 
 void
+TokenCollector::visit (AnonConst &expr)
+{
+  if (!expr.is_deferred ())
+    {
+      visit (expr.get_inner_expr ());
+      return;
+    }
+
+  push (Rust::Token::make_string (expr.get_locus (), "_"));
+}
+
+void
+TokenCollector::visit (ConstBlock &expr)
+{
+  push (Rust::Token::make (CONST, expr.get_locus ()));
+
+  // The inner expression is already a block expr, so we don't need to add
+  // curlies
+  visit (expr.get_const_expr ());
+}
+
+void
 TokenCollector::visit (ClosureExprInnerTyped &expr)
 {
   visit_closure_common (expr);
   push (Rust::Token::make (RETURN_TYPE, expr.get_locus ()));
   visit (expr.get_return_type ());
-  visit (expr.get_definition_block ());
+  visit (expr.get_definition_expr ());
 }
 
 void
@@ -1349,6 +1379,13 @@ TokenCollector::visit (ReturnExpr &expr)
 }
 
 void
+TokenCollector::visit (TryExpr &expr)
+{
+  push (Rust::Token::make (TRY, expr.get_locus ()));
+  visit (expr.get_block_expr ());
+}
+
+void
 TokenCollector::visit (UnsafeBlockExpr &expr)
 {
   push (Rust::Token::make (UNSAFE, expr.get_locus ()));
@@ -1518,7 +1555,95 @@ TokenCollector::visit (AsyncBlockExpr &expr)
 
 void
 TokenCollector::visit (InlineAsm &expr)
-{}
+{
+  push (Rust::Token::make_identifier (expr.get_locus (), "asm"));
+  push (Rust::Token::make (EXCLAM, expr.get_locus ()));
+  push (Rust::Token::make (LEFT_PAREN, expr.get_locus ()));
+
+  for (auto &template_str : expr.get_template_strs ())
+    push (Rust::Token::make_string (template_str.get_locus (),
+				    std::move (template_str.symbol)));
+
+  push (Rust::Token::make (COLON, expr.get_locus ()));
+
+  for (auto &operand : expr.get_operands ())
+    {
+      using RegisterType = AST::InlineAsmOperand::RegisterType;
+      switch (operand.get_register_type ())
+	{
+	case RegisterType::In:
+	  {
+	    visit (operand.get_in ().expr);
+	    break;
+	  }
+	case RegisterType::Out:
+	  {
+	    visit (operand.get_out ().expr);
+	    break;
+	  }
+	case RegisterType::InOut:
+	  {
+	    visit (operand.get_in_out ().expr);
+	    break;
+	  }
+	case RegisterType::SplitInOut:
+	  {
+	    auto split = operand.get_split_in_out ();
+	    visit (split.in_expr);
+	    visit (split.out_expr);
+	    break;
+	  }
+	case RegisterType::Const:
+	  {
+	    visit (operand.get_const ().anon_const.get_inner_expr ());
+	    break;
+	  }
+	case RegisterType::Sym:
+	  {
+	    visit (operand.get_sym ().expr);
+	    break;
+	  }
+	case RegisterType::Label:
+	  {
+	    visit (operand.get_label ().expr);
+	    break;
+	  }
+	}
+      push (Rust::Token::make (COMMA, expr.get_locus ()));
+    }
+  push (Rust::Token::make (COLON, expr.get_locus ()));
+
+  for (auto &clobber : expr.get_clobber_abi ())
+    {
+      push (Rust::Token::make_string (expr.get_locus (),
+				      std::move (clobber.symbol)));
+      push (Rust::Token::make (COMMA, expr.get_locus ()));
+    }
+  push (Rust::Token::make (COLON, expr.get_locus ()));
+
+  for (auto it = expr.named_args.begin (); it != expr.named_args.end (); ++it)
+    {
+      auto &arg = *it;
+      push (
+	Rust::Token::make_identifier (expr.get_locus (), arg.first.c_str ()));
+      push (Rust::Token::make (EQUAL, expr.get_locus ()));
+      push (Rust::Token::make_identifier (expr.get_locus (),
+					  std::to_string (arg.second)));
+
+      push (Rust::Token::make (COMMA, expr.get_locus ()));
+    }
+
+  push (Rust::Token::make (COLON, expr.get_locus ()));
+
+  for (auto &option : expr.get_options ())
+    {
+      push (Rust::Token::make_identifier (
+	expr.get_locus (), InlineAsm::option_to_string (option).c_str ()));
+      push (Rust::Token::make (COMMA, expr.get_locus ()));
+    }
+
+  push (Rust::Token::make (RIGHT_PAREN, expr.get_locus ()));
+}
 
 void
 TokenCollector::visit (LlvmInlineAsm &expr)
@@ -1695,7 +1820,8 @@ TokenCollector::visit (UseTreeGlob &use_tree)
 {
   switch (use_tree.get_glob_type ())
     {
-      case UseTreeGlob::PathType::PATH_PREFIXED: {
+    case UseTreeGlob::PathType::PATH_PREFIXED:
+      {
 	auto path = use_tree.get_path ();
 	visit (path);
 	push (Rust::Token::make (SCOPE_RESOLUTION, UNDEF_LOCATION));
@@ -1715,7 +1841,8 @@ TokenCollector::visit (UseTreeList &use_tree)
 {
   switch (use_tree.get_path_type ())
     {
-      case UseTreeList::PathType::PATH_PREFIXED: {
+    case UseTreeList::PathType::PATH_PREFIXED:
+      {
 	auto path = use_tree.get_path ();
 	visit (path);
 	push (Rust::Token::make (SCOPE_RESOLUTION, UNDEF_LOCATION));
@@ -1743,7 +1870,8 @@ TokenCollector::visit (UseTreeRebind &use_tree)
   visit (path);
   switch (use_tree.get_new_bind_type ())
     {
-      case UseTreeRebind::NewBindType::IDENTIFIER: {
+    case UseTreeRebind::NewBindType::IDENTIFIER:
+      {
 	push (Rust::Token::make (AS, UNDEF_LOCATION));
 	auto id = use_tree.get_identifier ().as_string ();
 	push (
@@ -1964,8 +2092,7 @@ TokenCollector::visit (ConstantItem &item)
     }
   else
     {
-      auto id = item.get_identifier ();
-      push (Rust::Token::make_identifier (UNDEF_LOCATION, std::move (id)));
+      push (Rust::Token::make_identifier (item.get_identifier ()));
     }
   push (Rust::Token::make (COLON, UNDEF_LOCATION));
   visit (item.get_type ());
@@ -2370,10 +2497,10 @@ TokenCollector::visit (IdentifierPattern &pattern)
   auto id = pattern.get_ident ().as_string ();
   push (Rust::Token::make_identifier (UNDEF_LOCATION, std::move (id)));
 
-  if (pattern.has_pattern_to_bind ())
+  if (pattern.has_subpattern ())
     {
       push (Rust::Token::make (PATTERN_BIND, UNDEF_LOCATION));
-      visit (pattern.get_pattern_to_bind ());
+      visit (pattern.get_subpattern ());
     }
 }
 
@@ -2591,10 +2718,34 @@ TokenCollector::visit (GroupedPattern &pattern)
 }
 
 void
+TokenCollector::visit (SlicePatternItemsNoRest &items)
+{
+  visit_items_joined_by_separator (items.get_patterns (), COMMA);
+}
+
+void
+TokenCollector::visit (SlicePatternItemsHasRest &items)
+{
+  if (!items.get_lower_patterns ().empty ())
+    {
+      visit_items_joined_by_separator (items.get_lower_patterns (), COMMA);
+      push (Rust::Token::make (COMMA, UNDEF_LOCATION));
+    }
+
+  push (Rust::Token::make (DOT_DOT, UNDEF_LOCATION));
+
+  if (!items.get_upper_patterns ().empty ())
+    {
+      push (Rust::Token::make (COMMA, UNDEF_LOCATION));
+      visit_items_joined_by_separator (items.get_upper_patterns (), COMMA);
+    }
+}
+
+void
 TokenCollector::visit (SlicePattern &pattern)
 {
   push (Rust::Token::make (LEFT_SQUARE, pattern.get_locus ()));
-  visit_items_joined_by_separator (pattern.get_items (), COMMA);
+  visit (pattern.get_items ());
   push (Rust::Token::make (RIGHT_SQUARE, UNDEF_LOCATION));
 }
 
@@ -2862,5 +3013,23 @@ TokenCollector::visit (AST::FormatArgs &fmt)
 		 __FILE__, __LINE__);
 }
 
+void
+TokenCollector::visit (AST::OffsetOf &offset_of)
+{
+  auto loc = offset_of.get_locus ();
+
+  push (Rust::Token::make_identifier (loc, "offset_of"));
+  push (Rust::Token::make (EXCLAM, loc));
+  push (Rust::Token::make (LEFT_PAREN, loc));
+
+  visit (offset_of.get_type ());
+
+  push (Rust::Token::make (COMMA, loc));
+
+  push (Rust::Token::make_identifier (offset_of.get_field ()));
+
+  push (Rust::Token::make (RIGHT_PAREN, loc));
+}
+
 } // namespace AST
 } // namespace Rust
diff --git a/gcc/rust/ast/rust-ast-collector.h b/gcc/rust/ast/rust-ast-collector.h
index f45e3cc..d3ab18a 100644
--- a/gcc/rust/ast/rust-ast-collector.h
+++ b/gcc/rust/ast/rust-ast-collector.h
@@ -246,7 +246,7 @@ public:
   void visit (AttrInputLiteral &attr_input);
   void visit (AttrInputMacro &attr_input);
   void visit (MetaItemLitExpr &meta_item);
-  void visit (MetaItemPathLit &meta_item);
+  void visit (MetaItemPathExpr &meta_item);
   void visit (BorrowExpr &expr);
   void visit (DereferenceExpr &expr);
   void visit (ErrorPropagationExpr &expr);
@@ -277,6 +277,8 @@ public:
   void visit (ClosureParam &param);
   void visit (ClosureExprInner &expr);
   void visit (BlockExpr &expr);
+  void visit (AnonConst &expr);
+  void visit (ConstBlock &expr);
   void visit (ClosureExprInnerTyped &expr);
   void visit (ContinueExpr &expr);
   void visit (BreakExpr &expr);
@@ -287,6 +289,7 @@ public:
   void visit (RangeFromToInclExpr &expr);
   void visit (RangeToInclExpr &expr);
   void visit (ReturnExpr &expr);
+  void visit (TryExpr &expr);
   void visit (BoxExpr &expr);
   void visit (UnsafeBlockExpr &expr);
   void visit (LoopExpr &expr);
@@ -375,6 +378,8 @@ public:
   void visit (TuplePatternItemsRanged &tuple_items);
   void visit (TuplePattern &pattern);
   void visit (GroupedPattern &pattern);
+  void visit (SlicePatternItemsNoRest &items);
+  void visit (SlicePatternItemsHasRest &items);
   void visit (SlicePattern &pattern);
   void visit (AltPattern &pattern);
 
@@ -400,6 +405,7 @@ public:
   void visit (BareFunctionType &type);
 
   void visit (FormatArgs &fmt);
+  void visit (OffsetOf &offset_of);
 };
 } // namespace AST
 
diff --git a/gcc/rust/ast/rust-ast-dump.h b/gcc/rust/ast/rust-ast-dump.h
index 02c99b7..0c3875e 100644
--- a/gcc/rust/ast/rust-ast-dump.h
+++ b/gcc/rust/ast/rust-ast-dump.h
@@ -49,7 +49,8 @@ public:
       {
 	switch (item.get_kind ())
 	  {
-	    case AST::CollectItem::Kind::Token: {
+	  case AST::CollectItem::Kind::Token:
+	    {
 	      TokenPtr current = item.get_token ();
 	      if (require_spacing (previous, current))
 		stream << " ";
@@ -90,7 +91,6 @@ private:
 } // namespace Rust
 
 // In the global namespace to make it easier to call from debugger
-void
-debug (Rust::AST::Visitable &v);
+void debug (Rust::AST::Visitable &v);
 
 #endif // !RUST_AST_DUMP_H
diff --git a/gcc/rust/ast/rust-ast-formatting.h b/gcc/rust/ast/rust-ast-formatting.h
index 3dfabbc..aace93f 100644
--- a/gcc/rust/ast/rust-ast-formatting.h
+++ b/gcc/rust/ast/rust-ast-formatting.h
@@ -35,23 +35,18 @@ enum AttrMode
   INNER
 };
 
-std::string
-indent_spaces (enum indent_mode mode);
+std::string indent_spaces (enum indent_mode mode);
 
 // Gets a string in a certain delim type.
-std::string
-get_string_in_delims (std::string str_input, DelimType delim_type);
+std::string get_string_in_delims (std::string str_input, DelimType delim_type);
 
-std::string
-get_mode_dump_desc (AttrMode mode);
+std::string get_mode_dump_desc (AttrMode mode);
 
 // Adds lines below adding attributes
-std::string
-append_attributes (std::vector<Attribute> attrs, AttrMode mode);
+std::string append_attributes (std::vector<Attribute> attrs, AttrMode mode);
 
 // Removes the beginning and end quotes of a quoted string.
-std::string
-unquote_string (std::string input);
+std::string unquote_string (std::string input);
 
 } // namespace AST
 } // namespace Rust
diff --git a/gcc/rust/ast/rust-ast-full-decls.h b/gcc/rust/ast/rust-ast-full-decls.h
index 9359248..09706ce 100644
--- a/gcc/rust/ast/rust-ast-full-decls.h
+++ b/gcc/rust/ast/rust-ast-full-decls.h
@@ -78,7 +78,7 @@ class LiteralExpr;
 class AttrInputLiteral;
 class AttrInputMacro;
 class MetaItemLitExpr;
-class MetaItemPathLit;
+class MetaItemPathExpr;
 class OperatorExpr;
 class BorrowExpr;
 class DereferenceExpr;
@@ -115,6 +115,8 @@ struct ClosureParam;
 class ClosureExpr;
 class ClosureExprInner;
 class BlockExpr;
+class AnonConst;
+class ConstBlock;
 class ClosureExprInnerTyped;
 class ContinueExpr;
 class BreakExpr;
@@ -126,6 +128,7 @@ class RangeFullExpr;
 class RangeFromToInclExpr;
 class RangeToInclExpr;
 class ReturnExpr;
+class TryExpr;
 class UnsafeBlockExpr;
 class LoopLabel;
 class BaseLoopExpr;
@@ -146,7 +149,6 @@ class MatchExpr;
 class AwaitExpr;
 class AsyncBlockExpr;
 enum class InlineAsmOption;
-struct AnonConst;
 struct InlineAsmRegOrRegClass;
 class InlineAsmOperand;
 struct InlineAsmPlaceHolder;
@@ -246,6 +248,8 @@ class TuplePatternItemsMultiple;
 class TuplePatternItemsRanged;
 class TuplePattern;
 class GroupedPattern;
+class SlicePatternItemsNoRest;
+class SlicePatternItemsHasRest;
 class SlicePattern;
 class AltPattern;
 
diff --git a/gcc/rust/ast/rust-ast-visitor.cc b/gcc/rust/ast/rust-ast-visitor.cc
index b6833f6..ab8cdbe 100644
--- a/gcc/rust/ast/rust-ast-visitor.cc
+++ b/gcc/rust/ast/rust-ast-visitor.cc
@@ -19,6 +19,7 @@
 #include "rust-ast-visitor.h"
 #include "rust-ast-full-decls.h"
 #include "rust-ast.h"
+#include "rust-builtin-ast-nodes.h"
 #include "rust-path.h"
 #include "rust-token.h"
 #include "rust-expr.h"
@@ -223,10 +224,10 @@ DefaultASTVisitor::visit (AST::SimplePath &path)
 }
 
 void
-DefaultASTVisitor::visit (AST::MetaItemPathLit &meta_item)
+DefaultASTVisitor::visit (AST::MetaItemPathExpr &meta_item)
 {
   visit (meta_item.get_path ());
-  visit (meta_item.get_literal ());
+  visit (meta_item.get_expr ());
 }
 
 void
@@ -449,20 +450,38 @@ DefaultASTVisitor::visit (AST::BlockExpr &expr)
 {
   visit_outer_attrs (expr);
   visit_inner_attrs (expr);
+
+  if (expr.has_label ())
+    visit (expr.get_label ());
+
   for (auto &stmt : expr.get_statements ())
     visit (stmt);
+
   if (expr.has_tail_expr ())
     visit (expr.get_tail_expr ());
 }
 
 void
+DefaultASTVisitor::visit (AST::ConstBlock &expr)
+{
+  visit (expr.get_const_expr ());
+}
+
+void
+DefaultASTVisitor::visit (AST::AnonConst &expr)
+{
+  if (!expr.is_deferred ())
+    visit (expr.get_inner_expr ());
+}
+
+void
 DefaultASTVisitor::visit (AST::ClosureExprInnerTyped &expr)
 {
   visit_outer_attrs (expr);
   for (auto &param : expr.get_params ())
     visit (param);
   visit (expr.get_return_type ());
-  visit (expr.get_definition_block ());
+  visit (expr.get_definition_expr ());
 }
 
 void
@@ -538,6 +557,13 @@ DefaultASTVisitor::visit (AST::ReturnExpr &expr)
 }
 
 void
+DefaultASTVisitor::visit (AST::TryExpr &expr)
+{
+  visit_outer_attrs (expr);
+  visit (expr.get_block_expr ());
+}
+
+void
 DefaultASTVisitor::visit (AST::BoxExpr &expr)
 {
   visit_outer_attrs (expr);
@@ -582,7 +608,10 @@ DefaultASTVisitor::visit (AST::WhileLetLoopExpr &expr)
   visit_outer_attrs (expr);
   for (auto &pattern : expr.get_patterns ())
     visit (pattern);
-  visit (expr.get_loop_label ());
+
+  if (expr.has_loop_label ())
+    visit (expr.get_loop_label ());
+
   visit (expr.get_scrutinee_expr ());
   visit (expr.get_loop_block ());
 }
@@ -680,33 +709,40 @@ DefaultASTVisitor::visit (AST::InlineAsm &expr)
     {
       switch (operand.get_register_type ())
 	{
-	  case RegisterType::In: {
+	case RegisterType::In:
+	  {
 	    visit (operand.get_in ().expr);
 	    break;
 	  }
-	  case RegisterType::Out: {
+	case RegisterType::Out:
+	  {
 	    visit (operand.get_out ().expr);
 	    break;
 	  }
-	  case RegisterType::InOut: {
+	case RegisterType::InOut:
+	  {
 	    visit (operand.get_in_out ().expr);
 	    break;
 	  }
-	  case RegisterType::SplitInOut: {
+	case RegisterType::SplitInOut:
+	  {
 	    auto split = operand.get_split_in_out ();
 	    visit (split.in_expr);
 	    visit (split.out_expr);
 	    break;
 	  }
-	  case RegisterType::Const: {
-	    visit (operand.get_const ().anon_const.expr);
+	case RegisterType::Const:
+	  {
+	    visit (operand.get_const ().anon_const.get_inner_expr ());
 	    break;
 	  }
-	  case RegisterType::Sym: {
+	case RegisterType::Sym:
+	  {
 	    visit (operand.get_sym ().expr);
 	    break;
 	  }
-	  case RegisterType::Label: {
+	case RegisterType::Label:
+	  {
 	    visit (operand.get_label ().expr);
 	    break;
 	  }
@@ -755,7 +791,8 @@ DefaultASTVisitor::visit (AST::TypeBoundWhereClauseItem &item)
 void
 DefaultASTVisitor::visit (AST::Visibility &vis)
 {
-  visit (vis.get_path ());
+  if (vis.has_path ())
+    visit (vis.get_path ());
 }
 
 void
@@ -922,7 +959,7 @@ DefaultASTVisitor::visit (AST::EnumItem &item)
 void
 DefaultASTVisitor::visit (AST::EnumItemTuple &item)
 {
-  visit (reinterpret_cast<EnumItem &> (item));
+  DefaultASTVisitor::visit (static_cast<EnumItem &> (item));
   for (auto &field : item.get_tuple_fields ())
     visit (field);
 }
@@ -930,7 +967,7 @@ DefaultASTVisitor::visit (AST::EnumItemTuple &item)
 void
 DefaultASTVisitor::visit (AST::EnumItemStruct &item)
 {
-  visit (reinterpret_cast<EnumItem &> (item));
+  DefaultASTVisitor::visit (static_cast<EnumItem &> (item));
   for (auto &field : item.get_struct_fields ())
     visit (field);
 }
@@ -938,7 +975,7 @@ DefaultASTVisitor::visit (AST::EnumItemStruct &item)
 void
 DefaultASTVisitor::visit (AST::EnumItemDiscriminant &item)
 {
-  visit (reinterpret_cast<EnumItem &> (item));
+  DefaultASTVisitor::visit (static_cast<EnumItem &> (item));
   visit (item.get_expr ());
 }
 
@@ -1179,8 +1216,8 @@ DefaultASTVisitor::visit (AST::LiteralPattern &pattern)
 void
 DefaultASTVisitor::visit (AST::IdentifierPattern &pattern)
 {
-  if (pattern.has_pattern_to_bind ())
-    visit (pattern.get_pattern_to_bind ());
+  if (pattern.has_subpattern ())
+    visit (pattern.get_subpattern ());
 }
 
 void
@@ -1310,13 +1347,28 @@ DefaultASTVisitor::visit (AST::GroupedPattern &pattern)
 }
 
 void
-DefaultASTVisitor::visit (AST::SlicePattern &pattern)
+DefaultASTVisitor::visit (AST::SlicePatternItemsNoRest &items)
+{
+  for (auto &item : items.get_patterns ())
+    visit (item);
+}
+
+void
+DefaultASTVisitor::visit (AST::SlicePatternItemsHasRest &items)
 {
-  for (auto &item : pattern.get_items ())
+  for (auto &item : items.get_lower_patterns ())
+    visit (item);
+  for (auto &item : items.get_upper_patterns ())
     visit (item);
 }
 
 void
+DefaultASTVisitor::visit (AST::SlicePattern &pattern)
+{
+  visit (pattern.get_items ());
+}
+
+void
 DefaultASTVisitor::visit (AST::AltPattern &pattern)
 {
   for (auto &alt : pattern.get_alts ())
@@ -1454,6 +1506,12 @@ DefaultASTVisitor::visit (AST::FormatArgs &)
 }
 
 void
+DefaultASTVisitor::visit (AST::OffsetOf &offset_of)
+{
+  visit (offset_of.get_type ());
+}
+
+void
 DefaultASTVisitor::visit (AST::VariadicParam &param)
 {
   if (param.has_pattern ())
diff --git a/gcc/rust/ast/rust-ast-visitor.h b/gcc/rust/ast/rust-ast-visitor.h
index b1fc504..2d81aa1 100644
--- a/gcc/rust/ast/rust-ast-visitor.h
+++ b/gcc/rust/ast/rust-ast-visitor.h
@@ -73,7 +73,7 @@ public:
   virtual void visit (AttrInputLiteral &attr_input) = 0;
   virtual void visit (AttrInputMacro &attr_input) = 0;
   virtual void visit (MetaItemLitExpr &meta_item) = 0;
-  virtual void visit (MetaItemPathLit &meta_item) = 0;
+  virtual void visit (MetaItemPathExpr &meta_item) = 0;
   virtual void visit (BorrowExpr &expr) = 0;
   virtual void visit (DereferenceExpr &expr) = 0;
   virtual void visit (ErrorPropagationExpr &expr) = 0;
@@ -104,6 +104,8 @@ public:
   virtual void visit (FieldAccessExpr &expr) = 0;
   virtual void visit (ClosureExprInner &expr) = 0;
   virtual void visit (BlockExpr &expr) = 0;
+  virtual void visit (AnonConst &expr) = 0;
+  virtual void visit (ConstBlock &expr) = 0;
   virtual void visit (ClosureExprInnerTyped &expr) = 0;
   virtual void visit (ContinueExpr &expr) = 0;
   virtual void visit (BreakExpr &expr) = 0;
@@ -114,6 +116,7 @@ public:
   virtual void visit (RangeFromToInclExpr &expr) = 0;
   virtual void visit (RangeToInclExpr &expr) = 0;
   virtual void visit (ReturnExpr &expr) = 0;
+  virtual void visit (TryExpr &expr) = 0;
   virtual void visit (BoxExpr &expr) = 0;
   virtual void visit (UnsafeBlockExpr &expr) = 0;
   virtual void visit (LoopExpr &expr) = 0;
@@ -209,6 +212,8 @@ public:
   virtual void visit (TuplePatternItemsRanged &tuple_items) = 0;
   virtual void visit (TuplePattern &pattern) = 0;
   virtual void visit (GroupedPattern &pattern) = 0;
+  virtual void visit (SlicePatternItemsNoRest &items) = 0;
+  virtual void visit (SlicePatternItemsHasRest &items) = 0;
   virtual void visit (SlicePattern &pattern) = 0;
   virtual void visit (AltPattern &pattern) = 0;
 
@@ -235,6 +240,7 @@ public:
 
   // special AST nodes for certain builtin macros such as `asm!()`
   virtual void visit (FormatArgs &fmt) = 0;
+  virtual void visit (OffsetOf &fmt) = 0;
 
   // TODO: rust-cond-compilation.h visiting? not currently used
 };
@@ -264,7 +270,7 @@ public:
   virtual void visit (AST::AttrInputLiteral &attr_input) override;
   virtual void visit (AST::AttrInputMacro &attr_input) override;
   virtual void visit (AST::MetaItemLitExpr &meta_item) override;
-  virtual void visit (AST::MetaItemPathLit &meta_item) override;
+  virtual void visit (AST::MetaItemPathExpr &meta_item) override;
   virtual void visit (AST::BorrowExpr &expr) override;
   virtual void visit (AST::DereferenceExpr &expr) override;
   virtual void visit (AST::ErrorPropagationExpr &expr) override;
@@ -293,6 +299,8 @@ public:
   virtual void visit (AST::FieldAccessExpr &expr) override;
   virtual void visit (AST::ClosureExprInner &expr) override;
   virtual void visit (AST::BlockExpr &expr) override;
+  virtual void visit (AST::AnonConst &expr) override;
+  virtual void visit (AST::ConstBlock &expr) override;
   virtual void visit (AST::ClosureExprInnerTyped &expr) override;
   virtual void visit (AST::ContinueExpr &expr) override;
   virtual void visit (AST::BreakExpr &expr) override;
@@ -303,6 +311,7 @@ public:
   virtual void visit (AST::RangeFromToInclExpr &expr) override;
   virtual void visit (AST::RangeToInclExpr &expr) override;
   virtual void visit (AST::ReturnExpr &expr) override;
+  virtual void visit (AST::TryExpr &expr) override;
   virtual void visit (AST::BoxExpr &expr) override;
   virtual void visit (AST::UnsafeBlockExpr &expr) override;
   virtual void visit (AST::LoopExpr &expr) override;
@@ -379,6 +388,8 @@ public:
   virtual void visit (AST::TuplePatternItemsRanged &tuple_items) override;
   virtual void visit (AST::TuplePattern &pattern) override;
   virtual void visit (AST::GroupedPattern &pattern) override;
+  virtual void visit (AST::SlicePatternItemsNoRest &items) override;
+  virtual void visit (AST::SlicePatternItemsHasRest &items) override;
   virtual void visit (AST::SlicePattern &pattern) override;
   virtual void visit (AST::AltPattern &pattern) override;
   virtual void visit (AST::EmptyStmt &stmt) override;
@@ -402,6 +413,7 @@ public:
   virtual void visit (AST::FunctionParam &param) override;
   virtual void visit (AST::VariadicParam &param) override;
   virtual void visit (AST::FormatArgs &fmt) override;
+  virtual void visit (AST::OffsetOf &fmt) override;
 
   template <typename T> void visit (T &node) { node.accept_vis (*this); }
 
diff --git a/gcc/rust/ast/rust-ast.cc b/gcc/rust/ast/rust-ast.cc
index 4e82be4..8e856fb 100644
--- a/gcc/rust/ast/rust-ast.cc
+++ b/gcc/rust/ast/rust-ast.cc
@@ -33,6 +33,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "rust-operators.h"
 #include "rust-dir-owner.h"
 #include "rust-attribute-values.h"
+#include "rust-macro-invoc-lexer.h"
 
 /* Compilation unit used for various AST-related functions that would make
  * the headers too long if they were defined inline and don't receive any
@@ -249,27 +250,31 @@ Attribute::get_traits_to_derive ()
   auto &input = get_attr_input ();
   switch (input.get_attr_input_type ())
     {
-      case AST::AttrInput::META_ITEM: {
+    case AST::AttrInput::META_ITEM:
+      {
 	auto &meta = static_cast<AST::AttrInputMetaItemContainer &> (input);
 	for (auto &current : meta.get_items ())
 	  {
 	    // HACK: Find a better way to achieve the downcast.
 	    switch (current->get_kind ())
 	      {
-		case AST::MetaItemInner::Kind::MetaItem: {
+	      case AST::MetaItemInner::Kind::MetaItem:
+		{
 		  // Let raw pointer go out of scope without freeing, it doesn't
 		  // own the data anyway
 		  auto meta_item
 		    = static_cast<AST::MetaItem *> (current.get ());
 		  switch (meta_item->get_item_kind ())
 		    {
-		      case AST::MetaItem::ItemKind::Path: {
+		    case AST::MetaItem::ItemKind::Path:
+		      {
 			auto path
 			  = static_cast<AST::MetaItemPath *> (meta_item);
 			result.push_back (path->get_path ());
 		      }
 		      break;
-		      case AST::MetaItem::ItemKind::Word: {
+		    case AST::MetaItem::ItemKind::Word:
+		      {
 			auto word = static_cast<AST::MetaWord *> (meta_item);
 			// Convert current word to path
 			current = std::make_unique<AST::MetaItemPath> (
@@ -283,7 +288,7 @@ Attribute::get_traits_to_derive ()
 		      break;
 		    case AST::MetaItem::ItemKind::ListPaths:
 		    case AST::MetaItem::ItemKind::NameValueStr:
-		    case AST::MetaItem::ItemKind::PathLit:
+		    case AST::MetaItem::ItemKind::PathExpr:
 		    case AST::MetaItem::ItemKind::Seq:
 		    case AST::MetaItem::ItemKind::ListNameValueStr:
 		    default:
@@ -620,7 +625,7 @@ ConstantItem::as_string () const
 {
   std::string str = VisItem::as_string ();
 
-  str += "const " + identifier;
+  str += "const " + identifier.as_string ();
 
   // DEBUG: null pointer check
   if (type == nullptr)
@@ -782,7 +787,8 @@ UseTreeGlob::as_string () const
       return "*";
     case GLOBAL:
       return "::*";
-      case PATH_PREFIXED: {
+    case PATH_PREFIXED:
+      {
 	std::string path_str = path.as_string ();
 	return path_str + "::*";
       }
@@ -805,7 +811,8 @@ UseTreeList::as_string () const
     case GLOBAL:
       path_str = "::{";
       break;
-      case PATH_PREFIXED: {
+    case PATH_PREFIXED:
+      {
 	path_str = path.as_string () + "::{";
 	break;
       }
@@ -1272,6 +1279,25 @@ BlockExpr::as_string () const
 }
 
 std::string
+AnonConst::as_string () const
+{
+  std::string str = "AnonConst: ";
+
+  if (kind == AnonConst::Kind::DeferredInference)
+    str += "_";
+  else
+    str += expr.value ()->as_string ();
+
+  return str;
+}
+
+std::string
+ConstBlock::as_string () const
+{
+  return "ConstBlock: " + expr.as_string ();
+}
+
+std::string
 TraitImpl::as_string () const
 {
   std::string str = VisItem::as_string ();
@@ -1619,6 +1645,19 @@ ReturnExpr::as_string () const
 }
 
 std::string
+TryExpr::as_string () const
+{
+  /* TODO: find way to incorporate outer attrs - may have to represent in
+   * different style (i.e. something more like BorrowExpr: \n outer attrs) */
+
+  std::string str ("try ");
+
+  str += block_expr->as_string ();
+
+  return str;
+}
+
+std::string
 RangeToExpr::as_string () const
 {
   return ".." + to->as_string ();
@@ -2714,7 +2753,7 @@ ImplTraitTypeOneBound::as_string () const
 {
   std::string str ("ImplTraitTypeOneBound: \n TraitBound: ");
 
-  return str + trait_bound.as_string ();
+  return str + trait_bound->as_string ();
 }
 
 std::string
@@ -2736,7 +2775,7 @@ std::string
 ArrayType::as_string () const
 {
   // TODO: rewrite to work with non-linearisable types and exprs
-  return "[" + elem_type->as_string () + "; " + size->as_string () + "]";
+  return "[" + elem_type->as_string () + "; " + size.as_string () + "]";
 }
 
 std::string
@@ -3477,13 +3516,24 @@ DelimTokenTree::parse_to_meta_item () const
   return new AttrInputMetaItemContainer (std::move (meta_items));
 }
 
+AttributeParser::AttributeParser (
+  std::vector<std::unique_ptr<Token>> token_stream, int stream_start_pos)
+  : lexer (new MacroInvocLexer (std::move (token_stream))),
+    parser (new Parser<MacroInvocLexer> (*lexer))
+{
+  if (stream_start_pos)
+    lexer->skip_token (stream_start_pos - 1);
+}
+
+AttributeParser::~AttributeParser () {}
+
 std::unique_ptr<MetaItemInner>
 AttributeParser::parse_meta_item_inner ()
 {
   // if first tok not identifier, not a "special" case one
-  if (peek_token ()->get_id () != IDENTIFIER)
+  if (lexer->peek_token ()->get_id () != IDENTIFIER)
     {
-      switch (peek_token ()->get_id ())
+      switch (lexer->peek_token ()->get_id ())
 	{
 	case CHAR_LITERAL:
 	case STRING_LITERAL:
@@ -3504,48 +3554,46 @@ AttributeParser::parse_meta_item_inner ()
 	  return parse_path_meta_item ();
 
 	default:
-	  rust_error_at (peek_token ()->get_locus (),
+	  rust_error_at (lexer->peek_token ()->get_locus (),
 			 "unrecognised token '%s' in meta item",
-			 get_token_description (peek_token ()->get_id ()));
+			 get_token_description (
+			   lexer->peek_token ()->get_id ()));
 	  return nullptr;
 	}
     }
 
   // else, check for path
-  if (peek_token (1)->get_id () == SCOPE_RESOLUTION)
+  if (lexer->peek_token (1)->get_id () == SCOPE_RESOLUTION)
     {
       // path
       return parse_path_meta_item ();
     }
 
-  auto ident = peek_token ()->as_string ();
-  auto ident_locus = peek_token ()->get_locus ();
+  auto ident = lexer->peek_token ()->get_str ();
+  auto ident_locus = lexer->peek_token ()->get_locus ();
 
-  if (is_end_meta_item_tok (peek_token (1)->get_id ()))
+  if (is_end_meta_item_tok (lexer->peek_token (1)->get_id ()))
     {
       // meta word syntax
-      skip_token ();
+      lexer->skip_token ();
       return std::unique_ptr<MetaWord> (new MetaWord (ident, ident_locus));
     }
 
-  if (peek_token (1)->get_id () == EQUAL)
+  if (lexer->peek_token (1)->get_id () == EQUAL)
     {
       // maybe meta name value str syntax - check next 2 tokens
-      if (peek_token (2)->get_id () == STRING_LITERAL
-	  && is_end_meta_item_tok (peek_token (3)->get_id ()))
+      if (lexer->peek_token (2)->get_id () == STRING_LITERAL
+	  && is_end_meta_item_tok (lexer->peek_token (3)->get_id ()))
 	{
 	  // meta name value str syntax
-	  auto &value_tok = peek_token (2);
-	  auto value = value_tok->as_string ();
+	  const_TokenPtr value_tok = lexer->peek_token (2);
+	  auto value = value_tok->get_str ();
 	  auto locus = value_tok->get_locus ();
 
-	  skip_token (2);
-
-	  // remove the quotes from the string value
-	  std::string raw_value = unquote_string (std::move (value));
+	  lexer->skip_token (2);
 
 	  return std::unique_ptr<MetaNameValueStr> (
-	    new MetaNameValueStr (ident, ident_locus, std::move (raw_value),
+	    new MetaNameValueStr (ident, ident_locus, std::move (value),
 				  locus));
 	}
       else
@@ -3555,16 +3603,16 @@ AttributeParser::parse_meta_item_inner ()
 	}
     }
 
-  if (peek_token (1)->get_id () != LEFT_PAREN)
+  if (lexer->peek_token (1)->get_id () != LEFT_PAREN)
     {
-      rust_error_at (peek_token (1)->get_locus (),
+      rust_error_at (lexer->peek_token (1)->get_locus (),
 		     "unexpected token '%s' after identifier in attribute",
-		     get_token_description (peek_token (1)->get_id ()));
+		     get_token_description (lexer->peek_token (1)->get_id ()));
       return nullptr;
     }
 
   // is it one of those special cases like not?
-  if (peek_token ()->get_id () == IDENTIFIER)
+  if (lexer->peek_token ()->get_id () == IDENTIFIER)
     {
       return parse_path_meta_item ();
     }
@@ -3643,49 +3691,46 @@ AttributeParser::is_end_meta_item_tok (TokenId id) const
 std::unique_ptr<MetaItem>
 AttributeParser::parse_path_meta_item ()
 {
-  SimplePath path = parse_simple_path ();
+  SimplePath path = parser->parse_simple_path ();
   if (path.is_empty ())
     {
-      rust_error_at (peek_token ()->get_locus (),
+      rust_error_at (lexer->peek_token ()->get_locus (),
 		     "failed to parse simple path in attribute");
       return nullptr;
     }
 
-  switch (peek_token ()->get_id ())
+  switch (lexer->peek_token ()->get_id ())
     {
-      case LEFT_PAREN: {
+    case LEFT_PAREN:
+      {
 	std::vector<std::unique_ptr<MetaItemInner>> meta_items
 	  = parse_meta_item_seq ();
 
 	return std::unique_ptr<MetaItemSeq> (
 	  new MetaItemSeq (std::move (path), std::move (meta_items)));
       }
-      case EQUAL: {
-	skip_token ();
+    case EQUAL:
+      {
+	lexer->skip_token ();
 
-	location_t locus = peek_token ()->get_locus ();
-	Literal lit = parse_literal ();
-	if (lit.is_error ())
-	  {
-	    rust_error_at (peek_token ()->get_locus (),
-			   "failed to parse literal in attribute");
-	    return nullptr;
-	  }
-	LiteralExpr expr (std::move (lit), {}, locus);
-	// stream_pos++;
-	/* shouldn't be required anymore due to parsing literal actually
-	 * skipping the token */
-	return std::unique_ptr<MetaItemPathLit> (
-	  new MetaItemPathLit (std::move (path), std::move (expr)));
+	std::unique_ptr<Expr> expr = parser->parse_expr ();
+
+	// handle error
+	// parse_expr should already emit an error and return nullptr
+	if (!expr)
+	  return nullptr;
+
+	return std::unique_ptr<MetaItemPathExpr> (
+	  new MetaItemPathExpr (std::move (path), std::move (expr)));
       }
     case COMMA:
       // just simple path
       return std::unique_ptr<MetaItemPath> (
 	new MetaItemPath (std::move (path)));
     default:
-      rust_error_at (peek_token ()->get_locus (),
+      rust_error_at (lexer->peek_token ()->get_locus (),
 		     "unrecognised token '%s' in meta item",
-		     get_token_description (peek_token ()->get_id ()));
+		     get_token_description (lexer->peek_token ()->get_id ()));
       return nullptr;
     }
 }
@@ -3695,41 +3740,41 @@ AttributeParser::parse_path_meta_item ()
 std::vector<std::unique_ptr<MetaItemInner>>
 AttributeParser::parse_meta_item_seq ()
 {
-  int vec_length = token_stream.size ();
   std::vector<std::unique_ptr<MetaItemInner>> meta_items;
 
-  if (peek_token ()->get_id () != LEFT_PAREN)
+  if (lexer->peek_token ()->get_id () != LEFT_PAREN)
     {
-      rust_error_at (peek_token ()->get_locus (),
+      rust_error_at (lexer->peek_token ()->get_locus (),
 		     "missing left paren in delim token tree");
       return {};
     }
-  skip_token ();
+  lexer->skip_token ();
 
-  while (stream_pos < vec_length && peek_token ()->get_id () != RIGHT_PAREN)
+  while (lexer->peek_token ()->get_id () != END_OF_FILE
+	 && lexer->peek_token ()->get_id () != RIGHT_PAREN)
     {
       std::unique_ptr<MetaItemInner> inner = parse_meta_item_inner ();
       if (inner == nullptr)
 	{
-	  rust_error_at (peek_token ()->get_locus (),
+	  rust_error_at (lexer->peek_token ()->get_locus (),
 			 "failed to parse inner meta item in attribute");
 	  return {};
 	}
       meta_items.push_back (std::move (inner));
 
-      if (peek_token ()->get_id () != COMMA)
+      if (lexer->peek_token ()->get_id () != COMMA)
 	break;
 
-      skip_token ();
+      lexer->skip_token ();
     }
 
-  if (peek_token ()->get_id () != RIGHT_PAREN)
+  if (lexer->peek_token ()->get_id () != RIGHT_PAREN)
     {
-      rust_error_at (peek_token ()->get_locus (),
+      rust_error_at (lexer->peek_token ()->get_locus (),
 		     "missing right paren in delim token tree");
       return {};
     }
-  skip_token ();
+  lexer->skip_token ();
 
   return meta_items;
 }
@@ -3752,130 +3797,19 @@ DelimTokenTree::to_token_stream () const
   return tokens;
 }
 
-Literal
-AttributeParser::parse_literal ()
-{
-  const std::unique_ptr<Token> &tok = peek_token ();
-  switch (tok->get_id ())
-    {
-    case CHAR_LITERAL:
-      skip_token ();
-      return Literal (tok->as_string (), Literal::CHAR, tok->get_type_hint ());
-    case STRING_LITERAL:
-      skip_token ();
-      return Literal (tok->as_string (), Literal::STRING,
-		      tok->get_type_hint ());
-    case BYTE_CHAR_LITERAL:
-      skip_token ();
-      return Literal (tok->as_string (), Literal::BYTE, tok->get_type_hint ());
-    case BYTE_STRING_LITERAL:
-      skip_token ();
-      return Literal (tok->as_string (), Literal::BYTE_STRING,
-		      tok->get_type_hint ());
-    case RAW_STRING_LITERAL:
-      skip_token ();
-      return Literal (tok->as_string (), Literal::RAW_STRING,
-		      tok->get_type_hint ());
-    case INT_LITERAL:
-      skip_token ();
-      return Literal (tok->as_string (), Literal::INT, tok->get_type_hint ());
-    case FLOAT_LITERAL:
-      skip_token ();
-      return Literal (tok->as_string (), Literal::FLOAT, tok->get_type_hint ());
-    case TRUE_LITERAL:
-      skip_token ();
-      return Literal ("true", Literal::BOOL, tok->get_type_hint ());
-    case FALSE_LITERAL:
-      skip_token ();
-      return Literal ("false", Literal::BOOL, tok->get_type_hint ());
-    default:
-      rust_error_at (tok->get_locus (), "expected literal - found '%s'",
-		     get_token_description (tok->get_id ()));
-      return Literal::create_error ();
-    }
-}
-
-SimplePath
-AttributeParser::parse_simple_path ()
-{
-  bool has_opening_scope_res = false;
-  if (peek_token ()->get_id () == SCOPE_RESOLUTION)
-    {
-      has_opening_scope_res = true;
-      skip_token ();
-    }
-
-  std::vector<SimplePathSegment> segments;
-
-  SimplePathSegment segment = parse_simple_path_segment ();
-  if (segment.is_error ())
-    {
-      rust_error_at (
-	peek_token ()->get_locus (),
-	"failed to parse simple path segment in attribute simple path");
-      return SimplePath::create_empty ();
-    }
-  segments.push_back (std::move (segment));
-
-  while (peek_token ()->get_id () == SCOPE_RESOLUTION)
-    {
-      skip_token ();
-
-      SimplePathSegment segment = parse_simple_path_segment ();
-      if (segment.is_error ())
-	{
-	  rust_error_at (
-	    peek_token ()->get_locus (),
-	    "failed to parse simple path segment in attribute simple path");
-	  return SimplePath::create_empty ();
-	}
-      segments.push_back (std::move (segment));
-    }
-  segments.shrink_to_fit ();
-
-  return SimplePath (std::move (segments), has_opening_scope_res);
-}
-
-SimplePathSegment
-AttributeParser::parse_simple_path_segment ()
-{
-  const std::unique_ptr<Token> &tok = peek_token ();
-  switch (tok->get_id ())
-    {
-    case IDENTIFIER:
-      skip_token ();
-      return SimplePathSegment (tok->as_string (), tok->get_locus ());
-    case SUPER:
-      skip_token ();
-      return SimplePathSegment ("super", tok->get_locus ());
-    case SELF:
-      skip_token ();
-      return SimplePathSegment ("self", tok->get_locus ());
-    case CRATE:
-      skip_token ();
-      return SimplePathSegment ("crate", tok->get_locus ());
-    case DOLLAR_SIGN:
-      if (peek_token (1)->get_id () == CRATE)
-	{
-	  skip_token (1);
-	  return SimplePathSegment ("$crate", tok->get_locus ());
-	}
-      gcc_fallthrough ();
-    default:
-      rust_error_at (tok->get_locus (),
-		     "unexpected token '%s' in simple path segment",
-		     get_token_description (tok->get_id ()));
-      return SimplePathSegment::create_error ();
-    }
-}
-
 std::unique_ptr<MetaItemLitExpr>
 AttributeParser::parse_meta_item_lit ()
 {
-  location_t locus = peek_token ()->get_locus ();
-  LiteralExpr lit_expr (parse_literal (), {}, locus);
+  std::unique_ptr<LiteralExpr> lit_expr = parser->parse_literal_expr ({});
+
+  // TODO: return nullptr instead?
+  if (!lit_expr)
+    lit_expr = std::unique_ptr<LiteralExpr> (
+      new LiteralExpr (Literal::create_error (), {},
+		       lexer->peek_token ()->get_locus ()));
+
   return std::unique_ptr<MetaItemLitExpr> (
-    new MetaItemLitExpr (std::move (lit_expr)));
+    new MetaItemLitExpr (std::move (*lit_expr)));
 }
 
 bool
@@ -4084,10 +4018,12 @@ MetaNameValueStr::check_cfg_predicate (const Session &session) const
 }
 
 bool
-MetaItemPathLit::check_cfg_predicate (const Session &session) const
+MetaItemPathExpr::check_cfg_predicate (const Session &session) const
 {
+  // FIXME: Accept path expressions
+  rust_assert (expr->is_literal ());
   return session.options.target_data.has_key_value_pair (path.as_string (),
-							 lit.as_string ());
+							 expr->as_string ());
 }
 
 std::vector<std::unique_ptr<Token>>
@@ -4175,8 +4111,10 @@ MetaListNameValueStr::to_attribute () const
 }
 
 Attribute
-MetaItemPathLit::to_attribute () const
+MetaItemPathExpr::to_attribute () const
 {
+  rust_assert (expr->is_literal ());
+  auto &lit = static_cast<LiteralExpr &> (*expr);
   return Attribute (path, std::unique_ptr<AttrInputLiteral> (
 			    new AttrInputLiteral (lit)));
 }
@@ -4279,11 +4217,12 @@ AttrInputMacro::AttrInputMacro (const AttrInputMacro &oth)
   : macro (oth.macro->clone_macro_invocation_impl ())
 {}
 
-void
+AttrInputMacro &
 AttrInputMacro::operator= (const AttrInputMacro &oth)
 {
   macro = std::unique_ptr<MacroInvocation> (
     oth.macro->clone_macro_invocation_impl ());
+  return *this;
 }
 
 /* Visitor implementations - these are short but inlining can't happen anyway
@@ -4345,7 +4284,7 @@ MetaItemLitExpr::accept_vis (ASTVisitor &vis)
 }
 
 void
-MetaItemPathLit::accept_vis (ASTVisitor &vis)
+MetaItemPathExpr::accept_vis (ASTVisitor &vis)
 {
   vis.visit (*this);
 }
@@ -4513,6 +4452,18 @@ BlockExpr::accept_vis (ASTVisitor &vis)
 }
 
 void
+AnonConst::accept_vis (ASTVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
+ConstBlock::accept_vis (ASTVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
 ClosureExprInnerTyped::accept_vis (ASTVisitor &vis)
 {
   vis.visit (*this);
@@ -4573,6 +4524,12 @@ ReturnExpr::accept_vis (ASTVisitor &vis)
 }
 
 void
+TryExpr::accept_vis (ASTVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
 UnsafeBlockExpr::accept_vis (ASTVisitor &vis)
 {
   vis.visit (*this);
@@ -5010,6 +4967,12 @@ FormatArgs::accept_vis (ASTVisitor &vis)
   vis.visit (*this);
 }
 
+void
+OffsetOf::accept_vis (ASTVisitor &vis)
+{
+  vis.visit (*this);
+}
+
 std::string
 FormatArgs::as_string () const
 {
@@ -5017,6 +4980,12 @@ FormatArgs::as_string () const
   return "FormatArgs";
 }
 
+std::string
+OffsetOf::as_string () const
+{
+  return "OffsetOf(" + type->as_string () + ", " + field.as_string () + ")";
+}
+
 location_t
 FormatArgs::get_locus () const
 {
@@ -5047,7 +5016,8 @@ FormatArgs::get_outer_attrs ()
   rust_unreachable ();
 }
 
-void FormatArgs::set_outer_attrs (std::vector<Attribute>)
+void
+FormatArgs::set_outer_attrs (std::vector<Attribute>)
 {
   rust_unreachable ();
 }
@@ -5060,6 +5030,24 @@ FormatArgs::clone_expr_impl () const
   return new FormatArgs (*this);
 }
 
+std::vector<Attribute> &
+OffsetOf::get_outer_attrs ()
+{
+  rust_unreachable ();
+}
+
+void
+OffsetOf::set_outer_attrs (std::vector<Attribute>)
+{
+  rust_unreachable ();
+}
+
+Expr *
+OffsetOf::clone_expr_impl () const
+{
+  return new OffsetOf (*this);
+}
+
 } // namespace AST
 
 std::ostream &
diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h
index aa6ad50..2d2c5d0 100644
--- a/gcc/rust/ast/rust-ast.h
+++ b/gcc/rust/ast/rust-ast.h
@@ -62,13 +62,14 @@ public:
     return ident == other.ident;
   }
 
+  operator const std::string & () const { return ident; }
+
 private:
   std::string ident;
   location_t loc;
 };
 
-std::ostream &
-operator<< (std::ostream &os, Identifier const &i);
+std::ostream &operator<< (std::ostream &os, Identifier const &i);
 
 namespace AST {
 // foward decl: ast visitor
@@ -82,6 +83,38 @@ public:
   virtual void accept_vis (ASTVisitor &vis) = 0;
 };
 
+/**
+ * Base function for reconstructing and asserting that the new NodeId is
+ * different from the old NodeId. It then wraps the given pointer into a unique
+ * pointer and returns it.
+ */
+template <typename T>
+std::unique_ptr<T>
+reconstruct_base (const T *instance)
+{
+  auto *reconstructed = instance->reconstruct_impl ();
+
+  rust_assert (reconstructed->get_node_id () != instance->get_node_id ());
+
+  return std::unique_ptr<T> (reconstructed);
+}
+
+/**
+ * Reconstruct multiple items in a vector
+ */
+template <typename T>
+std::vector<std::unique_ptr<T>>
+reconstruct_vec (const std::vector<std::unique_ptr<T>> &to_reconstruct)
+{
+  std::vector<std::unique_ptr<T>> reconstructed;
+  reconstructed.reserve (to_reconstruct.size ());
+
+  for (const auto &elt : to_reconstruct)
+    reconstructed.emplace_back (std::unique_ptr<T> (elt->reconstruct_impl ()));
+
+  return reconstructed;
+}
+
 // Delimiter types - used in macros and whatever.
 enum DelimType
 {
@@ -250,6 +283,7 @@ public:
   std::vector<std::unique_ptr<Token>> to_token_stream () const override;
 
   TokenId get_id () const { return tok_ref->get_id (); }
+  bool has_str () const { return tok_ref->has_str (); }
   const std::string &get_str () const { return tok_ref->get_str (); }
 
   location_t get_locus () const { return tok_ref->get_locus (); }
@@ -403,15 +437,15 @@ class SimplePath
 
 public:
   // Constructor
-  SimplePath (std::vector<SimplePathSegment> path_segments,
-	      bool has_opening_scope_resolution = false,
-	      location_t locus = UNDEF_LOCATION)
+  explicit SimplePath (std::vector<SimplePathSegment> path_segments,
+		       bool has_opening_scope_resolution = false,
+		       location_t locus = UNDEF_LOCATION)
     : opening_scope_resolution (has_opening_scope_resolution),
       segments (std::move (path_segments)), locus (locus),
       node_id (Analysis::Mappings::get ().get_next_node_id ())
   {}
 
-  SimplePath (Identifier ident)
+  explicit SimplePath (Identifier ident)
     : opening_scope_resolution (false),
       segments ({SimplePathSegment (ident.as_string (), ident.get_locus ())}),
       locus (ident.get_locus ()),
@@ -1039,7 +1073,7 @@ public:
     Path,
     Word,
     NameValueStr,
-    PathLit,
+    PathExpr,
     Seq,
     ListPaths,
     ListNameValueStr,
@@ -1057,7 +1091,7 @@ public:
 class MetaItemLitExpr;
 
 // Forward decl - defined in rust-expr.h
-class MetaItemPathLit;
+class MetaItemPathExpr;
 
 // Forward decl - defined in rust-macro.h
 class MetaItemPath;
@@ -1256,6 +1290,8 @@ public:
     FieldAccess,
     Closure,
     Block,
+    ConstExpr,
+    ConstBlock,
     Continue,
     Break,
     Range,
@@ -1272,6 +1308,7 @@ public:
     LlvmInlineAsm,
     Identifier,
     FormatArgs,
+    OffsetOf,
     MacroInvocation,
     Borrow,
     Dereference,
@@ -1283,6 +1320,7 @@ public:
     TypeCast,
     Assignment,
     CompoundAssignment,
+    Try,
   };
 
   virtual Kind get_expr_kind () const = 0;
@@ -1477,6 +1515,10 @@ public:
     return std::unique_ptr<Type> (clone_type_impl ());
   }
 
+  // Similar to `clone_type`, but generates a new instance of the node with a
+  // different NodeId
+  std::unique_ptr<Type> reconstruct () const { return reconstruct_base (this); }
+
   // virtual destructor
   virtual ~Type () {}
 
@@ -1495,11 +1537,13 @@ public:
   virtual location_t get_locus () const = 0;
 
   NodeId get_node_id () const { return node_id; }
+  virtual Type *reconstruct_impl () const = 0;
 
 protected:
   Type () : node_id (Analysis::Mappings::get ().get_next_node_id ()) {}
+  Type (NodeId node_id) : node_id (node_id) {}
 
-  // Clone function implementation as pure virtual method
+  // Clone and reconstruct function implementations as pure virtual methods
   virtual Type *clone_type_impl () const = 0;
 
   NodeId node_id;
@@ -1515,6 +1559,13 @@ public:
     return std::unique_ptr<TypeNoBounds> (clone_type_no_bounds_impl ());
   }
 
+  std::unique_ptr<TypeNoBounds> reconstruct () const
+  {
+    return reconstruct_base (this);
+  }
+
+  virtual TypeNoBounds *reconstruct_impl () const override = 0;
+
 protected:
   // Clone function implementation as pure virtual method
   virtual TypeNoBounds *clone_type_no_bounds_impl () const = 0;
@@ -1549,6 +1600,11 @@ public:
     return std::unique_ptr<TypeParamBound> (clone_type_param_bound_impl ());
   }
 
+  std::unique_ptr<TypeParamBound> reconstruct () const
+  {
+    return reconstruct_base (this);
+  }
+
   virtual std::string as_string () const = 0;
 
   NodeId get_node_id () const { return node_id; }
@@ -1557,10 +1613,14 @@ public:
 
   virtual TypeParamBoundType get_bound_type () const = 0;
 
+  virtual TypeParamBound *reconstruct_impl () const = 0;
+
 protected:
   // Clone function implementation as pure virtual method
   virtual TypeParamBound *clone_type_param_bound_impl () const = 0;
 
+  TypeParamBound () : node_id (Analysis::Mappings::get ().get_next_node_id ())
+  {}
   TypeParamBound (NodeId node_id) : node_id (node_id) {}
 
   NodeId node_id;
@@ -1622,6 +1682,10 @@ protected:
   {
     return new Lifetime (node_id, lifetime_type, lifetime_name, locus);
   }
+  Lifetime *reconstruct_impl () const override
+  {
+    return new Lifetime (lifetime_type, lifetime_name, locus);
+  }
 };
 
 /* Base generic parameter in AST. Abstract - can be represented by a Lifetime
diff --git a/gcc/rust/ast/rust-builtin-ast-nodes.h b/gcc/rust/ast/rust-builtin-ast-nodes.h
index 3684092..2893e7b 100644
--- a/gcc/rust/ast/rust-builtin-ast-nodes.h
+++ b/gcc/rust/ast/rust-builtin-ast-nodes.h
@@ -225,6 +225,59 @@ protected:
   virtual Expr *clone_expr_impl () const override;
 };
 
+/**
+ * The node associated with the builtin offset_of!() macro
+ */
+class OffsetOf : public Expr
+{
+public:
+  OffsetOf (std::unique_ptr<Type> &&type, Identifier field, location_t loc)
+    : type (std::move (type)), field (field), loc (loc)
+  {}
+
+  OffsetOf (const OffsetOf &other)
+    : type (other.type->clone_type ()), field (other.field), loc (other.loc),
+      marked_for_strip (other.marked_for_strip)
+  {}
+
+  OffsetOf &operator= (const OffsetOf &other)
+  {
+    type = other.type->clone_type ();
+    field = other.field;
+    loc = other.loc;
+    marked_for_strip = other.marked_for_strip;
+
+    return *this;
+  }
+
+  void accept_vis (AST::ASTVisitor &vis) override;
+
+  virtual location_t get_locus () const override { return loc; }
+  const Type &get_type () const { return *type; }
+  Type &get_type () { return *type; }
+  const Identifier &get_field () const { return field; }
+
+  bool is_expr_without_block () const override { return false; }
+
+  void mark_for_strip () override { marked_for_strip = true; }
+  bool is_marked_for_strip () const override { return marked_for_strip; }
+
+  std::string as_string () const override;
+
+  std::vector<Attribute> &get_outer_attrs () override;
+  void set_outer_attrs (std::vector<Attribute>) override;
+  Expr *clone_expr_impl () const override;
+
+  Expr::Kind get_expr_kind () const override { return Expr::Kind::OffsetOf; }
+
+private:
+  std::unique_ptr<Type> type;
+  Identifier field;
+
+  location_t loc;
+  bool marked_for_strip = false;
+};
+
 } // namespace AST
 } // namespace Rust
 
diff --git a/gcc/rust/ast/rust-collect-lang-items.cc b/gcc/rust/ast/rust-collect-lang-items.cc
index cd6be7f..306c6f7 100644
--- a/gcc/rust/ast/rust-collect-lang-items.cc
+++ b/gcc/rust/ast/rust-collect-lang-items.cc
@@ -109,5 +109,29 @@ CollectLangItems::visit (AST::EnumItem &item)
   DefaultASTVisitor::visit (item);
 }
 
+void
+CollectLangItems::visit (AST::EnumItemTuple &item)
+{
+  maybe_add_lang_item (item);
+
+  DefaultASTVisitor::visit (item);
+}
+
+void
+CollectLangItems::visit (AST::EnumItemStruct &item)
+{
+  maybe_add_lang_item (item);
+
+  DefaultASTVisitor::visit (item);
+}
+
+void
+CollectLangItems::visit (AST::EnumItemDiscriminant &item)
+{
+  maybe_add_lang_item (item);
+
+  DefaultASTVisitor::visit (item);
+}
+
 } // namespace AST
 } // namespace Rust
diff --git a/gcc/rust/ast/rust-collect-lang-items.h b/gcc/rust/ast/rust-collect-lang-items.h
index ddb34a9..ddc7b51 100644
--- a/gcc/rust/ast/rust-collect-lang-items.h
+++ b/gcc/rust/ast/rust-collect-lang-items.h
@@ -50,6 +50,9 @@ public:
   void visit (AST::Function &item) override;
   void visit (AST::StructStruct &item) override;
   void visit (AST::EnumItem &item) override;
+  void visit (AST::EnumItemTuple &item) override;
+  void visit (AST::EnumItemStruct &item) override;
+  void visit (AST::EnumItemDiscriminant &item) override;
 
 private:
   template <typename T> void maybe_add_lang_item (const T &item);
diff --git a/gcc/rust/ast/rust-cond-compilation.h b/gcc/rust/ast/rust-cond-compilation.h
index 610b904..56a5646 100644
--- a/gcc/rust/ast/rust-cond-compilation.h
+++ b/gcc/rust/ast/rust-cond-compilation.h
@@ -42,8 +42,8 @@ public:
 
 protected:
   // Clone function impl to be overriden in base classes
-  virtual ConfigurationPredicate *
-  clone_configuration_predicate_impl () const = 0;
+  virtual ConfigurationPredicate *clone_configuration_predicate_impl () const
+    = 0;
 };
 
 // A configuration option - true if option is set, false if option is not set.
diff --git a/gcc/rust/ast/rust-desugar-apit.cc b/gcc/rust/ast/rust-desugar-apit.cc
new file mode 100644
index 0000000..bca14ee
--- /dev/null
+++ b/gcc/rust/ast/rust-desugar-apit.cc
@@ -0,0 +1,522 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-desugar-apit.h"
+#include "rust-ast.h"
+#include "rust-type.h"
+
+namespace Rust {
+namespace AST {
+
+class DesugarApitType : public DefaultASTVisitor
+{
+  using DefaultASTVisitor::visit;
+
+public:
+  static std::pair<AST::Type *, std::vector<std::unique_ptr<GenericParam>>>
+  Desugar (AST::Type &type)
+  {
+    DesugarApitType visitor (&type);
+    type.accept_vis (visitor);
+    rust_assert (visitor.translated != nullptr);
+    return std::make_pair (visitor.translated,
+			   std::move (visitor.implicit_generic_params));
+  }
+
+  // Generate a unique impl trait parameter name
+  static Identifier get_impl_name ()
+  {
+    static size_t counter = 0;
+    return Identifier ("Impl_" + std::to_string (counter++));
+  }
+
+  // these can hold other types
+  void visit (AST::TupleType &tuple) override
+  {
+    for (auto &elem : tuple.get_elems ())
+      {
+	auto &type = *elem.get ();
+	auto desugar = Desugar (type);
+	auto tt = desugar.first;
+
+	auto &implicit_generics = desugar.second;
+	if (implicit_generics.empty ())
+	  continue;
+
+	if (tt != elem.get ())
+	  elem = std::unique_ptr<Type> (tt);
+
+	for (auto &implicit_generic : implicit_generics)
+	  implicit_generic_params.push_back (std::move (implicit_generic));
+      }
+  }
+
+  void visit (AST::ArrayType &type) override
+  {
+    auto &element_type = type.get_element_type ();
+    auto desugar = Desugar (*element_type);
+    auto tt = desugar.first;
+
+    auto &implicit_generics = desugar.second;
+    if (implicit_generics.empty ())
+      return;
+
+    if (tt != element_type.get ())
+      element_type = std::unique_ptr<AST::Type> (tt);
+
+    for (auto &implicit_generic : implicit_generics)
+      implicit_generic_params.push_back (std::move (implicit_generic));
+  }
+
+  void visit (AST::ReferenceType &type) override
+  {
+    // Get a reference to the current type for in-place modification
+    auto &referenced_type = type.get_type_referenced ();
+    auto desugar = Desugar (referenced_type);
+    auto tt = desugar.first;
+
+    auto &implicit_generics = desugar.second;
+    if (implicit_generics.empty ())
+      return;
+
+    // Update the reference type's contents rather than creating a new one
+    if (&referenced_type != tt)
+      {
+	std::unique_ptr<AST::TypeNoBounds> new_type_no_bounds (
+	  static_cast<AST::TypeNoBounds *> (tt));
+	type.get_type_ptr () = std::move (new_type_no_bounds);
+      }
+
+    // Collect all the implicit generic parameters we found
+    for (auto &implicit_generic : implicit_generics)
+      implicit_generic_params.push_back (std::move (implicit_generic));
+  }
+
+  void visit (AST::RawPointerType &type) override
+  {
+    auto &pointed_type = type.get_type_pointed_to ();
+    auto desugar = Desugar (pointed_type);
+    auto tt = desugar.first;
+
+    auto &implicit_generics = desugar.second;
+    if (implicit_generics.empty ())
+      return;
+
+    // Update the pointer's inner type directly using the new accessor
+    if (&pointed_type != tt)
+      {
+	std::unique_ptr<AST::TypeNoBounds> new_type_no_bounds (
+	  static_cast<AST::TypeNoBounds *> (tt));
+	type.get_type_ptr () = std::move (new_type_no_bounds);
+      }
+
+    // Collect all the implicit generic parameters we found
+    for (auto &implicit_generic : implicit_generics)
+      implicit_generic_params.push_back (std::move (implicit_generic));
+  }
+
+  void visit (AST::SliceType &type) override
+  {
+    auto &element_type = type.get_elem_type ();
+    auto desugar = Desugar (element_type);
+    auto tt = desugar.first;
+
+    auto &implicit_generics = desugar.second;
+    if (implicit_generics.empty ())
+      return;
+
+    if (&element_type != tt)
+      {
+	std::unique_ptr<AST::Type> new_elem_type (tt);
+	type.get_elem_type_ptr () = std::move (new_elem_type);
+      }
+
+    // Collect all the implicit generic parameters we found
+    for (auto &implicit_generic : implicit_generics)
+      implicit_generic_params.push_back (std::move (implicit_generic));
+  }
+
+  void visit (AST::ParenthesisedType &type) override
+  {
+    auto &inner_type_ptr = type.get_type_in_parens ();
+    auto desugar = Desugar (*inner_type_ptr);
+    auto tt = desugar.first;
+
+    auto &implicit_generics = desugar.second;
+    if (implicit_generics.empty ())
+      return;
+
+    if (inner_type_ptr.get () != tt)
+      {
+	std::unique_ptr<AST::Type> new_inner_type (tt);
+	inner_type_ptr = std::move (new_inner_type);
+      }
+
+    // Collect all the implicit generic parameters we found
+    for (auto &implicit_generic : implicit_generics)
+      implicit_generic_params.push_back (std::move (implicit_generic));
+  }
+
+  // this is where the desugar happens
+  void visit (AST::ImplTraitType &type) override
+  {
+    // Generate a unique name using the static method
+    auto ident = get_impl_name ();
+
+    // Create a type path for the new generic parameter
+    // Create a SimplePathSegment with the identifier string
+    auto simple_seg = SimplePathSegment (ident.as_string (), type.get_locus ());
+    // Create a vector of SimplePathSegments for SimplePath constructor
+    std::vector<SimplePathSegment> simple_segs = {simple_seg};
+    // Create a SimplePath
+    auto simple_path = SimplePath (simple_segs, false, type.get_locus ());
+
+    // Convert to TypePath by creating path segments
+    std::vector<std::unique_ptr<TypePathSegment>> segments;
+    segments.push_back (std::unique_ptr<TypePathSegment> (new TypePathSegment (
+      PathIdentSegment (ident.as_string (), type.get_locus ()), false,
+      type.get_locus ())));
+
+    // Create TypePath from segments
+    auto type_path
+      = new TypePath (std::move (segments), type.get_locus (), false);
+
+    // Convert bounds from impl trait to generic parameter bounds
+    std::vector<std::unique_ptr<TypeParamBound>> bounds;
+    for (auto &bound : type.get_type_param_bounds ())
+      bounds.push_back (bound->clone_type_param_bound ());
+
+    // Create the new generic parameter
+    auto generic_param = std::unique_ptr<TypeParam> (
+      new TypeParam (ident, type.get_locus (), std::move (bounds), nullptr, {},
+		     true /*from impl trait*/));
+
+    // Store the generic parameter to be added to the function signature
+    implicit_generic_params.push_back (std::move (generic_param));
+
+    // Replace impl trait with the new type parameter
+    translated = type_path;
+  }
+
+  void visit (AST::ImplTraitTypeOneBound &type) override
+  {
+    // Generate a unique name using the static method
+    auto ident = get_impl_name ();
+
+    // Create a type path for the new generic parameter
+    // Create a SimplePathSegment with the identifier string
+    auto simple_seg = SimplePathSegment (ident.as_string (), type.get_locus ());
+    // Create a vector of SimplePathSegments for SimplePath constructor
+    std::vector<SimplePathSegment> simple_segs = {simple_seg};
+    // Create a SimplePath
+    auto simple_path = SimplePath (simple_segs, false, type.get_locus ());
+
+    // Convert to TypePath by creating path segments
+    std::vector<std::unique_ptr<TypePathSegment>> segments;
+    segments.push_back (std::unique_ptr<TypePathSegment> (new TypePathSegment (
+      PathIdentSegment (ident.as_string (), type.get_locus ()), false,
+      type.get_locus ())));
+
+    // Create TypePath from segments
+    auto type_path
+      = new TypePath (std::move (segments), type.get_locus (), false);
+
+    // Convert the bound to a generic parameter bound
+    std::vector<std::unique_ptr<TypeParamBound>> bounds;
+    bounds.push_back (std::move (type.get_trait_bound ()));
+
+    // Create the new generic parameter
+    auto generic_param = std::unique_ptr<TypeParam> (
+      new TypeParam (ident, type.get_locus (), std::move (bounds), nullptr, {},
+		     true /*from impl trait*/));
+
+    // Store the generic parameter to be added to the function signature
+    implicit_generic_params.push_back (std::move (generic_param));
+
+    // Replace impl trait with the new type parameter
+    translated = type_path;
+  }
+
+private:
+  DesugarApitType (AST::Type *base)
+    : translated (base), implicit_generic_params ()
+  {}
+
+  AST::Type *translated;
+  std::vector<std::unique_ptr<GenericParam>> implicit_generic_params;
+};
+
+// ---------
+
+class ApitBoundProcessor
+{
+public:
+  ApitBoundProcessor (
+    WhereClause &where_clause,
+    std::vector<std::unique_ptr<GenericParam>> &generic_params)
+    : where_clause (where_clause), generic_params (generic_params)
+  {}
+
+  void go (std::vector<std::unique_ptr<GenericParam>> &implicit_generics)
+  {
+    // some desugars are more complex so imagine this case
+    //
+    // pub fn foo(_value: impl Bar<Baz = impl Foo>) -> i32 {
+    //   15
+    // }
+    //
+    // this needs to become:
+    //
+    // pub fn foo<T, U>(_value: T) -> i32
+    // where
+    //     T: Bar<Baz = U>,
+    //     U: Foo,
+    // {
+    //     15
+    // }
+    //
+    // so we need to walk all the implicit generics and the trait bounds paths
+    // for more generics
+
+    for (auto &implicit_generic : implicit_generics)
+      {
+	switch (implicit_generic->get_kind ())
+	  {
+	  case GenericParam::Kind::Type:
+	    {
+	      TypeParam &p
+		= *static_cast<TypeParam *> (implicit_generic.get ());
+
+	      process_type_param (p);
+	      generic_params.push_back (std::move (implicit_generic));
+	      for (auto &synth : synthetic_params)
+		generic_params.push_back (std::move (synth));
+	      synthetic_params.clear ();
+	    }
+	    break;
+
+	  default:
+	    generic_params.push_back (std::move (implicit_generic));
+	    break;
+	  }
+      }
+  }
+
+private:
+  void process_type_param (TypeParam &p)
+  {
+    auto &bounds = p.get_type_param_bounds ();
+    std::vector<size_t> bounds_to_remove;
+    for (size_t i = 0; i < bounds.size (); i++)
+      {
+	auto &tb = bounds[i];
+	switch (tb->get_bound_type ())
+	  {
+	  case TypeParamBound::TypeParamBoundType::TRAIT:
+	    {
+	      TraitBound &ttb = *static_cast<TraitBound *> (tb.get ());
+	      TypePath &path = ttb.get_type_path ();
+	      bool deusgared = process_type_path (p, ttb, path);
+	      if (deusgared)
+		bounds_to_remove.push_back (i);
+	    }
+
+	  default:
+	    break;
+	  }
+      }
+    for (auto it = bounds_to_remove.rbegin (); it != bounds_to_remove.rend ();
+	 ++it)
+      bounds.erase (bounds.begin () + *it);
+  }
+
+  bool process_type_path (TypeParam &p, TraitBound &parent, TypePath &path)
+  {
+    bool desugared = false;
+    for (auto &segment : path.get_segments ())
+      {
+	switch (segment->get_type ())
+	  {
+	  case TypePathSegment::SegmentType::GENERIC:
+	    {
+	      TypePathSegmentGeneric &seg
+		= *static_cast<TypePathSegmentGeneric *> (segment.get ());
+	      desugared |= process_generic_segment (p, parent, path, seg);
+	    }
+
+	  default:
+	    break;
+	  }
+      }
+    return desugared;
+  }
+
+  bool process_generic_segment (TypeParam &p, TraitBound &parent,
+				TypePath &path, TypePathSegmentGeneric &seg)
+  {
+    // we need to look for any impl types as default arguments in any generics
+    // and remove this index from the generic arguments by using a where
+    // constraint instead
+
+    std::vector<std::unique_ptr<WhereClauseItem>> new_clauses;
+    GenericArgs &generic_args = seg.get_generic_args ();
+    std::vector<std::reference_wrapper<const GenericArgsBinding>>
+      bindings_desugared;
+    std::vector<GenericArgsBinding> &bindings
+      = generic_args.get_binding_args ();
+
+    for (auto &generic : bindings)
+      {
+	auto &t = generic.get_type ();
+	auto translated = DesugarApitType::Desugar (t);
+	auto tt = translated.first;
+
+	auto &implicit_generics = translated.second;
+	if (implicit_generics.empty ())
+	  continue;
+
+	if (tt != &t)
+	  {
+	    bindings_desugared.push_back (generic);
+	    generic.get_type_ptr () = std::unique_ptr<Type> (tt);
+	  }
+
+	for (auto &implicit_generic : implicit_generics)
+	  {
+	    switch (implicit_generic->get_kind ())
+	      {
+	      case GenericParam::Kind::Type:
+		{
+		  TypeParam &tp
+		    = *static_cast<TypeParam *> (implicit_generic.get ());
+
+		  std::vector<std::unique_ptr<TypeParamBound>>
+		    type_param_bounds;
+		  for (auto &b : tp.get_type_param_bounds ())
+		    type_param_bounds.push_back (std::move (b));
+		  tp.get_type_param_bounds ().clear ();
+
+		  // add synthetic parameter for this
+		  synthetic_params.push_back (std::move (implicit_generic));
+
+		  auto bound_type_path
+		    = get_type_for_identifier (tp.get_type_representation ());
+
+		  auto clause = new TypeBoundWhereClauseItem (
+		    {}, std::move (bound_type_path),
+		    std::move (type_param_bounds), tp.get_locus ());
+		  std::unique_ptr<WhereClauseItem> clause_item
+		    = std::unique_ptr<WhereClauseItem> (clause);
+		  new_clauses.push_back (std::move (clause_item));
+		}
+		break;
+
+	      default:
+		synthetic_params.push_back (std::move (implicit_generic));
+		break;
+	      }
+	  }
+      }
+
+    std::vector<std::unique_ptr<TypeParamBound>> type_param_bounds;
+    auto bound = std::unique_ptr<TypeParamBound> (new TraitBound (parent));
+    type_param_bounds.push_back (std::move (bound));
+    auto parent_type_path
+      = get_type_for_identifier (p.get_type_representation ());
+    auto clause
+      = new TypeBoundWhereClauseItem ({}, std::move (parent_type_path),
+				      std::move (type_param_bounds),
+				      parent.get_locus ());
+    std::unique_ptr<WhereClauseItem> clause_item
+      = std::unique_ptr<WhereClauseItem> (clause);
+    where_clause.get_items ().push_back (std::move (clause_item));
+
+    for (auto &where_item : new_clauses)
+      where_clause.get_items ().push_back (std::move (where_item));
+
+    return !bindings_desugared.empty ();
+  }
+
+  static std::unique_ptr<Type> get_type_for_identifier (const Identifier &ident)
+  {
+    auto simple_seg
+      = SimplePathSegment (ident.as_string (), ident.get_locus ());
+    std::vector<SimplePathSegment> simple_segs = {simple_seg};
+    auto simple_path = SimplePath (simple_segs, false, ident.get_locus ());
+    std::vector<std::unique_ptr<TypePathSegment>> segments;
+    segments.push_back (std::unique_ptr<TypePathSegment> (new TypePathSegment (
+      PathIdentSegment (ident.as_string (), ident.get_locus ()), false,
+      ident.get_locus ())));
+    auto type_path = new TypePath (std::move (segments), ident.get_locus ());
+    return std::unique_ptr<Type> (type_path);
+  }
+
+private:
+  WhereClause &where_clause;
+  std::vector<std::unique_ptr<GenericParam>> &generic_params;
+
+  // mutates
+  std::vector<std::unique_ptr<GenericParam>> synthetic_params;
+};
+
+// ---------
+
+DesugarApit::DesugarApit () {}
+
+void
+DesugarApit::go (AST::Crate &crate)
+{
+  DefaultASTVisitor::visit (crate);
+}
+
+void
+DesugarApit::visit (AST::Function &function)
+{
+  if (!function.has_function_params ())
+    return;
+
+  auto &fn_params = function.get_function_params ();
+  for (auto &param : fn_params)
+    {
+      if (param->is_variadic () || param->is_self ())
+	continue;
+
+      auto *p = param.get ();
+      auto &fp = *static_cast<AST::FunctionParam *> (p);
+      auto &type = fp.get_type ();
+
+      auto translated = DesugarApitType::Desugar (type);
+      auto tt = translated.first;
+
+      auto &implicit_generics = translated.second;
+      if (implicit_generics.empty ())
+	continue;
+
+      if (fp.get_type_ptr ().get () != tt)
+	{
+	  fp.get_type_ptr () = std::unique_ptr<AST::Type> (tt);
+	}
+
+      ApitBoundProcessor processor (function.get_where_clause (),
+				    function.get_generic_params ());
+      processor.go (implicit_generics);
+    }
+}
+
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/ast/rust-desugar-apit.h b/gcc/rust/ast/rust-desugar-apit.h
new file mode 100644
index 0000000..07c25e2
--- /dev/null
+++ b/gcc/rust/ast/rust-desugar-apit.h
@@ -0,0 +1,42 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_DESUGAR_APIT_H
+#define RUST_DESUGAR_APIT_H
+
+#include "rust-ast-visitor.h"
+
+namespace Rust {
+namespace AST {
+
+class DesugarApit : public DefaultASTVisitor
+{
+  using DefaultASTVisitor::visit;
+
+public:
+  DesugarApit ();
+  void go (AST::Crate &);
+
+private:
+  void visit (AST::Function &) override;
+};
+
+} // namespace AST
+} // namespace Rust
+
+#endif // ! RUST_DESUGAR_APIT_H
diff --git a/gcc/rust/ast/rust-desugar-for-loops.cc b/gcc/rust/ast/rust-desugar-for-loops.cc
index ffc3470..5cc1c19 100644
--- a/gcc/rust/ast/rust-desugar-for-loops.cc
+++ b/gcc/rust/ast/rust-desugar-for-loops.cc
@@ -17,7 +17,6 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-desugar-for-loops.h"
-#include "rust-ast-visitor.h"
 #include "rust-ast.h"
 #include "rust-hir-map.h"
 #include "rust-path.h"
@@ -31,32 +30,10 @@ namespace AST {
 
 DesugarForLoops::DesugarForLoops () {}
 
-void
-DesugarForLoops::go (AST::Crate &crate)
-{
-  DefaultASTVisitor::visit (crate);
-}
-
-static void
-replace_for_loop (std::unique_ptr<Expr> &for_loop,
-		  std::unique_ptr<Expr> &&expanded)
-{
-  for_loop = std::move (expanded);
-}
-
-MatchArm
-DesugarForLoops::DesugarCtx::make_match_arm (std::unique_ptr<Pattern> &&path)
-{
-  auto patterns = std::vector<std::unique_ptr<Pattern>> ();
-  patterns.emplace_back (std::move (path));
-
-  return MatchArm (std::move (patterns), loc);
-}
-
 MatchCase
 DesugarForLoops::DesugarCtx::make_break_arm ()
 {
-  auto arm = make_match_arm (std::unique_ptr<Pattern> (new PathInExpression (
+  auto arm = builder.match_arm (std::unique_ptr<Pattern> (new PathInExpression (
     builder.path_in_expression (LangItem::Kind::OPTION_NONE))));
 
   auto break_expr
@@ -79,7 +56,7 @@ DesugarForLoops::DesugarCtx::make_continue_arm ()
     builder.path_in_expression (LangItem::Kind::OPTION_SOME),
     std::move (pattern_item)));
 
-  auto val_arm = make_match_arm (std::move (pattern));
+  auto val_arm = builder.match_arm (std::move (pattern));
 
   auto next = builder.identifier (DesugarCtx::next_value_id);
 
@@ -91,14 +68,8 @@ DesugarForLoops::DesugarCtx::make_continue_arm ()
   return MatchCase (std::move (val_arm), std::move (assignment));
 }
 
-std::unique_ptr<Stmt>
-DesugarForLoops::DesugarCtx::statementify (std::unique_ptr<Expr> &&expr)
-{
-  return std::unique_ptr<Stmt> (new ExprStmt (std::move (expr), loc, true));
-}
-
 std::unique_ptr<Expr>
-DesugarForLoops::desugar (AST::ForLoopExpr &expr)
+DesugarForLoops::desugar (ForLoopExpr &expr)
 {
   auto ctx = DesugarCtx (expr.get_locus ());
 
@@ -140,10 +111,10 @@ DesugarForLoops::desugar (AST::ForLoopExpr &expr)
 
   auto loop_stmts = std::vector<std::unique_ptr<Stmt>> ();
   loop_stmts.emplace_back (std::move (let_next));
-  loop_stmts.emplace_back (ctx.statementify (std::move (match_next)));
+  loop_stmts.emplace_back (ctx.builder.statementify (std::move (match_next)));
   loop_stmts.emplace_back (std::move (let_pat));
   loop_stmts.emplace_back (
-    ctx.statementify (expr.get_loop_block ().clone_expr ()));
+    ctx.builder.statementify (expr.get_loop_block ().clone_expr ()));
 
   // loop {
   //     <let_next>;
@@ -170,34 +141,18 @@ DesugarForLoops::desugar (AST::ForLoopExpr &expr)
 }
 
 void
-DesugarForLoops::maybe_desugar_expr (std::unique_ptr<Expr> &expr)
+DesugarForLoops::go (std::unique_ptr<Expr> &ptr)
 {
-  if (expr->get_expr_kind () == AST::Expr::Kind::Loop)
-    {
-      auto &loop = static_cast<AST::BaseLoopExpr &> (*expr);
+  rust_assert (ptr->get_expr_kind () == Expr::Kind::Loop);
 
-      if (loop.get_loop_kind () == AST::BaseLoopExpr::Kind::For)
-	{
-	  auto &for_loop = static_cast<AST::ForLoopExpr &> (loop);
+  auto &loop = static_cast<BaseLoopExpr &> (*ptr);
 
-	  auto desugared = desugar (for_loop);
-
-	  replace_for_loop (expr, std::move (desugared));
-	}
-    }
-}
-
-void
-DesugarForLoops::visit (AST::BlockExpr &block)
-{
-  for (auto &stmt : block.get_statements ())
-    if (stmt->get_stmt_kind () == AST::Stmt::Kind::Expr)
-      maybe_desugar_expr (static_cast<AST::ExprStmt &> (*stmt).get_expr_ptr ());
+  rust_assert (loop.get_loop_kind () == BaseLoopExpr::Kind::For);
 
-  if (block.has_tail_expr ())
-    maybe_desugar_expr (block.get_tail_expr_ptr ());
+  auto &for_loop = static_cast<ForLoopExpr &> (loop);
+  auto desugared = DesugarForLoops ().desugar (for_loop);
 
-  DefaultASTVisitor::visit (block);
+  ptr = std::move (desugared);
 }
 
 } // namespace AST
diff --git a/gcc/rust/ast/rust-desugar-for-loops.h b/gcc/rust/ast/rust-desugar-for-loops.h
index 7beb692..96b63ff 100644
--- a/gcc/rust/ast/rust-desugar-for-loops.h
+++ b/gcc/rust/ast/rust-desugar-for-loops.h
@@ -20,7 +20,6 @@
 #define RUST_DESUGAR_FOR_LOOPS_H
 
 #include "rust-ast-builder.h"
-#include "rust-ast-visitor.h"
 #include "rust-expr.h"
 
 namespace Rust {
@@ -69,15 +68,14 @@ namespace AST {
 // of the way the typechecker is currently structured, where it will fetch name
 // resolution information in order to typecheck paths - which technically isn't
 // necessary.
-class DesugarForLoops : public DefaultASTVisitor
+class DesugarForLoops
 {
-  using DefaultASTVisitor::visit;
-
 public:
-  DesugarForLoops ();
-  void go (AST::Crate &);
+  static void go (std::unique_ptr<Expr> &ptr);
 
 private:
+  DesugarForLoops ();
+
   struct DesugarCtx
   {
     DesugarCtx (location_t loc) : builder (Builder (loc)), loc (loc) {}
@@ -85,10 +83,8 @@ private:
     Builder builder;
     location_t loc;
 
-    MatchArm make_match_arm (std::unique_ptr<Pattern> &&pattern);
     MatchCase make_break_arm ();
     MatchCase make_continue_arm ();
-    std::unique_ptr<Stmt> statementify (std::unique_ptr<Expr> &&expr);
 
     constexpr static const char *continue_pattern_id = "#val";
     constexpr static const char *next_value_id = "#__next";
@@ -96,10 +92,7 @@ private:
     constexpr static const char *result_id = "#result";
   };
 
-  std::unique_ptr<Expr> desugar (AST::ForLoopExpr &expr);
-  void maybe_desugar_expr (std::unique_ptr<Expr> &expr);
-
-  void visit (AST::BlockExpr &) override;
+  std::unique_ptr<Expr> desugar (ForLoopExpr &expr);
 };
 
 } // namespace AST
diff --git a/gcc/rust/ast/rust-desugar-question-mark.cc b/gcc/rust/ast/rust-desugar-question-mark.cc
index 4d2933b..01400d8 100644
--- a/gcc/rust/ast/rust-desugar-question-mark.cc
+++ b/gcc/rust/ast/rust-desugar-question-mark.cc
@@ -18,7 +18,6 @@
 
 #include "rust-desugar-question-mark.h"
 #include "rust-ast-builder.h"
-#include "rust-ast-visitor.h"
 
 namespace Rust {
 namespace AST {
@@ -26,42 +25,14 @@ namespace AST {
 DesugarQuestionMark::DesugarQuestionMark () {}
 
 void
-DesugarQuestionMark::go (AST::Crate &crate)
+DesugarQuestionMark::go (std::unique_ptr<Expr> &ptr)
 {
-  DesugarQuestionMark::visit (crate);
-}
-
-void
-DesugarQuestionMark::visit (ExprStmt &stmt)
-{
-  if (stmt.get_expr ().get_expr_kind () == Expr::Kind::ErrorPropagation)
-    desugar_and_replace (stmt.get_expr_ptr ());
-
-  DefaultASTVisitor::visit (stmt);
-}
-
-void
-DesugarQuestionMark::visit (CallExpr &call)
-{
-  if (call.get_function_expr ().get_expr_kind ()
-      == Expr::Kind::ErrorPropagation)
-    desugar_and_replace (call.get_function_expr_ptr ());
-
-  for (auto &arg : call.get_params ())
-    if (arg->get_expr_kind () == Expr::Kind::ErrorPropagation)
-      desugar_and_replace (arg);
-
-  DefaultASTVisitor::visit (call);
-}
+  rust_assert (ptr->get_expr_kind () == Expr::Kind::ErrorPropagation);
 
-void
-DesugarQuestionMark::visit (LetStmt &stmt)
-{
-  if (stmt.has_init_expr ()
-      && stmt.get_init_expr ().get_expr_kind () == Expr::Kind::ErrorPropagation)
-    desugar_and_replace (stmt.get_init_expr_ptr ());
+  auto original = static_cast<ErrorPropagationExpr &> (*ptr);
+  auto desugared = DesugarQuestionMark ().desugar (original);
 
-  DefaultASTVisitor::visit (stmt);
+  ptr = std::move (desugared);
 }
 
 MatchArm
@@ -99,6 +70,12 @@ ok_case (Builder &builder)
 MatchCase
 err_case (Builder &builder)
 {
+  // TODO: We need to handle the case where there is an enclosing `try {}`
+  // block, as that will create an additional block label that we can break to.
+  // This allows try blocks to use the question mark operator without having the
+  // offending statement early return from the enclosing function
+  // FIXME: How to mark that there is an enclosing block label?
+
   auto val = builder.identifier_pattern ("err");
 
   auto patterns = std::vector<std::unique_ptr<Pattern>> ();
@@ -154,14 +131,5 @@ DesugarQuestionMark::desugar (ErrorPropagationExpr &expr)
 						    expr.get_locus ()));
 }
 
-void
-DesugarQuestionMark::desugar_and_replace (std::unique_ptr<Expr> &ptr)
-{
-  auto original = static_cast<ErrorPropagationExpr &> (*ptr);
-  auto desugared = desugar (original);
-
-  ptr = std::move (desugared);
-}
-
 } // namespace AST
 } // namespace Rust
diff --git a/gcc/rust/ast/rust-desugar-question-mark.h b/gcc/rust/ast/rust-desugar-question-mark.h
index e4c513f..542c52b 100644
--- a/gcc/rust/ast/rust-desugar-question-mark.h
+++ b/gcc/rust/ast/rust-desugar-question-mark.h
@@ -19,9 +19,7 @@
 #ifndef RUST_DESUGAR_QUESTION_MARK
 #define RUST_DESUGAR_QUESTION_MARK
 
-#include "rust-ast-visitor.h"
 #include "rust-expr.h"
-#include "rust-stmt.h"
 
 namespace Rust {
 namespace AST {
@@ -56,21 +54,15 @@ namespace AST {
 //   }
 // }
 // ```
-class DesugarQuestionMark : public DefaultASTVisitor
+class DesugarQuestionMark
 {
-  using DefaultASTVisitor::visit;
-
 public:
-  DesugarQuestionMark ();
-  void go (AST::Crate &);
+  static void go (std::unique_ptr<Expr> &ptr);
 
 private:
-  void desugar_and_replace (std::unique_ptr<Expr> &ptr);
-  std::unique_ptr<Expr> desugar (ErrorPropagationExpr &);
+  DesugarQuestionMark ();
 
-  void visit (AST::ExprStmt &) override;
-  void visit (AST::CallExpr &) override;
-  void visit (AST::LetStmt &) override;
+  std::unique_ptr<Expr> desugar (ErrorPropagationExpr &);
 };
 
 } // namespace AST
diff --git a/gcc/rust/ast/rust-desugar-try-block.cc b/gcc/rust/ast/rust-desugar-try-block.cc
new file mode 100644
index 0000000..07f06aa
--- /dev/null
+++ b/gcc/rust/ast/rust-desugar-try-block.cc
@@ -0,0 +1,62 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-desugar-try-block.h"
+#include "rust-ast-builder.h"
+#include "rust-expr.h"
+
+namespace Rust {
+namespace AST {
+
+DesugarTryBlock::DesugarTryBlock () {}
+
+void
+DesugarTryBlock::go (std::unique_ptr<Expr> &ptr)
+{
+  rust_assert (ptr->get_expr_kind () == Expr::Kind::Try);
+
+  auto original = static_cast<TryExpr &> (*ptr);
+  auto desugared = DesugarTryBlock ().desugar (original);
+
+  ptr = std::move (desugared);
+}
+
+std::unique_ptr<Expr>
+DesugarTryBlock::desugar (TryExpr &expr)
+{
+  auto builder = Builder (expr.get_locus ());
+  auto &block = expr.get_block_expr ();
+
+  if (block.has_statements ())
+    rust_sorry_at (expr.get_locus (),
+		   "cannot desugar try-blocks with statements");
+
+  auto tail_expr = builder.tuple ();
+
+  if (block.has_tail_expr ())
+    tail_expr = block.get_tail_expr ().clone_expr ();
+
+  // Wrap in Try::from_ok call
+  auto from_ok = builder.path_in_expression (LangItem::Kind::TRY_FROM_OK);
+  auto call = builder.call (ptrify (from_ok), std::move (tail_expr));
+
+  return builder.block (std::move (call));
+}
+
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/ast/rust-desugar-try-block.h b/gcc/rust/ast/rust-desugar-try-block.h
new file mode 100644
index 0000000..bfd0463
--- /dev/null
+++ b/gcc/rust/ast/rust-desugar-try-block.h
@@ -0,0 +1,42 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_DESUGAR_TRY_BLOCK
+#define RUST_DESUGAR_TRY_BLOCK
+
+#include "rust-expr.h"
+
+namespace Rust {
+namespace AST {
+
+// FIXME: Add documentation
+class DesugarTryBlock
+{
+public:
+  static void go (std::unique_ptr<Expr> &ptr);
+
+private:
+  DesugarTryBlock ();
+
+  std::unique_ptr<Expr> desugar (TryExpr &);
+};
+
+} // namespace AST
+} // namespace Rust
+
+#endif // ! RUST_DESUGAR_TRY_BLOCK
diff --git a/gcc/rust/ast/rust-desugar-while-let.cc b/gcc/rust/ast/rust-desugar-while-let.cc
new file mode 100644
index 0000000..5eadc59
--- /dev/null
+++ b/gcc/rust/ast/rust-desugar-while-let.cc
@@ -0,0 +1,104 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-desugar-while-let.h"
+#include "rust-ast.h"
+#include "rust-hir-map.h"
+#include "rust-path.h"
+#include "rust-pattern.h"
+#include "rust-stmt.h"
+#include "rust-expr.h"
+#include "rust-ast-builder.h"
+
+namespace Rust {
+namespace AST {
+
+DesugarWhileLet::DesugarWhileLet () {}
+
+MatchCase
+DesugarWhileLet::DesugarCtx::make_break_arm ()
+{
+  auto arm = builder.match_arm (builder.wildcard ());
+
+  auto break_expr
+    = std::unique_ptr<Expr> (new BreakExpr (tl::nullopt, nullptr, {}, loc));
+
+  return MatchCase (std::move (arm), std::move (break_expr));
+}
+
+MatchCase
+DesugarWhileLet::DesugarCtx::make_continue_arm (
+  std::unique_ptr<Pattern> &&pattern, std::unique_ptr<BlockExpr> &&body)
+{
+  auto arm = builder.match_arm (std::move (pattern));
+
+  return MatchCase (std::move (arm), std::move (body));
+}
+
+std::unique_ptr<Expr>
+DesugarWhileLet::desugar (WhileLetLoopExpr &expr)
+{
+  rust_assert (expr.get_patterns ().size () == 1);
+
+  auto pattern = expr.get_patterns ()[0]->clone_pattern ();
+  auto body = expr.get_loop_block ().clone_block_expr ();
+  auto scrutinee = expr.get_scrutinee_expr ().clone_expr ();
+
+  auto ctx = DesugarCtx (expr.get_locus ());
+
+  // _ => break,
+  auto break_arm = ctx.make_break_arm ();
+
+  // <pattern> => <body>,
+  auto continue_arm
+    = ctx.make_continue_arm (std::move (pattern), std::move (body));
+
+  // match <scrutinee> {
+  //     <continue_arm>
+  //     <break_arm>
+  // }
+  auto match_expr
+    = ctx.builder.match (std::move (scrutinee),
+			 {std::move (continue_arm), std::move (break_arm)});
+
+  auto loop_stmts = std::vector<std::unique_ptr<Stmt>> ();
+  loop_stmts.emplace_back (ctx.builder.statementify (std::move (match_expr)));
+
+  // loop {
+  //     <match_expr>
+  // }
+  return ctx.builder.loop (std::move (loop_stmts));
+}
+
+void
+DesugarWhileLet::go (std::unique_ptr<Expr> &ptr)
+{
+  rust_assert (ptr->get_expr_kind () == Expr::Kind::Loop);
+
+  auto &loop = static_cast<BaseLoopExpr &> (*ptr);
+
+  rust_assert (loop.get_loop_kind () == BaseLoopExpr::Kind::WhileLet);
+
+  auto &while_let = static_cast<WhileLetLoopExpr &> (loop);
+  auto desugared = DesugarWhileLet ().desugar (while_let);
+
+  ptr = std::move (desugared);
+}
+
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/ast/rust-desugar-while-let.h b/gcc/rust/ast/rust-desugar-while-let.h
new file mode 100644
index 0000000..60e0693
--- /dev/null
+++ b/gcc/rust/ast/rust-desugar-while-let.h
@@ -0,0 +1,71 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_DESUGAR_WHILE_LET_H
+#define RUST_DESUGAR_WHILE_LET_H
+
+#include "rust-ast-builder.h"
+#include "rust-expr.h"
+
+namespace Rust {
+namespace AST {
+
+// Desugar while-let into a set of other AST nodes. The desugar is of the
+// following form:
+//
+// ```
+// whilet let <pat> = <expr> <body>
+// ```
+//
+// becomes:
+//
+// ```
+// loop {
+//     match <expr> {
+//         <pat> => <body>,
+//         _ => break
+//     }
+// }
+// ```
+class DesugarWhileLet
+{
+public:
+  static void go (std::unique_ptr<Expr> &ptr);
+
+private:
+  DesugarWhileLet ();
+
+  struct DesugarCtx
+  {
+    DesugarCtx (location_t loc) : builder (Builder (loc)), loc (loc) {}
+
+    Builder builder;
+    location_t loc;
+
+    MatchCase make_break_arm ();
+    MatchCase make_continue_arm (std::unique_ptr<Pattern> &&pattern,
+				 std::unique_ptr<BlockExpr> &&body);
+  };
+
+  std::unique_ptr<Expr> desugar (WhileLetLoopExpr &expr);
+};
+
+} // namespace AST
+} // namespace Rust
+
+#endif // ! RUST_DESUGAR_WHILE_LET_H
diff --git a/gcc/rust/ast/rust-expr.h b/gcc/rust/ast/rust-expr.h
index fdb6360..7b0df25 100644
--- a/gcc/rust/ast/rust-expr.h
+++ b/gcc/rust/ast/rust-expr.h
@@ -1,6 +1,7 @@
 #ifndef RUST_AST_EXPR_H
 #define RUST_AST_EXPR_H
 
+#include "optional.h"
 #include "rust-ast.h"
 #include "rust-common.h"
 #include "rust-path.h"
@@ -182,9 +183,13 @@ public:
 
   AttrInputMacro (AttrInputMacro &&oth) : macro (std::move (oth.macro)) {}
 
-  void operator= (const AttrInputMacro &oth);
+  AttrInputMacro &operator= (const AttrInputMacro &oth);
 
-  void operator= (AttrInputMacro &&oth) { macro = std::move (oth.macro); }
+  AttrInputMacro &operator= (AttrInputMacro &&oth)
+  {
+    macro = std::move (oth.macro);
+    return *this;
+  }
 
   std::string as_string () const override;
 
@@ -244,36 +249,50 @@ protected:
   }
 };
 
-// more generic meta item "path = lit" form
-class MetaItemPathLit : public MetaItem
+// more generic meta item "path = expr" form
+class MetaItemPathExpr : public MetaItem
 {
   SimplePath path;
-  LiteralExpr lit;
+  std::unique_ptr<Expr> expr;
 
 public:
-  MetaItemPathLit (SimplePath path, LiteralExpr lit_expr)
-    : path (std::move (path)), lit (std::move (lit_expr))
+  MetaItemPathExpr (SimplePath path, std::unique_ptr<Expr> expr)
+    : path (std::move (path)), expr (std::move (expr))
   {}
 
+  MetaItemPathExpr (const MetaItemPathExpr &other)
+    : MetaItem (other), path (other.path), expr (other.expr->clone_expr ())
+  {}
+
+  MetaItemPathExpr (MetaItemPathExpr &&) = default;
+
+  MetaItemPathExpr &operator= (MetaItemPathExpr &&) = default;
+
+  MetaItemPathExpr operator= (const MetaItemPathExpr &other)
+  {
+    MetaItem::operator= (other);
+    path = other.path;
+    expr = other.expr->clone_expr ();
+    return *this;
+  }
+
   SimplePath get_path () const { return path; }
 
   SimplePath &get_path () { return path; }
 
-  LiteralExpr get_literal () const { return lit; }
-
-  LiteralExpr &get_literal () { return lit; }
+  Expr &get_expr () { return *expr; }
 
   std::string as_string () const override
   {
-    return path.as_string () + " = " + lit.as_string ();
+    return path.as_string () + " = " + expr->as_string ();
   }
 
   MetaItem::ItemKind get_item_kind () const override
   {
-    return MetaItem::ItemKind::PathLit;
+    return MetaItem::ItemKind::PathExpr;
   }
 
-  // There are two Locations in MetaItemPathLit (path and lit_expr),
+  // There are two Locations in MetaItemPathExpr (path and expr),
   //  we have no idea use which of them, just simply return UNKNOWN_LOCATION
   //  now.
   // Maybe we will figure out when we really need the location in the future.
@@ -289,9 +308,9 @@ public:
 
 protected:
   // Use covariance to implement clone function as returning this type
-  MetaItemPathLit *clone_meta_item_inner_impl () const override
+  MetaItemPathExpr *clone_meta_item_inner_impl () const override
   {
-    return new MetaItemPathLit (*this);
+    return new MetaItemPathExpr (*this);
   }
 };
 
@@ -395,6 +414,8 @@ public:
     return *main_or_left_expr;
   }
 
+  bool has_borrow_expr () const { return main_or_left_expr != nullptr; }
+
   bool get_is_mut () const { return mutability == Mutability::Mut; }
 
   Mutability get_mutability () const { return mutability; }
@@ -1160,11 +1181,11 @@ protected:
 // Value array elements
 class ArrayElemsValues : public ArrayElems
 {
-  std::vector<std::unique_ptr<Expr> > values;
+  std::vector<std::unique_ptr<Expr>> values;
   location_t locus;
 
 public:
-  ArrayElemsValues (std::vector<std::unique_ptr<Expr> > elems, location_t locus)
+  ArrayElemsValues (std::vector<std::unique_ptr<Expr>> elems, location_t locus)
     : ArrayElems (), values (std::move (elems)), locus (locus)
   {}
 
@@ -1192,14 +1213,16 @@ public:
 
   std::string as_string () const override;
 
+  location_t get_locus () const { return locus; }
+
   void accept_vis (ASTVisitor &vis) override;
 
   // TODO: this mutable getter seems really dodgy. Think up better way.
-  const std::vector<std::unique_ptr<Expr> > &get_values () const
+  const std::vector<std::unique_ptr<Expr>> &get_values () const
   {
     return values;
   }
-  std::vector<std::unique_ptr<Expr> > &get_values () { return values; }
+  std::vector<std::unique_ptr<Expr>> &get_values () { return values; }
 
   size_t get_num_values () const { return values.size (); }
 
@@ -1214,6 +1237,8 @@ protected:
 class ArrayElemsCopied : public ArrayElems
 {
   std::unique_ptr<Expr> elem_to_copy;
+
+  // TODO: This should be replaced by a ConstExpr
   std::unique_ptr<Expr> num_copies;
   location_t locus;
 
@@ -1246,6 +1271,8 @@ public:
 
   std::string as_string () const override;
 
+  location_t get_locus () const { return locus; }
+
   void accept_vis (ASTVisitor &vis) override;
 
   // TODO: is this better? Or is a "vis_block" better?
@@ -1472,7 +1499,7 @@ class TupleExpr : public ExprWithoutBlock
 {
   std::vector<Attribute> outer_attrs;
   std::vector<Attribute> inner_attrs;
-  std::vector<std::unique_ptr<Expr> > tuple_elems;
+  std::vector<std::unique_ptr<Expr>> tuple_elems;
   location_t locus;
 
   // TODO: find another way to store this to save memory?
@@ -1492,7 +1519,7 @@ public:
     outer_attrs = std::move (new_attrs);
   }
 
-  TupleExpr (std::vector<std::unique_ptr<Expr> > tuple_elements,
+  TupleExpr (std::vector<std::unique_ptr<Expr>> tuple_elements,
 	     std::vector<Attribute> inner_attribs,
 	     std::vector<Attribute> outer_attribs, location_t locus)
     : outer_attrs (std::move (outer_attribs)),
@@ -1543,14 +1570,11 @@ public:
   bool is_marked_for_strip () const override { return marked_for_strip; }
 
   // TODO: this mutable getter seems really dodgy. Think up better way.
-  const std::vector<std::unique_ptr<Expr> > &get_tuple_elems () const
-  {
-    return tuple_elems;
-  }
-  std::vector<std::unique_ptr<Expr> > &get_tuple_elems ()
+  const std::vector<std::unique_ptr<Expr>> &get_tuple_elems () const
   {
     return tuple_elems;
   }
+  std::vector<std::unique_ptr<Expr>> &get_tuple_elems () { return tuple_elems; }
 
   bool is_unit () const { return tuple_elems.size () == 0; }
 
@@ -1775,6 +1799,8 @@ public:
 
   std::string as_string () const;
 
+  location_t get_locus () const { return locus; }
+
   // TODO: is this better? Or is a "vis_block" better?
   Expr &get_base_struct ()
   {
@@ -1972,7 +1998,7 @@ protected:
 class StructExprStructFields : public StructExprStruct
 {
   // std::vector<StructExprField> fields;
-  std::vector<std::unique_ptr<StructExprField> > fields;
+  std::vector<std::unique_ptr<StructExprField>> fields;
 
   // bool has_struct_base;
   StructBase struct_base;
@@ -1985,8 +2011,8 @@ public:
   // Constructor for StructExprStructFields when no struct base is used
   StructExprStructFields (
     PathInExpression struct_path,
-    std::vector<std::unique_ptr<StructExprField> > expr_fields,
-    location_t locus, StructBase base_struct = StructBase::error (),
+    std::vector<std::unique_ptr<StructExprField>> expr_fields, location_t locus,
+    StructBase base_struct = StructBase::error (),
     std::vector<Attribute> inner_attribs = std::vector<Attribute> (),
     std::vector<Attribute> outer_attribs = std::vector<Attribute> ())
     : StructExprStruct (std::move (struct_path), std::move (inner_attribs),
@@ -2023,11 +2049,11 @@ public:
   void accept_vis (ASTVisitor &vis) override;
 
   // TODO: this mutable getter seems really dodgy. Think up better way.
-  std::vector<std::unique_ptr<StructExprField> > &get_fields ()
+  std::vector<std::unique_ptr<StructExprField>> &get_fields ()
   {
     return fields;
   }
-  const std::vector<std::unique_ptr<StructExprField> > &get_fields () const
+  const std::vector<std::unique_ptr<StructExprField>> &get_fields () const
   {
     return fields;
   }
@@ -2084,7 +2110,7 @@ class CallExpr : public ExprWithoutBlock
 {
   std::vector<Attribute> outer_attrs;
   std::unique_ptr<Expr> function;
-  std::vector<std::unique_ptr<Expr> > params;
+  std::vector<std::unique_ptr<Expr>> params;
   location_t locus;
 
 public:
@@ -2093,7 +2119,7 @@ public:
   std::string as_string () const override;
 
   CallExpr (std::unique_ptr<Expr> function_expr,
-	    std::vector<std::unique_ptr<Expr> > function_params,
+	    std::vector<std::unique_ptr<Expr>> function_params,
 	    std::vector<Attribute> outer_attribs, location_t locus)
     : outer_attrs (std::move (outer_attribs)),
       function (std::move (function_expr)),
@@ -2150,11 +2176,11 @@ public:
   bool is_marked_for_strip () const override { return function == nullptr; }
 
   // TODO: this mutable getter seems really dodgy. Think up better way.
-  const std::vector<std::unique_ptr<Expr> > &get_params () const
+  const std::vector<std::unique_ptr<Expr>> &get_params () const
   {
     return params;
   }
-  std::vector<std::unique_ptr<Expr> > &get_params () { return params; }
+  std::vector<std::unique_ptr<Expr>> &get_params () { return params; }
 
   // TODO: is this better? Or is a "vis_block" better?
   Expr &get_function_expr ()
@@ -2190,7 +2216,7 @@ class MethodCallExpr : public ExprWithoutBlock
   std::vector<Attribute> outer_attrs;
   std::unique_ptr<Expr> receiver;
   PathExprSegment method_name;
-  std::vector<std::unique_ptr<Expr> > params;
+  std::vector<std::unique_ptr<Expr>> params;
   location_t locus;
 
 public:
@@ -2198,7 +2224,7 @@ public:
 
   MethodCallExpr (std::unique_ptr<Expr> call_receiver,
 		  PathExprSegment method_path,
-		  std::vector<std::unique_ptr<Expr> > method_params,
+		  std::vector<std::unique_ptr<Expr>> method_params,
 		  std::vector<Attribute> outer_attribs, location_t locus)
     : outer_attrs (std::move (outer_attribs)),
       receiver (std::move (call_receiver)),
@@ -2254,11 +2280,11 @@ public:
   bool is_marked_for_strip () const override { return receiver == nullptr; }
 
   // TODO: this mutable getter seems really dodgy. Think up better way.
-  const std::vector<std::unique_ptr<Expr> > &get_params () const
+  const std::vector<std::unique_ptr<Expr>> &get_params () const
   {
     return params;
   }
-  std::vector<std::unique_ptr<Expr> > &get_params () { return params; }
+  std::vector<std::unique_ptr<Expr>> &get_params () { return params; }
 
   // TODO: is this better? Or is a "vis_block" better?
   Expr &get_receiver_expr ()
@@ -2504,6 +2530,8 @@ public:
   bool get_has_move () const { return has_move; }
 
   Expr::Kind get_expr_kind () const override { return Expr::Kind::Closure; }
+
+  virtual Expr &get_definition_expr () = 0;
 };
 
 // Represents a non-type-specified closure expression AST node
@@ -2563,7 +2591,7 @@ public:
     return closure_inner == nullptr;
   }
 
-  Expr &get_definition_expr ()
+  Expr &get_definition_expr () override
   {
     rust_assert (closure_inner != nullptr);
     return *closure_inner;
@@ -2583,7 +2611,7 @@ class BlockExpr : public ExprWithBlock
 {
   std::vector<Attribute> outer_attrs;
   std::vector<Attribute> inner_attrs;
-  std::vector<std::unique_ptr<Stmt> > statements;
+  std::vector<std::unique_ptr<Stmt>> statements;
   std::unique_ptr<Expr> expr;
   tl::optional<LoopLabel> label;
   location_t start_locus;
@@ -2599,7 +2627,7 @@ public:
   // Returns whether the block contains a final expression.
   bool has_tail_expr () const { return expr != nullptr; }
 
-  BlockExpr (std::vector<std::unique_ptr<Stmt> > block_statements,
+  BlockExpr (std::vector<std::unique_ptr<Stmt>> block_statements,
 	     std::unique_ptr<Expr> block_expr,
 	     std::vector<Attribute> inner_attribs,
 	     std::vector<Attribute> outer_attribs,
@@ -2678,11 +2706,11 @@ public:
   const std::vector<Attribute> &get_inner_attrs () const { return inner_attrs; }
   std::vector<Attribute> &get_inner_attrs () { return inner_attrs; }
 
-  const std::vector<std::unique_ptr<Stmt> > &get_statements () const
+  const std::vector<std::unique_ptr<Stmt>> &get_statements () const
   {
     return statements;
   }
-  std::vector<std::unique_ptr<Stmt> > &get_statements () { return statements; }
+  std::vector<std::unique_ptr<Stmt>> &get_statements () { return statements; }
 
   // TODO: is this better? Or is a "vis_block" better?
   Expr &get_tail_expr ()
@@ -2744,6 +2772,151 @@ protected:
   }
 };
 
+class AnonConst : public ExprWithBlock
+{
+public:
+  enum class Kind
+  {
+    Explicit,
+    DeferredInference,
+  };
+
+  AnonConst (std::unique_ptr<Expr> &&expr, location_t locus = UNKNOWN_LOCATION)
+    : ExprWithBlock (), locus (locus), kind (Kind::Explicit),
+      expr (std::move (expr))
+  {
+    rust_assert (this->expr.value ());
+  }
+
+  AnonConst (location_t locus = UNKNOWN_LOCATION)
+    : ExprWithBlock (), locus (locus), kind (Kind::DeferredInference),
+      expr (tl::nullopt)
+  {}
+
+  AnonConst (const AnonConst &other)
+  {
+    node_id = other.node_id;
+    locus = other.locus;
+    kind = other.kind;
+
+    if (other.expr)
+      expr = other.expr.value ()->clone_expr ();
+  }
+
+  AnonConst operator= (const AnonConst &other)
+  {
+    node_id = other.node_id;
+    locus = other.locus;
+    kind = other.kind;
+
+    if (other.expr)
+      expr = other.expr.value ()->clone_expr ();
+
+    return *this;
+  }
+
+  std::string as_string () const override;
+
+  Expr::Kind get_expr_kind () const override { return Expr::Kind::ConstExpr; }
+
+  location_t get_locus () const override { return locus; }
+
+  Expr &get_inner_expr ()
+  {
+    rust_assert (expr.has_value ());
+    return *expr.value ();
+  }
+
+  NodeId get_node_id () const override { return node_id; }
+
+  /* FIXME: AnonConst are always "internal" and should not have outer attributes
+   * - is that true? Or should we instead call
+   * expr->get_outer_attrs()/expr->set_outer_attrs() */
+
+  std::vector<Attribute> &get_outer_attrs () override
+  {
+    static auto attrs = std::vector<Attribute> ();
+    return attrs;
+  }
+
+  void set_outer_attrs (std::vector<Attribute>) override {}
+
+  /* FIXME: Likewise for mark_for_strip() ? */
+  void mark_for_strip () override {}
+  bool is_marked_for_strip () const override { return false; }
+
+  void accept_vis (ASTVisitor &vis) override;
+
+  bool is_deferred () const { return kind == Kind::DeferredInference; }
+
+private:
+  location_t locus;
+  Kind kind;
+  tl::optional<std::unique_ptr<Expr>> expr;
+
+  AnonConst *clone_expr_with_block_impl () const override
+  {
+    return new AnonConst (*this);
+  }
+};
+
+class ConstBlock : public ExprWithBlock
+{
+public:
+  ConstBlock (AnonConst &&expr, location_t locus = UNKNOWN_LOCATION,
+	      std::vector<Attribute> &&outer_attrs = {})
+    : ExprWithBlock (), expr (std::move (expr)),
+      outer_attrs (std::move (outer_attrs)), locus (locus)
+  {}
+
+  ConstBlock (const ConstBlock &other)
+    : ExprWithBlock (other), expr (other.expr), outer_attrs (other.outer_attrs),
+      locus (other.locus)
+  {}
+
+  ConstBlock operator= (const ConstBlock &other)
+  {
+    expr = other.expr;
+    node_id = other.node_id;
+    outer_attrs = other.outer_attrs;
+    locus = other.locus;
+
+    return *this;
+  }
+
+  std::string as_string () const override;
+
+  Expr::Kind get_expr_kind () const override { return Expr::Kind::ConstBlock; }
+
+  AnonConst &get_const_expr () { return expr; }
+
+  void accept_vis (ASTVisitor &vis) override;
+
+  std::vector<Attribute> &get_outer_attrs () override { return outer_attrs; }
+
+  void set_outer_attrs (std::vector<Attribute> new_attrs) override
+  {
+    outer_attrs = std::move (new_attrs);
+  }
+
+  location_t get_locus () const override { return locus; }
+
+  bool is_marked_for_strip () const override { return marked_for_strip; }
+  void mark_for_strip () override { marked_for_strip = true; }
+
+private:
+  AnonConst expr;
+
+  std::vector<Attribute> outer_attrs;
+  location_t locus;
+  bool marked_for_strip = false;
+
+  ConstBlock *clone_expr_with_block_impl () const override
+  {
+    return new ConstBlock (*this);
+  }
+};
+
 // Represents a type-specified closure expression AST node
 class ClosureExprInnerTyped : public ClosureExpr
 {
@@ -2812,7 +2985,7 @@ public:
   bool is_marked_for_strip () const override { return expr == nullptr; }
 
   // TODO: is this better? Or is a "vis_block" better?
-  BlockExpr &get_definition_block ()
+  BlockExpr &get_definition_expr () override
   {
     rust_assert (expr != nullptr);
     return *expr;
@@ -3572,6 +3745,82 @@ protected:
   }
 };
 
+// Try expression AST node representation
+class TryExpr : public ExprWithBlock
+{
+  std::vector<Attribute> outer_attrs;
+  std::unique_ptr<BlockExpr> block_expr;
+  location_t locus;
+
+  // TODO: find another way to store this to save memory?
+  bool marked_for_strip = false;
+
+public:
+  std::string as_string () const override;
+
+  // Constructor for ReturnExpr.
+  TryExpr (std::unique_ptr<BlockExpr> block_expr,
+	   std::vector<Attribute> outer_attribs, location_t locus)
+    : outer_attrs (std::move (outer_attribs)),
+      block_expr (std::move (block_expr)), locus (locus)
+  {
+    rust_assert (this->block_expr);
+  }
+
+  // Copy constructor with clone
+  TryExpr (TryExpr const &other)
+    : ExprWithBlock (other), outer_attrs (other.outer_attrs),
+      block_expr (other.block_expr->clone_block_expr ()), locus (other.locus),
+      marked_for_strip (other.marked_for_strip)
+  {}
+
+  // Overloaded assignment operator to clone return_expr pointer
+  TryExpr &operator= (TryExpr const &other)
+  {
+    ExprWithBlock::operator= (other);
+    locus = other.locus;
+    marked_for_strip = other.marked_for_strip;
+    outer_attrs = other.outer_attrs;
+
+    block_expr = other.block_expr->clone_block_expr ();
+
+    return *this;
+  }
+
+  // move constructors
+  TryExpr (TryExpr &&other) = default;
+  TryExpr &operator= (TryExpr &&other) = default;
+
+  location_t get_locus () const override final { return locus; }
+
+  void accept_vis (ASTVisitor &vis) override;
+
+  // Can't think of any invalid invariants, so store boolean.
+  void mark_for_strip () override { marked_for_strip = true; }
+  bool is_marked_for_strip () const override { return marked_for_strip; }
+
+  // TODO: is this better? Or is a "vis_block" better?
+  BlockExpr &get_block_expr () { return *block_expr; }
+
+  const std::vector<Attribute> &get_outer_attrs () const { return outer_attrs; }
+  std::vector<Attribute> &get_outer_attrs () override { return outer_attrs; }
+
+  void set_outer_attrs (std::vector<Attribute> new_attrs) override
+  {
+    outer_attrs = std::move (new_attrs);
+  }
+
+  Expr::Kind get_expr_kind () const override { return Expr::Kind::Try; }
+
+protected:
+  /* Use covariance to implement clone function as returning this object rather
+   * than base */
+  TryExpr *clone_expr_with_block_impl () const override
+  {
+    return new TryExpr (*this);
+  }
+};
+
 // Forward decl - defined in rust-macro.h
 class MacroInvocation;
 
@@ -3847,14 +4096,14 @@ protected:
 class WhileLetLoopExpr : public BaseLoopExpr
 {
   // MatchArmPatterns patterns;
-  std::vector<std::unique_ptr<Pattern> > match_arm_patterns; // inlined
+  std::vector<std::unique_ptr<Pattern>> match_arm_patterns; // inlined
   std::unique_ptr<Expr> scrutinee;
 
 public:
   std::string as_string () const override;
 
   // Constructor with a loop label
-  WhileLetLoopExpr (std::vector<std::unique_ptr<Pattern> > match_arm_patterns,
+  WhileLetLoopExpr (std::vector<std::unique_ptr<Pattern>> match_arm_patterns,
 		    std::unique_ptr<Expr> scrutinee,
 		    std::unique_ptr<BlockExpr> loop_block, location_t locus,
 		    tl::optional<LoopLabel> loop_label = tl::nullopt,
@@ -3908,11 +4157,11 @@ public:
   }
 
   // TODO: this mutable getter seems really dodgy. Think up better way.
-  const std::vector<std::unique_ptr<Pattern> > &get_patterns () const
+  const std::vector<std::unique_ptr<Pattern>> &get_patterns () const
   {
     return match_arm_patterns;
   }
-  std::vector<std::unique_ptr<Pattern> > &get_patterns ()
+  std::vector<std::unique_ptr<Pattern>> &get_patterns ()
   {
     return match_arm_patterns;
   }
@@ -4195,7 +4444,7 @@ protected:
 class IfLetExpr : public ExprWithBlock
 {
   std::vector<Attribute> outer_attrs;
-  std::vector<std::unique_ptr<Pattern> > match_arm_patterns; // inlined
+  std::vector<std::unique_ptr<Pattern>> match_arm_patterns; // inlined
   std::unique_ptr<Expr> value;
   std::unique_ptr<BlockExpr> if_block;
   location_t locus;
@@ -4203,7 +4452,7 @@ class IfLetExpr : public ExprWithBlock
 public:
   std::string as_string () const override;
 
-  IfLetExpr (std::vector<std::unique_ptr<Pattern> > match_arm_patterns,
+  IfLetExpr (std::vector<std::unique_ptr<Pattern>> match_arm_patterns,
 	     std::unique_ptr<Expr> value, std::unique_ptr<BlockExpr> if_block,
 	     std::vector<Attribute> outer_attrs, location_t locus)
     : outer_attrs (std::move (outer_attrs)),
@@ -4297,11 +4546,11 @@ public:
   }
 
   // TODO: this mutable getter seems really dodgy. Think up better way.
-  const std::vector<std::unique_ptr<Pattern> > &get_patterns () const
+  const std::vector<std::unique_ptr<Pattern>> &get_patterns () const
   {
     return match_arm_patterns;
   }
-  std::vector<std::unique_ptr<Pattern> > &get_patterns ()
+  std::vector<std::unique_ptr<Pattern>> &get_patterns ()
   {
     return match_arm_patterns;
   }
@@ -4341,11 +4590,11 @@ class IfLetExprConseqElse : public IfLetExpr
 public:
   std::string as_string () const override;
 
-  IfLetExprConseqElse (
-    std::vector<std::unique_ptr<Pattern> > match_arm_patterns,
-    std::unique_ptr<Expr> value, std::unique_ptr<BlockExpr> if_block,
-    std::unique_ptr<ExprWithBlock> else_block,
-    std::vector<Attribute> outer_attrs, location_t locus)
+  IfLetExprConseqElse (std::vector<std::unique_ptr<Pattern>> match_arm_patterns,
+		       std::unique_ptr<Expr> value,
+		       std::unique_ptr<BlockExpr> if_block,
+		       std::unique_ptr<ExprWithBlock> else_block,
+		       std::vector<Attribute> outer_attrs, location_t locus)
     : IfLetExpr (std::move (match_arm_patterns), std::move (value),
 		 std::move (if_block), std::move (outer_attrs), locus),
       else_block (std::move (else_block))
@@ -4398,7 +4647,7 @@ struct MatchArm
 private:
   std::vector<Attribute> outer_attrs;
   // MatchArmPatterns patterns;
-  std::vector<std::unique_ptr<Pattern> > match_arm_patterns; // inlined
+  std::vector<std::unique_ptr<Pattern>> match_arm_patterns; // inlined
 
   // bool has_match_arm_guard;
   // inlined from MatchArmGuard
@@ -4411,7 +4660,7 @@ public:
   bool has_match_arm_guard () const { return guard_expr != nullptr; }
 
   // Constructor for match arm with a guard expression
-  MatchArm (std::vector<std::unique_ptr<Pattern> > match_arm_patterns,
+  MatchArm (std::vector<std::unique_ptr<Pattern>> match_arm_patterns,
 	    location_t locus, std::unique_ptr<Expr> guard_expr = nullptr,
 	    std::vector<Attribute> outer_attrs = std::vector<Attribute> ())
     : outer_attrs (std::move (outer_attrs)),
@@ -4463,7 +4712,7 @@ public:
   static MatchArm create_error ()
   {
     location_t locus = UNDEF_LOCATION;
-    return MatchArm (std::vector<std::unique_ptr<Pattern> > (), locus);
+    return MatchArm (std::vector<std::unique_ptr<Pattern>> (), locus);
   }
 
   std::string as_string () const;
@@ -4485,11 +4734,11 @@ public:
   const std::vector<Attribute> &get_outer_attrs () const { return outer_attrs; }
   std::vector<Attribute> &get_outer_attrs () { return outer_attrs; }
 
-  const std::vector<std::unique_ptr<Pattern> > &get_patterns () const
+  const std::vector<std::unique_ptr<Pattern>> &get_patterns () const
   {
     return match_arm_patterns;
   }
-  std::vector<std::unique_ptr<Pattern> > &get_patterns ()
+  std::vector<std::unique_ptr<Pattern>> &get_patterns ()
   {
     return match_arm_patterns;
   }
@@ -4836,29 +5085,6 @@ enum class InlineAsmOption
   MAY_UNWIND = 1 << 8,
 };
 
-struct AnonConst
-{
-  NodeId id;
-  std::unique_ptr<Expr> expr;
-  AnonConst (NodeId id, std::unique_ptr<Expr> expr)
-    : id (id), expr (std::move (expr))
-  {
-    rust_assert (this->expr != nullptr);
-  }
-  AnonConst (const AnonConst &other)
-  {
-    id = other.id;
-    expr = other.expr->clone_expr ();
-  }
-
-  AnonConst operator= (const AnonConst &other)
-  {
-    id = other.id;
-    expr = other.expr->clone_expr ();
-    return *this;
-  }
-};
-
 struct InlineAsmRegOrRegClass
 {
   enum Type
@@ -5288,6 +5514,20 @@ struct TupleTemplateStr
 // Inline Assembly Node
 class InlineAsm : public ExprWithoutBlock
 {
+public:
+  enum class Option
+  {
+    PURE = 1 << 0,
+    NOMEM = 1 << 1,
+    READONLY = 1 << 2,
+    PRESERVES_FLAGS = 1 << 3,
+    NORETURN = 1 << 4,
+    NOSTACK = 1 << 5,
+    ATT_SYNTAX = 1 << 6,
+    RAW = 1 << 7,
+    MAY_UNWIND = 1 << 8,
+  };
+
 private:
   location_t locus;
   // TODO: Not sure how outer_attrs plays with InlineAsm, I put it here in order
@@ -5311,7 +5551,7 @@ public:
   std::map<std::string, int> named_args;
   std::set<int> reg_args;
   std::vector<TupleClobber> clobber_abi;
-  std::set<InlineAsmOption> options;
+  std::set<InlineAsm::Option> options;
 
   std::vector<location_t> line_spans;
 
@@ -5342,7 +5582,7 @@ public:
 
   std::vector<TupleClobber> get_clobber_abi () { return clobber_abi; }
 
-  std::set<InlineAsmOption> get_options () { return options; }
+  std::set<InlineAsm::Option> get_options () { return options; }
 
   InlineAsm *clone_expr_without_block_impl () const override
   {
@@ -5350,6 +5590,33 @@ public:
   }
 
   Expr::Kind get_expr_kind () const override { return Expr::Kind::InlineAsm; }
+
+  static std::string option_to_string (Option option)
+  {
+    switch (option)
+      {
+      case Option::PURE:
+	return "pure";
+      case Option::NOMEM:
+	return "nomem";
+      case Option::READONLY:
+	return "readonly";
+      case Option::PRESERVES_FLAGS:
+	return "preserves_flags";
+      case Option::NORETURN:
+	return "noreturn";
+      case Option::NOSTACK:
+	return "nostack";
+      case Option::ATT_SYNTAX:
+	return "att_syntax";
+      case Option::RAW:
+	return "raw";
+      case Option::MAY_UNWIND:
+	return "may_unwind";
+      default:
+	rust_unreachable ();
+      }
+  }
 };
 
 class LlvmInlineAsm : public ExprWithoutBlock
diff --git a/gcc/rust/ast/rust-expression-yeast.cc b/gcc/rust/ast/rust-expression-yeast.cc
new file mode 100644
index 0000000..9f6a62f
--- /dev/null
+++ b/gcc/rust/ast/rust-expression-yeast.cc
@@ -0,0 +1,118 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-expression-yeast.h"
+#include "rust-ast-visitor.h"
+#include "rust-desugar-question-mark.h"
+#include "rust-desugar-try-block.h"
+#include "rust-desugar-for-loops.h"
+#include "rust-ast-full.h"
+#include "rust-desugar-while-let.h"
+#include "rust-expr.h"
+#include "rust-stmt.h"
+
+namespace Rust {
+namespace AST {
+
+void
+ExpressionYeast::go (AST::Crate &crate)
+{
+  DefaultASTVisitor::visit (crate);
+}
+
+void
+ExpressionYeast::dispatch_loops (std::unique_ptr<Expr> &loop_expr)
+{
+  auto &loop = static_cast<BaseLoopExpr &> (*loop_expr.get ());
+
+  switch (loop.get_loop_kind ())
+    {
+    case BaseLoopExpr::Kind::For:
+      DesugarForLoops::go (loop_expr);
+      break;
+    case BaseLoopExpr::Kind::WhileLet:
+      DesugarWhileLet::go (loop_expr);
+      break;
+    default:
+      break;
+    }
+}
+
+void
+ExpressionYeast::dispatch (std::unique_ptr<Expr> &expr)
+{
+  switch (expr->get_expr_kind ())
+    {
+    case Expr::Kind::ErrorPropagation:
+      DesugarQuestionMark::go (expr);
+      break;
+    case Expr::Kind::Try:
+      DesugarTryBlock::go (expr);
+      break;
+    case Expr::Kind::Loop:
+      dispatch_loops (expr);
+      break;
+
+    default:
+      break;
+    }
+}
+
+void
+ExpressionYeast::visit (ExprStmt &stmt)
+{
+  dispatch (stmt.get_expr_ptr ());
+
+  DefaultASTVisitor::visit (stmt);
+}
+
+void
+ExpressionYeast::visit (CallExpr &call)
+{
+  dispatch (call.get_function_expr_ptr ());
+
+  for (auto &arg : call.get_params ())
+    dispatch (arg);
+
+  DefaultASTVisitor::visit (call);
+}
+
+void
+ExpressionYeast::visit (BlockExpr &block)
+{
+  for (auto &stmt : block.get_statements ())
+    if (stmt->get_stmt_kind () == Stmt::Kind::Expr)
+      dispatch (static_cast<ExprStmt &> (*stmt).get_expr_ptr ());
+
+  if (block.has_tail_expr ())
+    dispatch (block.get_tail_expr_ptr ());
+
+  DefaultASTVisitor::visit (block);
+}
+
+void
+ExpressionYeast::visit (LetStmt &stmt)
+{
+  if (stmt.has_init_expr ())
+    dispatch (stmt.get_init_expr_ptr ());
+
+  DefaultASTVisitor::visit (stmt);
+}
+
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/ast/rust-expression-yeast.h b/gcc/rust/ast/rust-expression-yeast.h
new file mode 100644
index 0000000..855918f
--- /dev/null
+++ b/gcc/rust/ast/rust-expression-yeast.h
@@ -0,0 +1,52 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_EXPRESSION_YEAST
+#define RUST_EXPRESSION_YEAST
+
+#include "rust-ast-visitor.h"
+#include "rust-ast.h"
+#include "rust-desugar-question-mark.h"
+
+namespace Rust {
+namespace AST {
+
+// This visitor takes care of all the expression desugars: try-blocks,
+// error-propagation, etc.
+class ExpressionYeast : public AST::DefaultASTVisitor
+{
+  using AST::DefaultASTVisitor::visit;
+
+public:
+  void go (AST::Crate &);
+
+private:
+  // Dispatch to the proper desugar
+  void dispatch (std::unique_ptr<Expr> &expr);
+  void dispatch_loops (std::unique_ptr<Expr> &loop_expr);
+
+  void visit (AST::ExprStmt &) override;
+  void visit (AST::CallExpr &) override;
+  void visit (AST::LetStmt &) override;
+  void visit (AST::BlockExpr &) override;
+};
+
+} // namespace AST
+} // namespace Rust
+
+#endif // ! RUST_EXPRESSION_YEAST
diff --git a/gcc/rust/ast/rust-fmt.h b/gcc/rust/ast/rust-fmt.h
index a54faec..3722db2 100644
--- a/gcc/rust/ast/rust-fmt.h
+++ b/gcc/rust/ast/rust-fmt.h
@@ -266,11 +266,10 @@ enum ParseMode
 
 extern "C" {
 
-FormatArgsHandle
-collect_pieces (const char *input, bool append_newline, ParseMode parse_mode);
+FormatArgsHandle collect_pieces (const char *input, bool append_newline,
+				 ParseMode parse_mode);
 
-FormatArgsHandle
-clone_pieces (const FormatArgsHandle &);
+FormatArgsHandle clone_pieces (const FormatArgsHandle &);
 
 void destroy_pieces (FormatArgsHandle);
 
diff --git a/gcc/rust/ast/rust-item.h b/gcc/rust/ast/rust-item.h
index 062f85d..d11eed7 100644
--- a/gcc/rust/ast/rust-item.h
+++ b/gcc/rust/ast/rust-item.h
@@ -51,21 +51,12 @@ class TypePath;
 // A type generic parameter (as opposed to a lifetime generic parameter)
 class TypeParam : public GenericParam
 {
-  // bool has_outer_attribute;
-  // std::unique_ptr<Attribute> outer_attr;
   AST::AttrVec outer_attrs;
-
   Identifier type_representation;
-
-  // bool has_type_param_bounds;
-  // TypeParamBounds type_param_bounds;
-  std::vector<std::unique_ptr<TypeParamBound>>
-    type_param_bounds; // inlined form
-
-  // bool has_type;
+  std::vector<std::unique_ptr<TypeParamBound>> type_param_bounds;
   std::unique_ptr<Type> type;
-
   location_t locus;
+  bool was_impl_trait;
 
 public:
   Identifier get_type_representation () const { return type_representation; }
@@ -85,18 +76,19 @@ public:
 	     std::vector<std::unique_ptr<TypeParamBound>> type_param_bounds
 	     = std::vector<std::unique_ptr<TypeParamBound>> (),
 	     std::unique_ptr<Type> type = nullptr,
-	     AST::AttrVec outer_attrs = {})
+	     AST::AttrVec outer_attrs = {}, bool was_impl_trait = false)
     : GenericParam (Analysis::Mappings::get ().get_next_node_id ()),
       outer_attrs (std::move (outer_attrs)),
       type_representation (std::move (type_representation)),
       type_param_bounds (std::move (type_param_bounds)),
-      type (std::move (type)), locus (locus)
+      type (std::move (type)), locus (locus), was_impl_trait (was_impl_trait)
   {}
 
   // Copy constructor uses clone
   TypeParam (TypeParam const &other)
     : GenericParam (other.node_id), outer_attrs (other.outer_attrs),
-      type_representation (other.type_representation), locus (other.locus)
+      type_representation (other.type_representation), locus (other.locus),
+      was_impl_trait (other.was_impl_trait)
   {
     // guard to prevent null pointer dereference
     if (other.type != nullptr)
@@ -114,6 +106,7 @@ public:
     outer_attrs = other.outer_attrs;
     locus = other.locus;
     node_id = other.node_id;
+    was_impl_trait = other.was_impl_trait;
 
     // guard to prevent null pointer dereference
     if (other.type != nullptr)
@@ -153,17 +146,19 @@ public:
     return type;
   }
 
-  // TODO: mutable getter seems kinda dodgy
   std::vector<std::unique_ptr<TypeParamBound>> &get_type_param_bounds ()
   {
     return type_param_bounds;
   }
+
   const std::vector<std::unique_ptr<TypeParamBound>> &
   get_type_param_bounds () const
   {
     return type_param_bounds;
   }
 
+  bool from_impl_trait () const { return was_impl_trait; }
+
 protected:
   // Clone function implementation as virtual method
   TypeParam *clone_generic_param_impl () const override
@@ -2455,7 +2450,7 @@ class ConstantItem : public VisItem, public AssociatedItem
   // either has an identifier or "_" - maybe handle in identifier?
   // bool identifier_is_underscore;
   // if no identifier declared, identifier will be "_"
-  std::string identifier;
+  Identifier identifier;
 
   std::unique_ptr<Type> type;
   std::unique_ptr<Expr> const_expr;
@@ -2465,7 +2460,7 @@ class ConstantItem : public VisItem, public AssociatedItem
 public:
   std::string as_string () const override;
 
-  ConstantItem (std::string ident, Visibility vis, std::unique_ptr<Type> type,
+  ConstantItem (Identifier ident, Visibility vis, std::unique_ptr<Type> type,
 		std::unique_ptr<Expr> const_expr,
 		std::vector<Attribute> outer_attrs, location_t locus)
     : VisItem (std::move (vis), std::move (outer_attrs)),
@@ -2473,7 +2468,7 @@ public:
       const_expr (std::move (const_expr)), locus (locus)
   {}
 
-  ConstantItem (std::string ident, Visibility vis, std::unique_ptr<Type> type,
+  ConstantItem (Identifier ident, Visibility vis, std::unique_ptr<Type> type,
 		std::vector<Attribute> outer_attrs, location_t locus)
     : VisItem (std::move (vis), std::move (outer_attrs)),
       identifier (std::move (ident)), type (std::move (type)),
@@ -2516,7 +2511,7 @@ public:
 
   /* Returns whether constant item is an "unnamed" (wildcard underscore used
    * as identifier) constant. */
-  bool is_unnamed () const { return identifier == "_"; }
+  bool is_unnamed () const { return identifier.as_string () == "_"; }
 
   location_t get_locus () const override final { return locus; }
 
@@ -2561,7 +2556,7 @@ public:
     return type;
   }
 
-  std::string get_identifier () const { return identifier; }
+  const Identifier &get_identifier () const { return identifier; }
 
   Item::Kind get_item_kind () const override
   {
diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h
index fc01e57..4165075 100644
--- a/gcc/rust/ast/rust-macro.h
+++ b/gcc/rust/ast/rust-macro.h
@@ -27,6 +27,11 @@
 #include "rust-macro-builtins.h"
 
 namespace Rust {
+
+// forward declarations for AttributeParser
+class MacroInvocLexer;
+template <typename ManagedTokenSource> class Parser;
+
 namespace AST {
 
 class MacroFragSpec
@@ -756,22 +761,16 @@ private:
   std::vector<std::unique_ptr<MacroInvocation>> pending_eager_invocs;
 
 protected:
-  /* Use covariance to implement clone function as returning this object rather
-   * than base */
   MacroInvocation *clone_pattern_impl () const final override
   {
     return clone_macro_invocation_impl ();
   }
 
-  /* Use covariance to implement clone function as returning this object rather
-   * than base */
   MacroInvocation *clone_expr_without_block_impl () const final override
   {
     return clone_macro_invocation_impl ();
   }
 
-  /* Use covariance to implement clone function as returning this object rather
-   * than base */
   MacroInvocation *clone_type_no_bounds_impl () const final override
   {
     return clone_macro_invocation_impl ();
@@ -788,6 +787,20 @@ public:
     return new MacroInvocation (*this);
   }
 
+  std::unique_ptr<MacroInvocation> reconstruct_macro_invocation () const
+  {
+    return nullptr;
+    //  return reconstruct (this,
+    // &MacroInvocation::reconstruct_macro_invocation_impl);
+  }
+
+  MacroInvocation *reconstruct_impl () const override
+  {
+    return new MacroInvocation (kind, builtin_kind, invoc_data, outer_attrs,
+				locus, is_semi_coloned,
+				reconstruct_vec (pending_eager_invocs));
+  }
+
   void add_semicolon () override { is_semi_coloned = true; }
 
   Pattern::Kind get_pattern_kind () override
@@ -1108,16 +1121,14 @@ struct AttributeParser
 {
 private:
   // TODO: might as well rewrite to use lexer tokens
-  std::vector<std::unique_ptr<Token>> token_stream;
-  int stream_pos;
+  std::unique_ptr<MacroInvocLexer> lexer;
+  std::unique_ptr<Parser<MacroInvocLexer>> parser;
 
 public:
   AttributeParser (std::vector<std::unique_ptr<Token>> token_stream,
-		   int stream_start_pos = 0)
-    : token_stream (std::move (token_stream)), stream_pos (stream_start_pos)
-  {}
+		   int stream_start_pos = 0);
 
-  ~AttributeParser () = default;
+  ~AttributeParser ();
 
   std::vector<std::unique_ptr<MetaItemInner>> parse_meta_item_seq ();
 
@@ -1126,24 +1137,10 @@ private:
   std::unique_ptr<MetaItemInner> parse_meta_item_inner ();
   // Returns whether token can end a meta item.
   bool is_end_meta_item_tok (TokenId id) const;
-  // Parses a simple path.
-  SimplePath parse_simple_path ();
-  // Parses a segment of a simple path (but not scope resolution operator).
-  SimplePathSegment parse_simple_path_segment ();
   // Parses a MetaItemLitExpr.
   std::unique_ptr<MetaItemLitExpr> parse_meta_item_lit ();
-  // Parses a literal.
-  Literal parse_literal ();
   // Parses a meta item that begins with a simple path.
   std::unique_ptr<MetaItem> parse_path_meta_item ();
-
-  // TODO: should this be const?
-  std::unique_ptr<Token> &peek_token (int i = 0)
-  {
-    return token_stream[stream_pos + i];
-  }
-
-  void skip_token (int i = 0) { stream_pos += 1 + i; }
 };
 } // namespace AST
 } // namespace Rust
diff --git a/gcc/rust/ast/rust-path.cc b/gcc/rust/ast/rust-path.cc
index 8e43ddf..793423a 100644
--- a/gcc/rust/ast/rust-path.cc
+++ b/gcc/rust/ast/rust-path.cc
@@ -266,6 +266,27 @@ TypePath::as_simple_path () const
 		     locus);
 }
 
+std::string
+TypePath::make_debug_string () const
+{
+  rust_assert (!segments.empty ());
+
+  std::string output;
+
+  for (const auto &segment : segments)
+    {
+      if (segment != nullptr && !segment->is_lang_item ()
+	  && !segment->is_error ())
+	{
+	  if (!output.empty () || has_opening_scope_resolution_op ())
+	    output.append ("::");
+	  output.append (segment->get_ident_segment ().as_string ());
+	}
+    }
+
+  return output;
+}
+
 // hopefully definition here will prevent circular dependency issue
 TraitBound *
 TypePath::to_trait_bound (bool in_parens) const
diff --git a/gcc/rust/ast/rust-path.h b/gcc/rust/ast/rust-path.h
index a4ba93b..a1b19d5 100644
--- a/gcc/rust/ast/rust-path.h
+++ b/gcc/rust/ast/rust-path.h
@@ -391,6 +391,13 @@ public:
     return default_value.value ();
   }
 
+  tl::optional<GenericArg> &get_default_value () { return default_value; }
+
+  const tl::optional<GenericArg> &get_default_value () const
+  {
+    return default_value;
+  }
+
   std::string as_string () const override;
 
   void accept_vis (ASTVisitor &vis) override;
@@ -779,6 +786,11 @@ public:
   {
     return new TypePathSegment (*this);
   }
+  virtual TypePathSegment *reconstruct_impl () const
+  {
+    return new TypePathSegment (lang_item, ident_segment,
+				has_separating_scope_resolution, locus);
+  }
 
 public:
   virtual ~TypePathSegment () {}
@@ -790,6 +802,11 @@ public:
   {
     return std::unique_ptr<TypePathSegment> (clone_type_path_segment_impl ());
   }
+  // Unique pointer custom reconstruct function
+  std::unique_ptr<TypePathSegment> reconstruct () const
+  {
+    return reconstruct_base (this);
+  }
 
   TypePathSegment (PathIdentSegment ident_segment,
 		   bool has_separating_scope_resolution, location_t locus)
@@ -814,6 +831,15 @@ public:
       node_id (Analysis::Mappings::get ().get_next_node_id ())
   {}
 
+  // General constructor
+  TypePathSegment (tl::optional<LangItem::Kind> lang_item,
+		   tl::optional<PathIdentSegment> ident_segment,
+		   bool has_separating_scope_resolution, location_t locus)
+    : lang_item (lang_item), ident_segment (ident_segment), locus (locus),
+      has_separating_scope_resolution (has_separating_scope_resolution),
+      node_id (Analysis::Mappings::get ().get_next_node_id ())
+  {}
+
   TypePathSegment (TypePathSegment const &other)
     : lang_item (other.lang_item), ident_segment (other.ident_segment),
       locus (other.locus),
@@ -968,11 +994,7 @@ public:
   void accept_vis (ASTVisitor &vis) override;
 
   // TODO: is this better? Or is a "vis_pattern" better?
-  GenericArgs &get_generic_args ()
-  {
-    rust_assert (has_generic_args ());
-    return generic_args;
-  }
+  GenericArgs &get_generic_args () { return generic_args; }
 
   // Use covariance to override base class method
   TypePathSegmentGeneric *clone_type_path_segment_impl () const override
@@ -1149,6 +1171,11 @@ protected:
   {
     return new TypePath (*this);
   }
+  TypePath *reconstruct_impl () const override
+  {
+    return new TypePath (reconstruct_vec (segments), locus,
+			 has_opening_scope_resolution);
+  }
 
 public:
   /* Returns whether the TypePath has an opening scope resolution operator
@@ -1215,6 +1242,8 @@ public:
 
   std::string as_string () const override;
 
+  std::string make_debug_string () const;
+
   /* Converts TypePath to SimplePath if possible (i.e. no generic or function
    * arguments). Otherwise returns an empty SimplePath. */
   SimplePath as_simple_path () const;
@@ -1438,6 +1467,12 @@ protected:
   {
     return new QualifiedPathInType (*this);
   }
+  QualifiedPathInType *reconstruct_impl () const override
+  {
+    return new QualifiedPathInType (path_type,
+				    associated_segment->reconstruct (),
+				    reconstruct_vec (segments), locus);
+  }
 
 public:
   QualifiedPathInType (
diff --git a/gcc/rust/ast/rust-pattern.cc b/gcc/rust/ast/rust-pattern.cc
index fc7b610..15ab0b7 100644
--- a/gcc/rust/ast/rust-pattern.cc
+++ b/gcc/rust/ast/rust-pattern.cc
@@ -65,8 +65,8 @@ IdentifierPattern::as_string () const
 
   str += variable_ident.as_string ();
 
-  if (has_pattern_to_bind ())
-    str += " @ " + to_bind->as_string ();
+  if (has_subpattern ())
+    str += " @ " + subpattern->as_string ();
 
   return str;
 }
@@ -327,17 +327,53 @@ GroupedExpr::as_string () const
 }
 
 std::string
-SlicePattern::as_string () const
+SlicePatternItemsNoRest::as_string () const
 {
-  std::string str ("SlicePattern: ");
+  std::string str;
 
-  for (const auto &pattern : items)
+  for (const auto &pattern : patterns)
     str += "\n " + pattern->as_string ();
 
   return str;
 }
 
 std::string
+SlicePatternItemsHasRest::as_string () const
+{
+  std::string str;
+
+  str += "\n Lower patterns: ";
+  if (lower_patterns.empty ())
+    {
+      str += "none";
+    }
+  else
+    {
+      for (const auto &lower : lower_patterns)
+	str += "\n  " + lower->as_string ();
+    }
+
+  str += "\n Upper patterns: ";
+  if (upper_patterns.empty ())
+    {
+      str += "none";
+    }
+  else
+    {
+      for (const auto &upper : upper_patterns)
+	str += "\n  " + upper->as_string ();
+    }
+
+  return str;
+}
+
+std::string
+SlicePattern::as_string () const
+{
+  return "SlicePattern: " + items->as_string ();
+}
+
+std::string
 AltPattern::as_string () const
 {
   std::string str ("AltPattern: ");
@@ -367,6 +403,18 @@ GroupedExpr::accept_vis (ASTVisitor &vis)
 }
 
 void
+SlicePatternItemsNoRest::accept_vis (ASTVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
+SlicePatternItemsHasRest::accept_vis (ASTVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
 SlicePattern::accept_vis (ASTVisitor &vis)
 {
   vis.visit (*this);
diff --git a/gcc/rust/ast/rust-pattern.h b/gcc/rust/ast/rust-pattern.h
index 69dbd98..4945ec4 100644
--- a/gcc/rust/ast/rust-pattern.h
+++ b/gcc/rust/ast/rust-pattern.h
@@ -74,7 +74,7 @@ class IdentifierPattern : public Pattern
   bool is_mut;
 
   // bool has_pattern;
-  std::unique_ptr<Pattern> to_bind;
+  std::unique_ptr<Pattern> subpattern;
   location_t locus;
   NodeId node_id;
 
@@ -82,22 +82,22 @@ public:
   std::string as_string () const override;
 
   // Returns whether the IdentifierPattern has a pattern to bind.
-  bool has_pattern_to_bind () const { return to_bind != nullptr; }
+  bool has_subpattern () const { return subpattern != nullptr; }
 
   // Constructor
   IdentifierPattern (Identifier ident, location_t locus, bool is_ref = false,
 		     bool is_mut = false,
-		     std::unique_ptr<Pattern> to_bind = nullptr)
+		     std::unique_ptr<Pattern> subpattern = nullptr)
     : Pattern (), variable_ident (std::move (ident)), is_ref (is_ref),
-      is_mut (is_mut), to_bind (std::move (to_bind)), locus (locus),
+      is_mut (is_mut), subpattern (std::move (subpattern)), locus (locus),
       node_id (Analysis::Mappings::get ().get_next_node_id ())
   {}
 
   IdentifierPattern (NodeId node_id, Identifier ident, location_t locus,
 		     bool is_ref = false, bool is_mut = false,
-		     std::unique_ptr<Pattern> to_bind = nullptr)
+		     std::unique_ptr<Pattern> subpattern = nullptr)
     : Pattern (), variable_ident (std::move (ident)), is_ref (is_ref),
-      is_mut (is_mut), to_bind (std::move (to_bind)), locus (locus),
+      is_mut (is_mut), subpattern (std::move (subpattern)), locus (locus),
       node_id (node_id)
   {}
 
@@ -107,8 +107,8 @@ public:
       is_mut (other.is_mut), locus (other.locus), node_id (other.node_id)
   {
     // fix to get prevent null pointer dereference
-    if (other.to_bind != nullptr)
-      to_bind = other.to_bind->clone_pattern ();
+    if (other.subpattern != nullptr)
+      subpattern = other.subpattern->clone_pattern ();
   }
 
   // Overload assignment operator to use clone
@@ -121,10 +121,10 @@ public:
     node_id = other.node_id;
 
     // fix to prevent null pointer dereference
-    if (other.to_bind != nullptr)
-      to_bind = other.to_bind->clone_pattern ();
+    if (other.subpattern != nullptr)
+      subpattern = other.subpattern->clone_pattern ();
     else
-      to_bind = nullptr;
+      subpattern = nullptr;
 
     return *this;
   }
@@ -137,11 +137,10 @@ public:
 
   void accept_vis (ASTVisitor &vis) override;
 
-  // TODO: is this better? Or is a "vis_pattern" better?
-  Pattern &get_pattern_to_bind ()
+  Pattern &get_subpattern ()
   {
-    rust_assert (has_pattern_to_bind ());
-    return *to_bind;
+    rust_assert (has_subpattern ());
+    return *subpattern;
   }
 
   Identifier get_ident () const { return variable_ident; }
@@ -375,8 +374,7 @@ enum class RangeKind
   EXCLUDED,
 };
 
-RangeKind
-tokenid_to_rangekind (TokenId id);
+RangeKind tokenid_to_rangekind (TokenId id);
 // AST node for matching within a certain range (range pattern)
 class RangePattern : public Pattern
 {
@@ -950,7 +948,7 @@ public:
    * is empty). */
   bool has_struct_pattern_elems () const { return !elems.is_empty (); }
 
-  location_t get_locus () const override { return path.get_locus (); }
+  location_t get_locus () const override { return locus; }
 
   void accept_vis (ASTVisitor &vis) override;
 
@@ -1523,41 +1521,217 @@ protected:
   }
 };
 
+// Base abstract class representing patterns in a SlicePattern
+class SlicePatternItems
+{
+public:
+  enum SlicePatternItemType
+  {
+    NO_REST,
+    HAS_REST,
+  };
+
+  virtual ~SlicePatternItems () {}
+
+  // TODO: should this store location data?
+
+  // Unique pointer custom clone function
+  std::unique_ptr<SlicePatternItems> clone_slice_pattern_items () const
+  {
+    return std::unique_ptr<SlicePatternItems> (
+      clone_slice_pattern_items_impl ());
+  }
+
+  virtual std::string as_string () const = 0;
+
+  virtual void accept_vis (ASTVisitor &vis) = 0;
+
+  virtual SlicePatternItemType get_pattern_type () const = 0;
+
+protected:
+  // pure virtual clone implementation
+  virtual SlicePatternItems *clone_slice_pattern_items_impl () const = 0;
+};
+
+// Class representing the patterns in a SlicePattern without `..`
+class SlicePatternItemsNoRest : public SlicePatternItems
+{
+  std::vector<std::unique_ptr<Pattern>> patterns;
+
+public:
+  SlicePatternItemsNoRest (std::vector<std::unique_ptr<Pattern>> patterns)
+    : patterns (std::move (patterns))
+  {}
+
+  // Copy constructor with vector clone
+  SlicePatternItemsNoRest (SlicePatternItemsNoRest const &other)
+  {
+    patterns.reserve (other.patterns.size ());
+    for (const auto &e : other.patterns)
+      patterns.push_back (e->clone_pattern ());
+  }
+
+  // Overloaded assignment operator to vector clone
+  SlicePatternItemsNoRest &operator= (SlicePatternItemsNoRest const &other)
+  {
+    patterns.clear ();
+    patterns.reserve (other.patterns.size ());
+    for (const auto &e : other.patterns)
+      patterns.push_back (e->clone_pattern ());
+
+    return *this;
+  }
+
+  // move constructors
+  SlicePatternItemsNoRest (SlicePatternItemsNoRest &&other) = default;
+  SlicePatternItemsNoRest &operator= (SlicePatternItemsNoRest &&other)
+    = default;
+
+  std::string as_string () const override;
+
+  void accept_vis (ASTVisitor &vis) override;
+
+  // TODO: seems kinda dodgy. Think of better way.
+  std::vector<std::unique_ptr<Pattern>> &get_patterns () { return patterns; }
+  const std::vector<std::unique_ptr<Pattern>> &get_patterns () const
+  {
+    return patterns;
+  }
+
+  SlicePatternItemType get_pattern_type () const override
+  {
+    return SlicePatternItemType::NO_REST;
+  }
+
+protected:
+  /* Use covariance to implement clone function as returning this object rather
+   * than base */
+  SlicePatternItemsNoRest *clone_slice_pattern_items_impl () const override
+  {
+    return new SlicePatternItemsNoRest (*this);
+  }
+};
+
+// Class representing the patterns in a SlicePattern that contains a `..`
+class SlicePatternItemsHasRest : public SlicePatternItems
+{
+  std::vector<std::unique_ptr<Pattern>> lower_patterns;
+  std::vector<std::unique_ptr<Pattern>> upper_patterns;
+
+public:
+  SlicePatternItemsHasRest (
+    std::vector<std::unique_ptr<Pattern>> lower_patterns,
+    std::vector<std::unique_ptr<Pattern>> upper_patterns)
+    : lower_patterns (std::move (lower_patterns)),
+      upper_patterns (std::move (upper_patterns))
+  {}
+
+  // Copy constructor with vector clone
+  SlicePatternItemsHasRest (SlicePatternItemsHasRest const &other)
+  {
+    lower_patterns.reserve (other.lower_patterns.size ());
+    for (const auto &e : other.lower_patterns)
+      lower_patterns.push_back (e->clone_pattern ());
+
+    upper_patterns.reserve (other.upper_patterns.size ());
+    for (const auto &e : other.upper_patterns)
+      upper_patterns.push_back (e->clone_pattern ());
+  }
+
+  // Overloaded assignment operator to clone
+  SlicePatternItemsHasRest &operator= (SlicePatternItemsHasRest const &other)
+  {
+    lower_patterns.clear ();
+    lower_patterns.reserve (other.lower_patterns.size ());
+    for (const auto &e : other.lower_patterns)
+      lower_patterns.push_back (e->clone_pattern ());
+
+    upper_patterns.clear ();
+    upper_patterns.reserve (other.upper_patterns.size ());
+    for (const auto &e : other.upper_patterns)
+      upper_patterns.push_back (e->clone_pattern ());
+
+    return *this;
+  }
+
+  // move constructors
+  SlicePatternItemsHasRest (SlicePatternItemsHasRest &&other) = default;
+  SlicePatternItemsHasRest &operator= (SlicePatternItemsHasRest &&other)
+    = default;
+
+  std::string as_string () const override;
+
+  void accept_vis (ASTVisitor &vis) override;
+
+  // TODO: seems kinda dodgy. Think of better way.
+  std::vector<std::unique_ptr<Pattern>> &get_lower_patterns ()
+  {
+    return lower_patterns;
+  }
+  const std::vector<std::unique_ptr<Pattern>> &get_lower_patterns () const
+  {
+    return lower_patterns;
+  }
+
+  // TODO: seems kinda dodgy. Think of better way.
+  std::vector<std::unique_ptr<Pattern>> &get_upper_patterns ()
+  {
+    return upper_patterns;
+  }
+  const std::vector<std::unique_ptr<Pattern>> &get_upper_patterns () const
+  {
+    return upper_patterns;
+  }
+
+  SlicePatternItemType get_pattern_type () const override
+  {
+    return SlicePatternItemType::HAS_REST;
+  }
+
+protected:
+  /* Use covariance to implement clone function as returning this object rather
+   * than base */
+  SlicePatternItemsHasRest *clone_slice_pattern_items_impl () const override
+  {
+    return new SlicePatternItemsHasRest (*this);
+  }
+};
+
 // AST node representing patterns that can match slices and arrays
 class SlicePattern : public Pattern
 {
-  std::vector<std::unique_ptr<Pattern>> items;
+  std::unique_ptr<SlicePatternItems> items;
   location_t locus;
   NodeId node_id;
 
 public:
   std::string as_string () const override;
 
-  SlicePattern (std::vector<std::unique_ptr<Pattern>> items, location_t locus)
+  SlicePattern (std::unique_ptr<SlicePatternItems> items, location_t locus)
     : items (std::move (items)), locus (locus),
       node_id (Analysis::Mappings::get ().get_next_node_id ())
   {}
 
-  // Copy constructor with vector clone
+  // Copy constructor requires clone
   SlicePattern (SlicePattern const &other) : locus (other.locus)
   {
+    // guard to prevent null dereference
+    rust_assert (other.items != nullptr);
+
     node_id = other.node_id;
-    items.reserve (other.items.size ());
-    for (const auto &e : other.items)
-      items.push_back (e->clone_pattern ());
+    items = other.items->clone_slice_pattern_items ();
   }
 
-  // Overloaded assignment operator to vector clone
+  // Overloaded assignment operator to clone
   SlicePattern &operator= (SlicePattern const &other)
   {
     locus = other.locus;
     node_id = other.node_id;
 
-    items.clear ();
-    items.reserve (other.items.size ());
-    for (const auto &e : other.items)
-      items.push_back (e->clone_pattern ());
+    // guard to prevent null dereference
+    rust_assert (other.items != nullptr);
 
+    items = other.items->clone_slice_pattern_items ();
     return *this;
   }
 
@@ -1570,10 +1744,10 @@ public:
   void accept_vis (ASTVisitor &vis) override;
 
   // TODO: seems kinda dodgy. Think of better way.
-  std::vector<std::unique_ptr<Pattern>> &get_items () { return items; }
-  const std::vector<std::unique_ptr<Pattern>> &get_items () const
+  SlicePatternItems &get_items ()
   {
-    return items;
+    rust_assert (items != nullptr);
+    return *items;
   }
 
   NodeId get_node_id () const override { return node_id; }
diff --git a/gcc/rust/ast/rust-type.h b/gcc/rust/ast/rust-type.h
index 1bb521d..2a3496b 100644
--- a/gcc/rust/ast/rust-type.h
+++ b/gcc/rust/ast/rust-type.h
@@ -19,7 +19,9 @@
 #ifndef RUST_AST_TYPE_H
 #define RUST_AST_TYPE_H
 
+#include "optional.h"
 #include "rust-ast.h"
+#include "rust-expr.h"
 #include "rust-path.h"
 
 namespace Rust {
@@ -73,6 +75,13 @@ public:
       type_path (std::move (type_path)), locus (locus)
   {}
 
+  TraitBound (TraitBound const &other)
+    : TypeParamBound (other.get_node_id ()), in_parens (other.in_parens),
+      opening_question_mark (other.opening_question_mark),
+      for_lifetimes (other.for_lifetimes), type_path (other.type_path),
+      locus (other.locus)
+  {}
+
   std::string as_string () const override;
 
   location_t get_locus () const override final { return locus; }
@@ -99,6 +108,11 @@ protected:
     return new TraitBound (node_id, type_path, locus, in_parens,
 			   opening_question_mark, for_lifetimes);
   }
+  TraitBound *reconstruct_impl () const override
+  {
+    return new TraitBound (type_path, locus, in_parens, opening_question_mark,
+			   for_lifetimes);
+  }
 };
 
 // definition moved to rust-ast.h
@@ -120,6 +134,10 @@ protected:
   {
     return new ImplTraitType (*this);
   }
+  ImplTraitType *reconstruct_impl () const override
+  {
+    return new ImplTraitType (reconstruct_vec (type_param_bounds), locus);
+  }
 
 public:
   ImplTraitType (
@@ -129,7 +147,8 @@ public:
   {}
 
   // copy constructor with vector clone
-  ImplTraitType (ImplTraitType const &other) : locus (other.locus)
+  ImplTraitType (ImplTraitType const &other)
+    : Type (other.node_id), locus (other.locus)
   {
     type_param_bounds.reserve (other.type_param_bounds.size ());
     for (const auto &e : other.type_param_bounds)
@@ -184,6 +203,11 @@ protected:
   {
     return new TraitObjectType (*this);
   }
+  TraitObjectType *reconstruct_impl () const override
+  {
+    return new TraitObjectType (reconstruct_vec (type_param_bounds), locus,
+				has_dyn);
+  }
 
 public:
   TraitObjectType (
@@ -195,7 +219,7 @@ public:
 
   // copy constructor with vector clone
   TraitObjectType (TraitObjectType const &other)
-    : has_dyn (other.has_dyn), locus (other.locus)
+    : Type (other.node_id), has_dyn (other.has_dyn), locus (other.locus)
   {
     type_param_bounds.reserve (other.type_param_bounds.size ());
     for (const auto &e : other.type_param_bounds)
@@ -251,6 +275,10 @@ protected:
   {
     return new ParenthesisedType (*this);
   }
+  ParenthesisedType *reconstruct_impl () const override
+  {
+    return new ParenthesisedType (type_in_parens->reconstruct (), locus);
+  }
 
 public:
   // Constructor uses Type pointer for polymorphism
@@ -305,33 +333,35 @@ public:
 // Impl trait with a single bound? Poor reference material here.
 class ImplTraitTypeOneBound : public TypeNoBounds
 {
-  TraitBound trait_bound;
+  std::unique_ptr<TypeParamBound> trait_bound;
   location_t locus;
 
-protected:
-  /* Use covariance to implement clone function as returning this object rather
-   * than base */
-  ImplTraitTypeOneBound *clone_type_no_bounds_impl () const override
-  {
-    return new ImplTraitTypeOneBound (*this);
-  }
-
 public:
-  ImplTraitTypeOneBound (TraitBound trait_bound, location_t locus)
+  ImplTraitTypeOneBound (std::unique_ptr<TypeParamBound> trait_bound,
+			 location_t locus)
     : trait_bound (std::move (trait_bound)), locus (locus)
   {}
 
+  ImplTraitTypeOneBound (ImplTraitTypeOneBound const &other)
+    : trait_bound (other.trait_bound->clone_type_param_bound ()),
+      locus (other.locus)
+  {}
+
   std::string as_string () const override;
 
   location_t get_locus () const override final { return locus; }
 
   void accept_vis (ASTVisitor &vis) override;
 
-  // TODO: would a "vis_type" be better?
-  TraitBound &get_trait_bound ()
+  std::unique_ptr<TypeParamBound> &get_trait_bound () { return trait_bound; }
+
+  TypeNoBounds *clone_type_no_bounds_impl () const override
   {
-    // TODO: check to ensure invariants are met?
-    return trait_bound;
+    return new ImplTraitTypeOneBound (*this);
+  }
+  TypeNoBounds *reconstruct_impl () const override
+  {
+    return new ImplTraitTypeOneBound (trait_bound->reconstruct (), locus);
   }
 };
 
@@ -350,6 +380,10 @@ protected:
   {
     return new TraitObjectTypeOneBound (*this);
   }
+  TraitObjectTypeOneBound *reconstruct_impl () const override
+  {
+    return new TraitObjectTypeOneBound (trait_bound, locus, has_dyn);
+  }
 
 public:
   TraitObjectTypeOneBound (TraitBound trait_bound, location_t locus,
@@ -443,6 +477,10 @@ protected:
   {
     return new TupleType (*this);
   }
+  TupleType *reconstruct_impl () const override
+  {
+    return new TupleType (reconstruct_vec (elems), locus);
+  }
 };
 
 /* A type with no values, representing the result of computations that never
@@ -459,6 +497,10 @@ protected:
   {
     return new NeverType (*this);
   }
+  NeverType *reconstruct_impl () const override
+  {
+    return new NeverType (locus);
+  }
 
 public:
   NeverType (location_t locus) : locus (locus) {}
@@ -529,6 +571,9 @@ public:
     return *type;
   }
 
+  // Getter for direct access to the type unique_ptr
+  std::unique_ptr<TypeNoBounds> &get_type_ptr () { return type; }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -536,6 +581,10 @@ protected:
   {
     return new RawPointerType (*this);
   }
+  RawPointerType *reconstruct_impl () const override
+  {
+    return new RawPointerType (pointer_type, type->reconstruct (), locus);
+  }
 };
 
 // A type pointing to memory owned by another value
@@ -604,6 +653,9 @@ public:
 
   TypeNoBounds &get_base_type () { return *type; }
 
+  // Getter for direct access to the type unique_ptr
+  std::unique_ptr<TypeNoBounds> &get_type_ptr () { return type; }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -611,33 +663,42 @@ protected:
   {
     return new ReferenceType (*this);
   }
+  ReferenceType *reconstruct_impl () const override
+  {
+    return new ReferenceType (has_mut, type->reconstruct (), locus,
+			      // TODO: Improve this - it's ugly!
+			      has_lifetime () ? tl::make_optional<Lifetime> (
+				lifetime->get_lifetime_type (),
+				lifetime->get_lifetime_name (),
+				lifetime->get_locus ())
+					      : tl::nullopt);
+  }
 };
 
 // A fixed-size sequence of elements of a specified type
 class ArrayType : public TypeNoBounds
 {
   std::unique_ptr<Type> elem_type;
-  std::unique_ptr<Expr> size;
+  AnonConst size;
   location_t locus;
 
 public:
   // Constructor requires pointers for polymorphism
-  ArrayType (std::unique_ptr<Type> type, std::unique_ptr<Expr> array_size,
-	     location_t locus)
+  ArrayType (std::unique_ptr<Type> type, AnonConst array_size, location_t locus)
     : elem_type (std::move (type)), size (std::move (array_size)), locus (locus)
   {}
 
   // Copy constructor requires deep copies of both unique pointers
   ArrayType (ArrayType const &other)
-    : elem_type (other.elem_type->clone_type ()),
-      size (other.size->clone_expr ()), locus (other.locus)
+    : elem_type (other.elem_type->clone_type ()), size (other.size),
+      locus (other.locus)
   {}
 
   // Overload assignment operator to deep copy pointers
   ArrayType &operator= (ArrayType const &other)
   {
     elem_type = other.elem_type->clone_type ();
-    size = other.size->clone_expr ();
+    size = other.size;
     locus = other.locus;
     return *this;
   }
@@ -660,12 +721,15 @@ public:
   }
 
   // TODO: would a "vis_expr" be better?
-  Expr &get_size_expr ()
+  AnonConst &get_size_expr ()
   {
-    rust_assert (size != nullptr);
-    return *size;
+    // rust_assert (size != nullptr);
+
+    return size;
   }
 
+  std::unique_ptr<Type> &get_element_type () { return elem_type; }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
@@ -673,6 +737,12 @@ protected:
   {
     return new ArrayType (*this);
   }
+  ArrayType *reconstruct_impl () const override
+  {
+    return new ArrayType (elem_type->reconstruct (),
+			  size /* FIXME: This should be `reconstruct_expr()` */,
+			  locus);
+  }
 };
 
 /* A dynamically-sized type representing a "view" into a sequence of elements of
@@ -719,13 +789,20 @@ public:
     return *elem_type;
   }
 
+  // Getter for direct access to the elem_type unique_ptr
+  std::unique_ptr<Type> &get_elem_type_ptr () { return elem_type; }
+
 protected:
-  /* Use covariance to implement clone function as returning this object rather
-   * than base */
+  /* Use covariance to implement clone function as returning this object
+   * rather than base */
   SliceType *clone_type_no_bounds_impl () const override
   {
     return new SliceType (*this);
   }
+  SliceType *reconstruct_impl () const override
+  {
+    return new SliceType (elem_type->reconstruct (), locus);
+  }
 };
 
 /* Type used in generic arguments to explicitly request type inference (wildcard
@@ -736,13 +813,21 @@ class InferredType : public TypeNoBounds
 
   // e.g. Vec<_> = whatever
 protected:
-  /* Use covariance to implement clone function as returning this object rather
-   * than base */
+  /* Use covariance to implement clone function as returning this object
+   * rather than base */
   InferredType *clone_type_no_bounds_impl () const override
   {
+    // This goes through the copy constructor
     return new InferredType (*this);
   }
 
+  InferredType *reconstruct_impl () const override
+  {
+    // This goes through the base constructor which calls the base
+    // TypeNoBounds constructor, which allocates a new NodeId
+    return new InferredType (locus);
+  }
+
 public:
   InferredType (location_t locus) : locus (locus) {}
 
@@ -961,9 +1046,17 @@ public:
 
   FunctionQualifiers &get_function_qualifiers () { return function_qualifiers; }
 
+  BareFunctionType *reconstruct_impl () const override
+  {
+    return new BareFunctionType (
+      for_lifetimes, function_qualifiers, params,
+      /* FIXME: Should params be reconstruct() as well? */
+      _is_variadic, variadic_attrs, return_type->reconstruct (), locus);
+  }
+
 protected:
-  /* Use covariance to implement clone function as returning this object rather
-   * than base */
+  /* Use covariance to implement clone function as returning this object
+   * rather than base */
   BareFunctionType *clone_type_no_bounds_impl () const override
   {
     return new BareFunctionType (*this);
@@ -980,13 +1073,13 @@ class MacroInvocation;
  * function item type?
  * closure expression types?
  * primitive types (bool, int, float, char, str (the slice))
- * Although supposedly TypePaths are used to reference these types (including
- * primitives) */
+ * Although supposedly TypePaths are used to reference these types
+ * (including primitives) */
 
 /* FIXME: Incomplete spec references:
- *  anonymous type parameters, aka "impl Trait in argument position" - impl then
- * trait bounds abstract return types, aka "impl Trait in return position" -
- * impl then trait bounds */
+ *  anonymous type parameters, aka "impl Trait in argument position" - impl
+ * then trait bounds abstract return types, aka "impl Trait in return
+ * position" - impl then trait bounds */
 } // namespace AST
 } // namespace Rust
 
diff --git a/gcc/rust/backend/rust-compile-asm.cc b/gcc/rust/backend/rust-compile-asm.cc
index 7351cf0..b7143a8 100644
--- a/gcc/rust/backend/rust-compile-asm.cc
+++ b/gcc/rust/backend/rust-compile-asm.cc
@@ -74,57 +74,94 @@ CompileAsm::asm_construct_string_tree (HIR::InlineAsm &expr)
   return Backend::string_constant_expression (result);
 }
 
+tl::optional<std::reference_wrapper<HIR::Expr>>
+get_out_expr (HIR::InlineAsmOperand &operand)
+{
+  switch (operand.get_register_type ())
+    {
+    case HIR::InlineAsmOperand::RegisterType::Out:
+      return *operand.get_out ().expr;
+    case HIR::InlineAsmOperand::RegisterType::InOut:
+      return *operand.get_in_out ().expr;
+    case HIR::InlineAsmOperand::RegisterType::SplitInOut:
+      return *operand.get_split_in_out ().out_expr;
+    case HIR::InlineAsmOperand::RegisterType::Const:
+    case HIR::InlineAsmOperand::RegisterType::Sym:
+    case HIR::InlineAsmOperand::RegisterType::Label:
+    case HIR::InlineAsmOperand::RegisterType::In:
+      break;
+    }
+  return tl::nullopt;
+}
+
 tree
 CompileAsm::asm_construct_outputs (HIR::InlineAsm &expr)
 {
   // TODO: Do i need to do this?
 
   tree head = NULL_TREE;
-  for (auto &output : expr.get_operands ())
+  for (auto &operand : expr.get_operands ())
     {
-      if (output.get_register_type ()
-	  == AST::InlineAsmOperand::RegisterType::Out)
-	{
-	  auto out = output.get_out ();
-
-	  tree out_tree = CompileExpr::Compile (*out.expr, this->ctx);
-	  // expects a tree list
-	  // TODO: This assumes that the output is a register
-	  std::string expr_name = "=r";
-	  auto name = build_string (expr_name.size () + 1, expr_name.c_str ());
-	  head
-	    = chainon (head, build_tree_list (build_tree_list (NULL_TREE, name),
-					      out_tree));
-
-	  /*Backend::debug (head);*/
-	  /*head = chainon (head, out_tree);*/
-	}
+      tl::optional<std::reference_wrapper<HIR::Expr>> out_expr
+	= get_out_expr (operand);
+      if (!out_expr.has_value ())
+	continue;
+
+      tree out_tree = CompileExpr::Compile (*out_expr, this->ctx);
+      // expects a tree list
+      // TODO: This assumes that the output is a register
+      std::string expr_name = "=r";
+      auto name = build_string (expr_name.size () + 1, expr_name.c_str ());
+      head = chainon (head, build_tree_list (build_tree_list (NULL_TREE, name),
+					     out_tree));
+
+      /*Backend::debug (head);*/
+      /*head = chainon (head, out_tree);*/
     }
   return head;
 }
 
+tl::optional<std::reference_wrapper<HIR::Expr>>
+get_in_expr (HIR::InlineAsmOperand &operand)
+{
+  switch (operand.get_register_type ())
+    {
+    case HIR::InlineAsmOperand::RegisterType::In:
+      return *operand.get_in ().expr;
+    case HIR::InlineAsmOperand::RegisterType::InOut:
+      return *operand.get_in_out ().expr;
+    case HIR::InlineAsmOperand::RegisterType::SplitInOut:
+      return *operand.get_split_in_out ().in_expr;
+    case HIR::InlineAsmOperand::RegisterType::Const:
+    case HIR::InlineAsmOperand::RegisterType::Sym:
+    case HIR::InlineAsmOperand::RegisterType::Label:
+    case HIR::InlineAsmOperand::RegisterType::Out:
+      break;
+    }
+  return tl::nullopt;
+}
+
 tree
 CompileAsm::asm_construct_inputs (HIR::InlineAsm &expr)
 {
   // TODO: Do i need to do this?
   tree head = NULL_TREE;
-  for (auto &input : expr.get_operands ())
+  for (auto &operand : expr.get_operands ())
     {
-      if (input.get_register_type () == AST::InlineAsmOperand::RegisterType::In)
-	{
-	  auto in = input.get_in ();
-
-	  tree in_tree = CompileExpr::Compile (*in.expr, this->ctx);
-	  // expects a tree list
-	  // TODO: This assumes that the input is a register
-	  std::string expr_name = "r";
-	  auto name = build_string (expr_name.size () + 1, expr_name.c_str ());
-	  head
-	    = chainon (head, build_tree_list (build_tree_list (NULL_TREE, name),
-					      in_tree));
-
-	  /*head = chainon (head, out_tree);*/
-	}
+      tl::optional<std::reference_wrapper<HIR::Expr>> in_expr
+	= get_in_expr (operand);
+      if (!in_expr.has_value ())
+	continue;
+
+      tree in_tree = CompileExpr::Compile (*in_expr, this->ctx);
+      // expects a tree list
+      // TODO: This assumes that the input is a register
+      std::string expr_name = "r";
+      auto name = build_string (expr_name.size () + 1, expr_name.c_str ());
+      head = chainon (head, build_tree_list (build_tree_list (NULL_TREE, name),
+					     in_tree));
+
+      /*head = chainon (head, out_tree);*/
     }
   return head;
 }
diff --git a/gcc/rust/backend/rust-compile-base.cc b/gcc/rust/backend/rust-compile-base.cc
index 12b9561..73c34b2 100644
--- a/gcc/rust/backend/rust-compile-base.cc
+++ b/gcc/rust/backend/rust-compile-base.cc
@@ -576,6 +576,25 @@ HIRCompileBase::compile_constant_expr (
 }
 
 tree
+HIRCompileBase::query_compile_const_expr (Context *ctx, TyTy::BaseType *expr_ty,
+					  HIR::Expr &const_value_expr)
+{
+  HIRCompileBase c (ctx);
+
+  ctx->push_const_context ();
+
+  HirId expr_id = const_value_expr.get_mappings ().get_hirid ();
+  location_t locus = const_value_expr.get_locus ();
+  tree capacity_expr = HIRCompileBase::compile_constant_expr (
+    ctx, expr_id, expr_ty, expr_ty, Resolver::CanonicalPath::create_empty (),
+    const_value_expr, locus, locus);
+
+  ctx->pop_const_context ();
+
+  return fold_expr (capacity_expr);
+}
+
+tree
 HIRCompileBase::indirect_expression (tree expr, location_t locus)
 {
   if (expr == error_mark_node)
@@ -677,8 +696,12 @@ HIRCompileBase::compile_function (
   std::string ir_symbol_name
     = canonical_path.get () + fntype->subst_as_string ();
 
+  rust_debug_loc (locus, "--> Compiling [%s] - %s", ir_symbol_name.c_str (),
+		  fntype->get_name ().c_str ());
+
   // we don't mangle the main fn since we haven't implemented the main shim
-  bool is_main_fn = fn_name.compare ("main") == 0 && is_root_item;
+  bool is_main_fn = fn_name.compare ("main") == 0 && is_root_item
+		    && canonical_path.size () <= 2;
   if (is_main_fn)
     {
       rust_assert (!main_identifier_node);
@@ -687,14 +710,9 @@ HIRCompileBase::compile_function (
     }
   std::string asm_name = fn_name;
 
-  auto &mappings = Analysis::Mappings::get ();
-
-  if (flag_name_resolution_2_0)
-    ir_symbol_name = mappings.get_current_crate_name () + "::" + ir_symbol_name;
-
   unsigned int flags = 0;
   tree fndecl = Backend::function (compiled_fn_type, ir_symbol_name,
-				   "" /* asm_name */, flags, locus);
+				   tl::nullopt /* asm_name */, flags, locus);
 
   setup_fndecl (fndecl, is_main_fn, fntype->has_substitutions_defined (),
 		visibility, qualifiers, outer_attrs);
@@ -812,11 +830,12 @@ HIRCompileBase::compile_constant_item (
   // machineary that we already have. This means the best approach is to
   // make a _fake_ function with a block so it can hold onto temps then
   // use our constexpr code to fold it completely or error_mark_node
-  Backend::typed_identifier receiver;
+  Backend::typed_identifier receiver ("", NULL_TREE, UNKNOWN_LOCATION);
   tree compiled_fn_type = Backend::function_type (
     receiver, {}, {Backend::typed_identifier ("_", const_type, locus)}, NULL,
     locus);
-  tree fndecl = Backend::function (compiled_fn_type, ident, "", 0, locus);
+  tree fndecl
+    = Backend::function (compiled_fn_type, ident, tl::nullopt, 0, locus);
   TREE_READONLY (fndecl) = 1;
 
   tree enclosing_scope = NULL_TREE;
diff --git a/gcc/rust/backend/rust-compile-base.h b/gcc/rust/backend/rust-compile-base.h
index 6814abc..e9b8596 100644
--- a/gcc/rust/backend/rust-compile-base.h
+++ b/gcc/rust/backend/rust-compile-base.h
@@ -38,6 +38,9 @@ public:
     const Resolver::CanonicalPath &canonical_path, HIR::Expr &const_value_expr,
     location_t locus, location_t expr_locus);
 
+  static tree query_compile_const_expr (Context *ctx, TyTy::BaseType *expr_ty,
+					HIR::Expr &const_value_expr);
+
 protected:
   HIRCompileBase (Context *ctx) : ctx (ctx) {}
 
diff --git a/gcc/rust/backend/rust-compile-block.cc b/gcc/rust/backend/rust-compile-block.cc
index f844a27..03c36d2 100644
--- a/gcc/rust/backend/rust-compile-block.cc
+++ b/gcc/rust/backend/rust-compile-block.cc
@@ -19,6 +19,7 @@
 #include "rust-compile-block.h"
 #include "rust-compile-stmt.h"
 #include "rust-compile-expr.h"
+#include "rust-hir-expr.h"
 
 namespace Rust {
 namespace Compile {
diff --git a/gcc/rust/backend/rust-compile-block.h b/gcc/rust/backend/rust-compile-block.h
index 3f38d08..f84bace 100644
--- a/gcc/rust/backend/rust-compile-block.h
+++ b/gcc/rust/backend/rust-compile-block.h
@@ -20,6 +20,7 @@
 #define RUST_COMPILE_BLOCK
 
 #include "rust-compile-base.h"
+#include "rust-hir-expr.h"
 #include "rust-hir-visitor.h"
 
 namespace Rust {
@@ -83,6 +84,8 @@ public:
   void visit (HIR::MethodCallExpr &) override {}
   void visit (HIR::FieldAccessExpr &) override {}
   void visit (HIR::BlockExpr &) override {}
+  void visit (HIR::AnonConst &) override {}
+  void visit (HIR::ConstBlock &) override {}
   void visit (HIR::ContinueExpr &) override {}
   void visit (HIR::BreakExpr &) override {}
   void visit (HIR::RangeFromToExpr &) override {}
@@ -101,6 +104,7 @@ public:
   void visit (HIR::AsyncBlockExpr &) override {}
   void visit (HIR::InlineAsm &) override {}
   void visit (HIR::LlvmInlineAsm &) override {}
+  void visit (HIR::OffsetOf &) override {}
 
 private:
   CompileConditionalBlocks (Context *ctx, Bvariable *result)
@@ -138,6 +142,12 @@ public:
     translated = CompileBlock::compile (expr, ctx, result);
   }
 
+  void visit (HIR::ConstBlock &expr) override
+  {
+    rust_unreachable ();
+    // translated = CompileExpr::compile (expr, ctx, result);
+  }
+
   // Empty visit for unused Expression HIR nodes.
   void visit (HIR::PathInExpression &) override {}
   void visit (HIR::QualifiedPathInExpression &) override {}
@@ -184,6 +194,8 @@ public:
   void visit (HIR::AsyncBlockExpr &) override {}
   void visit (HIR::InlineAsm &) override {}
   void visit (HIR::LlvmInlineAsm &) override {}
+  void visit (HIR::OffsetOf &) override {}
+  void visit (HIR::AnonConst &) override {}
 
 private:
   CompileExprWithBlock (Context *ctx, Bvariable *result)
diff --git a/gcc/rust/backend/rust-compile-context.cc b/gcc/rust/backend/rust-compile-context.cc
index 86f0894..349d492 100644
--- a/gcc/rust/backend/rust-compile-context.cc
+++ b/gcc/rust/backend/rust-compile-context.cc
@@ -22,9 +22,18 @@
 namespace Rust {
 namespace Compile {
 
+Context *
+Context::get ()
+{
+  static Context *instance;
+  if (instance == nullptr)
+    instance = new Context ();
+
+  return instance;
+}
+
 Context::Context ()
-  : resolver (Resolver::Resolver::get ()),
-    tyctx (Resolver::TypeCheckContext::get ()),
+  : tyctx (Resolver::TypeCheckContext::get ()),
     mappings (Analysis::Mappings::get ()), mangler (Mangler ())
 {
   setup_builtins ();
@@ -70,7 +79,8 @@ Context::type_hasher (tree type)
       hstate.add_object (TYPE_HASH (TYPE_OFFSET_BASETYPE (type)));
       break;
 
-      case ARRAY_TYPE: {
+    case ARRAY_TYPE:
+      {
 	if (TYPE_DOMAIN (type))
 	  hstate.add_object (TYPE_HASH (TYPE_DOMAIN (type)));
 	if (!AGGREGATE_TYPE_P (TREE_TYPE (type)))
@@ -81,7 +91,8 @@ Context::type_hasher (tree type)
       }
       break;
 
-      case INTEGER_TYPE: {
+    case INTEGER_TYPE:
+      {
 	tree t = TYPE_MAX_VALUE (type);
 	if (!t)
 	  t = TYPE_MIN_VALUE (type);
@@ -91,7 +102,8 @@ Context::type_hasher (tree type)
       }
 
     case REAL_TYPE:
-      case FIXED_POINT_TYPE: {
+    case FIXED_POINT_TYPE:
+      {
 	unsigned prec = TYPE_PRECISION (type);
 	hstate.add_object (prec);
 	break;
@@ -103,7 +115,8 @@ Context::type_hasher (tree type)
 
     case RECORD_TYPE:
     case UNION_TYPE:
-      case QUAL_UNION_TYPE: {
+    case QUAL_UNION_TYPE:
+      {
 	for (tree t = TYPE_FIELDS (type); t; t = TREE_CHAIN (t))
 	  {
 	    hashval_t name_hash = IDENTIFIER_HASH_VALUE (DECL_NAME (t));
@@ -118,7 +131,8 @@ Context::type_hasher (tree type)
       break;
 
     case REFERENCE_TYPE:
-      case POINTER_TYPE: {
+    case POINTER_TYPE:
+      {
 	hashval_t type_hash = type_hasher (TREE_TYPE (type));
 	hstate.add_object (type_hash);
       }
diff --git a/gcc/rust/backend/rust-compile-context.h b/gcc/rust/backend/rust-compile-context.h
index ce81a1d..d4a642b 100644
--- a/gcc/rust/backend/rust-compile-context.h
+++ b/gcc/rust/backend/rust-compile-context.h
@@ -49,7 +49,7 @@ struct CustomDeriveInfo
 class Context
 {
 public:
-  Context ();
+  static Context *get ();
 
   void setup_builtins ();
 
@@ -90,7 +90,6 @@ public:
     return type;
   }
 
-  Resolver::Resolver *get_resolver () { return resolver; }
   Resolver::TypeCheckContext *get_tyctx () { return tyctx; }
   Analysis::Mappings &get_mappings () { return mappings; }
 
@@ -391,7 +390,8 @@ public:
   }
 
 private:
-  Resolver::Resolver *resolver;
+  Context ();
+
   Resolver::TypeCheckContext *tyctx;
   Analysis::Mappings &mappings;
   Mangler mangler;
diff --git a/gcc/rust/backend/rust-compile-expr.cc b/gcc/rust/backend/rust-compile-expr.cc
index dd3420f..6433923 100644
--- a/gcc/rust/backend/rust-compile-expr.cc
+++ b/gcc/rust/backend/rust-compile-expr.cc
@@ -17,6 +17,8 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-compile-expr.h"
+#include "rust-backend.h"
+#include "rust-compile-type.h"
 #include "rust-compile-struct-field-expr.h"
 #include "rust-compile-pattern.h"
 #include "rust-compile-resolve-path.h"
@@ -30,8 +32,11 @@
 #include "realmpfr.h"
 #include "convert.h"
 #include "print-tree.h"
+#include "rust-hir-expr.h"
 #include "rust-system.h"
+#include "rust-tree.h"
 #include "rust-tyty.h"
+#include "tree-core.h"
 
 namespace Rust {
 namespace Compile {
@@ -375,6 +380,31 @@ CompileExpr::visit (HIR::LlvmInlineAsm &expr)
 }
 
 void
+CompileExpr::visit (HIR::OffsetOf &expr)
+{
+  TyTy::BaseType *type = nullptr;
+  if (!ctx->get_tyctx ()->lookup_type (
+	expr.get_type ().get_mappings ().get_hirid (), &type))
+    {
+      translated = error_mark_node;
+      return;
+    }
+
+  auto compiled_ty = TyTyResolveCompile::compile (ctx, type);
+
+  rust_assert (TREE_CODE (compiled_ty) == RECORD_TYPE);
+
+  // Create an identifier node for the field
+  auto field_id = Backend::get_identifier_node (expr.get_field ().as_string ());
+
+  // And now look it up and get its value for `byte_position`
+  auto field = Backend::lookup_field (compiled_ty, field_id);
+  auto field_value = TREE_VALUE (field);
+
+  translated = byte_position (field_value);
+}
+
+void
 CompileExpr::visit (HIR::IfExprConseqElse &expr)
 {
   TyTy::BaseType *if_type = nullptr;
@@ -441,6 +471,18 @@ CompileExpr::visit (HIR::BlockExpr &expr)
 }
 
 void
+CompileExpr::visit (HIR::AnonConst &expr)
+{
+  expr.get_inner_expr ().accept_vis (*this);
+}
+
+void
+CompileExpr::visit (HIR::ConstBlock &expr)
+{
+  expr.get_const_expr ().accept_vis (*this);
+}
+
+void
 CompileExpr::visit (HIR::UnsafeBlockExpr &expr)
 {
   expr.get_block_expr ().accept_vis (*this);
@@ -471,6 +513,8 @@ CompileExpr::visit (HIR::StructExprStructFields &struct_expr)
       rust_error_at (struct_expr.get_locus (), "unknown type");
       return;
     }
+  if (!tyty->is<TyTy::ADTType> ())
+    return;
 
   // it must be an ADT
   rust_assert (tyty->get_kind () == TyTy::TypeKind::ADT);
@@ -669,6 +713,15 @@ void
 CompileExpr::visit (HIR::LoopExpr &expr)
 {
   TyTy::BaseType *block_tyty = nullptr;
+  fncontext fnctx = ctx->peek_fn ();
+  if (ctx->const_context_p () && !DECL_DECLARED_CONSTEXPR_P (fnctx.fndecl))
+    {
+      rich_location r (line_table, expr.get_locus ());
+      rust_error_at (r, ErrorCode::E0658,
+		     "%<loop%> is not allowed in const context");
+      return;
+    }
+
   if (!ctx->get_tyctx ()->lookup_type (expr.get_mappings ().get_hirid (),
 				       &block_tyty))
     {
@@ -676,7 +729,6 @@ CompileExpr::visit (HIR::LoopExpr &expr)
       return;
     }
 
-  fncontext fnctx = ctx->peek_fn ();
   tree enclosing_scope = ctx->peek_enclosing_scope ();
   tree block_type = TyTyResolveCompile::compile (ctx, block_tyty);
 
@@ -701,7 +753,8 @@ CompileExpr::visit (HIR::LoopExpr &expr)
 	loop_label.get_lifetime ().get_mappings ().get_hirid (), label);
     }
 
-  tree loop_begin_label = Backend::label (fnctx.fndecl, "", expr.get_locus ());
+  tree loop_begin_label
+    = Backend::label (fnctx.fndecl, tl::nullopt, expr.get_locus ());
   tree loop_begin_label_decl
     = Backend::label_definition_statement (loop_begin_label);
   ctx->add_statement (loop_begin_label_decl);
@@ -743,7 +796,8 @@ CompileExpr::visit (HIR::WhileLoopExpr &expr)
 				    start_location, end_location);
   ctx->push_block (loop_block);
 
-  tree loop_begin_label = Backend::label (fnctx.fndecl, "", expr.get_locus ());
+  tree loop_begin_label
+    = Backend::label (fnctx.fndecl, tl::nullopt, expr.get_locus ());
   tree loop_begin_label_decl
     = Backend::label_definition_statement (loop_begin_label);
   ctx->add_statement (loop_begin_label_decl);
@@ -787,25 +841,16 @@ CompileExpr::visit (HIR::BreakExpr &expr)
 
   if (expr.has_label ())
     {
-      NodeId resolved_node_id = UNKNOWN_NODEID;
-      if (flag_name_resolution_2_0)
-	{
-	  auto &nr_ctx
-	    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+      auto &nr_ctx
+	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-	  if (auto id
-	      = nr_ctx.lookup (expr.get_label ().get_mappings ().get_nodeid ()))
-	    resolved_node_id = *id;
-	}
-      else
+      NodeId resolved_node_id;
+      if (auto id
+	  = nr_ctx.lookup (expr.get_label ().get_mappings ().get_nodeid ()))
 	{
-	  NodeId tmp = UNKNOWN_NODEID;
-	  if (ctx->get_resolver ()->lookup_resolved_label (
-		expr.get_label ().get_mappings ().get_nodeid (), &tmp))
-	    resolved_node_id = tmp;
+	  resolved_node_id = *id;
 	}
-
-      if (resolved_node_id == UNKNOWN_NODEID)
+      else
 	{
 	  rust_error_at (
 	    expr.get_label ().get_locus (),
@@ -849,26 +894,16 @@ CompileExpr::visit (HIR::ContinueExpr &expr)
   tree label = ctx->peek_loop_begin_label ();
   if (expr.has_label ())
     {
-      NodeId resolved_node_id = UNKNOWN_NODEID;
-      if (flag_name_resolution_2_0)
-	{
-	  auto &nr_ctx
-	    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+      auto &nr_ctx
+	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-	  if (auto id
-	      = nr_ctx.lookup (expr.get_label ().get_mappings ().get_nodeid ()))
-	    resolved_node_id = *id;
-	}
-      else
+      NodeId resolved_node_id;
+      if (auto id
+	  = nr_ctx.lookup (expr.get_label ().get_mappings ().get_nodeid ()))
 	{
-	  NodeId tmp = UNKNOWN_NODEID;
-
-	  if (ctx->get_resolver ()->lookup_resolved_label (
-		expr.get_label ().get_mappings ().get_nodeid (), &tmp))
-	    resolved_node_id = tmp;
+	  resolved_node_id = *id;
 	}
-
-      if (resolved_node_id == UNKNOWN_NODEID)
+      else
 	{
 	  rust_error_at (
 	    expr.get_label ().get_locus (),
@@ -1130,9 +1165,8 @@ CompileExpr::visit (HIR::MatchExpr &expr)
   // setup the end label so the cases can exit properly
   tree fndecl = fnctx.fndecl;
   location_t end_label_locus = expr.get_locus (); // FIXME
-  tree end_label
-    = Backend::label (fndecl, "" /* empty creates an artificial label */,
-		      end_label_locus);
+  // tl::nullopt creates an artificial label
+  tree end_label = Backend::label (fndecl, tl::nullopt, end_label_locus);
   tree end_label_decl_statement
     = Backend::label_definition_statement (end_label);
 
@@ -1325,6 +1359,28 @@ CompileExpr::visit (HIR::CallExpr &expr)
   };
 
   auto fn_address = CompileExpr::Compile (expr.get_fnexpr (), ctx);
+  if (ctx->const_context_p ())
+    {
+      if (!FUNCTION_POINTER_TYPE_P (TREE_TYPE (fn_address)))
+	{
+	  rust_error_at (expr.get_locus (),
+			 "calls in constants are limited to constant "
+			 "functions, tuple structs and tuple variants");
+	  return;
+	}
+
+      if (TREE_CODE (fn_address) == ADDR_EXPR)
+	{
+	  tree fndecl = TREE_OPERAND (fn_address, 0);
+	  if (!DECL_DECLARED_CONSTEXPR_P (fndecl))
+	    {
+	      rust_error_at (expr.get_locus (),
+			     "calls in constants are limited to constant "
+			     "functions, tuple structs and tuple variants");
+	      return;
+	    }
+	}
+    }
 
   // is this a closure call?
   bool possible_trait_call
@@ -1883,7 +1939,8 @@ CompileExpr::visit (HIR::ArrayExpr &expr)
   HIR::ArrayElems &elements = expr.get_internal_elements ();
   switch (elements.get_array_expr_type ())
     {
-      case HIR::ArrayElems::ArrayExprType::VALUES: {
+    case HIR::ArrayElems::ArrayExprType::VALUES:
+      {
 	HIR::ArrayElemsValues &elems
 	  = static_cast<HIR::ArrayElemsValues &> (elements);
 	translated
@@ -2032,7 +2089,8 @@ HIRCompileBase::resolve_adjustements (
 	  return error_mark_node;
 
 	case Resolver::Adjustment::AdjustmentType::IMM_REF:
-	  case Resolver::Adjustment::AdjustmentType::MUT_REF: {
+	case Resolver::Adjustment::AdjustmentType::MUT_REF:
+	  {
 	    if (!RS_DST_FLAG (TREE_TYPE (e)))
 	      {
 		e = address_expression (e, locus);
@@ -2474,23 +2532,12 @@ CompileExpr::generate_closure_function (HIR::ClosureExpr &expr,
   if (is_block_expr)
     {
       auto body_mappings = function_body.get_mappings ();
-      if (flag_name_resolution_2_0)
-	{
-	  auto &nr_ctx
-	    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+      auto &nr_ctx
+	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-	  auto candidate = nr_ctx.values.to_rib (body_mappings.get_nodeid ());
+      auto candidate = nr_ctx.values.to_rib (body_mappings.get_nodeid ());
 
-	  rust_assert (candidate.has_value ());
-	}
-      else
-	{
-	  Resolver::Rib *rib = nullptr;
-	  bool ok
-	    = ctx->get_resolver ()->find_name_rib (body_mappings.get_nodeid (),
-						   &rib);
-	  rust_assert (ok);
-	}
+      rust_assert (candidate.has_value ());
     }
 
   tree enclosing_scope = NULL_TREE;
diff --git a/gcc/rust/backend/rust-compile-expr.h b/gcc/rust/backend/rust-compile-expr.h
index 65ed4b3..b8b4e8d 100644
--- a/gcc/rust/backend/rust-compile-expr.h
+++ b/gcc/rust/backend/rust-compile-expr.h
@@ -48,6 +48,8 @@ public:
   void visit (HIR::IfExpr &expr) override;
   void visit (HIR::IfExprConseqElse &expr) override;
   void visit (HIR::BlockExpr &expr) override;
+  void visit (HIR::AnonConst &expr) override;
+  void visit (HIR::ConstBlock &expr) override;
   void visit (HIR::UnsafeBlockExpr &expr) override;
   void visit (HIR::StructExprStruct &struct_expr) override;
   void visit (HIR::StructExprStructFields &struct_expr) override;
@@ -70,6 +72,7 @@ public:
   void visit (HIR::ClosureExpr &expr) override;
   void visit (HIR::InlineAsm &expr) override;
   void visit (HIR::LlvmInlineAsm &expr) override;
+  void visit (HIR::OffsetOf &expr) override;
 
   // TODO
   void visit (HIR::ErrorPropagationExpr &) override {}
diff --git a/gcc/rust/backend/rust-compile-implitem.cc b/gcc/rust/backend/rust-compile-implitem.cc
index 1230c85..63df2f5 100644
--- a/gcc/rust/backend/rust-compile-implitem.cc
+++ b/gcc/rust/backend/rust-compile-implitem.cc
@@ -27,22 +27,11 @@ CompileTraitItem::visit (HIR::TraitItemConst &constant)
   rust_assert (concrete != nullptr);
   TyTy::BaseType *resolved_type = concrete;
 
-  tl::optional<Resolver::CanonicalPath> canonical_path;
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path = nr_ctx.values.to_canonical_path (
-	constant.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path = ctx->get_mappings ().lookup_canonical_path (
-	constant.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path);
+  Resolver::CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (constant.get_mappings ().get_nodeid ());
 
   HIR::Expr &const_value_expr = constant.get_expr ();
   TyTy::BaseType *expr_type = nullptr;
@@ -52,7 +41,7 @@ CompileTraitItem::visit (HIR::TraitItemConst &constant)
 
   tree const_expr
     = compile_constant_item (constant.get_mappings ().get_hirid (), expr_type,
-			     resolved_type, *canonical_path, const_value_expr,
+			     resolved_type, canonical_path, const_value_expr,
 			     constant.get_locus (),
 			     const_value_expr.get_locus ());
   ctx->push_const (const_expr);
@@ -96,22 +85,11 @@ CompileTraitItem::visit (HIR::TraitItemFunc &func)
       fntype->override_context ();
     }
 
-  tl::optional<Resolver::CanonicalPath> canonical_path;
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.values.to_canonical_path (func.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path = ctx->get_mappings ().lookup_canonical_path (
-	func.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path);
+  Resolver::CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (func.get_mappings ().get_nodeid ());
 
   // FIXME: How do we get the proper visibility here?
   auto vis = HIR::Visibility (HIR::Visibility::VisType::PUBLIC);
@@ -121,7 +99,7 @@ CompileTraitItem::visit (HIR::TraitItemFunc &func)
 			function.get_self (), function.get_function_params (),
 			function.get_qualifiers (), vis,
 			func.get_outer_attrs (), func.get_locus (),
-			&func.get_block_expr (), *canonical_path, fntype);
+			&func.get_block_expr (), canonical_path, fntype);
   reference = address_expression (fndecl, ref_locus);
 }
 
diff --git a/gcc/rust/backend/rust-compile-intrinsic.cc b/gcc/rust/backend/rust-compile-intrinsic.cc
index 4888e23..450a869 100644
--- a/gcc/rust/backend/rust-compile-intrinsic.cc
+++ b/gcc/rust/backend/rust-compile-intrinsic.cc
@@ -32,8 +32,7 @@
 
 // declaration taken from "stringpool.h"
 // the get_identifier macro causes compilation issues
-extern tree
-get_identifier (const char *);
+extern tree get_identifier (const char *);
 
 namespace Rust {
 namespace Compile {
@@ -70,28 +69,19 @@ check_for_basic_integer_type (const std::string &intrinsic_str,
   return is_basic_integer;
 }
 
-static tree
-offset_handler (Context *ctx, TyTy::FnType *fntype);
-static tree
-sizeof_handler (Context *ctx, TyTy::FnType *fntype);
-static tree
-transmute_handler (Context *ctx, TyTy::FnType *fntype);
-static tree
-rotate_handler (Context *ctx, TyTy::FnType *fntype, tree_code op);
-static tree
-wrapping_op_handler_inner (Context *ctx, TyTy::FnType *fntype, tree_code op);
-static tree
-op_with_overflow_inner (Context *ctx, TyTy::FnType *fntype, tree_code op);
-static tree
-uninit_handler (Context *ctx, TyTy::FnType *fntype);
-static tree
-move_val_init_handler (Context *ctx, TyTy::FnType *fntype);
-static tree
-assume_handler (Context *ctx, TyTy::FnType *fntype);
-static tree
-discriminant_value_handler (Context *ctx, TyTy::FnType *fntype);
-static tree
-variant_count_handler (Context *ctx, TyTy::FnType *fntype);
+static tree offset_handler (Context *ctx, TyTy::FnType *fntype);
+static tree sizeof_handler (Context *ctx, TyTy::FnType *fntype);
+static tree transmute_handler (Context *ctx, TyTy::FnType *fntype);
+static tree rotate_handler (Context *ctx, TyTy::FnType *fntype, tree_code op);
+static tree wrapping_op_handler_inner (Context *ctx, TyTy::FnType *fntype,
+				       tree_code op);
+static tree op_with_overflow_inner (Context *ctx, TyTy::FnType *fntype,
+				    tree_code op);
+static tree uninit_handler (Context *ctx, TyTy::FnType *fntype);
+static tree move_val_init_handler (Context *ctx, TyTy::FnType *fntype);
+static tree assume_handler (Context *ctx, TyTy::FnType *fntype);
+static tree discriminant_value_handler (Context *ctx, TyTy::FnType *fntype);
+static tree variant_count_handler (Context *ctx, TyTy::FnType *fntype);
 
 enum class Prefetch
 {
@@ -99,8 +89,8 @@ enum class Prefetch
   Write
 };
 
-static tree
-prefetch_data_handler (Context *ctx, TyTy::FnType *fntype, Prefetch kind);
+static tree prefetch_data_handler (Context *ctx, TyTy::FnType *fntype,
+				   Prefetch kind);
 
 static inline tree
 rotate_left_handler (Context *ctx, TyTy::FnType *fntype)
@@ -140,10 +130,10 @@ prefetch_write_data (Context *ctx, TyTy::FnType *fntype)
   return prefetch_data_handler (ctx, fntype, Prefetch::Write);
 }
 
-static tree
-atomic_store_handler_inner (Context *ctx, TyTy::FnType *fntype, int ordering);
-static tree
-atomic_load_handler_inner (Context *ctx, TyTy::FnType *fntype, int ordering);
+static tree atomic_store_handler_inner (Context *ctx, TyTy::FnType *fntype,
+					int ordering);
+static tree atomic_load_handler_inner (Context *ctx, TyTy::FnType *fntype,
+				       int ordering);
 
 static inline std::function<tree (Context *, TyTy::FnType *)>
 atomic_store_handler (int ordering)
@@ -161,8 +151,8 @@ atomic_load_handler (int ordering)
   };
 }
 
-static inline tree
-unchecked_op_inner (Context *ctx, TyTy::FnType *fntype, tree_code op);
+static inline tree unchecked_op_inner (Context *ctx, TyTy::FnType *fntype,
+				       tree_code op);
 
 const static std::function<tree (Context *, TyTy::FnType *)>
 unchecked_op_handler (tree_code op)
@@ -172,8 +162,8 @@ unchecked_op_handler (tree_code op)
   };
 }
 
-static inline tree
-copy_handler_inner (Context *ctx, TyTy::FnType *fntype, bool overlaps);
+static inline tree copy_handler_inner (Context *ctx, TyTy::FnType *fntype,
+				       bool overlaps);
 
 const static std::function<tree (Context *, TyTy::FnType *)>
 copy_handler (bool overlaps)
@@ -183,8 +173,8 @@ copy_handler (bool overlaps)
   };
 }
 
-static inline tree
-expect_handler_inner (Context *ctx, TyTy::FnType *fntype, bool likely);
+static inline tree expect_handler_inner (Context *ctx, TyTy::FnType *fntype,
+					 bool likely);
 
 const static std::function<tree (Context *, TyTy::FnType *)>
 expect_handler (bool likely)
@@ -194,8 +184,8 @@ expect_handler (bool likely)
   };
 }
 
-static tree
-try_handler_inner (Context *ctx, TyTy::FnType *fntype, bool is_new_api);
+static tree try_handler_inner (Context *ctx, TyTy::FnType *fntype,
+			       bool is_new_api);
 
 const static std::function<tree (Context *, TyTy::FnType *)>
 try_handler (bool is_new_api)
@@ -447,8 +437,8 @@ sizeof_handler (Context *ctx, TyTy::FnType *fntype)
   // get the template parameter type tree fn size_of<T>();
   rust_assert (fntype->get_num_substitutions () == 1);
   auto &param_mapping = fntype->get_substs ().at (0);
-  const TyTy::ParamType *param_tyty = param_mapping.get_param_ty ();
-  TyTy::BaseType *resolved_tyty = param_tyty->resolve ();
+  const auto param_tyty = param_mapping.get_param_ty ();
+  auto resolved_tyty = param_tyty->resolve ();
   tree template_parameter_type
     = TyTyResolveCompile::compile (ctx, resolved_tyty);
 
@@ -653,8 +643,8 @@ op_with_overflow_inner (Context *ctx, TyTy::FnType *fntype, tree_code op)
 
   rust_assert (fntype->get_num_substitutions () == 1);
   auto &param_mapping = fntype->get_substs ().at (0);
-  const TyTy::ParamType *param_tyty = param_mapping.get_param_ty ();
-  TyTy::BaseType *resolved_tyty = param_tyty->resolve ();
+  const auto param_tyty = param_mapping.get_param_ty ();
+  auto resolved_tyty = param_tyty->resolve ();
   tree template_parameter_type
     = TyTyResolveCompile::compile (ctx, resolved_tyty);
 
@@ -1089,8 +1079,8 @@ uninit_handler (Context *ctx, TyTy::FnType *fntype)
   // get the template parameter type tree fn uninit<T>();
   rust_assert (fntype->get_num_substitutions () == 1);
   auto &param_mapping = fntype->get_substs ().at (0);
-  const TyTy::ParamType *param_tyty = param_mapping.get_param_ty ();
-  TyTy::BaseType *resolved_tyty = param_tyty->resolve ();
+  const auto param_tyty = param_mapping.get_param_ty ();
+  auto resolved_tyty = param_tyty->resolve ();
   tree template_parameter_type
     = TyTyResolveCompile::compile (ctx, resolved_tyty);
 
@@ -1154,8 +1144,8 @@ move_val_init_handler (Context *ctx, TyTy::FnType *fntype)
   // get the template parameter type tree fn size_of<T>();
   rust_assert (fntype->get_num_substitutions () == 1);
   auto &param_mapping = fntype->get_substs ().at (0);
-  const TyTy::ParamType *param_tyty = param_mapping.get_param_ty ();
-  TyTy::BaseType *resolved_tyty = param_tyty->resolve ();
+  auto param_tyty = param_mapping.get_param_ty ();
+  auto resolved_tyty = param_tyty->resolve ();
   tree template_parameter_type
     = TyTyResolveCompile::compile (ctx, resolved_tyty);
 
diff --git a/gcc/rust/backend/rust-compile-item.cc b/gcc/rust/backend/rust-compile-item.cc
index 9666990..b72e70d 100644
--- a/gcc/rust/backend/rust-compile-item.cc
+++ b/gcc/rust/backend/rust-compile-item.cc
@@ -50,33 +50,21 @@ CompileItem::visit (HIR::StaticItem &var)
 
   tree type = TyTyResolveCompile::compile (ctx, resolved_type);
 
-  tl::optional<Resolver::CanonicalPath> canonical_path;
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.values.to_canonical_path (var.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path = ctx->get_mappings ().lookup_canonical_path (
-	var.get_mappings ().get_nodeid ());
-    }
-
-  rust_assert (canonical_path.has_value ());
+  Resolver::CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (var.get_mappings ().get_nodeid ());
 
   ctx->push_const_context ();
   tree value
     = compile_constant_item (var.get_mappings ().get_hirid (), expr_type,
-			     resolved_type, *canonical_path, const_value_expr,
+			     resolved_type, canonical_path, const_value_expr,
 			     var.get_locus (), const_value_expr.get_locus ());
   ctx->pop_const_context ();
 
-  std::string name = canonical_path->get ();
-  std::string asm_name = ctx->mangle_item (resolved_type, *canonical_path);
+  std::string name = canonical_path.get ();
+  std::string asm_name = ctx->mangle_item (resolved_type, canonical_path);
 
   bool is_external = false;
   bool is_hidden = false;
@@ -115,24 +103,12 @@ CompileItem::visit (HIR::ConstantItem &constant)
     const_value_expr.get_mappings ().get_hirid (), &expr_type);
   rust_assert (ok);
 
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+
   // canonical path
   Resolver::CanonicalPath canonical_path
-    = Resolver::CanonicalPath::create_empty ();
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.values.to_canonical_path (mappings.get_nodeid ()).value ();
-    }
-  else
-    {
-      canonical_path = ctx->get_mappings ()
-			 .lookup_canonical_path (mappings.get_nodeid ())
-			 .value ();
-    }
+    = nr_ctx.to_canonical_path (mappings.get_nodeid ());
 
   ctx->push_const_context ();
   tree const_expr
@@ -210,26 +186,11 @@ CompileItem::visit (HIR::Function &function)
 	}
     }
 
-  Resolver::CanonicalPath canonical_path
-    = Resolver::CanonicalPath::create_empty ();
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      auto path = nr_ctx.values.to_canonical_path (
-	function.get_mappings ().get_nodeid ());
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-      canonical_path = path.value ();
-    }
-  else
-    {
-      auto path = ctx->get_mappings ().lookup_canonical_path (
-	function.get_mappings ().get_nodeid ());
-
-      canonical_path = *path;
-    }
+  Resolver::CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (function.get_mappings ().get_nodeid ());
 
   const std::string asm_name = ctx->mangle_item (fntype, canonical_path);
 
@@ -300,5 +261,65 @@ CompileItem::visit (HIR::Module &module)
     CompileItem::compile (item.get (), ctx);
 }
 
+void
+CompileItem::visit (HIR::TupleStruct &tuple_struct_decl)
+{
+  TyTy::BaseType *lookup = nullptr;
+  if (!ctx->get_tyctx ()->lookup_type (
+	tuple_struct_decl.get_mappings ().get_hirid (), &lookup))
+    {
+      rust_error_at (tuple_struct_decl.get_locus (), "failed to resolve type");
+      return;
+    }
+
+  if (lookup->is_concrete ())
+    TyTyResolveCompile::compile (ctx, lookup);
+}
+
+void
+CompileItem::visit (HIR::Enum &enum_decl)
+{
+  TyTy::BaseType *lookup = nullptr;
+  if (!ctx->get_tyctx ()->lookup_type (enum_decl.get_mappings ().get_hirid (),
+				       &lookup))
+    {
+      rust_error_at (enum_decl.get_locus (), "failed to resolve type");
+      return;
+    }
+
+  if (lookup->is_concrete ())
+    TyTyResolveCompile::compile (ctx, lookup);
+}
+
+void
+CompileItem::visit (HIR::Union &union_decl)
+{
+  TyTy::BaseType *lookup = nullptr;
+  if (!ctx->get_tyctx ()->lookup_type (union_decl.get_mappings ().get_hirid (),
+				       &lookup))
+    {
+      rust_error_at (union_decl.get_locus (), "failed to resolve type");
+      return;
+    }
+
+  if (lookup->is_concrete ())
+    TyTyResolveCompile::compile (ctx, lookup);
+}
+
+void
+CompileItem::visit (HIR::StructStruct &struct_decl)
+{
+  TyTy::BaseType *lookup = nullptr;
+  if (!ctx->get_tyctx ()->lookup_type (struct_decl.get_mappings ().get_hirid (),
+				       &lookup))
+    {
+      rust_error_at (struct_decl.get_locus (), "failed to resolve type");
+      return;
+    }
+
+  if (lookup->is_concrete ())
+    TyTyResolveCompile::compile (ctx, lookup);
+}
+
 } // namespace Compile
 } // namespace Rust
diff --git a/gcc/rust/backend/rust-compile-item.h b/gcc/rust/backend/rust-compile-item.h
index d9d946d..56baaab 100644
--- a/gcc/rust/backend/rust-compile-item.h
+++ b/gcc/rust/backend/rust-compile-item.h
@@ -44,9 +44,12 @@ public:
   void visit (HIR::ImplBlock &impl_block) override;
   void visit (HIR::ExternBlock &extern_block) override;
   void visit (HIR::Module &module) override;
+  void visit (HIR::TupleStruct &tuple_struct) override;
+  void visit (HIR::Enum &enum_decl) override;
+  void visit (HIR::Union &union_decl) override;
+  void visit (HIR::StructStruct &struct_decl) override;
 
   // Empty visit for unused Stmt HIR nodes.
-  void visit (HIR::TupleStruct &) override {}
   void visit (HIR::EnumItem &) override {}
   void visit (HIR::EnumItemTuple &) override {}
   void visit (HIR::EnumItemStruct &) override {}
@@ -57,9 +60,6 @@ public:
   void visit (HIR::ExternCrate &) override {}
   void visit (HIR::UseDeclaration &) override {}
   void visit (HIR::TypeAlias &) override {}
-  void visit (HIR::StructStruct &) override {}
-  void visit (HIR::Enum &) override {}
-  void visit (HIR::Union &) override {}
   void visit (HIR::Trait &) override {}
   void visit (HIR::EmptyStmt &) override {}
   void visit (HIR::LetStmt &) override {}
diff --git a/gcc/rust/backend/rust-compile-pattern.cc b/gcc/rust/backend/rust-compile-pattern.cc
index e83717b..fe65921 100644
--- a/gcc/rust/backend/rust-compile-pattern.cc
+++ b/gcc/rust/backend/rust-compile-pattern.cc
@@ -22,6 +22,11 @@
 #include "rust-constexpr.h"
 #include "rust-compile-type.h"
 #include "print-tree.h"
+#include "rust-diagnostics.h"
+#include "rust-hir-pattern-abstract.h"
+#include "rust-hir-pattern.h"
+#include "rust-system.h"
+#include "rust-tyty.h"
 
 namespace Rust {
 namespace Compile {
@@ -107,7 +112,8 @@ compile_range_pattern_bound (HIR::RangePatternBound &bound,
   tree result = NULL_TREE;
   switch (bound.get_bound_type ())
     {
-      case HIR::RangePatternBound::RangePatternBoundType::LITERAL: {
+    case HIR::RangePatternBound::RangePatternBoundType::LITERAL:
+      {
 	auto &ref = static_cast<HIR::RangePatternBoundLiteral &> (bound);
 
 	HIR::LiteralExpr litexpr (mappings, ref.get_literal (), locus,
@@ -117,7 +123,8 @@ compile_range_pattern_bound (HIR::RangePatternBound &bound,
       }
       break;
 
-      case HIR::RangePatternBound::RangePatternBoundType::PATH: {
+    case HIR::RangePatternBound::RangePatternBoundType::PATH:
+      {
 	auto &ref = static_cast<HIR::RangePatternBoundPath &> (bound);
 
 	result = ResolvePathRef::Compile (ref.get_path (), ctx);
@@ -127,7 +134,8 @@ compile_range_pattern_bound (HIR::RangePatternBound &bound,
       }
       break;
 
-      case HIR::RangePatternBound::RangePatternBoundType::QUALPATH: {
+    case HIR::RangePatternBound::RangePatternBoundType::QUALPATH:
+      {
 	auto &ref = static_cast<HIR::RangePatternBoundQualPath &> (bound);
 
 	result = ResolvePathRef::Compile (ref.get_qualified_path (), ctx);
@@ -204,6 +212,7 @@ CompilePatternCheckExpr::visit (HIR::StructPattern &pattern)
 
   rust_assert (adt->number_of_variants () > 0);
   TyTy::VariantDef *variant = nullptr;
+  tree variant_accesser_expr = nullptr;
   if (adt->is_enum ())
     {
       // lookup the variant
@@ -218,9 +227,7 @@ CompilePatternCheckExpr::visit (HIR::StructPattern &pattern)
 
       // find expected discriminant
       // // need to access qualifier the field, if we use QUAL_UNION_TYPE this
-      // // would be DECL_QUALIFIER i think. For now this will just access the
-      // // first record field and its respective qualifier because it will
-      // // always be set because this is all a big special union
+      // // would be DECL_QUALIFIER i think.
       HIR::Expr &discrim_expr = variant->get_discriminant ();
       tree discrim_expr_node = CompileExpr::Compile (discrim_expr, ctx);
 
@@ -229,6 +236,14 @@ CompilePatternCheckExpr::visit (HIR::StructPattern &pattern)
 	= Backend::struct_field_expression (match_scrutinee_expr, 0,
 					    pattern.get_path ().get_locus ());
 
+      // access variant data
+      tree scrutinee_union_expr
+	= Backend::struct_field_expression (match_scrutinee_expr, 1,
+					    pattern.get_path ().get_locus ());
+      variant_accesser_expr
+	= Backend::struct_field_expression (scrutinee_union_expr, variant_index,
+					    pattern.get_path ().get_locus ());
+
       check_expr
 	= Backend::comparison_expression (ComparisonOperator::EQUAL,
 					  scrutinee_expr_qualifier_expr,
@@ -240,6 +255,7 @@ CompilePatternCheckExpr::visit (HIR::StructPattern &pattern)
   else
     {
       variant = adt->get_variants ().at (0);
+      variant_accesser_expr = match_scrutinee_expr;
       check_expr = boolean_true_node;
     }
 
@@ -248,13 +264,15 @@ CompilePatternCheckExpr::visit (HIR::StructPattern &pattern)
     {
       switch (field->get_item_type ())
 	{
-	  case HIR::StructPatternField::ItemType::TUPLE_PAT: {
+	case HIR::StructPatternField::ItemType::TUPLE_PAT:
+	  {
 	    // TODO
 	    rust_unreachable ();
 	  }
 	  break;
 
-	  case HIR::StructPatternField::ItemType::IDENT_PAT: {
+	case HIR::StructPatternField::ItemType::IDENT_PAT:
+	  {
 	    HIR::StructPatternFieldIdentPat &ident
 	      = static_cast<HIR::StructPatternFieldIdentPat &> (*field.get ());
 
@@ -263,11 +281,8 @@ CompilePatternCheckExpr::visit (HIR::StructPattern &pattern)
 					nullptr, &offs);
 	    rust_assert (ok);
 
-	    // we may be offsetting by + 1 here since the first field in the
-	    // record is always the discriminator
-	    offs += adt->is_enum ();
 	    tree field_expr
-	      = Backend::struct_field_expression (match_scrutinee_expr, offs,
+	      = Backend::struct_field_expression (variant_accesser_expr, offs,
 						  ident.get_locus ());
 
 	    tree check_expr_sub
@@ -279,7 +294,8 @@ CompilePatternCheckExpr::visit (HIR::StructPattern &pattern)
 	  }
 	  break;
 
-	  case HIR::StructPatternField::ItemType::IDENT: {
+	case HIR::StructPatternField::ItemType::IDENT:
+	  {
 	    // ident pattern always matches - do nothing
 	  }
 	  break;
@@ -338,44 +354,70 @@ CompilePatternCheckExpr::visit (HIR::TupleStructPattern &pattern)
   HIR::TupleStructItems &items = pattern.get_items ();
   switch (items.get_item_type ())
     {
-      case HIR::TupleStructItems::RANGED: {
+    case HIR::TupleStructItems::RANGED:
+      {
 	// TODO
 	rust_unreachable ();
       }
       break;
 
-      case HIR::TupleStructItems::MULTIPLE: {
+    case HIR::TupleStructItems::MULTIPLE:
+      {
 	HIR::TupleStructItemsNoRange &items_no_range
 	  = static_cast<HIR::TupleStructItemsNoRange &> (items);
 
 	rust_assert (items_no_range.get_patterns ().size ()
 		     == variant->num_fields ());
 
-	size_t tuple_field_index = 0;
-	for (auto &pattern : items_no_range.get_patterns ())
+	if (adt->is_enum ())
 	  {
-	    // find payload union field of scrutinee
-	    tree payload_ref
-	      = Backend::struct_field_expression (match_scrutinee_expr, 1,
-						  pattern->get_locus ());
+	    size_t tuple_field_index = 0;
+	    for (auto &pattern : items_no_range.get_patterns ())
+	      {
+		// find payload union field of scrutinee
+		tree payload_ref
+		  = Backend::struct_field_expression (match_scrutinee_expr, 1,
+						      pattern->get_locus ());
 
-	    tree variant_ref
-	      = Backend::struct_field_expression (payload_ref, variant_index,
-						  pattern->get_locus ());
+		tree variant_ref
+		  = Backend::struct_field_expression (payload_ref,
+						      variant_index,
+						      pattern->get_locus ());
 
-	    tree field_expr
-	      = Backend::struct_field_expression (variant_ref,
-						  tuple_field_index++,
-						  pattern->get_locus ());
+		tree field_expr
+		  = Backend::struct_field_expression (variant_ref,
+						      tuple_field_index++,
+						      pattern->get_locus ());
 
-	    tree check_expr_sub
-	      = CompilePatternCheckExpr::Compile (*pattern, field_expr, ctx);
-	    check_expr = Backend::arithmetic_or_logical_expression (
-	      ArithmeticOrLogicalOperator::BITWISE_AND, check_expr,
-	      check_expr_sub, pattern->get_locus ());
+		tree check_expr_sub
+		  = CompilePatternCheckExpr::Compile (*pattern, field_expr,
+						      ctx);
+		check_expr = Backend::arithmetic_or_logical_expression (
+		  ArithmeticOrLogicalOperator::BITWISE_AND, check_expr,
+		  check_expr_sub, pattern->get_locus ());
+	      }
 	  }
+	else
+	  {
+	    // For non-enum TupleStructPatterns
+	    size_t tuple_field_index = 0;
+	    for (auto &pattern : items_no_range.get_patterns ())
+	      {
+		tree field_expr
+		  = Backend::struct_field_expression (match_scrutinee_expr,
+						      tuple_field_index++,
+						      pattern->get_locus ());
+
+		tree check_expr_sub
+		  = CompilePatternCheckExpr::Compile (*pattern, field_expr,
+						      ctx);
+		check_expr = Backend::arithmetic_or_logical_expression (
+		  ArithmeticOrLogicalOperator::BITWISE_AND, check_expr,
+		  check_expr_sub, pattern->get_locus ());
+	      }
+	  }
+	break;
       }
-      break;
     }
 }
 
@@ -386,13 +428,57 @@ CompilePatternCheckExpr::visit (HIR::TuplePattern &pattern)
 
   switch (pattern.get_items ().get_item_type ())
     {
-      case HIR::TuplePatternItems::RANGED: {
-	// TODO
-	gcc_unreachable ();
+    case HIR::TuplePatternItems::RANGED:
+      {
+	auto &items
+	  = static_cast<HIR::TuplePatternItemsRanged &> (pattern.get_items ());
+	size_t tuple_field_index = 0;
+
+	// lookup the type to find out number of fields
+	TyTy::BaseType *ty = nullptr;
+	bool ok = ctx->get_tyctx ()->lookup_type (
+	  pattern.get_mappings ().get_hirid (), &ty);
+	rust_assert (ok);
+	rust_assert (ty->get_kind () == TyTy::TypeKind::TUPLE);
+
+	// compile check expr for lower patterns
+	for (auto &pat : items.get_lower_patterns ())
+	  {
+	    tree field_expr
+	      = Backend::struct_field_expression (match_scrutinee_expr,
+						  tuple_field_index++,
+						  pat->get_locus ());
+
+	    tree check_expr_sub
+	      = CompilePatternCheckExpr::Compile (*pat, field_expr, ctx);
+	    check_expr = Backend::arithmetic_or_logical_expression (
+	      ArithmeticOrLogicalOperator::BITWISE_AND, check_expr,
+	      check_expr_sub, pat->get_locus ());
+	  }
+
+	// skip the fields that are not checked
+	tuple_field_index = static_cast<TyTy::TupleType &> (*ty).num_fields ()
+			    - items.get_upper_patterns ().size ();
+
+	// compile check expr for upper patterns
+	for (auto &pat : items.get_upper_patterns ())
+	  {
+	    tree field_expr
+	      = Backend::struct_field_expression (match_scrutinee_expr,
+						  tuple_field_index++,
+						  pat->get_locus ());
+
+	    tree check_expr_sub
+	      = CompilePatternCheckExpr::Compile (*pat, field_expr, ctx);
+	    check_expr = Backend::arithmetic_or_logical_expression (
+	      ArithmeticOrLogicalOperator::BITWISE_AND, check_expr,
+	      check_expr_sub, pat->get_locus ());
+	  }
       }
       break;
 
-      case HIR::TuplePatternItems::MULTIPLE: {
+    case HIR::TuplePatternItems::MULTIPLE:
+      {
 	auto &items = static_cast<HIR::TuplePatternItemsMultiple &> (
 	  pattern.get_items ());
 	size_t tuple_field_index = 0;
@@ -414,6 +500,99 @@ CompilePatternCheckExpr::visit (HIR::TuplePattern &pattern)
     }
 }
 
+void
+CompilePatternCheckExpr::visit (HIR::IdentifierPattern &pattern)
+{
+  if (pattern.has_subpattern ())
+    {
+      check_expr = CompilePatternCheckExpr::Compile (pattern.get_subpattern (),
+						     match_scrutinee_expr, ctx);
+    }
+  else
+    {
+      check_expr = boolean_true_node;
+    }
+}
+
+void
+CompilePatternCheckExpr::visit (HIR::SlicePattern &pattern)
+{
+  check_expr = boolean_true_node;
+
+  // lookup the type
+  TyTy::BaseType *lookup = nullptr;
+  bool ok
+    = ctx->get_tyctx ()->lookup_type (pattern.get_mappings ().get_hirid (),
+				      &lookup);
+  rust_assert (ok);
+
+  // pattern must either be ArrayType or SliceType, should be already confirmed
+  // by type checking
+  rust_assert (lookup->get_kind () == TyTy::TypeKind::ARRAY
+	       || lookup->get_kind () == TyTy::TypeKind::SLICE
+	       || lookup->get_kind () == TyTy::REF);
+
+  size_t array_element_index = 0;
+  switch (lookup->get_kind ())
+    {
+    case TyTy::TypeKind::ARRAY:
+      for (auto &pattern_member : pattern.get_items ())
+	{
+	  tree array_index_tree
+	    = Backend::size_constant_expression (array_element_index++);
+	  tree element_expr
+	    = Backend::array_index_expression (match_scrutinee_expr,
+					       array_index_tree,
+					       pattern.get_locus ());
+	  tree check_expr_sub
+	    = CompilePatternCheckExpr::Compile (*pattern_member, element_expr,
+						ctx);
+	  check_expr = Backend::arithmetic_or_logical_expression (
+	    ArithmeticOrLogicalOperator::BITWISE_AND, check_expr,
+	    check_expr_sub, pattern.get_locus ());
+	}
+      break;
+    case TyTy::TypeKind::SLICE:
+      rust_sorry_at (
+	pattern.get_locus (),
+	"SlicePattern matching against non-ref slices are not yet supported");
+      break;
+    case TyTy::TypeKind::REF:
+      {
+	rust_assert (RS_DST_FLAG_P (TREE_TYPE (match_scrutinee_expr)));
+	tree size_field
+	  = Backend::struct_field_expression (match_scrutinee_expr, 1,
+					      pattern.get_locus ());
+
+	// First compare the size
+	check_expr = Backend::comparison_expression (
+	  ComparisonOperator::EQUAL, size_field,
+	  build_int_cst (size_type_node, pattern.get_items ().size ()),
+	  pattern.get_locus ());
+
+	// Then compare each element in the slice pattern
+	for (auto &pattern_member : pattern.get_items ())
+	  {
+	    tree slice_index_tree
+	      = Backend::size_constant_expression (array_element_index++);
+	    tree element_expr
+	      = Backend::slice_index_expression (match_scrutinee_expr,
+						 slice_index_tree,
+						 pattern.get_locus ());
+	    tree check_expr_sub
+	      = CompilePatternCheckExpr::Compile (*pattern_member, element_expr,
+						  ctx);
+	    check_expr = Backend::arithmetic_or_logical_expression (
+	      ArithmeticOrLogicalOperator::BITWISE_AND, check_expr,
+	      check_expr_sub, pattern.get_locus ());
+	  }
+      }
+      break;
+    default:
+      rust_unreachable ();
+    }
+}
+
 // setup the bindings
 
 void
@@ -449,13 +628,15 @@ CompilePatternBindings::visit (HIR::TupleStructPattern &pattern)
   HIR::TupleStructItems &items = pattern.get_items ();
   switch (items.get_item_type ())
     {
-      case HIR::TupleStructItems::RANGED: {
+    case HIR::TupleStructItems::RANGED:
+      {
 	// TODO
 	rust_unreachable ();
       }
       break;
 
-      case HIR::TupleStructItems::MULTIPLE: {
+    case HIR::TupleStructItems::MULTIPLE:
+      {
 	HIR::TupleStructItemsNoRange &items_no_range
 	  = static_cast<HIR::TupleStructItemsNoRange &> (items);
 
@@ -504,6 +685,71 @@ CompilePatternBindings::visit (HIR::TupleStructPattern &pattern)
     }
 }
 
+tree
+CompilePatternBindings::make_struct_access (TyTy::ADTType *adt,
+					    TyTy::VariantDef *variant,
+					    const Identifier &ident,
+					    int variant_index)
+{
+  size_t offs = 0;
+  auto ok = variant->lookup_field (ident.as_string (), nullptr, &offs);
+  rust_assert (ok);
+
+  if (adt->is_enum ())
+    {
+      tree payload_accessor_union
+	= Backend::struct_field_expression (match_scrutinee_expr, 1,
+					    ident.get_locus ());
+
+      tree variant_accessor
+	= Backend::struct_field_expression (payload_accessor_union,
+					    variant_index, ident.get_locus ());
+
+      return Backend::struct_field_expression (variant_accessor, offs,
+					       ident.get_locus ());
+    }
+  else
+    {
+      tree variant_accessor = match_scrutinee_expr;
+
+      return Backend::struct_field_expression (variant_accessor, offs,
+					       ident.get_locus ());
+    }
+}
+
+void
+CompilePatternBindings::handle_struct_pattern_ident (
+  HIR::StructPatternField &pat, TyTy::ADTType *adt, TyTy::VariantDef *variant,
+  int variant_index)
+{
+  HIR::StructPatternFieldIdent &ident
+    = static_cast<HIR::StructPatternFieldIdent &> (pat);
+
+  auto identifier = ident.get_identifier ();
+  tree binding = make_struct_access (adt, variant, identifier, variant_index);
+
+  ctx->insert_pattern_binding (ident.get_mappings ().get_hirid (), binding);
+}
+
+void
+CompilePatternBindings::handle_struct_pattern_ident_pat (
+  HIR::StructPatternField &pat, TyTy::ADTType *adt, TyTy::VariantDef *variant,
+  int variant_index)
+{
+  auto &pattern = static_cast<HIR::StructPatternFieldIdentPat &> (pat);
+
+  tree binding = make_struct_access (adt, variant, pattern.get_identifier (),
+				     variant_index);
+  CompilePatternBindings::Compile (pattern.get_pattern (), binding, ctx);
+}
+
+void
+CompilePatternBindings::handle_struct_pattern_tuple_pat (
+  HIR::StructPatternField &pat)
+{
+  rust_unreachable ();
+}
+
 void
 CompilePatternBindings::visit (HIR::StructPattern &pattern)
 {
@@ -539,54 +785,14 @@ CompilePatternBindings::visit (HIR::StructPattern &pattern)
     {
       switch (field->get_item_type ())
 	{
-	  case HIR::StructPatternField::ItemType::TUPLE_PAT: {
-	    // TODO
-	    rust_unreachable ();
-	  }
+	case HIR::StructPatternField::ItemType::TUPLE_PAT:
+	  handle_struct_pattern_tuple_pat (*field);
 	  break;
-
-	  case HIR::StructPatternField::ItemType::IDENT_PAT: {
-	    // TODO
-	    rust_unreachable ();
-	  }
+	case HIR::StructPatternField::ItemType::IDENT_PAT:
+	  handle_struct_pattern_ident_pat (*field, adt, variant, variant_index);
 	  break;
-
-	  case HIR::StructPatternField::ItemType::IDENT: {
-	    HIR::StructPatternFieldIdent &ident
-	      = static_cast<HIR::StructPatternFieldIdent &> (*field.get ());
-
-	    size_t offs = 0;
-	    ok = variant->lookup_field (ident.get_identifier ().as_string (),
-					nullptr, &offs);
-	    rust_assert (ok);
-
-	    tree binding = error_mark_node;
-	    if (adt->is_enum ())
-	      {
-		tree payload_accessor_union
-		  = Backend::struct_field_expression (match_scrutinee_expr, 1,
-						      ident.get_locus ());
-
-		tree variant_accessor
-		  = Backend::struct_field_expression (payload_accessor_union,
-						      variant_index,
-						      ident.get_locus ());
-
-		binding
-		  = Backend::struct_field_expression (variant_accessor, offs,
-						      ident.get_locus ());
-	      }
-	    else
-	      {
-		tree variant_accessor = match_scrutinee_expr;
-		binding
-		  = Backend::struct_field_expression (variant_accessor, offs,
-						      ident.get_locus ());
-	      }
-
-	    ctx->insert_pattern_binding (ident.get_mappings ().get_hirid (),
-					 binding);
-	  }
+	case HIR::StructPatternField::ItemType::IDENT:
+	  handle_struct_pattern_ident (*field, adt, variant, variant_index);
 	  break;
 	}
     }
@@ -605,6 +811,12 @@ CompilePatternBindings::visit (HIR::ReferencePattern &pattern)
 void
 CompilePatternBindings::visit (HIR::IdentifierPattern &pattern)
 {
+  if (pattern.has_subpattern ())
+    {
+      CompilePatternBindings::Compile (pattern.get_subpattern (),
+				       match_scrutinee_expr, ctx);
+    }
+
   if (!pattern.get_is_ref ())
     {
       ctx->insert_pattern_binding (pattern.get_mappings ().get_hirid (),
@@ -631,7 +843,8 @@ CompilePatternBindings::visit (HIR::TuplePattern &pattern)
 
   switch (pattern.get_items ().get_item_type ())
     {
-      case HIR::TuplePatternItems::ItemType::RANGED: {
+    case HIR::TuplePatternItems::ItemType::RANGED:
+      {
 	size_t tuple_idx = 0;
 	auto &items
 	  = static_cast<HIR::TuplePatternItemsRanged &> (pattern.get_items ());
@@ -674,7 +887,8 @@ CompilePatternBindings::visit (HIR::TuplePattern &pattern)
 
 	return;
       }
-      case HIR::TuplePatternItems::ItemType::MULTIPLE: {
+    case HIR::TuplePatternItems::ItemType::MULTIPLE:
+      {
 	size_t tuple_idx = 0;
 	auto &items = static_cast<HIR::TuplePatternItemsMultiple &> (
 	  pattern.get_items ());
@@ -695,12 +909,67 @@ CompilePatternBindings::visit (HIR::TuplePattern &pattern)
 
 	return;
       }
-      default: {
+    default:
+      {
 	rust_unreachable ();
       }
     }
 }
 
+void
+CompilePatternBindings::visit (HIR::SlicePattern &pattern)
+{
+  // lookup the type
+  TyTy::BaseType *lookup = nullptr;
+  bool ok
+    = ctx->get_tyctx ()->lookup_type (pattern.get_mappings ().get_hirid (),
+				      &lookup);
+  rust_assert (ok);
+
+  rust_assert (lookup->get_kind () == TyTy::TypeKind::ARRAY
+	       || lookup->get_kind () == TyTy::TypeKind::SLICE
+	       || lookup->get_kind () == TyTy::REF);
+
+  size_t array_element_index = 0;
+  switch (lookup->get_kind ())
+    {
+    case TyTy::TypeKind::ARRAY:
+      for (auto &pattern_member : pattern.get_items ())
+	{
+	  tree array_index_tree
+	    = Backend::size_constant_expression (array_element_index++);
+	  tree element_expr
+	    = Backend::array_index_expression (match_scrutinee_expr,
+					       array_index_tree,
+					       pattern.get_locus ());
+	  CompilePatternBindings::Compile (*pattern_member, element_expr, ctx);
+	}
+      break;
+    case TyTy::TypeKind::SLICE:
+      rust_sorry_at (
+	pattern.get_locus (),
+	"SlicePattern matching against non-ref slices are not yet supported");
+      break;
+    case TyTy::TypeKind::REF:
+      {
+	for (auto &pattern_member : pattern.get_items ())
+	  {
+	    tree slice_index_tree
+	      = Backend::size_constant_expression (array_element_index++);
+	    tree element_expr
+	      = Backend::slice_index_expression (match_scrutinee_expr,
+						 slice_index_tree,
+						 pattern.get_locus ());
+	    CompilePatternBindings::Compile (*pattern_member, element_expr,
+					     ctx);
+	  }
+	break;
+      }
+    default:
+      rust_unreachable ();
+    }
+}
+
 //
 
 void
@@ -755,7 +1024,8 @@ CompilePatternLet::visit (HIR::TuplePattern &pattern)
 
   switch (pattern.get_items ().get_item_type ())
     {
-      case HIR::TuplePatternItems::ItemType::RANGED: {
+    case HIR::TuplePatternItems::ItemType::RANGED:
+      {
 	size_t tuple_idx = 0;
 	auto &items
 	  = static_cast<HIR::TuplePatternItemsRanged &> (pattern.get_items ());
@@ -799,7 +1069,8 @@ CompilePatternLet::visit (HIR::TuplePattern &pattern)
 
 	return;
       }
-      case HIR::TuplePatternItems::ItemType::MULTIPLE: {
+    case HIR::TuplePatternItems::ItemType::MULTIPLE:
+      {
 	size_t tuple_idx = 0;
 	auto &items = static_cast<HIR::TuplePatternItemsMultiple &> (
 	  pattern.get_items ());
@@ -821,7 +1092,8 @@ CompilePatternLet::visit (HIR::TuplePattern &pattern)
 
 	return;
       }
-      default: {
+    default:
+      {
 	rust_unreachable ();
       }
     }
diff --git a/gcc/rust/backend/rust-compile-pattern.h b/gcc/rust/backend/rust-compile-pattern.h
index c7a62fc..233799e 100644
--- a/gcc/rust/backend/rust-compile-pattern.h
+++ b/gcc/rust/backend/rust-compile-pattern.h
@@ -17,7 +17,9 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-compile-base.h"
+#include "rust-hir-pattern.h"
 #include "rust-hir-visitor.h"
+#include "rust-tyty.h"
 
 namespace Rust {
 namespace Compile {
@@ -43,12 +45,10 @@ public:
   void visit (HIR::StructPattern &) override;
   void visit (HIR::TupleStructPattern &) override;
   void visit (HIR::TuplePattern &) override;
+  void visit (HIR::IdentifierPattern &) override;
+  void visit (HIR::SlicePattern &) override;
 
   // Always succeeds
-  void visit (HIR::IdentifierPattern &) override
-  {
-    check_expr = boolean_true_node;
-  }
   void visit (HIR::WildcardPattern &) override
   {
     check_expr = boolean_true_node;
@@ -56,7 +56,6 @@ public:
 
   // Empty visit for unused Pattern HIR nodes.
   void visit (HIR::QualifiedPathInExpression &) override {}
-  void visit (HIR::SlicePattern &) override {}
 
   CompilePatternCheckExpr (Context *ctx, tree match_scrutinee_expr)
     : HIRCompileBase (ctx), match_scrutinee_expr (match_scrutinee_expr),
@@ -78,11 +77,25 @@ public:
     pattern.accept_vis (compiler);
   }
 
+  tree make_struct_access (TyTy::ADTType *adt, TyTy::VariantDef *variant,
+			   const Identifier &ident, int variant_index);
+
+  void handle_struct_pattern_ident (HIR::StructPatternField &pat,
+				    TyTy::ADTType *adt,
+				    TyTy::VariantDef *variant,
+				    int variant_index);
+  void handle_struct_pattern_ident_pat (HIR::StructPatternField &pat,
+					TyTy::ADTType *adt,
+					TyTy::VariantDef *variant,
+					int variant_index);
+  void handle_struct_pattern_tuple_pat (HIR::StructPatternField &pat);
+
   void visit (HIR::StructPattern &pattern) override;
   void visit (HIR::TupleStructPattern &pattern) override;
   void visit (HIR::ReferencePattern &pattern) override;
   void visit (HIR::IdentifierPattern &) override;
   void visit (HIR::TuplePattern &pattern) override;
+  void visit (HIR::SlicePattern &) override;
 
   // Empty visit for unused Pattern HIR nodes.
   void visit (HIR::AltPattern &) override {}
@@ -90,7 +103,6 @@ public:
   void visit (HIR::PathInExpression &) override {}
   void visit (HIR::QualifiedPathInExpression &) override {}
   void visit (HIR::RangePattern &) override {}
-  void visit (HIR::SlicePattern &) override {}
   void visit (HIR::WildcardPattern &) override {}
 
 protected:
diff --git a/gcc/rust/backend/rust-compile-resolve-path.cc b/gcc/rust/backend/rust-compile-resolve-path.cc
index 81d2dbb..f3b9dc2 100644
--- a/gcc/rust/backend/rust-compile-resolve-path.cc
+++ b/gcc/rust/backend/rust-compile-resolve-path.cc
@@ -187,13 +187,18 @@ ResolvePathRef::resolve_with_node_id (
     }
 
   // Handle unit struct
+  tree resolved_item = error_mark_node;
   if (lookup->get_kind () == TyTy::TypeKind::ADT)
-    return attempt_constructor_expression_lookup (lookup, ctx, mappings,
-						  expr_locus);
+    resolved_item
+      = attempt_constructor_expression_lookup (lookup, ctx, mappings,
+					       expr_locus);
+
+  if (!error_operand_p (resolved_item))
+    return resolved_item;
 
   // let the query system figure it out
-  tree resolved_item = query_compile (ref, lookup, final_segment, mappings,
-				      expr_locus, is_qualified_path);
+  resolved_item = query_compile (ref, lookup, final_segment, mappings,
+				 expr_locus, is_qualified_path);
   if (resolved_item != error_mark_node)
     {
       TREE_USED (resolved_item) = 1;
@@ -209,36 +214,24 @@ ResolvePathRef::resolve (const HIR::PathIdentSegment &final_segment,
 {
   TyTy::BaseType *lookup = nullptr;
   bool ok = ctx->get_tyctx ()->lookup_type (mappings.get_hirid (), &lookup);
-  rust_assert (ok);
+  if (!ok)
+    return error_mark_node;
 
   // need to look up the reference for this identifier
 
   // this can fail because it might be a Constructor for something
   // in that case the caller should attempt ResolvePathType::Compile
-  NodeId ref_node_id = UNKNOWN_NODEID;
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      auto resolved = nr_ctx.lookup (mappings.get_nodeid ());
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-      if (!resolved)
-	return attempt_constructor_expression_lookup (lookup, ctx, mappings,
-						      expr_locus);
+  auto resolved = nr_ctx.lookup (mappings.get_nodeid ());
 
-      ref_node_id = *resolved;
-    }
-  else
-    {
-      if (!ctx->get_resolver ()->lookup_resolved_name (mappings.get_nodeid (),
-						       &ref_node_id))
-	return attempt_constructor_expression_lookup (lookup, ctx, mappings,
-						      expr_locus);
-    }
+  if (!resolved)
+    return attempt_constructor_expression_lookup (lookup, ctx, mappings,
+						  expr_locus);
 
   return resolve_with_node_id (final_segment, mappings, expr_locus,
-			       is_qualified_path, ref_node_id);
+			       is_qualified_path, *resolved);
 }
 
 tree
@@ -336,11 +329,18 @@ HIRCompileBase::query_compile (HirId ref, TyTy::BaseType *lookup,
 	  rust_assert (lookup->is<TyTy::FnType> ());
 	  auto fn = lookup->as<TyTy::FnType> ();
 	  rust_assert (fn->get_num_type_params () > 0);
-	  auto &self = fn->get_substs ().at (0);
-	  auto receiver = self.get_param_ty ();
+	  TyTy::SubstitutionParamMapping &self = fn->get_substs ().at (0);
+	  TyTy::BaseGeneric *receiver = self.get_param_ty ();
+	  TyTy::BaseType *r = receiver;
+	  if (!receiver->can_resolve ())
+	    {
+	      bool ok
+		= ctx->get_tyctx ()->lookup_type (receiver->get_ref (), &r);
+	      rust_assert (ok);
+	    }
+
 	  auto candidates
-	    = Resolver::PathProbeImplTrait::Probe (receiver, final_segment,
-						   trait_ref);
+	    = Resolver::PathProbeImplTrait::Probe (r, final_segment, trait_ref);
 	  if (candidates.size () == 0)
 	    {
 	      // this means we are defaulting back to the trait_item if
diff --git a/gcc/rust/backend/rust-compile-stmt.cc b/gcc/rust/backend/rust-compile-stmt.cc
index a4b5a98..b520baf 100644
--- a/gcc/rust/backend/rust-compile-stmt.cc
+++ b/gcc/rust/backend/rust-compile-stmt.cc
@@ -58,6 +58,9 @@ CompileStmt::visit (HIR::LetStmt &stmt)
       return;
     }
 
+  rust_debug_loc (stmt.get_locus (), "   -> LetStmt %s",
+		  ty->as_string ().c_str ());
+
   // setup var decl nodes
   fncontext fnctx = ctx->peek_fn ();
   tree fndecl = fnctx.fndecl;
diff --git a/gcc/rust/backend/rust-compile-type.cc b/gcc/rust/backend/rust-compile-type.cc
index 83e5756..0622954 100644
--- a/gcc/rust/backend/rust-compile-type.cc
+++ b/gcc/rust/backend/rust-compile-type.cc
@@ -17,11 +17,11 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-compile-type.h"
-#include "rust-compile-expr.h"
 #include "rust-constexpr.h"
-#include "rust-gcc.h"
+#include "rust-compile-base.h"
 
 #include "tree.h"
+#include "fold-const.h"
 #include "stor-layout.h"
 
 namespace Rust {
@@ -121,6 +121,13 @@ TyTyResolveCompile::visit (const TyTy::InferType &type)
 
   if (orig == lookup)
     {
+      TyTy::BaseType *def = nullptr;
+      if (type.default_type (&def))
+	{
+	  translated = TyTyResolveCompile::compile (ctx, def);
+	  return;
+	}
+
       translated = error_mark_node;
       return;
     }
@@ -135,6 +142,12 @@ TyTyResolveCompile::visit (const TyTy::ParamType &)
 }
 
 void
+TyTyResolveCompile::visit (const TyTy::ConstType &)
+{
+  translated = error_mark_node;
+}
+
+void
 TyTyResolveCompile::visit (const TyTy::ProjectionType &type)
 {
   translated = error_mark_node;
@@ -189,7 +202,7 @@ TyTyResolveCompile::visit (const TyTy::ClosureType &type)
 void
 TyTyResolveCompile::visit (const TyTy::FnType &type)
 {
-  Backend::typed_identifier receiver;
+  Backend::typed_identifier receiver ("", NULL_TREE, UNKNOWN_LOCATION);
   std::vector<Backend::typed_identifier> parameters;
   std::vector<Backend::typed_identifier> results;
 
@@ -454,7 +467,7 @@ TyTyResolveCompile::visit (const TyTy::TupleType &type)
     }
 
   tree struct_type_record = Backend::struct_type (fields);
-  translated = Backend::named_type (type.as_string (), struct_type_record,
+  translated = Backend::named_type (type.get_name (), struct_type_record,
 				    type.get_ident ().locus);
 }
 
@@ -463,26 +476,15 @@ TyTyResolveCompile::visit (const TyTy::ArrayType &type)
 {
   tree element_type
     = TyTyResolveCompile::compile (ctx, type.get_element_type ());
+  TyTy::ConstType *const_capacity = type.get_capacity ();
+  tree folded_capacity_expr = const_capacity->get_value ();
 
-  ctx->push_const_context ();
-
-  HIR::Expr &hir_capacity_expr = type.get_capacity_expr ();
-  TyTy::BaseType *capacity_expr_ty = nullptr;
-  bool ok = ctx->get_tyctx ()->lookup_type (
-    hir_capacity_expr.get_mappings ().get_hirid (), &capacity_expr_ty);
-  rust_assert (ok);
-  tree capacity_expr = HIRCompileBase::compile_constant_expr (
-    ctx, hir_capacity_expr.get_mappings ().get_hirid (), capacity_expr_ty,
-    capacity_expr_ty, Resolver::CanonicalPath::create_empty (),
-    hir_capacity_expr, type.get_locus (), hir_capacity_expr.get_locus ());
-
-  ctx->pop_const_context ();
-
-  tree folded_capacity_expr = fold_expr (capacity_expr);
+  // build_index_type takes the maximum index, which is one less than
+  // the length.
+  tree index_type_tree = build_index_type (
+    fold_build2 (MINUS_EXPR, sizetype, folded_capacity_expr, size_one_node));
 
-  translated = Backend::array_type (element_type, folded_capacity_expr);
-  if (translated != error_mark_node)
-    translated = ctx->insert_compiled_type (translated);
+  translated = build_array_type (element_type, index_type_tree, false);
 }
 
 void
@@ -755,7 +757,9 @@ TyTyResolveCompile::visit (const TyTy::DynamicObjectType &type)
 void
 TyTyResolveCompile::visit (const TyTy::OpaqueType &type)
 {
-  translated = error_mark_node;
+  rust_assert (type.can_resolve ());
+  auto underlying = type.resolve ();
+  translated = TyTyResolveCompile::compile (ctx, underlying, trait_object_mode);
 }
 
 tree
diff --git a/gcc/rust/backend/rust-compile-type.h b/gcc/rust/backend/rust-compile-type.h
index 7ebc4a6..0675343 100644
--- a/gcc/rust/backend/rust-compile-type.h
+++ b/gcc/rust/backend/rust-compile-type.h
@@ -50,6 +50,7 @@ public:
   void visit (const TyTy::ReferenceType &) override;
   void visit (const TyTy::PointerType &) override;
   void visit (const TyTy::ParamType &) override;
+  void visit (const TyTy::ConstType &) override;
   void visit (const TyTy::StrType &) override;
   void visit (const TyTy::NeverType &) override;
   void visit (const TyTy::PlaceholderType &) override;
diff --git a/gcc/rust/backend/rust-compile-var-decl.h b/gcc/rust/backend/rust-compile-var-decl.h
index 4c46a7b..5c6d145 100644
--- a/gcc/rust/backend/rust-compile-var-decl.h
+++ b/gcc/rust/backend/rust-compile-var-decl.h
@@ -70,7 +70,8 @@ public:
   {
     switch (pattern.get_items ().get_item_type ())
       {
-	case HIR::TuplePatternItems::ItemType::MULTIPLE: {
+      case HIR::TuplePatternItems::ItemType::MULTIPLE:
+	{
 	  rust_assert (TREE_CODE (translated_type) == RECORD_TYPE);
 	  auto &items = static_cast<HIR::TuplePatternItemsMultiple &> (
 	    pattern.get_items ());
diff --git a/gcc/rust/backend/rust-constexpr.cc b/gcc/rust/backend/rust-constexpr.cc
index dc2d6b1..d524d09 100644
--- a/gcc/rust/backend/rust-constexpr.cc
+++ b/gcc/rust/backend/rust-constexpr.cc
@@ -68,32 +68,24 @@ literal_type_p (tree t)
   return false;
 }
 
-static bool
-verify_constant (tree, bool, bool *, bool *);
-
-static HOST_WIDE_INT
-find_array_ctor_elt (tree ary, tree dindex, bool insert = false);
-static int
-array_index_cmp (tree key, tree index);
-static bool
-potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
-				 tsubst_flags_t flags, tree *jump_target);
-bool
-potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
-				 tsubst_flags_t flags);
-tree
-unshare_constructor (tree t MEM_STAT_DECL);
-void
-maybe_save_constexpr_fundef (tree fun);
-
-static bool
-returns (tree *jump_target);
-static bool
-breaks (tree *jump_target);
-static bool
-continues (tree *jump_target);
-static bool
-switches (tree *jump_target);
+static bool verify_constant (tree, bool, bool *, bool *);
+
+static HOST_WIDE_INT find_array_ctor_elt (tree ary, tree dindex,
+					  bool insert = false);
+static int array_index_cmp (tree key, tree index);
+static bool potential_constant_expression_1 (tree t, bool want_rval,
+					     bool strict, bool now,
+					     tsubst_flags_t flags,
+					     tree *jump_target);
+bool potential_constant_expression_1 (tree t, bool want_rval, bool strict,
+				      bool now, tsubst_flags_t flags);
+tree unshare_constructor (tree t MEM_STAT_DECL);
+void maybe_save_constexpr_fundef (tree fun);
+
+static bool returns (tree *jump_target);
+static bool breaks (tree *jump_target);
+static bool continues (tree *jump_target);
+static bool switches (tree *jump_target);
 
 struct constexpr_global_ctx
 {
@@ -463,60 +455,52 @@ save_fundef_copy (tree fun, tree copy)
   *slot = copy;
 }
 
-static tree
-constant_value_1 (tree decl, bool strict_p, bool return_aggregate_cst_ok_p,
-		  bool unshare_p);
-tree
-decl_constant_value (tree decl, bool unshare_p);
+static tree constant_value_1 (tree decl, bool strict_p,
+			      bool return_aggregate_cst_ok_p, bool unshare_p);
+tree decl_constant_value (tree decl, bool unshare_p);
 
-static void
-non_const_var_error (location_t loc, tree r);
+static void non_const_var_error (location_t loc, tree r);
 
-static tree
-eval_constant_expression (const constexpr_ctx *ctx, tree, bool, bool *, bool *,
-			  tree * = NULL);
+static tree eval_constant_expression (const constexpr_ctx *ctx, tree, bool,
+				      bool *, bool *, tree * = NULL);
 
-static tree
-constexpr_fn_retval (const constexpr_ctx *ctx, tree r);
+static tree constexpr_fn_retval (const constexpr_ctx *ctx, tree r);
 
-static tree
-eval_store_expression (const constexpr_ctx *ctx, tree r, bool, bool *, bool *);
+static tree eval_store_expression (const constexpr_ctx *ctx, tree r, bool,
+				   bool *, bool *);
 
-static tree
-eval_call_expression (const constexpr_ctx *ctx, tree r, bool, bool *, bool *);
+static tree eval_call_expression (const constexpr_ctx *ctx, tree r, bool,
+				  bool *, bool *);
 
-static tree
-eval_binary_expression (const constexpr_ctx *ctx, tree r, bool, bool *, bool *);
+static tree eval_binary_expression (const constexpr_ctx *ctx, tree r, bool,
+				    bool *, bool *);
 
-static tree
-get_function_named_in_call (tree t);
+static tree get_function_named_in_call (tree t);
 
-static tree
-eval_statement_list (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
-		     bool *overflow_p, tree *jump_target);
-static tree
-extract_string_elt (tree string, unsigned chars_per_elt, unsigned index);
+static tree eval_statement_list (const constexpr_ctx *ctx, tree t,
+				 bool *non_constant_p, bool *overflow_p,
+				 tree *jump_target);
+static tree extract_string_elt (tree string, unsigned chars_per_elt,
+				unsigned index);
 
-static tree
-eval_conditional_expression (const constexpr_ctx *ctx, tree t, bool lval,
-			     bool *non_constant_p, bool *overflow_p,
-			     tree *jump_target);
+static tree eval_conditional_expression (const constexpr_ctx *ctx, tree t,
+					 bool lval, bool *non_constant_p,
+					 bool *overflow_p, tree *jump_target);
 
-static tree
-eval_bit_field_ref (const constexpr_ctx *ctx, tree t, bool lval,
-		    bool *non_constant_p, bool *overflow_p);
+static tree eval_bit_field_ref (const constexpr_ctx *ctx, tree t, bool lval,
+				bool *non_constant_p, bool *overflow_p);
 
-static tree
-eval_loop_expr (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
-		bool *overflow_p, tree *jump_target);
+static tree eval_loop_expr (const constexpr_ctx *ctx, tree t,
+			    bool *non_constant_p, bool *overflow_p,
+			    tree *jump_target);
 
-static tree
-eval_switch_expr (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
-		  bool *overflow_p, tree *jump_target);
+static tree eval_switch_expr (const constexpr_ctx *ctx, tree t,
+			      bool *non_constant_p, bool *overflow_p,
+			      tree *jump_target);
 
-static tree
-eval_unary_expression (const constexpr_ctx *ctx, tree t, bool /*lval*/,
-		       bool *non_constant_p, bool *overflow_p);
+static tree eval_unary_expression (const constexpr_ctx *ctx, tree t,
+				   bool /*lval*/, bool *non_constant_p,
+				   bool *overflow_p);
 
 /* Variables and functions to manage constexpr call expansion context.
    These do not need to be marked for PCH or GC.  */
@@ -1235,7 +1219,8 @@ get_or_insert_ctor_field (tree ctor, tree index, int pos_hint = -1)
       /* We fell off the end of the CONSTRUCTOR, so insert a new
 	 entry at the end.  */
 
-      insert : {
+    insert:
+      {
 	constructor_elt ce = {index, NULL_TREE};
 
 	vec_safe_insert (CONSTRUCTOR_ELTS (ctor), idx, ce);
@@ -1568,10 +1553,9 @@ free_constructor (tree t)
     }
 }
 
-static tree
-eval_and_check_array_index (const constexpr_ctx *ctx, tree t,
-			    bool allow_one_past, bool *non_constant_p,
-			    bool *overflow_p);
+static tree eval_and_check_array_index (const constexpr_ctx *ctx, tree t,
+					bool allow_one_past,
+					bool *non_constant_p, bool *overflow_p);
 
 // forked from gcc/cp/constexpr.cc cxx_eval_array_reference
 
@@ -1901,6 +1885,9 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
 
   location_t loc = EXPR_LOCATION (t);
 
+  if (t == NULL_TREE)
+    return NULL_TREE;
+
   if (CONSTANT_CLASS_P (t))
     {
       if (TREE_OVERFLOW (t))
@@ -1936,8 +1923,9 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
 	  return eval_constant_expression (ctx, r, lval, non_constant_p,
 					   overflow_p);
 	}
-      /* fall through */
-      case CONST_DECL: {
+    /* fall through */
+    case CONST_DECL:
+      {
 	/* We used to not check lval for CONST_DECL, but darwin.cc uses
 	   CONST_DECL for aggregate constants.  */
 	if (lval)
@@ -2045,7 +2033,8 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
 				   overflow_p);
       break;
 
-      case TARGET_EXPR: {
+    case TARGET_EXPR:
+      {
 	tree type = TREE_TYPE (t);
 
 	if (!literal_type_p (type))
@@ -2129,7 +2118,8 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
 	}
       break;
 
-      case DECL_EXPR: {
+    case DECL_EXPR:
+      {
 	r = DECL_EXPR_DECL (t);
 
 	if (AGGREGATE_TYPE_P (TREE_TYPE (r)) || VECTOR_TYPE_P (TREE_TYPE (r)))
@@ -2201,7 +2191,8 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
       return eval_constant_expression (ctx, OBJ_TYPE_REF_EXPR (t), lval,
 				       non_constant_p, overflow_p);
 
-      case EXIT_EXPR: {
+    case EXIT_EXPR:
+      {
 	tree cond = TREE_OPERAND (t, 0);
 	cond = eval_constant_expression (ctx, cond, /*lval*/ false,
 					 non_constant_p, overflow_p);
@@ -2243,7 +2234,8 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
 	}
       break;
 
-      case ADDR_EXPR: {
+    case ADDR_EXPR:
+      {
 	tree oldop = TREE_OPERAND (t, 0);
 	tree op = eval_constant_expression (ctx, oldop,
 					    /*lval*/ true, non_constant_p,
@@ -2261,7 +2253,8 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
 	break;
       }
 
-      case COMPOUND_EXPR: {
+    case COMPOUND_EXPR:
+      {
 	/* check_return_expr sometimes wraps a TARGET_EXPR in a
 	   COMPOUND_EXPR; don't get confused.  Also handle EMPTY_CLASS_EXPR
 	   introduced by build_call_a.  */
@@ -2401,7 +2394,8 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
 				    non_constant_p, overflow_p, jump_target);
       break;
 
-      case CLEANUP_POINT_EXPR: {
+    case CLEANUP_POINT_EXPR:
+      {
 	auto_vec<tree, 2> cleanups;
 	vec<tree> *prev_cleanups = ctx->global->cleanups;
 	ctx->global->cleanups = &cleanups;
@@ -2441,7 +2435,8 @@ eval_constant_expression (const constexpr_ctx *ctx, tree t, bool lval,
       /* FALLTHROUGH.  */
     case NOP_EXPR:
     case CONVERT_EXPR:
-      case VIEW_CONVERT_EXPR: {
+    case VIEW_CONVERT_EXPR:
+      {
 	tree oldop = TREE_OPERAND (t, 0);
 
 	tree op = eval_constant_expression (ctx, oldop, lval, non_constant_p,
@@ -2688,7 +2683,8 @@ eval_store_expression (const constexpr_ctx *ctx, tree t, bool lval,
 	{
 	case BIT_FIELD_REF:
 	case COMPONENT_REF:
-	  case ARRAY_REF: {
+	case ARRAY_REF:
+	  {
 	    tree ob = TREE_OPERAND (probe, 0);
 	    tree elt = TREE_OPERAND (probe, 1);
 	    if (TREE_CODE (elt) == FIELD_DECL /*&& DECL_MUTABLE_P (elt)*/)
@@ -3940,7 +3936,8 @@ constexpr_fn_retval (const constexpr_ctx *ctx, tree body)
 {
   switch (TREE_CODE (body))
     {
-      case STATEMENT_LIST: {
+    case STATEMENT_LIST:
+      {
 	tree expr = NULL_TREE;
 	for (tree stmt : tsi_range (body))
 	  {
@@ -3958,13 +3955,15 @@ constexpr_fn_retval (const constexpr_ctx *ctx, tree body)
 	return expr;
       }
 
-      case RETURN_EXPR: {
+    case RETURN_EXPR:
+      {
 	bool non_constant_p = false;
 	bool overflow_p = false;
 	return eval_constant_expression (ctx, body, false, &non_constant_p,
 					 &overflow_p);
       }
-      case DECL_EXPR: {
+    case DECL_EXPR:
+      {
 	tree decl = DECL_EXPR_DECL (body);
 	if (TREE_CODE (decl) == USING_DECL
 	    /* Accept __func__, __FUNCTION__, and __PRETTY_FUNCTION__.  */
@@ -3976,7 +3975,8 @@ constexpr_fn_retval (const constexpr_ctx *ctx, tree body)
     case CLEANUP_POINT_EXPR:
       return constexpr_fn_retval (ctx, TREE_OPERAND (body, 0));
 
-      case BIND_EXPR: {
+    case BIND_EXPR:
+      {
 	tree b = BIND_EXPR_BODY (body);
 	return constexpr_fn_retval (ctx, b);
       }
@@ -4136,7 +4136,8 @@ array_index_cmp (tree key, tree index)
     {
     case INTEGER_CST:
       return tree_int_cst_compare (key, index);
-      case RANGE_EXPR: {
+    case RANGE_EXPR:
+      {
 	tree lo = TREE_OPERAND (index, 0);
 	tree hi = TREE_OPERAND (index, 1);
 	if (tree_int_cst_lt (key, lo))
@@ -5943,7 +5944,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
     case BIT_FIELD_REF:
       return RECUR (TREE_OPERAND (t, 0), want_rval);
 
-      case INDIRECT_REF: {
+    case INDIRECT_REF:
+      {
 	tree x = TREE_OPERAND (t, 0);
 	STRIP_NOPS (x);
 	return RECUR (x, rval);
@@ -6214,7 +6216,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
     case INIT_EXPR:
       return RECUR (TREE_OPERAND (t, 1), rval);
 
-      case CONSTRUCTOR: {
+    case CONSTRUCTOR:
+      {
 	vec<constructor_elt, va_gc> *v = CONSTRUCTOR_ELTS (t);
 	constructor_elt *ce;
 	for (i = 0; vec_safe_iterate (v, i, &ce); ++i)
@@ -6223,7 +6226,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	return true;
       }
 
-      case TREE_LIST: {
+    case TREE_LIST:
+      {
 	gcc_assert (TREE_PURPOSE (t) == NULL_TREE || DECL_P (TREE_PURPOSE (t)));
 	if (!RECUR (TREE_VALUE (t), want_rval))
 	  return false;
@@ -6238,7 +6242,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
     case ROUND_DIV_EXPR:
     case TRUNC_MOD_EXPR:
     case CEIL_MOD_EXPR:
-      case ROUND_MOD_EXPR: {
+    case ROUND_MOD_EXPR:
+      {
 	tree denom = TREE_OPERAND (t, 1);
 	if (!RECUR (denom, rval))
 	  return false;
@@ -6258,7 +6263,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	  }
       }
 
-      case COMPOUND_EXPR: {
+    case COMPOUND_EXPR:
+      {
 	/* check_return_expr sometimes wraps a TARGET_EXPR in a
 	   COMPOUND_EXPR; don't get confused.  */
 	tree op0 = TREE_OPERAND (t, 0);
@@ -6280,7 +6286,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
     case TRUTH_OR_EXPR:
     case TRUTH_ORIF_EXPR:
       tmp = boolean_false_node;
-      truth : {
+    truth:
+      {
 	tree op0 = TREE_OPERAND (t, 0);
 	tree op1 = TREE_OPERAND (t, 1);
 	if (!RECUR (op0, rval))
diff --git a/gcc/rust/backend/rust-constexpr.h b/gcc/rust/backend/rust-constexpr.h
index 77a0797..27f0a2e 100644
--- a/gcc/rust/backend/rust-constexpr.h
+++ b/gcc/rust/backend/rust-constexpr.h
@@ -24,8 +24,7 @@ namespace Rust {
 namespace Compile {
 
 extern tree fold_expr (tree);
-extern void
-maybe_save_constexpr_fundef (tree fun);
+extern void maybe_save_constexpr_fundef (tree fun);
 
 } // namespace Compile
 } // namespace Rust
diff --git a/gcc/rust/backend/rust-mangle-v0.cc b/gcc/rust/backend/rust-mangle-v0.cc
index d0df4ab..f6b1a4c 100644
--- a/gcc/rust/backend/rust-mangle-v0.cc
+++ b/gcc/rust/backend/rust-mangle-v0.cc
@@ -62,9 +62,9 @@ struct V0Path
   }
 };
 
-static std::string
-v0_path (Rust::Compile::Context *ctx, const TyTy::BaseType *ty,
-	 const Resolver::CanonicalPath &path);
+static std::string v0_path (Rust::Compile::Context *ctx,
+			    const TyTy::BaseType *ty,
+			    const Resolver::CanonicalPath &path);
 
 static std::string
 v0_tuple_prefix (const TyTy::BaseType *ty)
@@ -148,7 +148,8 @@ v0_complex_type_prefix (Context *ctx, const TyTy::BaseType *ty)
   // TODO: generics
   switch (ty->get_kind ())
     {
-      case TyTy::TypeKind::ADT: {
+    case TyTy::TypeKind::ADT:
+      {
 	const TyTy::ADTType *adt = static_cast<const TyTy::ADTType *> (ty);
 	return v0_path (ctx, ty, adt->get_ident ().path);
       }
@@ -387,7 +388,8 @@ v0_path (Rust::Compile::Context *ctx, const TyTy::BaseType *ty,
       {
 	switch (impl_item->first->get_impl_item_type ())
 	  {
-	    case HIR::ImplItem::FUNCTION: {
+	  case HIR::ImplItem::FUNCTION:
+	    {
 	      HIR::Function *fn
 		= static_cast<HIR::Function *> (impl_item->first);
 	      v0path
@@ -408,7 +410,8 @@ v0_path (Rust::Compile::Context *ctx, const TyTy::BaseType *ty,
       {
 	switch (trait_item.value ()->get_item_kind ())
 	  {
-	    case HIR::TraitItem::FUNC: {
+	  case HIR::TraitItem::FUNC:
+	    {
 	      auto fn = static_cast<HIR::TraitItemFunc *> (*trait_item);
 	      rust_unreachable ();
 	      v0path = v0_function_path (v0path, ctx, ty,
@@ -428,7 +431,8 @@ v0_path (Rust::Compile::Context *ctx, const TyTy::BaseType *ty,
     else if (auto item = mappings.lookup_hir_item (hir_id))
       switch (item.value ()->get_item_kind ())
 	{
-	  case HIR::Item::ItemKind::Function: {
+	case HIR::Item::ItemKind::Function:
+	  {
 	    HIR::Function *fn = static_cast<HIR::Function *> (*item);
 	    v0path
 	      = v0_function_path (v0path, ctx, ty, fn->get_generic_params (),
diff --git a/gcc/rust/backend/rust-mangle.h b/gcc/rust/backend/rust-mangle.h
index 2a84b6b..418f2bd 100644
--- a/gcc/rust/backend/rust-mangle.h
+++ b/gcc/rust/backend/rust-mangle.h
@@ -49,13 +49,12 @@ private:
   static enum MangleVersion version;
 };
 
-std::string
-legacy_mangle_item (const TyTy::BaseType *ty,
-		    const Resolver::CanonicalPath &path);
+std::string legacy_mangle_item (const TyTy::BaseType *ty,
+				const Resolver::CanonicalPath &path);
 
-std::string
-v0_mangle_item (Rust::Compile::Context *ctx, const TyTy::BaseType *ty,
-		const Resolver::CanonicalPath &path);
+std::string v0_mangle_item (Rust::Compile::Context *ctx,
+			    const TyTy::BaseType *ty,
+			    const Resolver::CanonicalPath &path);
 
 } // namespace Compile
 } // namespace Rust
diff --git a/gcc/rust/backend/rust-tree.cc b/gcc/rust/backend/rust-tree.cc
index 6cba04b..b86c3c8 100644
--- a/gcc/rust/backend/rust-tree.cc
+++ b/gcc/rust/backend/rust-tree.cc
@@ -268,7 +268,8 @@ convert_to_void (tree expr, impl_conv_void implicit)
     return expr;
   switch (TREE_CODE (expr))
     {
-      case COND_EXPR: {
+    case COND_EXPR:
+      {
 	/* The two parts of a cond expr might be separate lvalues.  */
 	tree op1 = TREE_OPERAND (expr, 1);
 	tree op2 = TREE_OPERAND (expr, 2);
@@ -294,7 +295,8 @@ convert_to_void (tree expr, impl_conv_void implicit)
 	break;
       }
 
-      case COMPOUND_EXPR: {
+    case COMPOUND_EXPR:
+      {
 	/* The second part of a compound expr contains the value.  */
 	tree op1 = TREE_OPERAND (expr, 1);
 	tree new_op1;
@@ -323,7 +325,8 @@ convert_to_void (tree expr, impl_conv_void implicit)
       maybe_warn_nodiscard (expr, implicit);
       break;
 
-      case INDIRECT_REF: {
+    case INDIRECT_REF:
+      {
 	tree type = TREE_TYPE (expr);
 	int is_reference = TYPE_REF_P (TREE_TYPE (TREE_OPERAND (expr, 0)));
 	int is_volatile = TYPE_VOLATILE (type);
@@ -518,7 +521,8 @@ convert_to_void (tree expr, impl_conv_void implicit)
 	break;
       }
 
-      case VAR_DECL: {
+    case VAR_DECL:
+      {
 	/* External variables might be incomplete.  */
 	tree type = TREE_TYPE (expr);
 	int is_complete = COMPLETE_TYPE_P (type);
@@ -1485,7 +1489,8 @@ find_parameter_packs_r (tree *tp, int *walk_subtrees, void *data)
      parameter pack.  ??? Should some of these be in cp_walk_subtrees?  */
   switch (TREE_CODE (t))
     {
-      case DECL_EXPR: {
+    case DECL_EXPR:
+      {
 	tree decl = DECL_EXPR_DECL (t);
 	if (is_typedef_decl (decl))
 	  /* Since we stop at typedefs above, we need to look through them at
@@ -1506,7 +1511,8 @@ find_parameter_packs_r (tree *tp, int *walk_subtrees, void *data)
       *walk_subtrees = 0;
       return NULL_TREE;
 
-      case DECLTYPE_TYPE: {
+    case DECLTYPE_TYPE:
+      {
 	/* When traversing a DECLTYPE_TYPE_EXPR, we need to set
 	   type_pack_expansion_p to false so that any placeholders
 	   within the expression don't get marked as parameter packs.  */
@@ -1970,7 +1976,8 @@ rs_tree_equal (tree t1, tree t2)
     case SAVE_EXPR:
       return rs_tree_equal (TREE_OPERAND (t1, 0), TREE_OPERAND (t2, 0));
 
-      case CALL_EXPR: {
+    case CALL_EXPR:
+      {
 	if (KOENIG_LOOKUP_P (t1) != KOENIG_LOOKUP_P (t2))
 	  return false;
 
@@ -1996,7 +2003,8 @@ rs_tree_equal (tree t1, tree t2)
 	return true;
       }
 
-      case TARGET_EXPR: {
+    case TARGET_EXPR:
+      {
 	tree o1 = TREE_OPERAND (t1, 0);
 	tree o2 = TREE_OPERAND (t2, 0);
 
@@ -2067,7 +2075,8 @@ rs_tree_equal (tree t1, tree t2)
     case tcc_expression:
     case tcc_vl_exp:
     case tcc_reference:
-      case tcc_statement: {
+    case tcc_statement:
+      {
 	int n = rs_tree_operand_length (t1);
 	if (TREE_CODE_CLASS (code1) == tcc_vl_exp
 	    && n != TREE_OPERAND_LENGTH (t2))
@@ -2095,7 +2104,11 @@ rs_tree_equal (tree t1, tree t2)
 
 /* TRUE iff TYPE is publicly & uniquely derived from PARENT.  */
 
-bool publicly_uniquely_derived_p (tree, tree) { return false; }
+bool
+publicly_uniquely_derived_p (tree, tree)
+{
+  return false;
+}
 
 // forked from gcc/cp/typeck.cc comp_except_types
 
@@ -3375,7 +3388,11 @@ release_tree_vector (vec<tree, va_gc> *vec)
 
 /* As above, but also check value-dependence of the expression as a whole.  */
 
-bool instantiation_dependent_expression_p (tree) { return false; }
+bool
+instantiation_dependent_expression_p (tree)
+{
+  return false;
+}
 
 // forked from gcc/cp/cvt.cc cp_get_callee
 
@@ -3425,7 +3442,11 @@ scalarish_type_p (const_tree t)
    constructors are deleted.  This function implements the ABI notion of
    non-trivial copy, which has diverged from the one in the standard.  */
 
-bool type_has_nontrivial_copy_init (const_tree) { return false; }
+bool
+type_has_nontrivial_copy_init (const_tree)
+{
+  return false;
+}
 
 // forked from gcc/cp/tree.cc build_local_temp
 
@@ -3448,7 +3469,11 @@ build_local_temp (tree type)
 /* Returns true iff DECL is a capture proxy for a normal capture
    (i.e. without explicit initializer).  */
 
-bool is_normal_capture_proxy (tree) { return false; }
+bool
+is_normal_capture_proxy (tree)
+{
+  return false;
+}
 
 // forked from gcc/cp/c-common.cc reject_gcc_builtin
 
@@ -3522,7 +3547,8 @@ is_bitfield_expr_with_lowered_type (const_tree exp)
     case BIT_NOT_EXPR:
       return is_bitfield_expr_with_lowered_type (TREE_OPERAND (exp, 0));
 
-      case COMPONENT_REF: {
+    case COMPONENT_REF:
+      {
 	tree field;
 
 	field = TREE_OPERAND (exp, 1);
@@ -3848,16 +3874,18 @@ strip_top_quals (tree t)
 /* Print an error message for invalid use of an incomplete type.
    VALUE is the expression that was used (or 0 if that isn't known)
    and TYPE is the type that was invalid.  DIAG_KIND indicates the
-   type of diagnostic (see diagnostic.def).  */
+   type of diagnostic (see diagnostics/kinds.def).  */
 
 void
 cxx_incomplete_type_diagnostic (location_t loc, const_tree value,
-				const_tree type, diagnostic_t diag_kind)
+				const_tree type,
+				enum diagnostics::kind diag_kind)
 {
   //  bool is_decl = false, complained = false;
 
-  gcc_assert (diag_kind == DK_WARNING || diag_kind == DK_PEDWARN
-	      || diag_kind == DK_ERROR);
+  gcc_assert (diag_kind == diagnostics::kind::warning
+	      || diag_kind == diagnostics::kind::pedwarn
+	      || diag_kind == diagnostics::kind::error);
 
   /* Avoid duplicate error message.  */
   if (TREE_CODE (type) == ERROR_MARK)
@@ -3907,7 +3935,8 @@ retry:
       break;
 
     case OFFSET_TYPE:
-      bad_member : {
+    bad_member:
+      {
 	tree member = TREE_OPERAND (value, 1);
 	if (is_overloaded_fn (member))
 	  member = get_first_fn (member);
@@ -3992,13 +4021,21 @@ decl_constant_var_p (tree decl)
 /* Returns true iff DECL is a variable or function declared with an auto type
    that has not yet been deduced to a real type.  */
 
-bool undeduced_auto_decl (tree) { return false; }
+bool
+undeduced_auto_decl (tree)
+{
+  return false;
+}
 
 // forked from gcc/cp/decl.cc require_deduced_type
 
 /* Complain if DECL has an undeduced return type.  */
 
-bool require_deduced_type (tree, tsubst_flags_t) { return true; }
+bool
+require_deduced_type (tree, tsubst_flags_t)
+{
+  return true;
+}
 
 /* Return the location of a tree passed to %+ formats.  */
 
@@ -4288,10 +4325,9 @@ struct GTY ((for_user)) source_location_table_entry
 
 } // namespace Rust
 
-extern void
-gt_pch_nx (Rust::source_location_table_entry &);
-extern void
-gt_pch_nx (Rust::source_location_table_entry *, gt_pointer_operator, void *);
+extern void gt_pch_nx (Rust::source_location_table_entry &);
+extern void gt_pch_nx (Rust::source_location_table_entry *, gt_pointer_operator,
+		       void *);
 
 namespace Rust {
 
@@ -4419,7 +4455,8 @@ lvalue_kind (const_tree ref)
     case VIEW_CONVERT_EXPR:
       return lvalue_kind (TREE_OPERAND (ref, 0));
 
-      case ARRAY_REF: {
+    case ARRAY_REF:
+      {
 	tree op1 = TREE_OPERAND (ref, 0);
 	if (TREE_CODE (TREE_TYPE (op1)) == ARRAY_TYPE)
 	  {
@@ -4516,7 +4553,8 @@ lvalue_kind (const_tree ref)
       op2_lvalue_kind = lvalue_kind (TREE_OPERAND (ref, 1));
       break;
 
-      case COND_EXPR: {
+    case COND_EXPR:
+      {
 	tree op1 = TREE_OPERAND (ref, 1);
 	if (!op1)
 	  op1 = TREE_OPERAND (ref, 0);
@@ -5131,7 +5169,7 @@ complete_type_or_maybe_complain (tree type, tree value, tsubst_flags_t complain)
   else if (!COMPLETE_TYPE_P (type))
     {
       if (complain & tf_error)
-	cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+	cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
       note_failed_type_completion_for_satisfaction (type);
       return NULL_TREE;
     }
diff --git a/gcc/rust/backend/rust-tree.h b/gcc/rust/backend/rust-tree.h
index bb99684..9e859d4 100644
--- a/gcc/rust/backend/rust-tree.h
+++ b/gcc/rust/backend/rust-tree.h
@@ -1543,7 +1543,7 @@ extern GTY (()) tree cp_global_trees[CPTI_MAX];
 #if defined ENABLE_TREE_CHECKING
 
 #define LANG_DECL_MIN_CHECK(NODE)                                              \
-  __extension__({                                                              \
+  __extension__ ({                                                             \
     struct lang_decl *lt = DECL_LANG_SPECIFIC (NODE);                          \
     if (!LANG_DECL_HAS_MIN (NODE))                                             \
       lang_check_failed (__FILE__, __LINE__, __FUNCTION__);                    \
@@ -1554,7 +1554,7 @@ extern GTY (()) tree cp_global_trees[CPTI_MAX];
    template, not just on a FUNCTION_DECL.  So when looking for things in
    lang_decl_fn, look down through a TEMPLATE_DECL into its result.  */
 #define LANG_DECL_FN_CHECK(NODE)                                               \
-  __extension__({                                                              \
+  __extension__ ({                                                             \
     struct lang_decl *lt = DECL_LANG_SPECIFIC (NODE);                          \
     if (!DECL_DECLARES_FUNCTION_P (NODE) || lt->u.base.selector != lds_fn)     \
       lang_check_failed (__FILE__, __LINE__, __FUNCTION__);                    \
@@ -1562,7 +1562,7 @@ extern GTY (()) tree cp_global_trees[CPTI_MAX];
   })
 
 #define LANG_DECL_NS_CHECK(NODE)                                               \
-  __extension__({                                                              \
+  __extension__ ({                                                             \
     struct lang_decl *lt = DECL_LANG_SPECIFIC (NODE);                          \
     if (TREE_CODE (NODE) != NAMESPACE_DECL || lt->u.base.selector != lds_ns)   \
       lang_check_failed (__FILE__, __LINE__, __FUNCTION__);                    \
@@ -1570,7 +1570,7 @@ extern GTY (()) tree cp_global_trees[CPTI_MAX];
   })
 
 #define LANG_DECL_PARM_CHECK(NODE)                                             \
-  __extension__({                                                              \
+  __extension__ ({                                                             \
     struct lang_decl *lt = DECL_LANG_SPECIFIC (NODE);                          \
     if (TREE_CODE (NODE) != PARM_DECL || lt->u.base.selector != lds_parm)      \
       lang_check_failed (__FILE__, __LINE__, __FUNCTION__);                    \
@@ -1578,7 +1578,7 @@ extern GTY (()) tree cp_global_trees[CPTI_MAX];
   })
 
 #define LANG_DECL_DECOMP_CHECK(NODE)                                           \
-  __extension__({                                                              \
+  __extension__ ({                                                             \
     struct lang_decl *lt = DECL_LANG_SPECIFIC (NODE);                          \
     if (!VAR_P (NODE) || lt->u.base.selector != lds_decomp)                    \
       lang_check_failed (__FILE__, __LINE__, __FUNCTION__);                    \
@@ -2060,8 +2060,8 @@ struct GTY (()) rust_cxx_saved_binding
 // forked from gcc/cp/name-lookup.h resort_type_member_vec
 
 /* needed for GTY annotation */
-extern void
-resort_type_member_vec (void *, void *, gt_pointer_operator, void *);
+extern void resort_type_member_vec (void *, void *, gt_pointer_operator,
+				    void *);
 
 // forked from gcc/cp/cp-tree.h saved_scope
 
@@ -2895,8 +2895,7 @@ enum compare_bounds_t
   bounds_first
 };
 
-extern tree
-convert_to_void (tree expr, impl_conv_void implicit);
+extern tree convert_to_void (tree expr, impl_conv_void implicit);
 
 // The lvalue-to-rvalue conversion (7.1) is applied if and only if the
 // expression is a glvalue of volatile-qualified type and it is one of the
@@ -2911,63 +2910,52 @@ convert_to_void (tree expr, impl_conv_void implicit);
 //   operands are one of these expressions, or
 // * comma expression (8.19) where the right operand is one of these
 //   expressions.
-extern tree
-mark_discarded_use (tree expr);
+extern tree mark_discarded_use (tree expr);
 
 // Mark EXP as read, not just set, for set but not used -Wunused warning
 // purposes.
-extern void
-mark_exp_read (tree exp);
+extern void mark_exp_read (tree exp);
 
 // We've seen an actual use of EXPR.  Possibly replace an outer variable
 // reference inside with its constant value or a lambda capture.
-extern tree
-mark_use (tree expr, bool rvalue_p, bool read_p, location_t loc,
-	  bool reject_builtin);
+extern tree mark_use (tree expr, bool rvalue_p, bool read_p, location_t loc,
+		      bool reject_builtin);
 
 // Called whenever the expression EXPR is used in an rvalue context.
 // When REJECT_BUILTIN is true the expression is checked to make sure
 // it doesn't make it possible to obtain the address of a GCC built-in
 // function with no library fallback (or any of its bits, such as in
 // a conversion to bool).
-extern tree
-mark_rvalue_use (tree, location_t = UNKNOWN_LOCATION,
-		 bool reject_builtin = true);
+extern tree mark_rvalue_use (tree, location_t = UNKNOWN_LOCATION,
+			     bool reject_builtin = true);
 
 // Called whenever an expression is used in an lvalue context.
-extern tree
-mark_lvalue_use (tree expr);
+extern tree mark_lvalue_use (tree expr);
 
 // As above, but don't consider this use a read.
-extern tree
-mark_lvalue_use_nonread (tree expr);
+extern tree mark_lvalue_use_nonread (tree expr);
 
 // We are using a reference VAL for its value. Bash that reference all the way
 // down to its lowest form.
-extern tree
-convert_from_reference (tree val);
+extern tree convert_from_reference (tree val);
 
 // Subroutine of convert_to_void.  Warn if we're discarding something with
 // attribute [[nodiscard]].
-extern void
-maybe_warn_nodiscard (tree expr, impl_conv_void implicit);
+extern void maybe_warn_nodiscard (tree expr, impl_conv_void implicit);
 
-extern location_t
-expr_loc_or_loc (const_tree t, location_t or_loc);
+extern location_t expr_loc_or_loc (const_tree t, location_t or_loc);
 
-extern location_t
-expr_loc_or_input_loc (const_tree t);
+extern location_t expr_loc_or_input_loc (const_tree t);
 
 // FN is the callee of a CALL_EXPR or AGGR_INIT_EXPR; return the FUNCTION_DECL
 // if we can.
-extern tree
-get_fndecl_from_callee (tree fn);
+extern tree get_fndecl_from_callee (tree fn);
 
 // FIXME some helpers from HIRCompileBase could probably be moved here over time
 
 // Return an expression for the address of BASE[INDEX], used in offset intrinsic
-extern tree
-pointer_offset_expression (tree base_tree, tree index_tree, location_t locus);
+extern tree pointer_offset_expression (tree base_tree, tree index_tree,
+				       location_t locus);
 
 /* A tree node, together with a location, so that we can track locations
    (and ranges) during parsing.
@@ -2978,11 +2966,9 @@ pointer_offset_expression (tree base_tree, tree index_tree, location_t locus);
 
 extern location_t rs_expr_location (const_tree);
 
-extern int
-is_empty_class (tree type);
+extern int is_empty_class (tree type);
 
-extern bool
-is_really_empty_class (tree, bool);
+extern bool is_really_empty_class (tree, bool);
 
 extern bool builtin_valid_in_constant_expr_p (const_tree);
 
@@ -2990,15 +2976,13 @@ extern bool maybe_constexpr_fn (tree);
 
 extern bool var_in_maybe_constexpr_fn (tree);
 
-extern int
-rs_type_quals (const_tree type);
+extern int rs_type_quals (const_tree type);
 
 inline bool type_unknown_p (const_tree);
 
 extern bool decl_maybe_constant_var_p (tree);
 
-extern void
-init_modules ();
+extern void init_modules ();
 
 extern bool var_in_constexpr_fn (tree);
 
@@ -3006,11 +2990,9 @@ inline tree ovl_first (tree) ATTRIBUTE_PURE;
 
 inline bool type_unknown_p (const_tree);
 
-extern tree
-lookup_add (tree fns, tree lookup);
+extern tree lookup_add (tree fns, tree lookup);
 
-extern tree
-ovl_make (tree fn, tree next = NULL_TREE);
+extern tree ovl_make (tree fn, tree next = NULL_TREE);
 
 extern int is_overloaded_fn (tree) ATTRIBUTE_PURE;
 
@@ -3024,19 +3006,15 @@ extern tree make_conv_op_name (tree);
 
 extern int type_memfn_quals (const_tree);
 
-struct c_fileinfo *
-get_fileinfo (const char *);
+struct c_fileinfo *get_fileinfo (const char *);
 
-extern tree
-cxx_make_type (enum tree_code CXX_MEM_STAT_INFO);
+extern tree cxx_make_type (enum tree_code CXX_MEM_STAT_INFO);
 
-extern tree
-build_cplus_array_type (tree, tree, int is_dep = -1);
+extern tree build_cplus_array_type (tree, tree, int is_dep = -1);
 
 extern bool is_byte_access_type (tree);
 
-extern bool
-comptypes (tree, tree, int);
+extern bool comptypes (tree, tree, int);
 
 extern tree canonical_eh_spec (tree);
 
@@ -3046,8 +3024,7 @@ extern bool rs_tree_equal (tree, tree);
 
 extern bool compparms (const_tree, const_tree);
 
-extern tree
-rs_build_qualified_type_real (tree, int, tsubst_flags_t);
+extern tree rs_build_qualified_type_real (tree, int, tsubst_flags_t);
 #define rs_build_qualified_type(TYPE, QUALS)                                   \
   rs_build_qualified_type_real ((TYPE), (QUALS), tf_warning_or_error)
 extern bool cv_qualified_p (const_tree);
@@ -3056,21 +3033,18 @@ extern bool similar_type_p (tree, tree);
 
 extern bool rs_tree_equal (tree, tree);
 
-extern bool
-vector_targets_convertible_p (const_tree t1, const_tree t2);
+extern bool vector_targets_convertible_p (const_tree t1, const_tree t2);
 
 extern bool same_type_ignoring_top_level_qualifiers_p (tree, tree);
 
 extern bool comp_ptr_ttypes_const (tree, tree, compare_bounds_t);
 
-extern tree
-get_class_binding_direct (tree, tree, bool want_type = false);
+extern tree get_class_binding_direct (tree, tree, bool want_type = false);
 
 extern tree skip_artificial_parms_for (const_tree, tree);
 
-extern void
-lang_check_failed (const char *, int,
-		   const char *) ATTRIBUTE_NORETURN ATTRIBUTE_COLD;
+extern void lang_check_failed (const char *, int,
+			       const char *) ATTRIBUTE_NORETURN ATTRIBUTE_COLD;
 
 extern tree default_init_uninitialized_part (tree);
 
@@ -3088,8 +3062,7 @@ extern tree in_class_defaulted_default_constructor (tree);
 
 extern bool is_instantiation_of_constexpr (tree);
 
-extern bool
-check_for_uninitialized_const_var (tree, bool, tsubst_flags_t);
+extern bool check_for_uninitialized_const_var (tree, bool, tsubst_flags_t);
 
 extern bool reduced_constant_expression_p (tree);
 
@@ -3108,19 +3081,17 @@ extern tree is_bitfield_expr_with_lowered_type (const_tree);
 
 extern tree convert_bitfield_to_declared_type (tree);
 
-extern tree
-cp_fold_maybe_rvalue (tree, bool);
+extern tree cp_fold_maybe_rvalue (tree, bool);
 
 extern tree maybe_undo_parenthesized_ref (tree);
 
-extern tree
-fold_offsetof (tree, tree = size_type_node, tree_code ctx = ERROR_MARK);
+extern tree fold_offsetof (tree, tree = size_type_node,
+			   tree_code ctx = ERROR_MARK);
 
 extern tree cp_truthvalue_conversion (tree, tsubst_flags_t);
 
-extern tree
-fold_non_dependent_expr (tree, tsubst_flags_t = tf_warning_or_error,
-			 bool = false, tree = NULL_TREE);
+extern tree fold_non_dependent_expr (tree, tsubst_flags_t = tf_warning_or_error,
+				     bool = false, tree = NULL_TREE);
 
 extern int char_type_p (tree);
 
@@ -3137,7 +3108,7 @@ extern bool reject_gcc_builtin (const_tree, location_t = UNKNOWN_LOCATION);
 extern tree resolve_nondeduced_context (tree, tsubst_flags_t);
 
 extern void cxx_incomplete_type_diagnostic (location_t, const_tree, const_tree,
-					    diagnostic_t);
+					    enum diagnostics::kind);
 
 extern void cxx_incomplete_type_error (location_t, const_tree, const_tree);
 
@@ -3163,13 +3134,11 @@ extern tree build_new_constexpr_heap_type (tree, tree, tree);
 
 extern bool is_empty_field (tree);
 
-extern bool
-in_immediate_context ();
+extern bool in_immediate_context ();
 
 extern tree cp_get_callee_fndecl_nofold (tree);
 
-extern bool
-cxx_mark_addressable (tree, bool = false);
+extern bool cxx_mark_addressable (tree, bool = false);
 
 extern tree fold_builtin_source_location (location_t);
 
@@ -3183,25 +3152,22 @@ extern bool glvalue_p (const_tree);
 
 extern cp_lvalue_kind lvalue_kind (const_tree);
 
-extern tree
-decl_constant_value (tree, bool);
+extern tree decl_constant_value (tree, bool);
 
 extern tree lookup_enumerator (tree, tree);
 
-extern int
-is_class_type (tree, int);
+extern int is_class_type (tree, int);
 
 extern tree braced_lists_to_strings (tree, tree);
 
-extern tree
-fold_builtin_is_pointer_inverconvertible_with_class (location_t, int, tree *);
+extern tree fold_builtin_is_pointer_inverconvertible_with_class (location_t,
+								 int, tree *);
 
 extern bool layout_compatible_type_p (tree, tree);
 
 extern tree finish_underlying_type (tree);
 
-extern tree
-c_common_type_for_mode (machine_mode, int);
+extern tree c_common_type_for_mode (machine_mode, int);
 
 extern bool std_layout_type_p (const_tree);
 
@@ -3213,25 +3179,21 @@ extern void note_failed_type_completion_for_satisfaction (tree);
 
 extern tree complete_type_or_maybe_complain (tree, tree, tsubst_flags_t);
 
-extern bool
-next_common_initial_seqence (tree &, tree &);
+extern bool next_common_initial_seqence (tree &, tree &);
 
 extern bool null_member_pointer_value_p (tree);
 
-extern tree
-fold_builtin_is_corresponding_member (location_t, int, tree *);
+extern tree fold_builtin_is_corresponding_member (location_t, int, tree *);
 
 extern tree cp_fold_rvalue (tree);
 
-extern tree
-maybe_constant_value (tree, tree = NULL_TREE, bool = false);
+extern tree maybe_constant_value (tree, tree = NULL_TREE, bool = false);
 
 extern tree lvalue_type (tree);
 
 extern void lvalue_error (location_t, enum lvalue_use);
 
-extern tree
-cp_fold_maybe_rvalue (tree, bool);
+extern tree cp_fold_maybe_rvalue (tree, bool);
 
 extern tree get_first_fn (tree) ATTRIBUTE_PURE;
 
@@ -3253,13 +3215,12 @@ enum
   ce_exact
 };
 
-extern tree
-rs_build_qualified_type_real (tree, int, tsubst_flags_t);
+extern tree rs_build_qualified_type_real (tree, int, tsubst_flags_t);
 #define rs_build_qualified_type(TYPE, QUALS)                                   \
   rs_build_qualified_type_real ((TYPE), (QUALS), tf_warning_or_error)
 
-extern tree
-rs_walk_subtrees (tree *, int *, walk_tree_fn, void *, hash_set<tree> *);
+extern tree rs_walk_subtrees (tree *, int *, walk_tree_fn, void *,
+			      hash_set<tree> *);
 #define rs_walk_tree(tp, func, data, pset)                                     \
   walk_tree_1 (tp, func, data, pset, rs_walk_subtrees)
 #define rs_walk_tree_without_duplicates(tp, func, data)                        \
@@ -3351,11 +3312,9 @@ gnu_vector_type_p (const_tree type)
   return TREE_CODE (type) == VECTOR_TYPE && !TYPE_INDIVISIBLE_P (type);
 }
 
-extern vec<tree, va_gc> *
-make_tree_vector (void);
+extern vec<tree, va_gc> *make_tree_vector (void);
 
-extern void
-release_tree_vector (vec<tree, va_gc> *);
+extern void release_tree_vector (vec<tree, va_gc> *);
 
 /* Simplified unique_ptr clone to release a tree vec on exit.  */
 
@@ -3373,7 +3332,7 @@ public:
   releasing_vec &operator= (const releasing_vec &);
 
   vec_t &operator* () const { return *v; }
-  vec_t *operator-> () const { return v; }
+  vec_t *operator->() const { return v; }
   vec_t *get () const { return v; }
   operator vec_t * () const { return v; }
   vec_t **operator& () { return &v; }
@@ -3430,7 +3389,7 @@ null_node_p (const_tree expr)
 
 inline void
 cxx_incomplete_type_diagnostic (const_tree value, const_tree type,
-				diagnostic_t diag_kind)
+				enum diagnostics::kind diag_kind)
 {
   cxx_incomplete_type_diagnostic (rs_expr_loc_or_input_loc (value), value, type,
 				  diag_kind);
@@ -3439,11 +3398,10 @@ cxx_incomplete_type_diagnostic (const_tree value, const_tree type,
 inline void
 cxx_incomplete_type_error (const_tree value, const_tree type)
 {
-  cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+  cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
 }
 
-extern location_t
-location_of (tree t);
+extern location_t location_of (tree t);
 
 /* Helpers for IMPLICIT_RVALUE_P to look through automatic dereference.  */
 
@@ -3465,23 +3423,18 @@ set_implicit_rvalue_p (tree ot)
 }
 
 namespace Compile {
-extern tree
-maybe_constant_init (tree, tree = NULL_TREE, bool = false);
+extern tree maybe_constant_init (tree, tree = NULL_TREE, bool = false);
 
-extern void
-explain_invalid_constexpr_fn (tree fun);
+extern void explain_invalid_constexpr_fn (tree fun);
 
 extern bool potential_constant_expression (tree);
 
-extern bool
-literal_type_p (tree t);
+extern bool literal_type_p (tree t);
 
-extern bool
-maybe_constexpr_fn (tree t);
+extern bool maybe_constexpr_fn (tree t);
 
-extern tree
-fold_non_dependent_init (tree, tsubst_flags_t = tf_warning_or_error,
-			 bool = false, tree = NULL_TREE);
+extern tree fold_non_dependent_init (tree, tsubst_flags_t = tf_warning_or_error,
+				     bool = false, tree = NULL_TREE);
 } // namespace Compile
 
 } // namespace Rust
diff --git a/gcc/rust/checks/errors/borrowck/polonius/rust-polonius.h b/gcc/rust/checks/errors/borrowck/polonius/rust-polonius.h
index 0ce2142..0434bcf 100644
--- a/gcc/rust/checks/errors/borrowck/polonius/rust-polonius.h
+++ b/gcc/rust/checks/errors/borrowck/polonius/rust-polonius.h
@@ -239,31 +239,25 @@ struct Facts
  *
  * Output is not yet implemented and is only dumped to stdout.
  */
-extern "C" FFI::Output
-polonius_run (FFI::FactsView input, bool dump_enabled);
+extern "C" FFI::Output polonius_run (FFI::FactsView input, bool dump_enabled);
 
 // Helper functions for FFIVector to be used on Rust side
 extern "C" {
 
-FFI::FFIVector<size_t> *
-FFIVector__new ();
+FFI::FFIVector<size_t> *FFIVector__new ();
 
-FFI::FFIVectorPair *
-FFIVector__new_vec_pair ();
+FFI::FFIVectorPair *FFIVector__new_vec_pair ();
 
-FFI::FFIVectorTriple *
-FFIVector__new_vec_triple ();
+FFI::FFIVectorTriple *FFIVector__new_vec_triple ();
 
-void
-FFIVector__push (FFI::FFIVector<size_t> *vector, size_t element);
+void FFIVector__push (FFI::FFIVector<size_t> *vector, size_t element);
 
 void
 FFIVector__push_vec_pair (FFI::FFIVectorPair *vector,
 			  FFI::Pair<size_t, FFI::FFIVector<size_t> *> element);
 
-void
-FFIVector__push_vec_triple (FFI::FFIVectorTriple *vector,
-			    FFI::Triple<size_t, size_t, size_t> element);
+void FFIVector__push_vec_triple (FFI::FFIVectorTriple *vector,
+				 FFI::Triple<size_t, size_t, size_t> element);
 }
 
 } // namespace Polonius
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.cc b/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.cc
index 6b8b2e9..5b22c1a 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.cc
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.cc
@@ -240,7 +240,8 @@ ExprStmtBuilder::visit (HIR::ArrayExpr &expr)
   auto &elems = expr.get_internal_elements ();
   switch (elems.get_array_expr_type ())
     {
-      case HIR::ArrayElems::VALUES: {
+    case HIR::ArrayElems::VALUES:
+      {
 	auto &elem_vals = (static_cast<HIR::ArrayElemsValues &> (elems));
 	auto init_values = visit_list (elem_vals.get_values ());
 	// collect locations
@@ -254,7 +255,8 @@ ExprStmtBuilder::visit (HIR::ArrayExpr &expr)
 		     lookup_type (expr), expr.get_locus ());
 	break;
       }
-      case HIR::ArrayElems::COPIED: {
+    case HIR::ArrayElems::COPIED:
+      {
 	auto &elem_copied = (static_cast<HIR::ArrayElemsCopied &> (elems));
 	auto init = visit_expr (elem_copied.get_elem_to_copy ());
 	return_expr (new InitializerExpr ({init}), lookup_type (expr),
@@ -329,6 +331,10 @@ ExprStmtBuilder::visit (HIR::LlvmInlineAsm &expr)
 {}
 
 void
+ExprStmtBuilder::visit (HIR::OffsetOf &expr)
+{}
+
+void
 ExprStmtBuilder::visit (HIR::MethodCallExpr &expr)
 {}
 
@@ -415,6 +421,18 @@ ExprStmtBuilder::visit (HIR::BlockExpr &block)
 }
 
 void
+ExprStmtBuilder::visit (HIR::AnonConst &block)
+{
+  rust_unreachable ();
+}
+
+void
+ExprStmtBuilder::visit (HIR::ConstBlock &block)
+{
+  rust_unreachable ();
+}
+
+void
 ExprStmtBuilder::visit (HIR::ContinueExpr &cont)
 {
   LoopAndLabelCtx info = cont.has_label () ? get_label_ctx (cont.get_label ())
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.h b/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.h
index 5cab3c4..ba5db8b 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.h
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-builder-expr-stmt.h
@@ -19,6 +19,7 @@
 #ifndef RUST_BIR_BUILDER_EXPR_H
 #define RUST_BIR_BUILDER_EXPR_H
 
+#include "rust-hir-expr.h"
 #include "rust-hir-visitor.h"
 #include "rust-bir-builder-internal.h"
 
@@ -84,6 +85,8 @@ protected: // Expr
   void visit (HIR::MethodCallExpr &expr) override;
   void visit (HIR::FieldAccessExpr &expr) override;
   void visit (HIR::BlockExpr &block) override;
+  void visit (HIR::AnonConst &block) override;
+  void visit (HIR::ConstBlock &block) override;
   void visit (HIR::ContinueExpr &cont) override;
   void visit (HIR::BreakExpr &brk) override;
   void visit (HIR::RangeFromToExpr &range) override;
@@ -101,6 +104,7 @@ protected: // Expr
   void visit (HIR::IfExprConseqElse &expr) override;
   void visit (HIR::InlineAsm &expr) override;
   void visit (HIR::LlvmInlineAsm &expr) override;
+  void visit (HIR::OffsetOf &expr) override;
 
   void visit (HIR::MatchExpr &expr) override;
   void visit (HIR::AwaitExpr &expr) override;
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-builder-internal.h b/gcc/rust/checks/errors/borrowck/rust-bir-builder-internal.h
index 4df0e14..e2cc2dd 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-builder-internal.h
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-builder-internal.h
@@ -27,6 +27,8 @@
 #include "rust-name-resolver.h"
 #include "rust-bir.h"
 #include "rust-bir-free-region.h"
+#include "rust-immutable-name-resolution-context.h"
+#include "options.h"
 
 namespace Rust {
 
@@ -402,19 +404,40 @@ protected: // HIR resolution helpers
   template <typename T> NodeId resolve_label (T &expr)
   {
     NodeId resolved_label;
-    bool ok
-      = ctx.resolver.lookup_resolved_label (expr.get_mappings ().get_nodeid (),
-					    &resolved_label);
-    rust_assert (ok);
+    if (flag_name_resolution_2_0)
+      {
+	auto &nr_ctx
+	  = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+	auto res = nr_ctx.lookup (expr.get_mappings ().get_nodeid ());
+	rust_assert (res.has_value ());
+	resolved_label = res.value ();
+      }
+    else
+      {
+	bool ok = ctx.resolver.lookup_resolved_label (
+	  expr.get_mappings ().get_nodeid (), &resolved_label);
+	rust_assert (ok);
+      }
     return resolved_label;
   }
 
   template <typename T> PlaceId resolve_variable (T &variable)
   {
     NodeId variable_id;
-    bool ok = ctx.resolver.lookup_resolved_name (
-      variable.get_mappings ().get_nodeid (), &variable_id);
-    rust_assert (ok);
+    if (flag_name_resolution_2_0)
+      {
+	auto &nr_ctx
+	  = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+	auto res = nr_ctx.lookup (variable.get_mappings ().get_nodeid ());
+	rust_assert (res.has_value ());
+	variable_id = res.value ();
+      }
+    else
+      {
+	bool ok = ctx.resolver.lookup_resolved_name (
+	  variable.get_mappings ().get_nodeid (), &variable_id);
+	rust_assert (ok);
+      }
     return ctx.place_db.lookup_variable (variable_id);
   }
 
@@ -425,9 +448,20 @@ protected: // HIR resolution helpers
     // Unlike variables,
     // functions do not have to be declared in PlaceDB before use.
     NodeId variable_id;
-    bool ok = ctx.resolver.lookup_resolved_name (
-      variable.get_mappings ().get_nodeid (), &variable_id);
-    rust_assert (ok);
+    if (flag_name_resolution_2_0)
+      {
+	auto &nr_ctx
+	  = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+	auto res = nr_ctx.lookup (variable.get_mappings ().get_nodeid ());
+	rust_assert (res.has_value ());
+	variable_id = res.value ();
+      }
+    else
+      {
+	bool ok = ctx.resolver.lookup_resolved_name (
+	  variable.get_mappings ().get_nodeid (), &variable_id);
+	rust_assert (ok);
+      }
     if (ty->is<TyTy::FnType> ())
       return ctx.place_db.get_constant (ty);
     else
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-builder-lazyboolexpr.h b/gcc/rust/checks/errors/borrowck/rust-bir-builder-lazyboolexpr.h
index b7a1555..9108009 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-builder-lazyboolexpr.h
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-builder-lazyboolexpr.h
@@ -169,6 +169,14 @@ public:
   {
     return_place (ExprStmtBuilder (ctx).build (expr), expr.get_locus ());
   }
+  void visit (HIR::AnonConst &expr) override
+  {
+    return_place (ExprStmtBuilder (ctx).build (expr), expr.get_locus ());
+  }
+  void visit (HIR::ConstBlock &expr) override
+  {
+    return_place (ExprStmtBuilder (ctx).build (expr), expr.get_locus ());
+  }
   void visit (HIR::UnsafeBlockExpr &expr) override
   {
     return_place (ExprStmtBuilder (ctx).build (expr), expr.get_locus ());
@@ -208,6 +216,7 @@ public:
 
   void visit (HIR::InlineAsm &expr) override {}
   void visit (HIR::LlvmInlineAsm &expr) override {}
+  void visit (HIR::OffsetOf &expr) override {}
 
 protected: // Illegal at this position.
   void visit (HIR::StructExprFieldIdentifier &field) override
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-builder-pattern.cc b/gcc/rust/checks/errors/borrowck/rust-bir-builder-pattern.cc
index ee37bb0..2d655f9 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-builder-pattern.cc
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-builder-pattern.cc
@@ -101,7 +101,8 @@ PatternBindingBuilder::visit (HIR::StructPattern &pattern)
     {
       switch (field->get_item_type ())
 	{
-	  case HIR::StructPatternField::TUPLE_PAT: {
+	case HIR::StructPatternField::TUPLE_PAT:
+	  {
 	    auto tuple
 	      = static_cast<HIR::StructPatternFieldTuplePat *> (field.get ());
 
@@ -123,7 +124,8 @@ PatternBindingBuilder::visit (HIR::StructPattern &pattern)
 	    tuple->get_tuple_pattern ().accept_vis (*this);
 	    break;
 	  }
-	  case HIR::StructPatternField::IDENT_PAT: {
+	case HIR::StructPatternField::IDENT_PAT:
+	  {
 	    auto ident_field
 	      = static_cast<HIR::StructPatternFieldIdentPat *> (field.get ());
 	    TyTy::StructFieldType *field_ty = nullptr;
@@ -139,7 +141,8 @@ PatternBindingBuilder::visit (HIR::StructPattern &pattern)
 	    ident_field->get_pattern ().accept_vis (*this);
 	    break;
 	  }
-	  case HIR::StructPatternField::IDENT: {
+	case HIR::StructPatternField::IDENT:
+	  {
 	    auto ident_field
 	      = static_cast<HIR::StructPatternFieldIdent *> (field.get ());
 	    TyTy::StructFieldType *field_ty = nullptr;
@@ -199,13 +202,15 @@ PatternBindingBuilder::visit (HIR::TuplePattern &pattern)
   size_t index = 0;
   switch (pattern.get_items ().get_item_type ())
     {
-      case HIR::TuplePatternItems::MULTIPLE: {
+    case HIR::TuplePatternItems::MULTIPLE:
+      {
 	auto &items = static_cast<HIR::TuplePatternItemsMultiple &> (
 	  pattern.get_items ());
 	visit_tuple_fields (items.get_patterns (), saved, index);
 	break;
       }
-      case HIR::TuplePatternItems::RANGED: {
+    case HIR::TuplePatternItems::RANGED:
+      {
 	auto &items
 	  = static_cast<HIR::TuplePatternItemsRanged &> (pattern.get_items ());
 
@@ -244,7 +249,8 @@ PatternBindingBuilder::visit (HIR::TupleStructPattern &pattern)
   size_t index = 0;
   switch (pattern.get_items ().get_item_type ())
     {
-      case HIR::TupleStructItems::RANGED: {
+    case HIR::TupleStructItems::RANGED:
+      {
 	auto &items
 	  = static_cast<HIR::TupleStructItemsRange &> (pattern.get_items ());
 
@@ -261,7 +267,8 @@ PatternBindingBuilder::visit (HIR::TupleStructPattern &pattern)
 	visit_tuple_fields (items.get_upper_patterns (), saved, index);
 	break;
       }
-      case HIR::TupleStructItems::MULTIPLE: {
+    case HIR::TupleStructItems::MULTIPLE:
+      {
 	auto &items
 	  = static_cast<HIR::TupleStructItemsNoRange &> (pattern.get_items ());
 	visit_tuple_fields (items.get_patterns (), saved, index);
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-builder-struct.h b/gcc/rust/checks/errors/borrowck/rust-bir-builder-struct.h
index 84311cc..d87ff8c 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-builder-struct.h
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-builder-struct.h
@@ -21,6 +21,7 @@
 
 #include "rust-bir-builder-internal.h"
 #include "rust-bir-builder-expr-stmt.h"
+#include "rust-hir-expr.h"
 
 namespace Rust {
 namespace BIR {
@@ -133,6 +134,8 @@ protected:
   void visit (HIR::MethodCallExpr &expr) override { rust_unreachable (); }
   void visit (HIR::FieldAccessExpr &expr) override { rust_unreachable (); }
   void visit (HIR::BlockExpr &expr) override { rust_unreachable (); }
+  void visit (HIR::AnonConst &expr) override { rust_unreachable (); }
+  void visit (HIR::ConstBlock &expr) override { rust_unreachable (); }
   void visit (HIR::ClosureExpr &expr) override { rust_unreachable (); }
   void visit (HIR::ContinueExpr &expr) override { rust_unreachable (); }
   void visit (HIR::BreakExpr &expr) override { rust_unreachable (); }
@@ -154,6 +157,7 @@ protected:
   void visit (HIR::AsyncBlockExpr &expr) override { rust_unreachable (); }
   void visit (HIR::InlineAsm &expr) override { rust_unreachable (); }
   void visit (HIR::LlvmInlineAsm &expr) override { rust_unreachable (); }
+  void visit (HIR::OffsetOf &expr) override { rust_unreachable (); }
   void visit (HIR::TypeParam &param) override { rust_unreachable (); }
   void visit (HIR::ConstGenericParam &param) override { rust_unreachable (); }
   void visit (HIR::LifetimeWhereClauseItem &item) override
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-dump.cc b/gcc/rust/checks/errors/borrowck/rust-bir-dump.cc
index 3864b81..9a7bb20 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-dump.cc
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-dump.cc
@@ -182,7 +182,8 @@ Dump::visit (const Statement &stmt)
   statement_place = stmt.get_place ();
   switch (stmt.get_kind ())
     {
-      case Statement::Kind::ASSIGNMENT: {
+    case Statement::Kind::ASSIGNMENT:
+      {
 	visit_place (stmt.get_place ());
 	stream << " = ";
 	stmt.get_expr ().accept_vis (*this);
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-fact-collector.h b/gcc/rust/checks/errors/borrowck/rust-bir-fact-collector.h
index 32a4cd7..4462d77 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-fact-collector.h
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-fact-collector.h
@@ -232,42 +232,50 @@ protected: // Main collection entry points (for different categories).
   {
     switch (stmt.get_kind ())
       {
-	case Statement::Kind::ASSIGNMENT: {
+      case Statement::Kind::ASSIGNMENT:
+	{
 	  // TODO: for unwind, must had hadning for non-panic-only assignements
 	  issue_write_deep (stmt.get_place ());
 	  visit_assignment_expr (stmt.get_place (), stmt.get_expr ());
 	  break;
 	}
-	case Statement::Kind::SWITCH: {
+      case Statement::Kind::SWITCH:
+	{
 	  issue_read_move (stmt.get_place ());
 	  issue_jumps ();
 	}
 	break;
-	case Statement::Kind::GOTO: {
+      case Statement::Kind::GOTO:
+	{
 	  issue_jumps ();
 	}
 	break;
-	case Statement::Kind::RETURN: {
+      case Statement::Kind::RETURN:
+	{
 	  issue_place_access (RETURN_VALUE_PLACE);
 	  issue_locals_dealloc ();
 	  break;
 	}
-	case Statement::Kind::STORAGE_DEAD: {
+      case Statement::Kind::STORAGE_DEAD:
+	{
 	  facts.path_moved_at_base.emplace_back (stmt.get_place ().value,
 						 get_current_point_mid ());
 	  facts.var_defined_at.emplace_back (stmt.get_place ().value,
 					     get_current_point_mid ());
 	  break;
 	}
-	case Statement::Kind::STORAGE_LIVE: {
+      case Statement::Kind::STORAGE_LIVE:
+	{
 	  issue_write_deep (stmt.get_place (), true);
 	  break;
 	}
-	case Statement::Kind::USER_TYPE_ASCRIPTION: {
+      case Statement::Kind::USER_TYPE_ASCRIPTION:
+	{
 	  issue_user_type_constraints (stmt.get_place (), stmt.get_type ());
 	  break;
 	}
-	case Statement::Kind::FAKE_READ: {
+      case Statement::Kind::FAKE_READ:
+	{
 	  issue_place_access (stmt.get_place ());
 	  break;
 	}
@@ -791,7 +799,8 @@ protected: // Subset helpers.
 	  type->as<const TyTy::SliceType> ()->get_element_type (), region_start,
 	  regions);
       case TyTy::FNDEF:
-	case TyTy::TUPLE: {
+      case TyTy::TUPLE:
+	{
 	  for (auto &field : type->as<const TyTy::TupleType> ()->get_fields ())
 	    sanitize_constraints (field.get_tyty (), region_start, regions);
 	}
@@ -815,6 +824,7 @@ protected: // Subset helpers.
       case TyTy::PLACEHOLDER:
       case TyTy::INFER:
       case TyTy::PARAM:
+      case TyTy::CONST:
       case TyTy::OPAQUE:
 	rust_unreachable ();
       }
@@ -874,7 +884,7 @@ protected: // Subset helpers.
 
     return region_end;
   }
-}; // namespace BIR
+};
 
 } // namespace BIR
 } // namespace Rust
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-place.h b/gcc/rust/checks/errors/borrowck/rust-bir-place.h
index dd9e672..2e9103f 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-place.h
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-place.h
@@ -53,7 +53,7 @@ using Variance = TyTy::VarianceAnalysis::Variance;
 /** A unique identifier for a loan in the BIR. */
 struct LoanId
 {
-  uint32_t value;
+  size_t value;
   // some overloads for comparision
   bool operator== (const LoanId &rhs) const { return value == rhs.value; }
   bool operator!= (const LoanId &rhs) const { return !(operator== (rhs)); }
@@ -217,7 +217,7 @@ public:
   const T &operator[] (I pid) const { return internal_vector[pid.value]; }
 
   void push_back (T &&param) { internal_vector.push_back (std::move (param)); }
-  template <typename... Args> void emplace_back (Args &&... args)
+  template <typename... Args> void emplace_back (Args &&...args)
   {
     internal_vector.emplace_back (std::forward<Args> (args)...);
   }
@@ -471,14 +471,16 @@ private:
       case TyTy::FNDEF:
       case TyTy::NEVER:
 	return true;
-	case TyTy::TUPLE: {
+      case TyTy::TUPLE:
+	{
 	  auto &fields = ty->as<TyTy::TupleType> ()->get_fields ();
 	  return std::all_of (fields.begin (), fields.end (),
 			      [] (const TyTy::TyVar &field) {
 				return is_type_copy (field.get_tyty ());
 			      });
 	}
-	case TyTy::ARRAY: {
+      case TyTy::ARRAY:
+	{
 	  return is_type_copy (ty->as<TyTy::ArrayType> ()->get_element_type ());
 	}
       case TyTy::INFER:
@@ -491,6 +493,7 @@ private:
       case TyTy::PROJECTION: // TODO: DUNNO
       case TyTy::CLOSURE:    // TODO: DUNNO
       case TyTy::DYNAMIC:    // TODO: dunno
+      case TyTy::CONST:
       case TyTy::OPAQUE:
 	return false;
       }
diff --git a/gcc/rust/checks/errors/borrowck/rust-bir-visitor.h b/gcc/rust/checks/errors/borrowck/rust-bir-visitor.h
index 5dac89e..d405569 100644
--- a/gcc/rust/checks/errors/borrowck/rust-bir-visitor.h
+++ b/gcc/rust/checks/errors/borrowck/rust-bir-visitor.h
@@ -51,7 +51,7 @@ template <typename BASE, typename T> class VisitableImpl : public BASE
 {
 public:
   template <typename... Args>
-  explicit VisitableImpl (Args &&... args) : BASE (std::forward<Args> (args)...)
+  explicit VisitableImpl (Args &&...args) : BASE (std::forward<Args> (args)...)
   {}
 
   void accept_vis (Visitor &visitor) override
diff --git a/gcc/rust/checks/errors/borrowck/rust-function-collector.h b/gcc/rust/checks/errors/borrowck/rust-function-collector.h
index 7cf0952..86f96c1 100644
--- a/gcc/rust/checks/errors/borrowck/rust-function-collector.h
+++ b/gcc/rust/checks/errors/borrowck/rust-function-collector.h
@@ -19,6 +19,7 @@
 #ifndef RUST_HIR_FUNCTION_COLLECTOR_H
 #define RUST_HIR_FUNCTION_COLLECTOR_H
 
+#include "rust-hir-expr.h"
 #include "rust-hir-item.h"
 #include "rust-hir-visitor.h"
 #include "rust-hir.h"
@@ -104,6 +105,8 @@ public:
   void visit (HIR::MethodCallExpr &expr) override {}
   void visit (HIR::FieldAccessExpr &expr) override {}
   void visit (HIR::BlockExpr &expr) override {}
+  void visit (HIR::AnonConst &expr) override {}
+  void visit (HIR::ConstBlock &expr) override {}
   void visit (HIR::ContinueExpr &expr) override {}
   void visit (HIR::BreakExpr &expr) override {}
   void visit (HIR::RangeFromToExpr &expr) override {}
@@ -124,6 +127,7 @@ public:
   void visit (HIR::AsyncBlockExpr &expr) override {}
   void visit (HIR::InlineAsm &expr) override {}
   void visit (HIR::LlvmInlineAsm &expr) override {}
+  void visit (HIR::OffsetOf &expr) override {}
   void visit (HIR::TypeParam &param) override {}
   void visit (HIR::ConstGenericParam &param) override {}
   void visit (HIR::LifetimeWhereClauseItem &item) override {}
diff --git a/gcc/rust/checks/errors/privacy/rust-privacy-check.cc b/gcc/rust/checks/errors/privacy/rust-privacy-check.cc
index 3d25459..5291276 100644
--- a/gcc/rust/checks/errors/privacy/rust-privacy-check.cc
+++ b/gcc/rust/checks/errors/privacy/rust-privacy-check.cc
@@ -25,8 +25,7 @@
 #include "rust-pub-restricted-visitor.h"
 #include "rust-privacy-reporter.h"
 
-extern bool
-saw_errors (void);
+extern bool saw_errors (void);
 
 namespace Rust {
 namespace Privacy {
diff --git a/gcc/rust/checks/errors/privacy/rust-privacy-ctx.h b/gcc/rust/checks/errors/privacy/rust-privacy-ctx.h
index a506613f..9699ac4 100644
--- a/gcc/rust/checks/errors/privacy/rust-privacy-ctx.h
+++ b/gcc/rust/checks/errors/privacy/rust-privacy-ctx.h
@@ -71,8 +71,7 @@ private:
 
 #if CHECKING_P
 namespace selftest {
-void
-rust_privacy_ctx_test (void);
+void rust_privacy_ctx_test (void);
 }
 #endif // !CHECKING_P
 
diff --git a/gcc/rust/checks/errors/privacy/rust-privacy-reporter.cc b/gcc/rust/checks/errors/privacy/rust-privacy-reporter.cc
index 2a10053..4af9639 100644
--- a/gcc/rust/checks/errors/privacy/rust-privacy-reporter.cc
+++ b/gcc/rust/checks/errors/privacy/rust-privacy-reporter.cc
@@ -157,7 +157,8 @@ PrivacyReporter::check_for_privacy_violation (const NodeId &use_id,
     {
     case ModuleVisibility::Public:
       break;
-      case ModuleVisibility::Restricted: {
+    case ModuleVisibility::Restricted:
+      {
 	// If we are in the crate, everything is restricted correctly, but we
 	// can't get a module for it
 	if (!current_module.has_value ())
@@ -215,12 +216,14 @@ PrivacyReporter::check_base_type_privacy (Analysis::NodeMapping &node_mappings,
     case TyTy::USIZE:
     case TyTy::ISIZE:
     case TyTy::ADT:
-      case TyTy::STR: {
+    case TyTy::STR:
+      {
 	auto ref_id = ty->get_ref ();
 	if (auto lookup_id = mappings.lookup_hir_to_node (ref_id))
 	  return check_for_privacy_violation (*lookup_id, locus);
-	rust_unreachable ();
       }
+      break;
+
     case TyTy::REF:
       return recursive_check (
 	static_cast<const TyTy::ReferenceType *> (ty)->get_base ());
@@ -243,7 +246,8 @@ PrivacyReporter::check_base_type_privacy (Analysis::NodeMapping &node_mappings,
 	   static_cast<const TyTy::TupleType *> (ty)->get_fields ())
 	recursive_check (param.get_tyty ());
       return;
-      case TyTy::PLACEHOLDER: {
+    case TyTy::PLACEHOLDER:
+      {
 	const auto p = static_cast<const TyTy::PlaceholderType *> (ty);
 	if (!p->can_resolve ())
 	  return;
@@ -273,6 +277,8 @@ PrivacyReporter::check_base_type_privacy (Analysis::NodeMapping &node_mappings,
       return;
     case TyTy::OPAQUE:
       return;
+    case TyTy::CONST:
+      return;
     case TyTy::ERROR:
       return;
     }
@@ -310,6 +316,12 @@ PrivacyReporter::visit (HIR::LlvmInlineAsm &)
 {}
 
 void
+PrivacyReporter::visit (HIR::OffsetOf &expr)
+{
+  // TODO: Do we have to do anything?
+}
+
+void
 PrivacyReporter::visit (HIR::TypePath &path)
 {
   check_for_privacy_violation (path.get_mappings ().get_nodeid (),
@@ -413,7 +425,8 @@ PrivacyReporter::visit (HIR::ArrayExpr &expr)
   HIR::ArrayElems &elements = expr.get_internal_elements ();
   switch (elements.get_array_expr_type ())
     {
-      case HIR::ArrayElems::ArrayExprType::VALUES: {
+    case HIR::ArrayElems::ArrayExprType::VALUES:
+      {
 	auto &elems = static_cast<HIR::ArrayElemsValues &> (elements);
 	for (auto &value : elems.get_values ())
 	  value->accept_vis (*this);
@@ -518,6 +531,18 @@ PrivacyReporter::visit (HIR::BlockExpr &expr)
 }
 
 void
+PrivacyReporter::visit (HIR::AnonConst &expr)
+{
+  expr.get_inner_expr ().accept_vis (*this);
+}
+
+void
+PrivacyReporter::visit (HIR::ConstBlock &expr)
+{
+  expr.get_const_expr ().accept_vis (*this);
+}
+
+void
 PrivacyReporter::visit (HIR::ContinueExpr &)
 {}
 
diff --git a/gcc/rust/checks/errors/privacy/rust-privacy-reporter.h b/gcc/rust/checks/errors/privacy/rust-privacy-reporter.h
index 7df2cf4..72716a6 100644
--- a/gcc/rust/checks/errors/privacy/rust-privacy-reporter.h
+++ b/gcc/rust/checks/errors/privacy/rust-privacy-reporter.h
@@ -19,6 +19,7 @@
 #ifndef RUST_PRIVACY_REPORTER_H
 #define RUST_PRIVACY_REPORTER_H
 
+#include "rust-hir-expr.h"
 #include "rust-hir-map.h"
 #include "rust-hir-visitor.h"
 #include "rust-mapping-common.h"
@@ -106,6 +107,8 @@ types
   virtual void visit (HIR::MethodCallExpr &expr);
   virtual void visit (HIR::FieldAccessExpr &expr);
   virtual void visit (HIR::BlockExpr &expr);
+  virtual void visit (HIR::AnonConst &expr);
+  virtual void visit (HIR::ConstBlock &expr);
   virtual void visit (HIR::ContinueExpr &expr);
   virtual void visit (HIR::BreakExpr &expr);
   virtual void visit (HIR::RangeFromToExpr &expr);
@@ -126,6 +129,7 @@ types
   virtual void visit (HIR::AsyncBlockExpr &expr);
   virtual void visit (HIR::InlineAsm &expr);
   virtual void visit (HIR::LlvmInlineAsm &expr);
+  virtual void visit (HIR::OffsetOf &expr);
 
   virtual void visit (HIR::EnumItemTuple &);
   virtual void visit (HIR::EnumItemStruct &);
diff --git a/gcc/rust/checks/errors/privacy/rust-reachability.cc b/gcc/rust/checks/errors/privacy/rust-reachability.cc
index 1e57674..223c77b 100644
--- a/gcc/rust/checks/errors/privacy/rust-reachability.cc
+++ b/gcc/rust/checks/errors/privacy/rust-reachability.cc
@@ -158,7 +158,8 @@ ReachabilityVisitor::visit (HIR::Enum &enum_item)
 
       switch (variant->get_enum_item_kind ())
 	{
-	  case HIR::EnumItem::Tuple: {
+	case HIR::EnumItem::Tuple:
+	  {
 	    // Should we update the fields only if they are public? Similarly to
 	    // what we do in the ReachabilityVisitor for HIR::TupleStruct?
 	    auto tuple_variant
@@ -167,7 +168,8 @@ ReachabilityVisitor::visit (HIR::Enum &enum_item)
 	      ctx.update_reachability (field.get_mappings (), variant_reach);
 	    break;
 	  }
-	  case HIR::EnumItem::Struct: {
+	case HIR::EnumItem::Struct:
+	  {
 	    // Should we update the fields only if they are public? Similarly to
 	    // what we do in the ReachabilityVisitor for HIR::StructStruct?
 	    auto struct_variant
diff --git a/gcc/rust/checks/errors/privacy/rust-visibility-resolver.cc b/gcc/rust/checks/errors/privacy/rust-visibility-resolver.cc
index f0da745..c59763d 100644
--- a/gcc/rust/checks/errors/privacy/rust-visibility-resolver.cc
+++ b/gcc/rust/checks/errors/privacy/rust-visibility-resolver.cc
@@ -127,7 +127,8 @@ VisibilityResolver::resolve_visibility (const HIR::Visibility &visibility,
     case HIR::Visibility::PUBLIC:
       to_resolve = ModuleVisibility::create_public ();
       return true;
-      case HIR::Visibility::RESTRICTED: {
+    case HIR::Visibility::RESTRICTED:
+      {
 	// FIXME: We also need to handle 2015 vs 2018 edition conflicts
 	auto id = UNKNOWN_DEFID;
 	auto result = resolve_module_path (visibility.get_path (), id);
diff --git a/gcc/rust/checks/errors/rust-const-checker.cc b/gcc/rust/checks/errors/rust-const-checker.cc
index 3716ea5..c40f9db 100644
--- a/gcc/rust/checks/errors/rust-const-checker.cc
+++ b/gcc/rust/checks/errors/rust-const-checker.cc
@@ -417,6 +417,26 @@ ConstChecker::visit (BlockExpr &expr)
 }
 
 void
+ConstChecker::visit (AnonConst &expr)
+{
+  const_context.enter (expr.get_mappings ().get_hirid ());
+
+  expr.get_inner_expr ().accept_vis (*this);
+
+  const_context.exit ();
+}
+
+void
+ConstChecker::visit (ConstBlock &expr)
+{
+  const_context.enter (expr.get_mappings ().get_hirid ());
+
+  expr.get_const_expr ().accept_vis (*this);
+
+  const_context.exit ();
+}
+
+void
 ConstChecker::visit (ContinueExpr &)
 {}
 
@@ -541,6 +561,10 @@ ConstChecker::visit (LlvmInlineAsm &)
 {}
 
 void
+ConstChecker::visit (OffsetOf &)
+{}
+
+void
 ConstChecker::visit (TypeParam &)
 {}
 
diff --git a/gcc/rust/checks/errors/rust-const-checker.h b/gcc/rust/checks/errors/rust-const-checker.h
index b954330..eb63095 100644
--- a/gcc/rust/checks/errors/rust-const-checker.h
+++ b/gcc/rust/checks/errors/rust-const-checker.h
@@ -113,6 +113,8 @@ private:
   virtual void visit (FieldAccessExpr &expr) override;
   virtual void visit (ClosureExpr &expr) override;
   virtual void visit (BlockExpr &expr) override;
+  virtual void visit (AnonConst &expr) override;
+  virtual void visit (ConstBlock &expr) override;
   virtual void visit (ContinueExpr &expr) override;
   virtual void visit (BreakExpr &expr) override;
   virtual void visit (RangeFromToExpr &expr) override;
@@ -133,6 +135,7 @@ private:
   virtual void visit (AsyncBlockExpr &expr) override;
   virtual void visit (InlineAsm &expr) override;
   virtual void visit (LlvmInlineAsm &expr) override;
+  virtual void visit (OffsetOf &expr) override;
 
   virtual void visit (TypeParam &param) override;
   virtual void visit (ConstGenericParam &param) override;
diff --git a/gcc/rust/checks/errors/rust-feature.cc b/gcc/rust/checks/errors/rust-feature.cc
index 441a1b2..071d3f8 100644
--- a/gcc/rust/checks/errors/rust-feature.cc
+++ b/gcc/rust/checks/errors/rust-feature.cc
@@ -84,7 +84,7 @@ const std::map<std::string, Feature::Name> Feature::name_hash_map = {
   {"exclusive_range_pattern", Feature::Name::EXCLUSIVE_RANGE_PATTERN},
   {"prelude_import", Feature::Name::PRELUDE_IMPORT},
   {"min_specialization", Feature::Name::MIN_SPECIALIZATION},
-}; // namespace Rust
+};
 
 tl::optional<Feature::Name>
 Feature::as_name (const std::string &name)
diff --git a/gcc/rust/checks/errors/rust-hir-pattern-analysis.cc b/gcc/rust/checks/errors/rust-hir-pattern-analysis.cc
index 648bc07..2566971 100644
--- a/gcc/rust/checks/errors/rust-hir-pattern-analysis.cc
+++ b/gcc/rust/checks/errors/rust-hir-pattern-analysis.cc
@@ -295,6 +295,18 @@ PatternChecker::visit (BlockExpr &expr)
 }
 
 void
+PatternChecker::visit (AnonConst &expr)
+{
+  expr.get_inner_expr ().accept_vis (*this);
+}
+
+void
+PatternChecker::visit (ConstBlock &expr)
+{
+  expr.get_const_expr ().accept_vis (*this);
+}
+
+void
 PatternChecker::visit (ContinueExpr &)
 {}
 
@@ -427,6 +439,10 @@ PatternChecker::visit (LlvmInlineAsm &expr)
 {}
 
 void
+PatternChecker::visit (OffsetOf &expr)
+{}
+
+void
 PatternChecker::visit (TypeParam &)
 {}
 
@@ -728,23 +744,27 @@ Constructor::is_covered_by (const Constructor &o) const
 
   switch (kind)
     {
-      case ConstructorKind::VARIANT: {
+    case ConstructorKind::VARIANT:
+      {
 	rust_assert (kind == ConstructorKind::VARIANT);
 	return variant_idx == o.variant_idx;
       }
       break;
-      case ConstructorKind::INT_RANGE: {
+    case ConstructorKind::INT_RANGE:
+      {
 	rust_assert (kind == ConstructorKind::INT_RANGE);
 	return int_range.lo >= o.int_range.lo && int_range.hi <= o.int_range.hi;
       }
       break;
-      case ConstructorKind::WILDCARD: {
+    case ConstructorKind::WILDCARD:
+      {
 	// TODO: wildcard is covered by a variant of enum with a single
 	// variant
 	return false;
       }
       break;
-      case ConstructorKind::STRUCT: {
+    case ConstructorKind::STRUCT:
+      {
 	// Struct pattern is always covered by a other struct constructor.
 	return true;
       }
@@ -900,19 +920,22 @@ PlaceInfo::specialize (const Constructor &c) const
   switch (c.get_kind ())
     {
     case Constructor::ConstructorKind::WILDCARD:
-      case Constructor::ConstructorKind::INT_RANGE: {
+    case Constructor::ConstructorKind::INT_RANGE:
+      {
 	return {};
       }
       break;
     case Constructor::ConstructorKind::STRUCT:
-      case Constructor::ConstructorKind::VARIANT: {
+    case Constructor::ConstructorKind::VARIANT:
+      {
 	rust_assert (ty->get_kind () == TyTy::TypeKind::ADT);
 	TyTy::ADTType *adt = static_cast<TyTy::ADTType *> (ty);
 	switch (adt->get_adt_kind ())
 	  {
 	  case TyTy::ADTType::ADTKind::ENUM:
 	  case TyTy::ADTType::ADTKind::STRUCT_STRUCT:
-	    case TyTy::ADTType::ADTKind::TUPLE_STRUCT: {
+	  case TyTy::ADTType::ADTKind::TUPLE_STRUCT:
+	    {
 	      TyTy::VariantDef *variant
 		= adt->get_variants ().at (c.get_variant_index ());
 	      if (variant->get_variant_type ()
@@ -926,14 +949,16 @@ PlaceInfo::specialize (const Constructor &c) const
 	      return new_place_infos;
 	    }
 	    break;
-	    case TyTy::ADTType::ADTKind::UNION: {
+	  case TyTy::ADTType::ADTKind::UNION:
+	    {
 	      // TODO: support unions
 	      rust_unreachable ();
 	    }
 	  }
       }
       break;
-      default: {
+    default:
+      {
 	rust_unreachable ();
       }
       break;
@@ -991,7 +1016,8 @@ WitnessPat::to_string () const
 {
   switch (ctor.get_kind ())
     {
-      case Constructor::ConstructorKind::STRUCT: {
+    case Constructor::ConstructorKind::STRUCT:
+      {
 	TyTy::ADTType *adt = static_cast<TyTy::ADTType *> (ty);
 	TyTy::VariantDef *variant
 	  = adt->get_variants ().at (ctor.get_variant_index ());
@@ -1016,7 +1042,8 @@ WitnessPat::to_string () const
 	return buf;
       }
       break;
-      case Constructor::ConstructorKind::VARIANT: {
+    case Constructor::ConstructorKind::VARIANT:
+      {
 	std::string buf;
 	TyTy::ADTType *adt = static_cast<TyTy::ADTType *> (ty);
 	buf += adt->get_identifier ();
@@ -1026,11 +1053,13 @@ WitnessPat::to_string () const
 
 	switch (variant->get_variant_type ())
 	  {
-	    case TyTy::VariantDef::VariantType::NUM: {
+	  case TyTy::VariantDef::VariantType::NUM:
+	    {
 	      return buf;
 	    }
 	    break;
-	    case TyTy::VariantDef::VariantType::TUPLE: {
+	  case TyTy::VariantDef::VariantType::TUPLE:
+	    {
 	      buf += "(";
 	      for (size_t i = 0; i < fields.size (); i++)
 		{
@@ -1042,7 +1071,8 @@ WitnessPat::to_string () const
 	      return buf;
 	    }
 	    break;
-	    case TyTy::VariantDef::VariantType::STRUCT: {
+	  case TyTy::VariantDef::VariantType::STRUCT:
+	    {
 	      buf += " {";
 	      if (!fields.empty ())
 		buf += " ";
@@ -1061,7 +1091,8 @@ WitnessPat::to_string () const
 	      buf += "}";
 	    }
 	    break;
-	    default: {
+	  default:
+	    {
 	      rust_unreachable ();
 	    }
 	    break;
@@ -1069,21 +1100,25 @@ WitnessPat::to_string () const
 	return buf;
       }
       break;
-      case Constructor::ConstructorKind::INT_RANGE: {
+    case Constructor::ConstructorKind::INT_RANGE:
+      {
 	// TODO: implement
 	rust_unreachable ();
       }
       break;
-      case Constructor::ConstructorKind::WILDCARD: {
+    case Constructor::ConstructorKind::WILDCARD:
+      {
 	return "_";
       }
       break;
-      case Constructor::ConstructorKind::REFERENCE: {
+    case Constructor::ConstructorKind::REFERENCE:
+      {
 	// TODO: implement
 	rust_unreachable ();
       }
       break;
-      default: {
+    default:
+      {
 	rust_unreachable ();
       }
       break;
@@ -1100,12 +1135,14 @@ WitnessMatrix::apply_constructor (const Constructor &ctor,
   // TODO: only support struct and variant ctor for now.
   switch (ctor.get_kind ())
     {
-      case Constructor::ConstructorKind::WILDCARD: {
+    case Constructor::ConstructorKind::WILDCARD:
+      {
 	arity = 0;
       }
       break;
     case Constructor::ConstructorKind::STRUCT:
-      case Constructor::ConstructorKind::VARIANT: {
+    case Constructor::ConstructorKind::VARIANT:
+      {
 	if (ty->get_kind () == TyTy::TypeKind::ADT)
 	  {
 	    TyTy::ADTType *adt = static_cast<TyTy::ADTType *> (ty);
@@ -1118,7 +1155,8 @@ WitnessMatrix::apply_constructor (const Constructor &ctor,
 	  }
       }
       break;
-      default: {
+    default:
+      {
 	rust_unreachable ();
       }
     }
@@ -1160,9 +1198,9 @@ WitnessMatrix::extend (const WitnessMatrix &other)
 }
 
 // forward declarations
-static DeconstructedPat
-lower_pattern (Resolver::TypeCheckContext *ctx, HIR::Pattern &pattern,
-	       TyTy::BaseType *scrutinee_ty);
+static DeconstructedPat lower_pattern (Resolver::TypeCheckContext *ctx,
+				       HIR::Pattern &pattern,
+				       TyTy::BaseType *scrutinee_ty);
 
 static DeconstructedPat
 lower_tuple_pattern (Resolver::TypeCheckContext *ctx,
@@ -1175,7 +1213,8 @@ lower_tuple_pattern (Resolver::TypeCheckContext *ctx,
   std::vector<DeconstructedPat> fields;
   switch (elems.get_item_type ())
     {
-      case HIR::TupleStructItems::ItemType::MULTIPLE: {
+    case HIR::TupleStructItems::ItemType::MULTIPLE:
+      {
 	HIR::TupleStructItemsNoRange &multiple
 	  = static_cast<HIR::TupleStructItemsNoRange &> (elems);
 
@@ -1191,12 +1230,14 @@ lower_tuple_pattern (Resolver::TypeCheckContext *ctx,
 	return DeconstructedPat (ctor, arity, fields, pattern.get_locus ());
       }
       break;
-      case HIR::TupleStructItems::ItemType::RANGED: {
+    case HIR::TupleStructItems::ItemType::RANGED:
+      {
 	// TODO: ranged tuple struct items
 	rust_unreachable ();
       }
       break;
-      default: {
+    default:
+      {
 	rust_unreachable ();
       }
     }
@@ -1227,7 +1268,8 @@ lower_struct_pattern (Resolver::TypeCheckContext *ctx,
     {
       switch (elem->get_item_type ())
 	{
-	  case HIR::StructPatternField::ItemType::IDENT: {
+	case HIR::StructPatternField::ItemType::IDENT:
+	  {
 	    HIR::StructPatternFieldIdent *ident
 	      = static_cast<HIR::StructPatternFieldIdent *> (elem.get ());
 	    int field_idx
@@ -1236,7 +1278,8 @@ lower_struct_pattern (Resolver::TypeCheckContext *ctx,
 	      = DeconstructedPat::make_wildcard (pattern.get_locus ());
 	  }
 	  break;
-	  case HIR::StructPatternField::ItemType::IDENT_PAT: {
+	case HIR::StructPatternField::ItemType::IDENT_PAT:
+	  {
 	    HIR::StructPatternFieldIdentPat *ident_pat
 	      = static_cast<HIR::StructPatternFieldIdentPat *> (elem.get ());
 	    int field_idx
@@ -1246,12 +1289,14 @@ lower_struct_pattern (Resolver::TypeCheckContext *ctx,
 	      variant->get_fields ().at (field_idx)->get_field_type ());
 	  }
 	  break;
-	  case HIR::StructPatternField::ItemType::TUPLE_PAT: {
+	case HIR::StructPatternField::ItemType::TUPLE_PAT:
+	  {
 	    // TODO: tuple: pat
 	    rust_unreachable ();
 	  }
 	  break;
-	  default: {
+	default:
+	  {
 	    rust_unreachable ();
 	  }
 	}
@@ -1268,11 +1313,13 @@ lower_pattern (Resolver::TypeCheckContext *ctx, HIR::Pattern &pattern,
   switch (pat_type)
     {
     case HIR::Pattern::PatternType::WILDCARD:
-      case HIR::Pattern::PatternType::IDENTIFIER: {
+    case HIR::Pattern::PatternType::IDENTIFIER:
+      {
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      case HIR::Pattern::PatternType::PATH: {
+    case HIR::Pattern::PatternType::PATH:
+      {
 	// TODO: support constants, associated constants, enum variants and
 	// structs
 	// https://doc.rust-lang.org/reference/patterns.html#path-patterns
@@ -1280,13 +1327,15 @@ lower_pattern (Resolver::TypeCheckContext *ctx, HIR::Pattern &pattern,
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      case HIR::Pattern::PatternType::REFERENCE: {
+    case HIR::Pattern::PatternType::REFERENCE:
+      {
 	// TODO: unimplemented. Treat this pattern as wildcard for now.
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
     case HIR::Pattern::PatternType::STRUCT:
-      case HIR::Pattern::PatternType::TUPLE_STRUCT: {
+    case HIR::Pattern::PatternType::TUPLE_STRUCT:
+      {
 	HirId path_id = UNKNOWN_HIRID;
 	if (pat_type == HIR::Pattern::PatternType::STRUCT)
 	  {
@@ -1343,37 +1392,44 @@ lower_pattern (Resolver::TypeCheckContext *ctx, HIR::Pattern &pattern,
 	  }
       }
       break;
-      case HIR::Pattern::PatternType::TUPLE: {
+    case HIR::Pattern::PatternType::TUPLE:
+      {
 	// TODO: unimplemented. Treat this pattern as wildcard for now.
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      case HIR::Pattern::PatternType::SLICE: {
+    case HIR::Pattern::PatternType::SLICE:
+      {
 	// TODO: unimplemented. Treat this pattern as wildcard for now.
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      case HIR::Pattern::PatternType::ALT: {
+    case HIR::Pattern::PatternType::ALT:
+      {
 	// TODO: unimplemented. Treat this pattern as wildcard for now.
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      case HIR::Pattern::PatternType::LITERAL: {
+    case HIR::Pattern::PatternType::LITERAL:
+      {
 	// TODO: unimplemented. Treat this pattern as wildcard for now.
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      case HIR::Pattern::PatternType::RANGE: {
+    case HIR::Pattern::PatternType::RANGE:
+      {
 	// TODO: unimplemented. Treat this pattern as wildcard for now.
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      case HIR::Pattern::PatternType::GROUPED: {
+    case HIR::Pattern::PatternType::GROUPED:
+      {
 	// TODO: unimplemented. Treat this pattern as wildcard for now.
 	return DeconstructedPat::make_wildcard (pattern.get_locus ());
       }
       break;
-      default: {
+    default:
+      {
 	rust_unreachable ();
       }
     }
diff --git a/gcc/rust/checks/errors/rust-hir-pattern-analysis.h b/gcc/rust/checks/errors/rust-hir-pattern-analysis.h
index 6d60ced..dd44abc 100644
--- a/gcc/rust/checks/errors/rust-hir-pattern-analysis.h
+++ b/gcc/rust/checks/errors/rust-hir-pattern-analysis.h
@@ -33,9 +33,9 @@ namespace Analysis {
 
 using namespace HIR;
 
-void
-check_match_usefulness (Resolver::TypeCheckContext *ctx,
-			TyTy::BaseType *scrutinee_ty, HIR::MatchExpr &expr);
+void check_match_usefulness (Resolver::TypeCheckContext *ctx,
+			     TyTy::BaseType *scrutinee_ty,
+			     HIR::MatchExpr &expr);
 
 class PatternChecker : public HIR::HIRFullVisitor
 {
@@ -86,6 +86,8 @@ private:
   virtual void visit (MethodCallExpr &expr) override;
   virtual void visit (FieldAccessExpr &expr) override;
   virtual void visit (BlockExpr &expr) override;
+  virtual void visit (AnonConst &expr) override;
+  virtual void visit (ConstBlock &expr) override;
   virtual void visit (ClosureExpr &expr) override;
   virtual void visit (ContinueExpr &expr) override;
   virtual void visit (BreakExpr &expr) override;
@@ -107,6 +109,7 @@ private:
   virtual void visit (AsyncBlockExpr &expr) override;
   virtual void visit (InlineAsm &expr) override;
   virtual void visit (LlvmInlineAsm &expr) override;
+  virtual void visit (OffsetOf &expr) override;
   virtual void visit (TypeParam &param) override;
   virtual void visit (ConstGenericParam &param) override;
   virtual void visit (LifetimeWhereClauseItem &item) override;
diff --git a/gcc/rust/checks/errors/rust-readonly-check2.cc b/gcc/rust/checks/errors/rust-readonly-check2.cc
new file mode 100644
index 0000000..2fa92ae
--- /dev/null
+++ b/gcc/rust/checks/errors/rust-readonly-check2.cc
@@ -0,0 +1,253 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-readonly-check2.h"
+#include "rust-hir-expr.h"
+#include "rust-hir-node.h"
+#include "rust-hir-path.h"
+#include "rust-hir-map.h"
+#include "rust-hir-pattern.h"
+#include "rust-mapping-common.h"
+#include "rust-system.h"
+#include "rust-immutable-name-resolution-context.h"
+#include "rust-tyty.h"
+
+namespace Rust {
+namespace HIR {
+
+static std::set<HirId> already_assigned_variables = {};
+
+ReadonlyChecker::ReadonlyChecker ()
+  : resolver (*Resolver::Resolver::get ()),
+    mappings (Analysis::Mappings::get ()),
+    context (*Resolver::TypeCheckContext::get ())
+{}
+
+void
+ReadonlyChecker::go (Crate &crate)
+{
+  for (auto &item : crate.get_items ())
+    item->accept_vis (*this);
+}
+
+void
+ReadonlyChecker::visit (AssignmentExpr &expr)
+{
+  Expr &lhs = expr.get_lhs ();
+  mutable_context.enter (expr.get_mappings ().get_hirid ());
+  lhs.accept_vis (*this);
+  mutable_context.exit ();
+}
+
+void
+ReadonlyChecker::visit (PathInExpression &expr)
+{
+  if (!mutable_context.is_in_context ())
+    return;
+
+  NodeId ast_node_id = expr.get_mappings ().get_nodeid ();
+  NodeId def_id;
+
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+  if (auto id = nr_ctx.lookup (ast_node_id))
+    def_id = *id;
+  else
+    return;
+
+  auto hir_id = mappings.lookup_node_to_hir (def_id);
+  if (!hir_id)
+    return;
+
+  // Check if the local variable is mutable.
+  auto maybe_pattern = mappings.lookup_hir_pattern (*hir_id);
+  if (maybe_pattern
+      && maybe_pattern.value ()->get_pattern_type ()
+	   == HIR::Pattern::PatternType::IDENTIFIER)
+    check_variable (static_cast<IdentifierPattern *> (maybe_pattern.value ()),
+		    expr.get_locus ());
+
+  // Check if the static item is mutable.
+  auto maybe_item = mappings.lookup_hir_item (*hir_id);
+  if (maybe_item
+      && maybe_item.value ()->get_item_kind () == HIR::Item::ItemKind::Static)
+    {
+      auto static_item = static_cast<HIR::StaticItem *> (*maybe_item);
+      if (!static_item->is_mut ())
+	rust_error_at (expr.get_locus (),
+		       "assignment of read-only location '%s'",
+		       static_item->get_identifier ().as_string ().c_str ());
+    }
+
+  // Check if the constant item is mutable.
+  if (maybe_item
+      && maybe_item.value ()->get_item_kind () == HIR::Item::ItemKind::Constant)
+    {
+      auto const_item = static_cast<HIR::ConstantItem *> (*maybe_item);
+      rust_error_at (expr.get_locus (), "assignment of read-only location '%s'",
+		     const_item->get_identifier ().as_string ().c_str ());
+    }
+}
+
+void
+ReadonlyChecker::check_variable (IdentifierPattern *pattern,
+				 location_t assigned_loc)
+{
+  if (!mutable_context.is_in_context ())
+    return;
+  if (pattern->is_mut ())
+    return;
+
+  auto hir_id = pattern->get_mappings ().get_hirid ();
+  if (already_assigned_variables.count (hir_id) > 0)
+    rust_error_at (assigned_loc, "assignment of read-only variable '%s'",
+		   pattern->as_string ().c_str ());
+  already_assigned_variables.insert (hir_id);
+}
+
+void
+ReadonlyChecker::collect_assignment_identifier (IdentifierPattern &pattern,
+						bool has_init_expr)
+{
+  if (has_init_expr)
+    {
+      HirId pattern_id = pattern.get_mappings ().get_hirid ();
+      already_assigned_variables.insert (pattern_id);
+    }
+}
+
+void
+ReadonlyChecker::collect_assignment_tuple (TuplePattern &tuple_pattern,
+					   bool has_init_expr)
+{
+  switch (tuple_pattern.get_items ().get_item_type ())
+    {
+    case HIR::TuplePatternItems::ItemType::MULTIPLE:
+      {
+	auto &items = static_cast<HIR::TuplePatternItemsMultiple &> (
+	  tuple_pattern.get_items ());
+	for (auto &sub : items.get_patterns ())
+	  {
+	    collect_assignment (*sub, has_init_expr);
+	  }
+      }
+      break;
+    default:
+      break;
+    }
+}
+
+void
+ReadonlyChecker::collect_assignment (Pattern &pattern, bool has_init_expr)
+{
+  switch (pattern.get_pattern_type ())
+    {
+    case HIR::Pattern::PatternType::IDENTIFIER:
+      {
+	collect_assignment_identifier (static_cast<IdentifierPattern &> (
+					 pattern),
+				       has_init_expr);
+      }
+      break;
+    case HIR::Pattern::PatternType::TUPLE:
+      {
+	auto &tuple_pattern = static_cast<HIR::TuplePattern &> (pattern);
+	collect_assignment_tuple (tuple_pattern, has_init_expr);
+      }
+      break;
+    default:
+      break;
+    }
+}
+
+void
+ReadonlyChecker::visit (LetStmt &stmt)
+{
+  HIR::Pattern &pattern = stmt.get_pattern ();
+  collect_assignment (pattern, stmt.has_init_expr ());
+}
+
+void
+ReadonlyChecker::visit (FieldAccessExpr &expr)
+{
+  if (mutable_context.is_in_context ())
+    {
+      expr.get_receiver_expr ().accept_vis (*this);
+    }
+}
+
+void
+ReadonlyChecker::visit (TupleIndexExpr &expr)
+{
+  if (mutable_context.is_in_context ())
+    {
+      expr.get_tuple_expr ().accept_vis (*this);
+    }
+}
+
+void
+ReadonlyChecker::visit (ArrayIndexExpr &expr)
+{
+  if (mutable_context.is_in_context ())
+    {
+      expr.get_array_expr ().accept_vis (*this);
+    }
+}
+
+void
+ReadonlyChecker::visit (TupleExpr &expr)
+{
+  if (mutable_context.is_in_context ())
+    {
+      // TODO: Add check for tuple expression
+    }
+}
+
+void
+ReadonlyChecker::visit (LiteralExpr &expr)
+{
+  if (mutable_context.is_in_context ())
+    {
+      rust_error_at (expr.get_locus (), "assignment of read-only location");
+    }
+}
+
+void
+ReadonlyChecker::visit (DereferenceExpr &expr)
+{
+  if (!mutable_context.is_in_context ())
+    return;
+  TyTy::BaseType *to_deref_type;
+  auto to_deref = expr.get_expr ().get_mappings ().get_hirid ();
+  if (!context.lookup_type (to_deref, &to_deref_type))
+    return;
+  if (to_deref_type->get_kind () == TyTy::TypeKind::REF)
+    {
+      auto ref_type = static_cast<TyTy::ReferenceType *> (to_deref_type);
+      if (!ref_type->is_mutable ())
+	rust_error_at (expr.get_locus (), "assignment of read-only location");
+    }
+  if (to_deref_type->get_kind () == TyTy::TypeKind::POINTER)
+    {
+      auto ptr_type = static_cast<TyTy::PointerType *> (to_deref_type);
+      if (!ptr_type->is_mutable ())
+	rust_error_at (expr.get_locus (), "assignment of read-only location");
+    }
+}
+} // namespace HIR
+} // namespace Rust
diff --git a/gcc/rust/checks/errors/rust-readonly-check2.h b/gcc/rust/checks/errors/rust-readonly-check2.h
new file mode 100644
index 0000000..06af9db
--- /dev/null
+++ b/gcc/rust/checks/errors/rust-readonly-check2.h
@@ -0,0 +1,67 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-hir-visitor.h"
+#include "rust-name-resolver.h"
+#include "rust-stacked-contexts.h"
+#include "rust-hir-type-check.h"
+
+namespace Rust {
+namespace HIR {
+class ReadonlyChecker : public DefaultHIRVisitor
+{
+public:
+  ReadonlyChecker ();
+
+  void go (HIR::Crate &crate);
+
+private:
+  enum class lvalue_use
+  {
+    assign,
+    increment,
+    decrement,
+  };
+
+  Resolver::Resolver &resolver;
+  Analysis::Mappings &mappings;
+  Resolver::TypeCheckContext &context;
+  StackedContexts<HirId> mutable_context;
+
+  using DefaultHIRVisitor::visit;
+
+  virtual void visit (AssignmentExpr &expr) override;
+  virtual void visit (PathInExpression &expr) override;
+  virtual void visit (FieldAccessExpr &expr) override;
+  virtual void visit (ArrayIndexExpr &expr) override;
+  virtual void visit (TupleExpr &expr) override;
+  virtual void visit (TupleIndexExpr &expr) override;
+  virtual void visit (LetStmt &stmt) override;
+  virtual void visit (LiteralExpr &expr) override;
+  virtual void visit (DereferenceExpr &expr) override;
+
+  void collect_assignment (Pattern &pattern, bool has_init_expr);
+  void collect_assignment_identifier (IdentifierPattern &pattern,
+				      bool has_init_expr);
+  void collect_assignment_tuple (TuplePattern &pattern, bool has_init_expr);
+
+  void check_variable (IdentifierPattern *pattern, location_t assigned_loc);
+};
+
+} // namespace HIR
+} // namespace Rust
\ No newline at end of file
diff --git a/gcc/rust/checks/errors/rust-unsafe-checker.cc b/gcc/rust/checks/errors/rust-unsafe-checker.cc
index 46eef11..405c59b 100644
--- a/gcc/rust/checks/errors/rust-unsafe-checker.cc
+++ b/gcc/rust/checks/errors/rust-unsafe-checker.cc
@@ -540,6 +540,18 @@ UnsafeChecker::visit (BlockExpr &expr)
 }
 
 void
+UnsafeChecker::visit (AnonConst &expr)
+{
+  expr.get_inner_expr ().accept_vis (*this);
+}
+
+void
+UnsafeChecker::visit (ConstBlock &expr)
+{
+  expr.get_const_expr ().accept_vis (*this);
+}
+
+void
 UnsafeChecker::visit (ContinueExpr &)
 {}
 
@@ -682,6 +694,12 @@ UnsafeChecker::visit (LlvmInlineAsm &expr)
 }
 
 void
+UnsafeChecker::visit (OffsetOf &expr)
+{
+  // nothing to do, offset_of!() is safe
+}
+
+void
 UnsafeChecker::visit (TypeParam &)
 {}
 
diff --git a/gcc/rust/checks/errors/rust-unsafe-checker.h b/gcc/rust/checks/errors/rust-unsafe-checker.h
index 9a8fb7c..dc3b482 100644
--- a/gcc/rust/checks/errors/rust-unsafe-checker.h
+++ b/gcc/rust/checks/errors/rust-unsafe-checker.h
@@ -95,6 +95,8 @@ private:
   virtual void visit (FieldAccessExpr &expr) override;
   virtual void visit (ClosureExpr &expr) override;
   virtual void visit (BlockExpr &expr) override;
+  virtual void visit (AnonConst &expr) override;
+  virtual void visit (ConstBlock &expr) override;
   virtual void visit (ContinueExpr &expr) override;
   virtual void visit (BreakExpr &expr) override;
   virtual void visit (RangeFromToExpr &expr) override;
@@ -115,6 +117,7 @@ private:
   virtual void visit (AsyncBlockExpr &expr) override;
   virtual void visit (InlineAsm &expr) override;
   virtual void visit (LlvmInlineAsm &expr) override;
+  virtual void visit (OffsetOf &expr) override;
   virtual void visit (TypeParam &param) override;
   virtual void visit (ConstGenericParam &param) override;
   virtual void visit (LifetimeWhereClauseItem &item) override;
diff --git a/gcc/rust/checks/lints/rust-lint-unused-var.cc b/gcc/rust/checks/lints/rust-lint-unused-var.cc
index 896eeb0..08596f1 100644
--- a/gcc/rust/checks/lints/rust-lint-unused-var.cc
+++ b/gcc/rust/checks/lints/rust-lint-unused-var.cc
@@ -31,6 +31,7 @@ check_decl (tree *t)
   tree var_name = DECL_NAME (*t);
   const char *var_name_ptr = IDENTIFIER_POINTER (var_name);
   bool starts_with_under_score = strncmp (var_name_ptr, "_", 1) == 0;
+  bool is_self = strcmp (var_name_ptr, "self") == 0;
 
   bool is_constant = TREE_CODE (*t) == CONST_DECL;
   // if (!is_constant)
@@ -43,7 +44,8 @@ check_decl (tree *t)
   //       	  starts_with_under_score ? "true" : "false", var_name_ptr);
   //   }
 
-  if (!TREE_USED (*t) && !DECL_ARTIFICIAL (*t) && !starts_with_under_score)
+  if (!TREE_USED (*t) && !DECL_ARTIFICIAL (*t) && !starts_with_under_score
+      && !is_self)
     {
       warning_at (DECL_SOURCE_LOCATION (*t),
 		  is_constant ? OPT_Wunused_const_variable_
diff --git a/gcc/rust/expand/rust-cfg-strip.cc b/gcc/rust/expand/rust-cfg-strip.cc
index a8c3ca5..58d8071 100644
--- a/gcc/rust/expand/rust-cfg-strip.cc
+++ b/gcc/rust/expand/rust-cfg-strip.cc
@@ -289,7 +289,8 @@ CfgStrip::maybe_strip_generic_args (AST::GenericArgs &args)
     {
       switch (arg.get_kind ())
 	{
-	  case AST::GenericArg::Kind::Type: {
+	case AST::GenericArg::Kind::Type:
+	  {
 	    auto &type = arg.get_type ();
 	    type.accept_vis (*this);
 
@@ -298,7 +299,8 @@ CfgStrip::maybe_strip_generic_args (AST::GenericArgs &args)
 			     "cannot strip type in this position");
 	    break;
 	  }
-	  case AST::GenericArg::Kind::Const: {
+	case AST::GenericArg::Kind::Const:
+	  {
 	    auto &expr = arg.get_expression ();
 	    expr.accept_vis (*this);
 
@@ -1190,7 +1192,7 @@ CfgStrip::visit (AST::ClosureExprInnerTyped &expr)
     rust_error_at (type.get_locus (), "cannot strip type in this position");
 
   // can't strip expression itself, but can strip sub-expressions
-  auto &definition_block = expr.get_definition_block ();
+  auto &definition_block = expr.get_definition_expr ();
   definition_block.accept_vis (*this);
   if (definition_block.is_marked_for_strip ())
     rust_error_at (definition_block.get_locus (),
@@ -1763,16 +1765,17 @@ CfgStrip::visit (AST::Module &module)
       return;
     }
 
-  // A loaded module might have inner attributes
-  if (module.get_kind () == AST::Module::ModuleKind::LOADED)
+  if (module.get_kind () == AST::Module::UNLOADED)
     {
-      // strip test based on inner attrs
-      expand_cfg_attrs (module.get_inner_attrs ());
-      if (fails_cfg_with_expand (module.get_inner_attrs ()))
-	{
-	  module.mark_for_strip ();
-	  return;
-	}
+      module.load_items ();
+    }
+
+  // strip test based on inner attrs
+  expand_cfg_attrs (module.get_inner_attrs ());
+  if (fails_cfg_with_expand (module.get_inner_attrs ()))
+    {
+      module.mark_for_strip ();
+      return;
     }
 
   // strip items if required
@@ -2260,12 +2263,12 @@ void
 CfgStrip::visit (AST::IdentifierPattern &pattern)
 {
   // can only strip sub-patterns of the inner pattern to bind
-  if (!pattern.has_pattern_to_bind ())
+  if (!pattern.has_subpattern ())
     return;
 
   AST::DefaultASTVisitor::visit (pattern);
 
-  auto &sub_pattern = pattern.get_pattern_to_bind ();
+  auto &sub_pattern = pattern.get_subpattern ();
   if (sub_pattern.is_marked_for_strip ())
     rust_error_at (sub_pattern.get_locus (),
 		   "cannot strip pattern in this position");
@@ -2475,20 +2478,44 @@ CfgStrip::visit (AST::GroupedPattern &pattern)
 }
 
 void
-CfgStrip::visit (AST::SlicePattern &pattern)
+CfgStrip::visit (AST::SlicePatternItemsNoRest &items)
 {
-  AST::DefaultASTVisitor::visit (pattern);
+  AST::DefaultASTVisitor::visit (items);
   // can't strip individual patterns, only sub-patterns
-  for (auto &item : pattern.get_items ())
+  for (auto &pattern : items.get_patterns ())
     {
-      if (item->is_marked_for_strip ())
-	rust_error_at (item->get_locus (),
+      if (pattern->is_marked_for_strip ())
+	rust_error_at (pattern->get_locus (),
 		       "cannot strip pattern in this position");
-      // TODO: quit stripping now? or keep going?
     }
 }
 
 void
+CfgStrip::visit (AST::SlicePatternItemsHasRest &items)
+{
+  AST::DefaultASTVisitor::visit (items);
+  // can't strip individual patterns, only sub-patterns
+  for (auto &pattern : items.get_lower_patterns ())
+    {
+      if (pattern->is_marked_for_strip ())
+	rust_error_at (pattern->get_locus (),
+		       "cannot strip pattern in this position");
+    }
+  for (auto &pattern : items.get_upper_patterns ())
+    {
+      if (pattern->is_marked_for_strip ())
+	rust_error_at (pattern->get_locus (),
+		       "cannot strip pattern in this position");
+    }
+}
+
+void
+CfgStrip::visit (AST::SlicePattern &pattern)
+{
+  AST::DefaultASTVisitor::visit (pattern);
+}
+
+void
 CfgStrip::visit (AST::AltPattern &pattern)
 {
   AST::DefaultASTVisitor::visit (pattern);
diff --git a/gcc/rust/expand/rust-cfg-strip.h b/gcc/rust/expand/rust-cfg-strip.h
index 4900ae8..767cf28 100644
--- a/gcc/rust/expand/rust-cfg-strip.h
+++ b/gcc/rust/expand/rust-cfg-strip.h
@@ -172,6 +172,8 @@ public:
   void visit (AST::TuplePatternItemsMultiple &tuple_items) override;
   void visit (AST::TuplePatternItemsRanged &tuple_items) override;
   void visit (AST::GroupedPattern &pattern) override;
+  void visit (AST::SlicePatternItemsNoRest &items) override;
+  void visit (AST::SlicePatternItemsHasRest &items) override;
   void visit (AST::SlicePattern &pattern) override;
   void visit (AST::AltPattern &pattern) override;
 
diff --git a/gcc/rust/expand/rust-derive-clone.h b/gcc/rust/expand/rust-derive-clone.h
index 61224ba..a3320c7 100644
--- a/gcc/rust/expand/rust-derive-clone.h
+++ b/gcc/rust/expand/rust-derive-clone.h
@@ -29,7 +29,7 @@ class DeriveClone : DeriveVisitor
 public:
   DeriveClone (location_t loc);
 
-  std::unique_ptr<AST::Item> go (Item &item);
+  std::unique_ptr<Item> go (Item &item);
 
 private:
   std::unique_ptr<Item> expanded;
@@ -80,10 +80,10 @@ private:
   MatchCase clone_enum_struct (PathInExpression variant_path,
 			       const EnumItemStruct &variant);
 
-  virtual void visit_struct (StructStruct &item);
-  virtual void visit_tuple (TupleStruct &item);
-  virtual void visit_enum (Enum &item);
-  virtual void visit_union (Union &item);
+  virtual void visit_struct (StructStruct &item) override;
+  virtual void visit_tuple (TupleStruct &item) override;
+  virtual void visit_enum (Enum &item) override;
+  virtual void visit_union (Union &item) override;
 };
 
 } // namespace AST
diff --git a/gcc/rust/expand/rust-derive-cmp-common.cc b/gcc/rust/expand/rust-derive-cmp-common.cc
new file mode 100644
index 0000000..22ca16f
--- /dev/null
+++ b/gcc/rust/expand/rust-derive-cmp-common.cc
@@ -0,0 +1,191 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-derive-cmp-common.h"
+#include "rust-ast-builder.h"
+#include "rust-item.h"
+
+namespace Rust {
+namespace AST {
+
+SelfOther
+SelfOther::index (Builder builder, int idx)
+{
+  return SelfOther{
+    builder.tuple_idx ("self", idx),
+    builder.tuple_idx ("other", idx),
+  };
+}
+
+std::vector<SelfOther>
+SelfOther::indexes (Builder builder, const std::vector<TupleField> &fields)
+{
+  std::vector<SelfOther> vec;
+
+  for (size_t i = 0; i < fields.size (); i++)
+    vec.emplace_back (SelfOther::index (builder, i));
+
+  return vec;
+}
+
+SelfOther
+SelfOther::field (Builder builder, const std::string &field_name)
+{
+  return SelfOther{
+    builder.field_access (builder.identifier ("self"), field_name),
+    builder.field_access (builder.identifier ("other"), field_name),
+  };
+}
+
+std::vector<SelfOther>
+SelfOther::fields (Builder builder, const std::vector<StructField> &fields)
+{
+  std::vector<SelfOther> vec;
+
+  for (const auto &field : fields)
+    vec.emplace_back (
+      SelfOther::field (builder, field.get_field_name ().as_string ()));
+
+  return vec;
+}
+
+MatchCase
+EnumMatchBuilder::tuple (EnumItem &variant_raw)
+{
+  auto &variant = static_cast<EnumItemTuple &> (variant_raw);
+
+  auto self_patterns = std::vector<std::unique_ptr<Pattern>> ();
+  auto other_patterns = std::vector<std::unique_ptr<Pattern>> ();
+
+  auto self_other_exprs = std::vector<SelfOther> ();
+
+  for (size_t i = 0; i < variant.get_tuple_fields ().size (); i++)
+    {
+      // The patterns we're creating for each field are `self_<i>` and
+      // `other_<i>` where `i` is the index of the field. It doesn't actually
+      // matter what we use, as long as it's ordered, unique, and that we can
+      // reuse it in the match case's return expression to check that they are
+      // equal.
+
+      auto self_pattern_str = "__self_" + std::to_string (i);
+      auto other_pattern_str = "__other_" + std::to_string (i);
+
+      self_patterns.emplace_back (
+	builder.identifier_pattern (self_pattern_str));
+      other_patterns.emplace_back (
+	builder.identifier_pattern (other_pattern_str));
+
+      self_other_exprs.emplace_back (SelfOther{
+	builder.identifier (self_pattern_str),
+	builder.identifier (other_pattern_str),
+      });
+    }
+
+  // TODO: Replace with `reconstruct()` instead of building these twice
+  auto self_variant_path = builder.variant_path (enum_path, variant_path);
+  auto other_variant_path = builder.variant_path (enum_path, variant_path);
+
+  auto self_pattern_items = std::unique_ptr<TupleStructItems> (
+    new TupleStructItemsNoRange (std::move (self_patterns)));
+  auto other_pattern_items = std::unique_ptr<TupleStructItems> (
+    new TupleStructItemsNoRange (std::move (other_patterns)));
+
+  auto self_pattern = std::unique_ptr<Pattern> (
+    new ReferencePattern (std::unique_ptr<Pattern> (new TupleStructPattern (
+			    self_variant_path, std::move (self_pattern_items))),
+			  false, false, builder.loc));
+  auto other_pattern = std::unique_ptr<Pattern> (new ReferencePattern (
+    std::unique_ptr<Pattern> (new TupleStructPattern (
+      other_variant_path, std::move (other_pattern_items))),
+    false, false, builder.loc));
+
+  auto tuple_items = std::make_unique<TuplePatternItemsMultiple> (
+    vec (std::move (self_pattern), std::move (other_pattern)));
+
+  auto pattern
+    = std::make_unique<TuplePattern> (std::move (tuple_items), builder.loc);
+
+  auto expr = fn (std::move (self_other_exprs));
+
+  return builder.match_case (std::move (pattern), std::move (expr));
+}
+
+MatchCase
+EnumMatchBuilder::strukt (EnumItem &variant_raw)
+{
+  auto &variant = static_cast<EnumItemStruct &> (variant_raw);
+
+  auto self_fields = std::vector<std::unique_ptr<StructPatternField>> ();
+  auto other_fields = std::vector<std::unique_ptr<StructPatternField>> ();
+
+  auto self_other_exprs = std::vector<SelfOther> ();
+
+  for (auto &field : variant.get_struct_fields ())
+    {
+      // The patterns we're creating for each field are `self_<field>` and
+      // `other_<field>` where `field` is the name of the field. It doesn't
+      // actually matter what we use, as long as it's ordered, unique, and that
+      // we can reuse it in the match case's return expression to check that
+      // they are equal.
+
+      auto field_name = field.get_field_name ().as_string ();
+
+      auto self_pattern_str = "__self_" + field_name;
+      auto other_pattern_str = "__other_" + field_name;
+
+      self_fields.emplace_back (builder.struct_pattern_ident_pattern (
+	field_name, builder.identifier_pattern (self_pattern_str)));
+      other_fields.emplace_back (builder.struct_pattern_ident_pattern (
+	field_name, builder.identifier_pattern (other_pattern_str)));
+
+      self_other_exprs.emplace_back (SelfOther{
+	builder.identifier (self_pattern_str),
+	builder.identifier (other_pattern_str),
+      });
+    }
+
+  // TODO: Replace with `reconstruct()` instead of building these twice
+  auto self_variant_path = builder.variant_path (enum_path, variant_path);
+  auto other_variant_path = builder.variant_path (enum_path, variant_path);
+
+  auto self_elts = StructPatternElements (std::move (self_fields));
+  auto other_elts = StructPatternElements (std::move (other_fields));
+
+  auto self_pattern = std::unique_ptr<Pattern> (new ReferencePattern (
+    std::unique_ptr<Pattern> (new StructPattern (self_variant_path, builder.loc,
+						 std::move (self_elts))),
+    false, false, builder.loc));
+  auto other_pattern = std::unique_ptr<Pattern> (
+    new ReferencePattern (std::unique_ptr<Pattern> (
+			    new StructPattern (other_variant_path, builder.loc,
+					       std::move (other_elts))),
+			  false, false, builder.loc));
+
+  auto tuple_items = std::make_unique<TuplePatternItemsMultiple> (
+    vec (std::move (self_pattern), std::move (other_pattern)));
+
+  auto pattern
+    = std::make_unique<TuplePattern> (std::move (tuple_items), builder.loc);
+
+  auto expr = fn (std::move (self_other_exprs));
+
+  return builder.match_case (std::move (pattern), std::move (expr));
+}
+
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/expand/rust-derive-cmp-common.h b/gcc/rust/expand/rust-derive-cmp-common.h
new file mode 100644
index 0000000..4efbed8
--- /dev/null
+++ b/gcc/rust/expand/rust-derive-cmp-common.h
@@ -0,0 +1,99 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_DERIVE_CMP_COMMON_H
+#define RUST_DERIVE_CMP_COMMON_H
+
+#include "rust-ast.h"
+#include "rust-ast-builder.h"
+#include "rust-item.h"
+#include "rust-path.h"
+
+namespace Rust {
+namespace AST {
+
+/**
+ * A pair of two expressions from each instance being compared. E.g. this
+ * could be `self.0` and `other.0`, or `self.field` and `other.field`
+ */
+struct SelfOther
+{
+  std::unique_ptr<Expr> self_expr;
+  std::unique_ptr<Expr> other_expr;
+
+  /* Create a <self.i> and an <other.i> expression */
+  static SelfOther index (Builder builder, int idx);
+  static std::vector<SelfOther> indexes (Builder builder,
+					 const std::vector<TupleField> &fields);
+
+  /* Create a <self.field> and an <other.field> expression */
+  static SelfOther field (Builder builder, const std::string &field_name);
+  static std::vector<SelfOther> fields (Builder builder,
+					const std::vector<StructField> &fields);
+};
+
+/**
+ * Builder for common match cases used when comparing two enum instances. This
+ * builder takes care of creating the unique patterns for the `self` instance
+ * and `other` instance, as well as the entire `MatchCase` required for building
+ * a proper comparision expression for an implementation of a comparision trait
+ * for an enum type. The functions take a lambda to use when creating the
+ * expression of the generated `MatchCase`.
+ */
+class EnumMatchBuilder
+{
+public:
+  /**
+   * The type of functions to call when creating the resulting expression in the
+   * generated `MatchCase`
+   */
+  using ExprFn
+    = std::function<std::unique_ptr<Expr> (std::vector<SelfOther> &&)>;
+
+  EnumMatchBuilder (const std::string &enum_path,
+		    const std::string &variant_path, ExprFn fn,
+		    Builder &builder)
+    : enum_path (enum_path), variant_path (variant_path), fn (fn),
+      builder (builder)
+  {}
+
+  /**
+   * Generate a `MatchCase` for an enum tuple variant
+   *
+   * (&Enum::Tuple(self0, self1), &Enum::Tuple(other0, other1)) => <fn>
+   */
+  MatchCase tuple (EnumItem &variant);
+
+  /**
+   * Generate a `MatchCase` for an enum struct variant
+   *
+   * (&Enum::Struct { a: self_a }, &Enum::Struct { a: other_a }) => <fn>
+   */
+  MatchCase strukt (EnumItem &variant);
+
+private:
+  const std::string &enum_path;
+  const std::string &variant_path;
+  ExprFn fn;
+  Builder &builder;
+};
+
+} // namespace AST
+} // namespace Rust
+
+#endif // ! RUST_DERIVE_CMP_COMMON_H
diff --git a/gcc/rust/expand/rust-derive-copy.h b/gcc/rust/expand/rust-derive-copy.h
index 71972eb..664a8e0 100644
--- a/gcc/rust/expand/rust-derive-copy.h
+++ b/gcc/rust/expand/rust-derive-copy.h
@@ -44,10 +44,10 @@ private:
   copy_impl (std::string name,
 	     const std::vector<std::unique_ptr<GenericParam>> &type_generics);
 
-  virtual void visit_struct (StructStruct &item);
-  virtual void visit_tuple (TupleStruct &item);
-  virtual void visit_enum (Enum &item);
-  virtual void visit_union (Union &item);
+  virtual void visit_struct (StructStruct &item) override;
+  virtual void visit_tuple (TupleStruct &item) override;
+  virtual void visit_enum (Enum &item) override;
+  virtual void visit_union (Union &item) override;
 };
 
 } // namespace AST
diff --git a/gcc/rust/expand/rust-derive-default.cc b/gcc/rust/expand/rust-derive-default.cc
index 2e8b456..26ee546 100644
--- a/gcc/rust/expand/rust-derive-default.cc
+++ b/gcc/rust/expand/rust-derive-default.cc
@@ -98,7 +98,8 @@ DeriveDefault::visit_struct (StructStruct &item)
   for (auto &field : item.get_fields ())
     {
       auto name = field.get_field_name ().as_string ();
-      auto expr = default_call (field.get_field_type ().clone_type ());
+      auto type = field.get_field_type ().reconstruct ();
+      auto expr = default_call (std::move (type));
 
       cloned_fields.emplace_back (
 	builder.struct_expr_field (std::move (name), std::move (expr)));
@@ -119,7 +120,7 @@ DeriveDefault::visit_tuple (TupleStruct &tuple_item)
 
   for (auto &field : tuple_item.get_fields ())
     {
-      auto type = field.get_field_type ().clone_type ();
+      auto type = field.get_field_type ().reconstruct ();
 
       defaulted_fields.emplace_back (default_call (std::move (type)));
     }
diff --git a/gcc/rust/expand/rust-derive-eq.cc b/gcc/rust/expand/rust-derive-eq.cc
index 5e7a894..7da137f 100644
--- a/gcc/rust/expand/rust-derive-eq.cc
+++ b/gcc/rust/expand/rust-derive-eq.cc
@@ -142,7 +142,7 @@ DeriveEq::visit_tuple (TupleStruct &item)
   auto types = std::vector<std::unique_ptr<Type>> ();
 
   for (auto &field : item.get_fields ())
-    types.emplace_back (field.get_field_type ().clone_type ());
+    types.emplace_back (field.get_field_type ().reconstruct ());
 
   expanded = eq_impls (assert_receiver_is_total_eq_fn (std::move (types)),
 		       item.get_identifier ().as_string (),
@@ -155,7 +155,7 @@ DeriveEq::visit_struct (StructStruct &item)
   auto types = std::vector<std::unique_ptr<Type>> ();
 
   for (auto &field : item.get_fields ())
-    types.emplace_back (field.get_field_type ().clone_type ());
+    types.emplace_back (field.get_field_type ().reconstruct ());
 
   expanded = eq_impls (assert_receiver_is_total_eq_fn (std::move (types)),
 		       item.get_identifier ().as_string (),
@@ -175,19 +175,20 @@ DeriveEq::visit_enum (Enum &item)
 	case EnumItem::Kind::Discriminant:
 	  // nothing to do as they contain no inner types
 	  continue;
-	  case EnumItem::Kind::Tuple: {
+	case EnumItem::Kind::Tuple:
+	  {
 	    auto &tuple = static_cast<EnumItemTuple &> (*variant);
 
 	    for (auto &field : tuple.get_tuple_fields ())
-	      types.emplace_back (field.get_field_type ().clone_type ());
-
+	      types.emplace_back (field.get_field_type ().reconstruct ());
 	    break;
 	  }
-	  case EnumItem::Kind::Struct: {
+	case EnumItem::Kind::Struct:
+	  {
 	    auto &tuple = static_cast<EnumItemStruct &> (*variant);
 
 	    for (auto &field : tuple.get_struct_fields ())
-	      types.emplace_back (field.get_field_type ().clone_type ());
+	      types.emplace_back (field.get_field_type ().reconstruct ());
 
 	    break;
 	  }
@@ -205,7 +206,7 @@ DeriveEq::visit_union (Union &item)
   auto types = std::vector<std::unique_ptr<Type>> ();
 
   for (auto &field : item.get_variants ())
-    types.emplace_back (field.get_field_type ().clone_type ());
+    types.emplace_back (field.get_field_type ().reconstruct ());
 
   expanded = eq_impls (assert_receiver_is_total_eq_fn (std::move (types)),
 		       item.get_identifier ().as_string (),
diff --git a/gcc/rust/expand/rust-derive-eq.h b/gcc/rust/expand/rust-derive-eq.h
index 17af526..fb187cc 100644
--- a/gcc/rust/expand/rust-derive-eq.h
+++ b/gcc/rust/expand/rust-derive-eq.h
@@ -31,7 +31,7 @@ class DeriveEq : DeriveVisitor
 public:
   DeriveEq (location_t loc);
 
-  std::vector<std::unique_ptr<AST::Item>> go (Item &item);
+  std::vector<std::unique_ptr<Item>> go (Item &item);
 
 private:
   std::vector<std::unique_ptr<Item>> expanded;
@@ -70,10 +70,10 @@ private:
    */
   std::unique_ptr<Stmt> assert_type_is_eq (std::unique_ptr<Type> &&type);
 
-  virtual void visit_struct (StructStruct &item);
-  virtual void visit_tuple (TupleStruct &item);
-  virtual void visit_enum (Enum &item);
-  virtual void visit_union (Union &item);
+  virtual void visit_struct (StructStruct &item) override;
+  virtual void visit_tuple (TupleStruct &item) override;
+  virtual void visit_enum (Enum &item) override;
+  virtual void visit_union (Union &item) override;
 };
 
 } // namespace AST
diff --git a/gcc/rust/expand/rust-derive-hash.cc b/gcc/rust/expand/rust-derive-hash.cc
index 0c9b0f7..94aede2 100644
--- a/gcc/rust/expand/rust-derive-hash.cc
+++ b/gcc/rust/expand/rust-derive-hash.cc
@@ -231,14 +231,7 @@ DeriveHash::visit_enum (Enum &item)
   auto cases = std::vector<MatchCase> ();
   auto type_name = item.get_identifier ().as_string ();
 
-  auto intrinsic = ptrify (
-    builder.path_in_expression ({"core", "intrinsics", "discriminant_value"},
-				true));
-
-  auto let_discr
-    = builder.let (builder.identifier_pattern (DeriveHash::discr), nullptr,
-		   builder.call (std::move (intrinsic),
-				 builder.identifier ("self")));
+  auto let_discr = builder.discriminant_value (DeriveHash::discr);
 
   auto discr_hash = builder.statementify (
     hash_call (builder.ref (builder.identifier (DeriveHash::discr))));
diff --git a/gcc/rust/expand/rust-derive-hash.h b/gcc/rust/expand/rust-derive-hash.h
index 02b0bee..67170d0 100644
--- a/gcc/rust/expand/rust-derive-hash.h
+++ b/gcc/rust/expand/rust-derive-hash.h
@@ -29,7 +29,7 @@ class DeriveHash : DeriveVisitor
 public:
   DeriveHash (location_t loc);
 
-  std::unique_ptr<AST::Item> go (Item &item);
+  std::unique_ptr<Item> go (Item &item);
 
 private:
   std::unique_ptr<Item> expanded;
@@ -49,10 +49,10 @@ private:
   MatchCase match_enum_struct (PathInExpression variant_path,
 			       const EnumItemStruct &variant);
 
-  virtual void visit_struct (StructStruct &item);
-  virtual void visit_tuple (TupleStruct &item);
-  virtual void visit_enum (Enum &item);
-  virtual void visit_union (Union &item);
+  virtual void visit_struct (StructStruct &item) override;
+  virtual void visit_tuple (TupleStruct &item) override;
+  virtual void visit_enum (Enum &item) override;
+  virtual void visit_union (Union &item) override;
 };
 
 } // namespace AST
diff --git a/gcc/rust/expand/rust-derive-ord.cc b/gcc/rust/expand/rust-derive-ord.cc
new file mode 100644
index 0000000..afc4b71
--- /dev/null
+++ b/gcc/rust/expand/rust-derive-ord.cc
@@ -0,0 +1,323 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-derive-ord.h"
+#include "rust-ast.h"
+#include "rust-derive-cmp-common.h"
+#include "rust-derive.h"
+#include "rust-item.h"
+#include "rust-system.h"
+
+namespace Rust {
+namespace AST {
+
+DeriveOrd::DeriveOrd (Ordering ordering, location_t loc)
+  : DeriveVisitor (loc), ordering (ordering)
+{}
+
+std::unique_ptr<Item>
+DeriveOrd::go (Item &item)
+{
+  item.accept_vis (*this);
+
+  return std::move (expanded);
+}
+
+std::unique_ptr<Expr>
+DeriveOrd::cmp_call (std::unique_ptr<Expr> &&self_expr,
+		     std::unique_ptr<Expr> &&other_expr)
+{
+  auto cmp_fn_path = builder.path_in_expression (
+    {"core", "cmp", trait (ordering), fn (ordering)}, true);
+
+  return builder.call (ptrify (cmp_fn_path),
+		       vec (builder.ref (std::move (self_expr)),
+			    builder.ref (std::move (other_expr))));
+}
+
+std::unique_ptr<Item>
+DeriveOrd::cmp_impl (
+  std::unique_ptr<BlockExpr> &&fn_block, Identifier type_name,
+  const std::vector<std::unique_ptr<GenericParam>> &type_generics)
+{
+  auto fn = cmp_fn (std::move (fn_block), type_name);
+
+  auto trait = ordering == Ordering::Partial ? "PartialOrd" : "Ord";
+  auto trait_path = builder.type_path ({"core", "cmp", trait}, true);
+
+  auto trait_bound
+    = builder.trait_bound (builder.type_path ({"core", "cmp", trait}, true));
+
+  auto trait_items = vec (std::move (fn));
+
+  auto cmp_generics
+    = setup_impl_generics (type_name.as_string (), type_generics,
+			   std::move (trait_bound));
+
+  return builder.trait_impl (trait_path, std::move (cmp_generics.self_type),
+			     std::move (trait_items),
+			     std::move (cmp_generics.impl));
+}
+
+std::unique_ptr<AssociatedItem>
+DeriveOrd::cmp_fn (std::unique_ptr<BlockExpr> &&block, Identifier type_name)
+{
+  // Ordering
+  auto return_type = builder.type_path ({"core", "cmp", "Ordering"}, true);
+
+  // In the case of PartialOrd, we return an Option<Ordering>
+  if (ordering == Ordering::Partial)
+    {
+      auto generic = GenericArg::create_type (ptrify (return_type));
+
+      auto generic_seg = builder.type_path_segment_generic (
+	"Option", GenericArgs ({}, {generic}, {}, loc));
+      auto core = builder.type_path_segment ("core");
+      auto option = builder.type_path_segment ("option");
+
+      return_type
+	= builder.type_path (vec (std::move (core), std::move (option),
+				  std::move (generic_seg)),
+			     true);
+    }
+
+  // &self, other: &Self
+  auto params = vec (
+    builder.self_ref_param (),
+    builder.function_param (builder.identifier_pattern ("other"),
+			    builder.reference_type (ptrify (
+			      builder.type_path (type_name.as_string ())))));
+
+  auto function_name = fn (ordering);
+
+  return builder.function (function_name, std::move (params),
+			   ptrify (return_type), std::move (block));
+}
+
+std::unique_ptr<Pattern>
+DeriveOrd::make_equal ()
+{
+  std::unique_ptr<Pattern> equal = ptrify (
+    builder.path_in_expression ({"core", "cmp", "Ordering", "Equal"}, true));
+
+  // We need to wrap the pattern in Option::Some if we are doing partial
+  // ordering
+  if (ordering == Ordering::Partial)
+    {
+      auto pattern_items = std::unique_ptr<TupleStructItems> (
+	new TupleStructItemsNoRange (vec (std::move (equal))));
+
+      equal
+	= std::make_unique<TupleStructPattern> (builder.path_in_expression (
+						  LangItem::Kind::OPTION_SOME),
+						std::move (pattern_items));
+    }
+
+  return equal;
+}
+
+std::pair<MatchArm, MatchArm>
+DeriveOrd::make_cmp_arms ()
+{
+  // All comparison results other than Ordering::Equal
+  auto non_equal = builder.identifier_pattern (DeriveOrd::not_equal);
+  auto equal = make_equal ();
+
+  return {builder.match_arm (std::move (equal)),
+	  builder.match_arm (std::move (non_equal))};
+}
+
+std::unique_ptr<Expr>
+DeriveOrd::recursive_match (std::vector<SelfOther> &&members)
+{
+  if (members.empty ())
+    {
+      std::unique_ptr<Expr> value = ptrify (
+	builder.path_in_expression ({"core", "cmp", "Ordering", "Equal"},
+				    true));
+
+      if (ordering == Ordering::Partial)
+	value = builder.call (ptrify (builder.path_in_expression (
+				LangItem::Kind::OPTION_SOME)),
+			      std::move (value));
+
+      return value;
+    }
+
+  std::unique_ptr<Expr> final_expr = nullptr;
+
+  for (auto it = members.rbegin (); it != members.rend (); it++)
+    {
+      auto &member = *it;
+
+      auto call = cmp_call (std::move (member.self_expr),
+			    std::move (member.other_expr));
+
+      // For the last member (so the first iterator), we just create a call
+      // expression
+      if (it == members.rbegin ())
+	{
+	  final_expr = std::move (call);
+	  continue;
+	}
+
+      // If we aren't dealing with the last member, then we need to wrap all of
+      // that in a big match expression and keep going
+      auto match_arms = make_cmp_arms ();
+
+      auto match_cases
+	= {builder.match_case (std::move (match_arms.first),
+			       std::move (final_expr)),
+	   builder.match_case (std::move (match_arms.second),
+			       builder.identifier (DeriveOrd::not_equal))};
+
+      final_expr = builder.match (std::move (call), std::move (match_cases));
+    }
+
+  return final_expr;
+}
+
+// we need to do a recursive match expression for all of the fields used in a
+// struct so for something like struct Foo { a: i32, b: i32, c: i32 } we must
+// first compare each `a` field, then `b`, then `c`, like this:
+//
+// match cmp_fn(self.<field>, other.<field>) {
+//     Ordering::Equal => <recurse>,
+//     cmp => cmp,
+// }
+//
+// and the recurse will be the exact same expression, on the next field. so that
+// our result looks like this:
+//
+// match cmp_fn(self.a, other.a) {
+//     Ordering::Equal => match cmp_fn(self.b, other.b) {
+//         Ordering::Equal =>cmp_fn(self.c, other.c),
+//         cmp => cmp,
+//     }
+//     cmp => cmp,
+// }
+//
+// the last field comparison needs not to be a match but just the function call.
+// this is going to be annoying lol
+void
+DeriveOrd::visit_struct (StructStruct &item)
+{
+  auto fields = SelfOther::fields (builder, item.get_fields ());
+
+  auto match_expr = recursive_match (std::move (fields));
+
+  expanded = cmp_impl (builder.block (std::move (match_expr)),
+		       item.get_identifier (), item.get_generic_params ());
+}
+
+// same as structs, but for each field index instead of each field name -
+// straightforward once we have `visit_struct` working
+void
+DeriveOrd::visit_tuple (TupleStruct &item)
+{
+  auto fields = SelfOther::indexes (builder, item.get_fields ());
+
+  auto match_expr = recursive_match (std::move (fields));
+
+  expanded = cmp_impl (builder.block (std::move (match_expr)),
+		       item.get_identifier (), item.get_generic_params ());
+}
+
+// for enums, we need to generate a match for each of the enum's variant that
+// contains data and then do the same thing as visit_struct or visit_enum. if
+// the two aren't the same variant, then compare the two discriminant values for
+// all the dataless enum variants and in the general case.
+//
+// so for enum Foo { A(i32, i32), B, C } we need to do the following
+//
+// match (self, other) {
+//     (A(self_0, self_1), A(other_0, other_1)) => {
+//         match cmp_fn(self_0, other_0) {
+//             Ordering::Equal => cmp_fn(self_1, other_1),
+//             cmp => cmp,
+//         },
+//     _ => cmp_fn(discr_value(self), discr_value(other))
+// }
+void
+DeriveOrd::visit_enum (Enum &item)
+{
+  // NOTE: We can factor this even further with DerivePartialEq, but this is
+  // getting out of scope for this PR surely
+
+  auto cases = std::vector<MatchCase> ();
+  auto type_name = item.get_identifier ().as_string ();
+
+  auto let_sd = builder.discriminant_value (DeriveOrd::self_discr, "self");
+  auto let_od = builder.discriminant_value (DeriveOrd::other_discr, "other");
+
+  auto discr_cmp = cmp_call (builder.identifier (DeriveOrd::self_discr),
+			     builder.identifier (DeriveOrd::other_discr));
+
+  auto recursive_match_fn = [this] (std::vector<SelfOther> &&fields) {
+    return recursive_match (std::move (fields));
+  };
+
+  for (auto &variant : item.get_variants ())
+    {
+      auto enum_builder
+	= EnumMatchBuilder (type_name, variant->get_identifier ().as_string (),
+			    recursive_match_fn, builder);
+
+      switch (variant->get_enum_item_kind ())
+	{
+	case EnumItem::Kind::Struct:
+	  cases.emplace_back (enum_builder.strukt (*variant));
+	  break;
+	case EnumItem::Kind::Tuple:
+	  cases.emplace_back (enum_builder.tuple (*variant));
+	  break;
+	case EnumItem::Kind::Identifier:
+	case EnumItem::Kind::Discriminant:
+	  // We don't need to do anything for these, as they are handled by the
+	  // discriminant value comparison
+	  break;
+	}
+    }
+
+  // Add the last case which compares the discriminant values in case `self` and
+  // `other` are actually different variants of the enum
+  cases.emplace_back (
+    builder.match_case (builder.wildcard (), std::move (discr_cmp)));
+
+  auto match
+    = builder.match (builder.tuple (vec (builder.identifier ("self"),
+					 builder.identifier ("other"))),
+		     std::move (cases));
+
+  expanded
+    = cmp_impl (builder.block (vec (std::move (let_sd), std::move (let_od)),
+			       std::move (match)),
+		type_name, item.get_generic_params ());
+}
+
+void
+DeriveOrd::visit_union (Union &item)
+{
+  auto trait_name = trait (ordering);
+
+  rust_error_at (item.get_locus (), "derive(%s) cannot be used on unions",
+		 trait_name.c_str ());
+}
+
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/expand/rust-derive-ord.h b/gcc/rust/expand/rust-derive-ord.h
new file mode 100644
index 0000000..90ce9c8
--- /dev/null
+++ b/gcc/rust/expand/rust-derive-ord.h
@@ -0,0 +1,122 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_DERIVE_ORD_H
+#define RUST_DERIVE_ORD_H
+
+#include "rust-ast.h"
+#include "rust-derive-cmp-common.h"
+#include "rust-derive.h"
+
+namespace Rust {
+namespace AST {
+
+/**
+ * DeriveOrd is a bit special as the expansion of both `PartialOrd` and `Ord`
+ * is extremely similar. The only difference is that `PartialOrd` concerns
+ * partial-ordering, and thus its main method returns an `Option<Ordering>`,
+ * while `Ord` concerns total-ordering, and its main method returns an
+ * `Ordering`. Otherwise, the expansion logic is the same, so we factor both
+ * derives into one.
+ */
+class DeriveOrd : public DeriveVisitor
+{
+public:
+  enum class Ordering
+  {
+    Total,
+    Partial
+  };
+
+  std::string fn (Ordering ordering)
+  {
+    if (ordering == Ordering::Total)
+      return "cmp";
+    else
+      return "partial_cmp";
+  }
+
+  std::string trait (Ordering ordering)
+  {
+    if (ordering == Ordering::Total)
+      return "Ord";
+    else
+      return "PartialOrd";
+  }
+
+  DeriveOrd (Ordering ordering, location_t loc);
+
+  std::unique_ptr<Item> go (Item &item);
+
+private:
+  std::unique_ptr<Item> expanded;
+
+  Ordering ordering;
+
+  /* Identifier patterns for the non-equal match arms */
+  constexpr static const char *not_equal = "#non_eq";
+  constexpr static const char *self_discr = "#self_discr";
+  constexpr static const char *other_discr = "#other_discr";
+
+  /**
+   * Create the recursive matching structure used when implementing the
+   * comparison function on multiple sub items (fields, tuple indexes...)
+   */
+  std::unique_ptr<Expr> recursive_match (std::vector<SelfOther> &&members);
+
+  /**
+   * Create a pattern for the `Ordering::Equal` case. In the case of partial
+   * ordering, `Option::Some(Ordering::Equal)`.
+   */
+  std::unique_ptr<Pattern> make_equal ();
+
+  /**
+   * Make the match arms for one inner match in a comparison function block.
+   * This returns the "equal" match arm and the "rest" match arm, so something
+   * like `Ordering::Equal` and `non_eq` in the following match expression:
+   *
+   * match cmp(...) {
+   *     Ordering::Equal => match cmp(...) { ... }
+   *     non_eq => non_eq,
+   * }
+   */
+  std::pair<MatchArm, MatchArm> make_cmp_arms ();
+
+  /**
+   * Generate a call to the proper trait function, based on the ordering, in
+   * order to compare two given expressions
+   */
+  std::unique_ptr<Expr> cmp_call (std::unique_ptr<Expr> &&self_expr,
+				  std::unique_ptr<Expr> &&other_expr);
+
+  std::unique_ptr<Item>
+  cmp_impl (std::unique_ptr<BlockExpr> &&fn_block, Identifier type_name,
+	    const std::vector<std::unique_ptr<GenericParam>> &type_generics);
+  std::unique_ptr<AssociatedItem> cmp_fn (std::unique_ptr<BlockExpr> &&block,
+					  Identifier type_name);
+
+  virtual void visit_struct (StructStruct &item) override;
+  virtual void visit_tuple (TupleStruct &item) override;
+  virtual void visit_enum (Enum &item) override;
+  virtual void visit_union (Union &item) override;
+};
+
+} // namespace AST
+} // namespace Rust
+
+#endif // ! RUST_DERIVE_ORD_H
diff --git a/gcc/rust/expand/rust-derive-partial-eq.cc b/gcc/rust/expand/rust-derive-partial-eq.cc
index ff66faa..a0bf87a 100644
--- a/gcc/rust/expand/rust-derive-partial-eq.cc
+++ b/gcc/rust/expand/rust-derive-partial-eq.cc
@@ -64,11 +64,9 @@ DerivePartialEq::partialeq_impls (
 }
 
 std::unique_ptr<AssociatedItem>
-DerivePartialEq::eq_fn (std::unique_ptr<Expr> &&cmp_expression,
+DerivePartialEq::eq_fn (std::unique_ptr<BlockExpr> &&block,
 			std::string type_name)
 {
-  auto block = builder.block (tl::nullopt, std::move (cmp_expression));
-
   auto self_type
     = std::unique_ptr<TypeNoBounds> (new TypePath (builder.type_path ("Self")));
 
@@ -83,24 +81,6 @@ DerivePartialEq::eq_fn (std::unique_ptr<Expr> &&cmp_expression,
 			   std::move (block));
 }
 
-DerivePartialEq::SelfOther
-DerivePartialEq::tuple_indexes (int idx)
-{
-  return SelfOther{
-    builder.tuple_idx ("self", idx),
-    builder.tuple_idx ("other", idx),
-  };
-}
-
-DerivePartialEq::SelfOther
-DerivePartialEq::field_acccesses (const std::string &field_name)
-{
-  return SelfOther{
-    builder.field_access (builder.identifier ("self"), field_name),
-    builder.field_access (builder.identifier ("other"), field_name),
-  };
-}
-
 std::unique_ptr<Expr>
 DerivePartialEq::build_eq_expression (
   std::vector<SelfOther> &&field_expressions)
@@ -134,12 +114,10 @@ void
 DerivePartialEq::visit_tuple (TupleStruct &item)
 {
   auto type_name = item.get_struct_name ().as_string ();
-  auto fields = std::vector<SelfOther> ();
-
-  for (size_t idx = 0; idx < item.get_fields ().size (); idx++)
-    fields.emplace_back (tuple_indexes (idx));
+  auto fields = SelfOther::indexes (builder, item.get_fields ());
 
-  auto fn = eq_fn (build_eq_expression (std::move (fields)), type_name);
+  auto fn = eq_fn (builder.block (build_eq_expression (std::move (fields))),
+		   type_name);
 
   expanded
     = partialeq_impls (std::move (fn), type_name, item.get_generic_params ());
@@ -149,13 +127,10 @@ void
 DerivePartialEq::visit_struct (StructStruct &item)
 {
   auto type_name = item.get_struct_name ().as_string ();
-  auto fields = std::vector<SelfOther> ();
+  auto fields = SelfOther::fields (builder, item.get_fields ());
 
-  for (auto &field : item.get_fields ())
-    fields.emplace_back (
-      field_acccesses (field.get_field_name ().as_string ()));
-
-  auto fn = eq_fn (build_eq_expression (std::move (fields)), type_name);
+  auto fn = eq_fn (builder.block (build_eq_expression (std::move (fields))),
+		   type_name);
 
   expanded
     = partialeq_impls (std::move (fn), type_name, item.get_generic_params ());
@@ -199,8 +174,6 @@ DerivePartialEq::match_enum_tuple (PathInExpression variant_path,
       auto self_pattern_str = "__self_" + std::to_string (i);
       auto other_pattern_str = "__other_" + std::to_string (i);
 
-      rust_debug ("]ARTHUR[ %s", self_pattern_str.c_str ());
-
       self_patterns.emplace_back (
 	builder.identifier_pattern (self_pattern_str));
       other_patterns.emplace_back (
@@ -240,15 +213,55 @@ MatchCase
 DerivePartialEq::match_enum_struct (PathInExpression variant_path,
 				    const EnumItemStruct &variant)
 {
-  // NOTE: We currently do not support compiling struct patterns where an
-  // identifier is assigned a new pattern, e.g. Bloop { f0: x }
-  // This is what we should be using to compile PartialEq for enum struct
-  // variants, as we need to be comparing the field of each instance meaning we
-  // need to give two different names to two different instances of the same
-  // field. We cannot just use the field's name like we do when deriving
-  // `Clone`.
-
-  rust_unreachable ();
+  auto self_fields = std::vector<std::unique_ptr<StructPatternField>> ();
+  auto other_fields = std::vector<std::unique_ptr<StructPatternField>> ();
+
+  auto self_other_exprs = std::vector<SelfOther> ();
+
+  for (auto &field : variant.get_struct_fields ())
+    {
+      // The patterns we're creating for each field are `self_<field>` and
+      // `other_<field>` where `field` is the name of the field. It doesn't
+      // actually matter what we use, as long as it's ordered, unique, and that
+      // we can reuse it in the match case's return expression to check that
+      // they are equal.
+
+      auto field_name = field.get_field_name ().as_string ();
+
+      auto self_pattern_str = "__self_" + field_name;
+      auto other_pattern_str = "__other_" + field_name;
+
+      self_fields.emplace_back (builder.struct_pattern_ident_pattern (
+	field_name, builder.identifier_pattern (self_pattern_str)));
+      other_fields.emplace_back (builder.struct_pattern_ident_pattern (
+	field_name, builder.identifier_pattern (other_pattern_str)));
+
+      self_other_exprs.emplace_back (SelfOther{
+	builder.identifier (self_pattern_str),
+	builder.identifier (other_pattern_str),
+      });
+    }
+
+  auto self_elts = StructPatternElements (std::move (self_fields));
+  auto other_elts = StructPatternElements (std::move (other_fields));
+
+  auto self_pattern = std::unique_ptr<Pattern> (
+    new ReferencePattern (std::unique_ptr<Pattern> (new StructPattern (
+			    variant_path, loc, std::move (self_elts))),
+			  false, false, loc));
+  auto other_pattern = std::unique_ptr<Pattern> (
+    new ReferencePattern (std::unique_ptr<Pattern> (new StructPattern (
+			    variant_path, loc, std::move (other_elts))),
+			  false, false, loc));
+
+  auto tuple_items = std::make_unique<TuplePatternItemsMultiple> (
+    vec (std::move (self_pattern), std::move (other_pattern)));
+
+  auto pattern = std::make_unique<TuplePattern> (std::move (tuple_items), loc);
+
+  auto expr = build_eq_expression (std::move (self_other_exprs));
+
+  return builder.match_case (std::move (pattern), std::move (expr));
 }
 
 void
@@ -257,46 +270,56 @@ DerivePartialEq::visit_enum (Enum &item)
   auto cases = std::vector<MatchCase> ();
   auto type_name = item.get_identifier ().as_string ();
 
+  auto eq_expr_fn = [this] (std::vector<SelfOther> &&fields) {
+    return build_eq_expression (std::move (fields));
+  };
+
+  auto let_sd
+    = builder.discriminant_value (DerivePartialEq::self_discr, "self");
+  auto let_od
+    = builder.discriminant_value (DerivePartialEq::other_discr, "other");
+
+  auto discr_cmp
+    = builder.comparison_expr (builder.identifier (DerivePartialEq::self_discr),
+			       builder.identifier (
+				 DerivePartialEq::other_discr),
+			       ComparisonOperator::EQUAL);
+
   for (auto &variant : item.get_variants ())
     {
-      auto variant_path
-	= builder.variant_path (type_name,
-				variant->get_identifier ().as_string ());
+      auto enum_builder
+	= EnumMatchBuilder (type_name, variant->get_identifier ().as_string (),
+			    eq_expr_fn, builder);
 
       switch (variant->get_enum_item_kind ())
 	{
-	case EnumItem::Kind::Identifier:
-	case EnumItem::Kind::Discriminant:
-	  cases.emplace_back (match_enum_identifier (variant_path, variant));
-	  break;
 	case EnumItem::Kind::Tuple:
-	  cases.emplace_back (
-	    match_enum_tuple (variant_path,
-			      static_cast<EnumItemTuple &> (*variant)));
+	  cases.emplace_back (enum_builder.tuple (*variant));
 	  break;
 	case EnumItem::Kind::Struct:
-	  rust_sorry_at (
-	    item.get_locus (),
-	    "cannot derive(PartialEq) for enum struct variants yet");
+	  cases.emplace_back (enum_builder.strukt (*variant));
+	  break;
+	case EnumItem::Kind::Identifier:
+	case EnumItem::Kind::Discriminant:
+	  // We don't need to do anything for these, as they are handled by the
+	  // discriminant value comparison
 	  break;
 	}
     }
 
-  // NOTE: Mention using discriminant_value and skipping that last case, and
-  // instead skipping all identifiers/discriminant enum items and returning
-  // `true` in the wildcard case
-
   // In case the two instances of `Self` don't have the same discriminant,
   // automatically return false.
   cases.emplace_back (
-    builder.match_case (builder.wildcard (), builder.literal_bool (false)));
+    builder.match_case (builder.wildcard (), std::move (discr_cmp)));
 
   auto match
     = builder.match (builder.tuple (vec (builder.identifier ("self"),
 					 builder.identifier ("other"))),
 		     std::move (cases));
 
-  auto fn = eq_fn (std::move (match), type_name);
+  auto fn = eq_fn (builder.block (vec (std::move (let_sd), std::move (let_od)),
+				  std::move (match)),
+		   type_name);
 
   expanded
     = partialeq_impls (std::move (fn), type_name, item.get_generic_params ());
diff --git a/gcc/rust/expand/rust-derive-partial-eq.h b/gcc/rust/expand/rust-derive-partial-eq.h
index ac963a6..7985414 100644
--- a/gcc/rust/expand/rust-derive-partial-eq.h
+++ b/gcc/rust/expand/rust-derive-partial-eq.h
@@ -21,6 +21,7 @@
 
 #include "rust-derive.h"
 #include "rust-path.h"
+#include "rust-derive-cmp-common.h"
 
 namespace Rust {
 namespace AST {
@@ -30,7 +31,7 @@ class DerivePartialEq : DeriveVisitor
 public:
   DerivePartialEq (location_t loc);
 
-  std::vector<std::unique_ptr<AST::Item>> go (Item &item);
+  std::vector<std::unique_ptr<Item>> go (Item &item);
 
 private:
   std::vector<std::unique_ptr<Item>> expanded;
@@ -43,23 +44,10 @@ private:
     std::unique_ptr<AssociatedItem> &&eq_fn, std::string name,
     const std::vector<std::unique_ptr<GenericParam>> &type_generics);
 
-  std::unique_ptr<AssociatedItem> eq_fn (std::unique_ptr<Expr> &&cmp_expression,
+  std::unique_ptr<AssociatedItem> eq_fn (std::unique_ptr<BlockExpr> &&block,
 					 std::string type_name);
 
   /**
-   * A pair of two expressions from each instance being compared. E.g. this
-   * could be `self.0` and `other.0`, or `self.field` and `other.field`
-   */
-  struct SelfOther
-  {
-    std::unique_ptr<Expr> self_expr;
-    std::unique_ptr<Expr> other_expr;
-  };
-
-  SelfOther tuple_indexes (int idx);
-  SelfOther field_acccesses (const std::string &field_name);
-
-  /**
    * Build a suite of equality arithmetic expressions chained together by a
    * boolean AND operator
    */
@@ -73,10 +61,13 @@ private:
   MatchCase match_enum_struct (PathInExpression variant_path,
 			       const EnumItemStruct &variant);
 
-  virtual void visit_struct (StructStruct &item);
-  virtual void visit_tuple (TupleStruct &item);
-  virtual void visit_enum (Enum &item);
-  virtual void visit_union (Union &item);
+  constexpr static const char *self_discr = "#self_discr";
+  constexpr static const char *other_discr = "#other_discr";
+
+  virtual void visit_struct (StructStruct &item) override;
+  virtual void visit_tuple (TupleStruct &item) override;
+  virtual void visit_enum (Enum &item) override;
+  virtual void visit_union (Union &item) override;
 };
 
 } // namespace AST
diff --git a/gcc/rust/expand/rust-derive.cc b/gcc/rust/expand/rust-derive.cc
index 015b81e..55147df 100644
--- a/gcc/rust/expand/rust-derive.cc
+++ b/gcc/rust/expand/rust-derive.cc
@@ -22,6 +22,7 @@
 #include "rust-derive-debug.h"
 #include "rust-derive-default.h"
 #include "rust-derive-eq.h"
+#include "rust-derive-ord.h"
 #include "rust-derive-partial-eq.h"
 #include "rust-derive-hash.h"
 
@@ -59,10 +60,11 @@ DeriveVisitor::derive (Item &item, const Attribute &attr,
     case BuiltinMacro::Hash:
       return vec (DeriveHash (loc).go (item));
     case BuiltinMacro::Ord:
+      return vec (DeriveOrd (DeriveOrd::Ordering::Total, loc).go (item));
     case BuiltinMacro::PartialOrd:
+      return vec (DeriveOrd (DeriveOrd::Ordering::Partial, loc).go (item));
     default:
-      rust_sorry_at (loc, "unimplemented builtin derive macro");
-      return {};
+      rust_unreachable ();
     };
 }
 
@@ -79,7 +81,8 @@ DeriveVisitor::setup_impl_generics (
     {
       switch (generic->get_kind ())
 	{
-	  case GenericParam::Kind::Lifetime: {
+	case GenericParam::Kind::Lifetime:
+	  {
 	    LifetimeParam &lifetime_param = (LifetimeParam &) *generic.get ();
 
 	    Lifetime l = builder.new_lifetime (lifetime_param.get_lifetime ());
@@ -91,7 +94,8 @@ DeriveVisitor::setup_impl_generics (
 	  }
 	  break;
 
-	  case GenericParam::Kind::Type: {
+	case GenericParam::Kind::Type:
+	  {
 	    TypeParam &type_param = (TypeParam &) *generic.get ();
 
 	    std::unique_ptr<Type> associated_type = builder.single_type_path (
@@ -104,7 +108,8 @@ DeriveVisitor::setup_impl_generics (
 	    std::vector<std::unique_ptr<TypeParamBound>> extra_bounds;
 
 	    if (extra_bound)
-	      extra_bounds.emplace_back (std::move (*extra_bound));
+	      extra_bounds.emplace_back (
+		extra_bound.value ()->clone_type_param_bound ());
 
 	    auto impl_type_param
 	      = builder.new_type_param (type_param, std::move (extra_bounds));
@@ -113,17 +118,20 @@ DeriveVisitor::setup_impl_generics (
 	  }
 	  break;
 
-	  case GenericParam::Kind::Const: {
-	    rust_unreachable ();
+	case GenericParam::Kind::Const:
+	  {
+	    ConstGenericParam &const_param
+	      = (ConstGenericParam &) *generic.get ();
 
-	    // TODO
-	    // const ConstGenericParam *const_param
-	    //   = (const ConstGenericParam *) generic.get ();
-	    // std::unique_ptr<Expr> const_expr = nullptr;
+	    std::unique_ptr<Type> associated_type
+	      = builder.single_type_path (const_param.get_name ().as_string ());
 
-	    // GenericArg type_arg
-	    //   = GenericArg::create_const (std::move (const_expr));
-	    // generic_args.push_back (std::move (type_arg));
+	    GenericArg type_arg
+	      = GenericArg::create_type (std::move (associated_type));
+	    generic_args.push_back (std::move (type_arg));
+
+	    auto impl_const_param = builder.new_const_param (const_param);
+	    impl_generics.push_back (std::move (impl_const_param));
 	  }
 	  break;
 	}
diff --git a/gcc/rust/expand/rust-derive.h b/gcc/rust/expand/rust-derive.h
index 5fca49c..10c146c 100644
--- a/gcc/rust/expand/rust-derive.h
+++ b/gcc/rust/expand/rust-derive.h
@@ -118,7 +118,7 @@ private:
   virtual void visit (LiteralExpr &expr) override final{};
   virtual void visit (AttrInputLiteral &attr_input) override final{};
   virtual void visit (MetaItemLitExpr &meta_item) override final{};
-  virtual void visit (MetaItemPathLit &meta_item) override final{};
+  virtual void visit (MetaItemPathExpr &meta_item) override final{};
   virtual void visit (BorrowExpr &expr) override final{};
   virtual void visit (DereferenceExpr &expr) override final{};
   virtual void visit (ErrorPropagationExpr &expr) override final{};
@@ -147,6 +147,8 @@ private:
   virtual void visit (FieldAccessExpr &expr) override final{};
   virtual void visit (ClosureExprInner &expr) override final{};
   virtual void visit (BlockExpr &expr) override final{};
+  virtual void visit (AnonConst &expr) override final{};
+  virtual void visit (ConstBlock &expr) override final{};
   virtual void visit (ClosureExprInnerTyped &expr) override final{};
   virtual void visit (ContinueExpr &expr) override final{};
   virtual void visit (BreakExpr &expr) override final{};
@@ -157,6 +159,7 @@ private:
   virtual void visit (RangeFromToInclExpr &expr) override final{};
   virtual void visit (RangeToInclExpr &expr) override final{};
   virtual void visit (ReturnExpr &expr) override final{};
+  virtual void visit (TryExpr &expr) override final{};
   virtual void visit (BoxExpr &expr) override final{};
   virtual void visit (UnsafeBlockExpr &expr) override final{};
   virtual void visit (LoopExpr &expr) override final{};
@@ -228,6 +231,8 @@ private:
   virtual void visit (TuplePatternItemsRanged &tuple_items) override final{};
   virtual void visit (TuplePattern &pattern) override final{};
   virtual void visit (GroupedPattern &pattern) override final{};
+  virtual void visit (SlicePatternItemsNoRest &items) override final{};
+  virtual void visit (SlicePatternItemsHasRest &items) override final{};
   virtual void visit (SlicePattern &pattern) override final{};
   virtual void visit (AltPattern &pattern) override final{};
   virtual void visit (EmptyStmt &stmt) override final{};
@@ -251,6 +256,7 @@ private:
   virtual void visit (FunctionParam &param) override final{};
   virtual void visit (VariadicParam &param) override final{};
   virtual void visit (FormatArgs &param) override final{};
+  virtual void visit (OffsetOf &param) override final{};
 };
 
 } // namespace AST
diff --git a/gcc/rust/expand/rust-expand-format-args.cc b/gcc/rust/expand/rust-expand-format-args.cc
index af6182f..bda28dd 100644
--- a/gcc/rust/expand/rust-expand-format-args.cc
+++ b/gcc/rust/expand/rust-expand-format-args.cc
@@ -85,11 +85,13 @@ expand_format_args (AST::FormatArgs &fmt,
 	  static_pieces.emplace_back (
 	    builder.literal_string (node.string._0.to_string ()));
 	  break;
-	  case ffi::Piece::Tag::NextArgument: {
+	case ffi::Piece::Tag::NextArgument:
+	  {
 	    auto next_argument = node.next_argument._0;
 	    switch (node.next_argument._0.position.tag)
 	      {
-		case ffi::Position::Tag::ArgumentImplicitlyIs: {
+	      case ffi::Position::Tag::ArgumentImplicitlyIs:
+		{
 		  auto idx = next_argument.position.argument_implicitly_is._0;
 		  auto trait = next_argument.format;
 		  auto arg = arguments.at (idx);
diff --git a/gcc/rust/expand/rust-expand-visitor.cc b/gcc/rust/expand/rust-expand-visitor.cc
index 42df5e1..8f6e7fa 100644
--- a/gcc/rust/expand/rust-expand-visitor.cc
+++ b/gcc/rust/expand/rust-expand-visitor.cc
@@ -233,10 +233,7 @@ ExpandVisitor::expand_inner_items (
 	}
     }
 
-  std::function<std::unique_ptr<AST::Item> (AST::SingleASTNode)> extractor
-    = [] (AST::SingleASTNode node) { return node.take_item (); };
-
-  expand_macro_children (items, extractor);
+  expand_macro_children (items, &AST::SingleASTNode::take_item);
 
   expander.pop_context ();
 }
@@ -324,10 +321,7 @@ ExpandVisitor::expand_inner_stmts (AST::BlockExpr &expr)
   if (!expr.has_tail_expr ())
     expr.normalize_tail_expr ();
 
-  std::function<std::unique_ptr<AST::Stmt> (AST::SingleASTNode)> extractor
-    = [] (AST::SingleASTNode node) { return node.take_stmt (); };
-
-  expand_macro_children (stmts, extractor);
+  expand_macro_children (stmts, &AST::SingleASTNode::take_stmt);
 
   expander.pop_context ();
 }
@@ -544,7 +538,7 @@ ExpandVisitor::visit (AST::MetaItemLitExpr &)
 {}
 
 void
-ExpandVisitor::visit (AST::MetaItemPathLit &)
+ExpandVisitor::visit (AST::MetaItemPathExpr &)
 {}
 
 void
@@ -641,7 +635,7 @@ ExpandVisitor::visit (AST::ClosureExprInnerTyped &expr)
 
   maybe_expand_type (expr.get_return_type_ptr ());
 
-  visit (expr.get_definition_block ());
+  visit (expr.get_definition_expr ());
 }
 
 void
@@ -866,12 +860,9 @@ ExpandVisitor::visit (AST::Trait &trait)
 
   expander.push_context (MacroExpander::ContextType::TRAIT);
 
-  std::function<std::unique_ptr<AST::AssociatedItem> (AST::SingleASTNode)>
-    extractor
-    = [] (AST::SingleASTNode node) { return node.take_assoc_item (); };
-
   expand_macro_children (MacroExpander::ContextType::TRAIT,
-			 trait.get_trait_items (), extractor);
+			 trait.get_trait_items (),
+			 &AST::SingleASTNode::take_assoc_item);
 
   expander.pop_context ();
 }
@@ -894,12 +885,9 @@ ExpandVisitor::visit (AST::InherentImpl &impl)
   if (impl.has_where_clause ())
     expand_where_clause (impl.get_where_clause ());
 
-  std::function<std::unique_ptr<AST::AssociatedItem> (AST::SingleASTNode)>
-    extractor
-    = [] (AST::SingleASTNode node) { return node.take_assoc_item (); };
-
   expand_macro_children (MacroExpander::ContextType::IMPL,
-			 impl.get_impl_items (), extractor);
+			 impl.get_impl_items (),
+			 &AST::SingleASTNode::take_assoc_item);
 }
 
 void
@@ -922,12 +910,9 @@ ExpandVisitor::visit (AST::TraitImpl &impl)
   if (impl.has_where_clause ())
     expand_where_clause (impl.get_where_clause ());
 
-  std::function<std::unique_ptr<AST::AssociatedItem> (AST::SingleASTNode)>
-    extractor
-    = [] (AST::SingleASTNode node) { return node.take_assoc_item (); };
-
   expand_macro_children (MacroExpander::ContextType::TRAIT_IMPL,
-			 impl.get_impl_items (), extractor);
+			 impl.get_impl_items (),
+			 &AST::SingleASTNode::take_assoc_item);
 }
 
 void
@@ -944,12 +929,10 @@ void
 ExpandVisitor::visit (AST::ExternBlock &block)
 {
   visit_inner_attrs (block);
-  std::function<std::unique_ptr<AST::ExternalItem> (AST::SingleASTNode)>
-    extractor
-    = [] (AST::SingleASTNode node) { return node.take_external_item (); };
 
   expand_macro_children (MacroExpander::ContextType::EXTERN,
-			 block.get_extern_items (), extractor);
+			 block.get_extern_items (),
+			 &AST::SingleASTNode::take_external_item);
 }
 
 void
diff --git a/gcc/rust/expand/rust-expand-visitor.h b/gcc/rust/expand/rust-expand-visitor.h
index ad237c0..845e10c 100644
--- a/gcc/rust/expand/rust-expand-visitor.h
+++ b/gcc/rust/expand/rust-expand-visitor.h
@@ -28,14 +28,12 @@ namespace Rust {
 /**
  * Whether or not an attribute is a derive attribute
  */
-bool
-is_derive (AST::Attribute &attr);
+bool is_derive (AST::Attribute &attr);
 
 /**
  * Whether or not an attribute is builtin
  */
-bool
-is_builtin (AST::Attribute &attr);
+bool is_builtin (AST::Attribute &attr);
 
 class ExpandVisitor : public AST::DefaultASTVisitor
 {
@@ -107,7 +105,7 @@ public:
    */
   template <typename T, typename U>
   void expand_macro_children (MacroExpander::ContextType ctx, T &values,
-			      std::function<U (AST::SingleASTNode)> extractor)
+			      U (AST::SingleASTNode::*extractor) (void))
   {
     expander.push_context (ctx);
 
@@ -123,7 +121,7 @@ public:
    */
   template <typename T, typename U>
   void expand_macro_children (T &values,
-			      std::function<U (AST::SingleASTNode)> extractor)
+			      U (AST::SingleASTNode::*extractor) (void))
   {
     for (auto it = values.begin (); it != values.end ();)
       {
@@ -140,7 +138,7 @@ public:
 	    it = values.erase (it);
 	    for (auto &node : final_fragment.get_nodes ())
 	      {
-		U new_node = extractor (node);
+		U new_node = (node.*extractor) ();
 		if (new_node != nullptr)
 		  {
 		    it = values.insert (it, std::move (new_node));
@@ -211,7 +209,7 @@ public:
   void visit (AST::AttrInputLiteral &) override;
   void visit (AST::AttrInputMacro &) override;
   void visit (AST::MetaItemLitExpr &) override;
-  void visit (AST::MetaItemPathLit &) override;
+  void visit (AST::MetaItemPathExpr &) override;
   void visit (AST::ErrorPropagationExpr &expr) override;
   void visit (AST::ArithmeticOrLogicalExpr &expr) override;
   void visit (AST::ComparisonExpr &expr) override;
diff --git a/gcc/rust/expand/rust-macro-builtins-asm.cc b/gcc/rust/expand/rust-macro-builtins-asm.cc
index e255729..61222db 100644
--- a/gcc/rust/expand/rust-macro-builtins-asm.cc
+++ b/gcc/rust/expand/rust-macro-builtins-asm.cc
@@ -25,18 +25,6 @@
 #include "rust-parse.h"
 
 namespace Rust {
-std::map<AST::InlineAsmOption, std::string> InlineAsmOptionMap{
-  {AST::InlineAsmOption::PURE, "pure"},
-  {AST::InlineAsmOption::NOMEM, "nomem"},
-  {AST::InlineAsmOption::READONLY, "readonly"},
-  {AST::InlineAsmOption::PRESERVES_FLAGS, "preserves_flags"},
-  {AST::InlineAsmOption::NORETURN, "noreturn"},
-  {AST::InlineAsmOption::NOSTACK, "nostack"},
-  {AST::InlineAsmOption::MAY_UNWIND, "may_unwind"},
-  {AST::InlineAsmOption::ATT_SYNTAX, "att_syntax"},
-  {AST::InlineAsmOption::RAW, "raw"},
-};
-
 std::set<std::string> potentially_nonpromoted_keywords
   = {"in", "out", "lateout", "inout", "inlateout", "const", "sym", "label"};
 
@@ -396,6 +384,7 @@ parse_reg_operand_inout (InlineAsmContext inline_asm_ctx)
 {
   auto &parser = inline_asm_ctx.parser;
   auto token = parser.peek_current_token ();
+  location_t locus = token->get_locus ();
 
   if (!inline_asm_ctx.is_global_asm () && check_identifier (parser, "inout"))
     {
@@ -413,10 +402,8 @@ parse_reg_operand_inout (InlineAsmContext inline_asm_ctx)
 
       // TODO: Is error propogation our top priority, the ? in rust's asm.rs is
       // doing a lot of work.
-      // TODO: Not sure how to use parse_expr
-      if (!check_identifier (parser, ""))
-	rust_unreachable ();
-      // auto expr = parse_format_string (inline_asm_ctx);
+      std::unique_ptr<AST::Expr> in_expr = parser.parse_expr ();
+      rust_assert (in_expr != nullptr);
 
       std::unique_ptr<AST::Expr> out_expr;
 
@@ -426,11 +413,19 @@ parse_reg_operand_inout (InlineAsmContext inline_asm_ctx)
 	    {
 	      // auto result = parse_format_string (inline_asm_ctx);
 
-	      if (!check_identifier (parser, ""))
-		rust_unreachable ();
-	      // out_expr = parser.parse_expr();
+	      out_expr = parser.parse_expr ();
+
+	      AST::InlineAsmOperand::SplitInOut splitinout (
+		reg, false, std::move (in_expr), std::move (out_expr));
+
+	      inline_asm_ctx.inline_asm.operands.emplace_back (splitinout,
+							       locus);
+
+	      return inline_asm_ctx;
 	    }
 
+	  rust_unreachable ();
+
 	  // TODO: Rembmer to pass in clone_expr() instead of nullptr
 	  // https://github.com/rust-lang/rust/blob/a3167859f2fd8ff2241295469876a2b687280bdc/compiler/rustc_builtin_macros/src/asm.rs#L135
 	  // RUST VERSION: ast::InlineAsmOperand::SplitInOut { reg, in_expr:
@@ -444,6 +439,8 @@ parse_reg_operand_inout (InlineAsmContext inline_asm_ctx)
 	}
       else
 	{
+	  AST::InlineAsmOperand::InOut inout (reg, false, std::move (in_expr));
+	  inline_asm_ctx.inline_asm.operands.emplace_back (inout, locus);
 	  // https://github.com/rust-lang/rust/blob/a3167859f2fd8ff2241295469876a2b687280bdc/compiler/rustc_builtin_macros/src/asm.rs#L137
 	  // RUST VERSION: ast::InlineAsmOperand::InOut { reg, expr, late: false
 	  // }
@@ -500,7 +497,7 @@ parse_reg_operand_unexpected (InlineAsmContext inline_asm_ctx)
 }
 
 void
-check_and_set (InlineAsmContext &inline_asm_ctx, AST::InlineAsmOption option)
+check_and_set (InlineAsmContext &inline_asm_ctx, AST::InlineAsm::Option option)
 {
   auto &parser = inline_asm_ctx.parser;
   auto &inline_asm = inline_asm_ctx.inline_asm;
@@ -509,7 +506,7 @@ check_and_set (InlineAsmContext &inline_asm_ctx, AST::InlineAsmOption option)
       // TODO: report an error of duplication
       rust_error_at (parser.peek_current_token ()->get_locus (),
 		     "the %qs option was already provided",
-		     InlineAsmOptionMap[option].c_str ());
+		     AST::InlineAsm::option_to_string (option).c_str ());
       return;
     }
   else
@@ -536,39 +533,40 @@ parse_options (InlineAsmContext &inline_asm_ctx)
     {
       if (!is_global_asm && check_identifier (parser, "pure"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::PURE);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::PURE);
 	}
       else if (!is_global_asm && check_identifier (parser, "nomem"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::NOMEM);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::NOMEM);
 	}
       else if (!is_global_asm && check_identifier (parser, "readonly"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::READONLY);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::READONLY);
 	}
       else if (!is_global_asm && check_identifier (parser, "preserves_flags"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::PRESERVES_FLAGS);
+	  check_and_set (inline_asm_ctx,
+			 AST::InlineAsm::Option::PRESERVES_FLAGS);
 	}
       else if (!is_global_asm && check_identifier (parser, "noreturn"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::NORETURN);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::NORETURN);
 	}
       else if (!is_global_asm && check_identifier (parser, "nostack"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::NOSTACK);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::NOSTACK);
 	}
       else if (!is_global_asm && check_identifier (parser, "may_unwind"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::MAY_UNWIND);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::MAY_UNWIND);
 	}
       else if (check_identifier (parser, "att_syntax"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::ATT_SYNTAX);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::ATT_SYNTAX);
 	}
       else if (check_identifier (parser, "raw"))
 	{
-	  check_and_set (inline_asm_ctx, AST::InlineAsmOption::RAW);
+	  check_and_set (inline_asm_ctx, AST::InlineAsm::Option::RAW);
 	}
       else
 	{
@@ -807,7 +805,8 @@ expand_inline_asm_strings (InlineAsmContext inline_asm_ctx)
 	      auto next_argument = piece.next_argument._0;
 	      switch (piece.next_argument._0.position.tag)
 		{
-		  case Fmt::ffi::Position::Tag::ArgumentImplicitlyIs: {
+		case Fmt::ffi::Position::Tag::ArgumentImplicitlyIs:
+		  {
 		    auto idx = next_argument.position.argument_implicitly_is._0;
 		    /*auto trait = next_argument.format;*/
 		    /*auto arg = arguments.at (idx);*/
@@ -829,6 +828,11 @@ expand_inline_asm_strings (InlineAsmContext inline_asm_ctx)
 		  }
 		  break;
 		case Fmt::ffi::Position::Tag::ArgumentIs:
+		  {
+		    auto idx = next_argument.position.argument_is._0;
+		    transformed_template_str += "%" + std::to_string (idx);
+		    break;
+		  }
 		case Fmt::ffi::Position::Tag::ArgumentNamed:
 		  rust_sorry_at (inline_asm.get_locus (),
 				 "unhandled argument position specifier");
@@ -933,7 +937,9 @@ parse_format_strings (InlineAsmContext inline_asm_ctx)
     {
       if (!parser.skip_token (COMMA))
 	{
-	  break;
+	  rust_error_at (parser.peek_current_token ()->get_locus (),
+			 "expected token %qs", ";");
+	  return tl::unexpected<InlineAsmParseError> (COMMITTED);
 	}
       // Ok after the comma is good, we better be parsing correctly
       // everything in here, which is formatted string in ABNF
diff --git a/gcc/rust/expand/rust-macro-builtins-asm.h b/gcc/rust/expand/rust-macro-builtins-asm.h
index bd64a7f..3196a5a 100644
--- a/gcc/rust/expand/rust-macro-builtins-asm.h
+++ b/gcc/rust/expand/rust-macro-builtins-asm.h
@@ -142,16 +142,16 @@ tl::expected<InlineAsmContext, InlineAsmParseError>
 parse_reg_operand_unexpected (InlineAsmContext inline_asm_ctx);
 
 WARN_UNUSED_RESULT
-tl::optional<AST::Fragment>
-parse_asm (location_t invoc_locus, AST::MacroInvocData &invoc,
-	   AST::InvocKind semicolon, AST::AsmKind is_global_asm);
+tl::optional<AST::Fragment> parse_asm (location_t invoc_locus,
+				       AST::MacroInvocData &invoc,
+				       AST::InvocKind semicolon,
+				       AST::AsmKind is_global_asm);
 
 WARN_UNUSED_RESULT
-bool
-check_identifier (Parser<MacroInvocLexer> &parser, std::string ident);
+bool check_identifier (Parser<MacroInvocLexer> &parser, std::string ident);
 
-void
-check_and_set (InlineAsmContext &inline_asm_ctx, AST::InlineAsmOption option);
+void check_and_set (InlineAsmContext &inline_asm_ctx,
+		    AST::InlineAsm::Option option);
 
 // From rustc
 WARN_UNUSED_RESULT
@@ -168,9 +168,9 @@ tl::optional<std::string>
 parse_format_string (InlineAsmContext &inline_asm_ctx);
 
 WARN_UNUSED_RESULT
-tl::optional<std::string>
-parse_label (Parser<MacroInvocLexer> &parser, TokenId last_token_id,
-	     InlineAsmContext &inline_asm_ctx);
+tl::optional<std::string> parse_label (Parser<MacroInvocLexer> &parser,
+				       TokenId last_token_id,
+				       InlineAsmContext &inline_asm_ctx);
 
 // LLVM ASM bits
 
@@ -188,17 +188,13 @@ public:
   {}
 };
 
-void
-parse_llvm_outputs (LlvmAsmContext &ctx);
+void parse_llvm_outputs (LlvmAsmContext &ctx);
 
-void
-parse_llvm_inputs (LlvmAsmContext &ctx);
+void parse_llvm_inputs (LlvmAsmContext &ctx);
 
-void
-parse_llvm_clobbers (LlvmAsmContext &ctx);
+void parse_llvm_clobbers (LlvmAsmContext &ctx);
 
-void
-parse_llvm_options (LlvmAsmContext &ctx);
+void parse_llvm_options (LlvmAsmContext &ctx);
 
 WARN_UNUSED_RESULT tl::optional<AST::Fragment>
 parse_llvm_asm (location_t invoc_locus, AST::MacroInvocData &invoc,
diff --git a/gcc/rust/expand/rust-macro-builtins-format-args.cc b/gcc/rust/expand/rust-macro-builtins-format-args.cc
index 3e1249d..b20c849 100644
--- a/gcc/rust/expand/rust-macro-builtins-format-args.cc
+++ b/gcc/rust/expand/rust-macro-builtins-format-args.cc
@@ -52,8 +52,15 @@ format_args_parse_arguments (AST::MacroInvocData &invoc)
 
   // TODO: Handle the case where we're not parsing a string literal (macro
   // invocation for e.g.)
-  if (parser.peek_current_token ()->get_id () == STRING_LITERAL)
-    format_expr = parser.parse_literal_expr ();
+  switch (parser.peek_current_token ()->get_id ())
+    {
+    case STRING_LITERAL:
+    case RAW_STRING_LITERAL:
+      format_expr = parser.parse_literal_expr ();
+    default:
+      // do nothing
+      ;
+    }
 
   rust_assert (format_expr);
 
diff --git a/gcc/rust/expand/rust-macro-builtins-helpers.cc b/gcc/rust/expand/rust-macro-builtins-helpers.cc
index 864379a..ee01f65 100644
--- a/gcc/rust/expand/rust-macro-builtins-helpers.cc
+++ b/gcc/rust/expand/rust-macro-builtins-helpers.cc
@@ -68,6 +68,7 @@ make_eager_builtin_invocation (
 {
   auto path_str = make_macro_path_str (kind);
 
+  auto token_stream = arguments.to_token_stream ();
   std::unique_ptr<AST::Expr> node = AST::MacroInvocation::Builtin (
     kind,
     AST::MacroInvocData (AST::SimplePath (
@@ -76,7 +77,7 @@ make_eager_builtin_invocation (
     {}, locus, std::move (pending_invocations));
 
   return AST::Fragment ({AST::SingleASTNode (std::move (node))},
-			arguments.to_token_stream ());
+			std::move (token_stream));
 }
 
 /* Match the end token of a macro given the start delimiter of the macro */
@@ -110,9 +111,9 @@ std::unique_ptr<AST::LiteralExpr>
 try_extract_string_literal_from_fragment (const location_t &parent_locus,
 					  std::unique_ptr<AST::Expr> &node)
 {
-  auto maybe_lit = static_cast<AST::LiteralExpr *> (node.get ());
   if (!node || !node->is_literal ()
-      || maybe_lit->get_lit_type () != AST::Literal::STRING)
+      || static_cast<AST::LiteralExpr &> (*node).get_lit_type ()
+	   != AST::Literal::STRING)
     {
       rust_error_at (parent_locus, "argument must be a string literal");
       if (node)
diff --git a/gcc/rust/expand/rust-macro-builtins-helpers.h b/gcc/rust/expand/rust-macro-builtins-helpers.h
index 429537e..32cf58f 100644
--- a/gcc/rust/expand/rust-macro-builtins-helpers.h
+++ b/gcc/rust/expand/rust-macro-builtins-helpers.h
@@ -33,29 +33,23 @@
 #include "rust-token.h"
 namespace Rust {
 
-std::string
-make_macro_path_str (BuiltinMacro kind);
+std::string make_macro_path_str (BuiltinMacro kind);
 
-std::vector<std::unique_ptr<AST::MacroInvocation>>
-check_for_eager_invocations (
+std::vector<std::unique_ptr<AST::MacroInvocation>> check_for_eager_invocations (
   std::vector<std::unique_ptr<AST::Expr>> &expressions);
 
 // Shorthand function for creating unique_ptr tokens
 
-std::unique_ptr<AST::Token>
-make_token (const TokenPtr tok);
+std::unique_ptr<AST::Token> make_token (const TokenPtr tok);
 
-std::unique_ptr<AST::Expr>
-make_string (location_t locus, std::string value);
+std::unique_ptr<AST::Expr> make_string (location_t locus, std::string value);
 // TODO: Is this correct?
-AST::Fragment
-make_eager_builtin_invocation (
+AST::Fragment make_eager_builtin_invocation (
   BuiltinMacro kind, location_t locus, AST::DelimTokenTree arguments,
   std::vector<std::unique_ptr<AST::MacroInvocation>> &&pending_invocations);
 // Match the end token of a macro given the start delimiter of the macro
-TokenId
-macro_end_token (AST::DelimTokenTree &invoc_token_tree,
-		 Parser<MacroInvocLexer> &parser);
+TokenId macro_end_token (AST::DelimTokenTree &invoc_token_tree,
+			 Parser<MacroInvocLexer> &parser);
 // Expand and then extract a string literal from the macro
 std::unique_ptr<AST::LiteralExpr>
 try_extract_string_literal_from_fragment (const location_t &parent_locus,
@@ -70,21 +64,18 @@ try_expand_many_expr (Parser<MacroInvocLexer> &parser,
 //   and return the LiteralExpr for it. Allow for an optional trailing comma,
 //   but otherwise enforce that these are the only tokens.
 
-std::unique_ptr<AST::Expr>
-parse_single_string_literal (BuiltinMacro kind,
-			     AST::DelimTokenTree &invoc_token_tree,
-			     location_t invoc_locus, MacroExpander *expander,
-			     bool is_semicoloned = false);
+std::unique_ptr<AST::Expr> parse_single_string_literal (
+  BuiltinMacro kind, AST::DelimTokenTree &invoc_token_tree,
+  location_t invoc_locus, MacroExpander *expander, bool is_semicoloned = false);
 
 // Treat PATH as a path relative to the source file currently being
 // compiled, and return the absolute path for it.
 
-std::string
-source_relative_path (std::string path, location_t locus);
+std::string source_relative_path (std::string path, location_t locus);
 
 // Read the full contents of the file FILENAME and return them in a vector.
 // FIXME: platform specific.
-tl::optional<std::vector<uint8_t>>
-load_file_bytes (location_t invoc_locus, const char *filename);
+tl::optional<std::vector<uint8_t>> load_file_bytes (location_t invoc_locus,
+						    const char *filename);
 } // namespace Rust
 #endif // GCCRS_RUST_MACRO_BUILTINS_HELPERS_H
diff --git a/gcc/rust/expand/rust-macro-builtins-offset-of.cc b/gcc/rust/expand/rust-macro-builtins-offset-of.cc
new file mode 100644
index 0000000..53efe74
--- /dev/null
+++ b/gcc/rust/expand/rust-macro-builtins-offset-of.cc
@@ -0,0 +1,78 @@
+// Copyright (C) 2020-2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "optional.h"
+#include "rust-ast-fragment.h"
+#include "rust-ast.h"
+#include "rust-builtin-ast-nodes.h"
+#include "rust-diagnostics.h"
+#include "rust-macro-builtins-helpers.h"
+#include "rust-macro-builtins.h"
+#include "rust-macro-invoc-lexer.h"
+#include "rust-parse.h"
+
+namespace Rust {
+
+tl::optional<AST::Fragment>
+MacroBuiltin::offset_of_handler (location_t invoc_locus,
+				 AST::MacroInvocData &invoc,
+				 AST::InvocKind semicolon)
+{
+  MacroInvocLexer lex (invoc.get_delim_tok_tree ().to_token_stream ());
+  Parser<MacroInvocLexer> parser (lex);
+
+  auto last_token = macro_end_token (invoc.get_delim_tok_tree (), parser);
+
+  auto type = parser.parse_type ();
+
+  // if we don't see a type, there might be an eager macro expansion missing
+  // FIXME: handle that
+  if (!type)
+    {
+      rust_error_at (invoc_locus, "could not parse type argument for %qs",
+		     "offset_of");
+
+      // we skip so we can still parse the field arg and check if it is correct
+      while (parser.peek_current_token ()->get_id () != COMMA
+	     && parser.peek_current_token ()->get_id () != last_token)
+	parser.skip_token ();
+    }
+
+  parser.skip_token (COMMA);
+
+  auto field_tok = parser.parse_identifier_or_keyword_token ();
+  auto invalid_field = !field_tok || !field_tok->has_str ();
+
+  if (invalid_field)
+    rust_error_at (invoc_locus, "could not parse field argument for %qs",
+		   "offset_of");
+
+  if (!type || invalid_field)
+    return tl::nullopt;
+
+  auto field = Identifier (field_tok->get_str ());
+
+  // FIXME: Do we need to do anything to handle the optional comma at the end?
+  parser.maybe_skip_token (COMMA);
+
+  return AST::Fragment ({AST::SingleASTNode (std::make_unique<AST::OffsetOf> (
+			  std::move (type), field, invoc_locus))},
+			invoc.get_delim_tok_tree ().to_token_stream ());
+}
+
+} // namespace Rust
diff --git a/gcc/rust/expand/rust-macro-builtins.cc b/gcc/rust/expand/rust-macro-builtins.cc
index b58ed71..a7ae220 100644
--- a/gcc/rust/expand/rust-macro-builtins.cc
+++ b/gcc/rust/expand/rust-macro-builtins.cc
@@ -162,6 +162,9 @@ std::unordered_map<std::string, AST::MacroTranscriberFunc>
     {"Ord", MacroBuiltin::proc_macro_builtin},
     {"PartialOrd", MacroBuiltin::proc_macro_builtin},
     {"Hash", MacroBuiltin::proc_macro_builtin},
+    /* offset_of is not declared in Rust 1.49 but still needed for
+       Rust-for-Linux, so we still create a transcriber and warn the user */
+    {"offset_of", MacroBuiltin::offset_of_handler},
 };
 
 tl::optional<BuiltinMacro>
diff --git a/gcc/rust/expand/rust-macro-builtins.h b/gcc/rust/expand/rust-macro-builtins.h
index 541e956..b6c2907 100644
--- a/gcc/rust/expand/rust-macro-builtins.h
+++ b/gcc/rust/expand/rust-macro-builtins.h
@@ -19,6 +19,7 @@
 #ifndef RUST_MACRO_BUILTINS_H
 #define RUST_MACRO_BUILTINS_H
 
+#include "optional.h"
 #include "rust-ast.h"
 #include "rust-builtin-ast-nodes.h"
 #include "rust-ast-fragment.h"
@@ -188,6 +189,9 @@ public:
   format_args_handler (location_t invoc_locus, AST::MacroInvocData &invoc,
 		       AST::InvocKind semicolon, AST::FormatArgs::Newline nl);
 
+  static tl::optional<AST::Fragment>
+  offset_of_handler (location_t, AST::MacroInvocData &, AST::InvocKind);
+
   static tl::optional<AST::Fragment> sorry (location_t invoc_locus,
 					    AST::MacroInvocData &invoc,
 					    AST::InvocKind semicolon);
diff --git a/gcc/rust/expand/rust-macro-expand.cc b/gcc/rust/expand/rust-macro-expand.cc
index 673b8fb..4c54cef 100644
--- a/gcc/rust/expand/rust-macro-expand.cc
+++ b/gcc/rust/expand/rust-macro-expand.cc
@@ -19,6 +19,7 @@
 #include "rust-macro-expand.h"
 #include "optional.h"
 #include "rust-ast-fragment.h"
+#include "rust-macro-builtins.h"
 #include "rust-macro-substitute-ctx.h"
 #include "rust-ast-full.h"
 #include "rust-ast-visitor.h"
@@ -287,6 +288,26 @@ MacroExpander::expand_invoc (AST::MacroInvocation &invoc,
   // lookup the rules
   auto rules_def = mappings.lookup_macro_invocation (invoc);
 
+  // We special case the `offset_of!()` macro if the flag is here and manually
+  // resolve to the builtin transcriber we have specified
+  auto assume_builtin_offset_of
+    = flag_assume_builtin_offset_of
+      && (invoc.get_invoc_data ().get_path ().as_string () == "offset_of")
+      && !rules_def;
+
+  // TODO: This is *massive hack* which should be removed as we progress to
+  // Rust 1.71 when offset_of gets added to core
+  if (assume_builtin_offset_of)
+    {
+      fragment = MacroBuiltin::offset_of_handler (invoc.get_locus (),
+						  invoc_data, semicolon)
+		   .value_or (AST::Fragment::create_empty ());
+
+      set_expanded_fragment (std::move (fragment));
+
+      return;
+    }
+
   // If there's no rule associated with the invocation, we can simply return
   // early. The early name resolver will have already emitted an error.
   if (!rules_def)
@@ -430,7 +451,8 @@ MacroExpander::match_fragment (Parser<MacroInvocLexer> &parser,
       parser.parse_visibility ();
       break;
 
-      case AST::MacroFragSpec::STMT: {
+    case AST::MacroFragSpec::STMT:
+      {
 	auto restrictions = ParseRestrictions ();
 	restrictions.consume_semi = false;
 	parser.parse_stmt (restrictions);
@@ -480,19 +502,22 @@ MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
   // this is used so we can check that we delimit the stream correctly.
   switch (delimiter->get_id ())
     {
-      case LEFT_PAREN: {
+    case LEFT_PAREN:
+      {
 	if (!check_delim (AST::DelimType::PARENS))
 	  return false;
       }
       break;
 
-      case LEFT_SQUARE: {
+    case LEFT_SQUARE:
+      {
 	if (!check_delim (AST::DelimType::SQUARE))
 	  return false;
       }
       break;
 
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	if (!check_delim (AST::DelimType::CURLY))
 	  return false;
       }
@@ -510,7 +535,8 @@ MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
 
       switch (match->get_macro_match_type ())
 	{
-	  case AST::MacroMatch::MacroMatchType::Fragment: {
+	case AST::MacroMatch::MacroMatchType::Fragment:
+	  {
 	    AST::MacroMatchFragment *fragment
 	      = static_cast<AST::MacroMatchFragment *> (match.get ());
 	    if (!match_fragment (parser, *fragment))
@@ -524,14 +550,16 @@ MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
 	  }
 	  break;
 
-	  case AST::MacroMatch::MacroMatchType::Tok: {
+	case AST::MacroMatch::MacroMatchType::Tok:
+	  {
 	    AST::Token *tok = static_cast<AST::Token *> (match.get ());
 	    if (!match_token (parser, *tok))
 	      return false;
 	  }
 	  break;
 
-	  case AST::MacroMatch::MacroMatchType::Repetition: {
+	case AST::MacroMatch::MacroMatchType::Repetition:
+	  {
 	    AST::MacroMatchRepetition *rep
 	      = static_cast<AST::MacroMatchRepetition *> (match.get ());
 	    if (!match_repetition (parser, *rep))
@@ -539,7 +567,8 @@ MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
 	  }
 	  break;
 
-	  case AST::MacroMatch::MacroMatchType::Matcher: {
+	case AST::MacroMatch::MacroMatchType::Matcher:
+	  {
 	    AST::MacroMatcher *m
 	      = static_cast<AST::MacroMatcher *> (match.get ());
 	    expansion_depth++;
@@ -556,19 +585,22 @@ MacroExpander::match_matcher (Parser<MacroInvocLexer> &parser,
 
   switch (delimiter->get_id ())
     {
-      case LEFT_PAREN: {
+    case LEFT_PAREN:
+      {
 	if (!parser.skip_token (RIGHT_PAREN))
 	  return false;
       }
       break;
 
-      case LEFT_SQUARE: {
+    case LEFT_SQUARE:
+      {
 	if (!parser.skip_token (RIGHT_SQUARE))
 	  return false;
       }
       break;
 
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	if (!parser.skip_token (RIGHT_CURLY))
 	  return false;
       }
@@ -617,7 +649,8 @@ MacroExpander::match_n_matches (Parser<MacroInvocLexer> &parser,
 	  size_t offs_begin = source.get_offs ();
 	  switch (match->get_macro_match_type ())
 	    {
-	      case AST::MacroMatch::MacroMatchType::Fragment: {
+	    case AST::MacroMatch::MacroMatchType::Fragment:
+	      {
 		AST::MacroMatchFragment *fragment
 		  = static_cast<AST::MacroMatchFragment *> (match.get ());
 		valid_current_match = match_fragment (parser, *fragment);
@@ -632,20 +665,23 @@ MacroExpander::match_n_matches (Parser<MacroInvocLexer> &parser,
 	      }
 	      break;
 
-	      case AST::MacroMatch::MacroMatchType::Tok: {
+	    case AST::MacroMatch::MacroMatchType::Tok:
+	      {
 		AST::Token *tok = static_cast<AST::Token *> (match.get ());
 		valid_current_match = match_token (parser, *tok);
 	      }
 	      break;
 
-	      case AST::MacroMatch::MacroMatchType::Repetition: {
+	    case AST::MacroMatch::MacroMatchType::Repetition:
+	      {
 		AST::MacroMatchRepetition *rep
 		  = static_cast<AST::MacroMatchRepetition *> (match.get ());
 		valid_current_match = match_repetition (parser, *rep);
 	      }
 	      break;
 
-	      case AST::MacroMatch::MacroMatchType::Matcher: {
+	    case AST::MacroMatch::MacroMatchType::Matcher:
+	      {
 		AST::MacroMatcher *m
 		  = static_cast<AST::MacroMatcher *> (match.get ());
 		valid_current_match = match_matcher (parser, *m, true);
diff --git a/gcc/rust/expand/rust-macro-substitute-ctx.cc b/gcc/rust/expand/rust-macro-substitute-ctx.cc
index 02e4e3b..ac36ed8 100644
--- a/gcc/rust/expand/rust-macro-substitute-ctx.cc
+++ b/gcc/rust/expand/rust-macro-substitute-ctx.cc
@@ -273,7 +273,8 @@ SubstituteCtx::substitute_token (size_t token_idx)
       // don't substitute, dollar sign is alone/metavar is unknown
       return {std::vector<std::unique_ptr<AST::Token>> (), 0};
 
-      case LEFT_PAREN: {
+    case LEFT_PAREN:
+      {
 	// We need to parse up until the closing delimiter and expand this
 	// fragment->n times.
 	rust_debug ("expanding repetition");
diff --git a/gcc/rust/expand/rust-proc-macro.h b/gcc/rust/expand/rust-proc-macro.h
index 6ffaaf6..058c93a 100644
--- a/gcc/rust/expand/rust-proc-macro.h
+++ b/gcc/rust/expand/rust-proc-macro.h
@@ -82,11 +82,9 @@ public:
  *
  * @param The path to the shared object file to load.
  */
-const std::vector<ProcMacro::Procmacro>
-load_macros (std::string path);
+const std::vector<ProcMacro::Procmacro> load_macros (std::string path);
 
-std::string
-generate_proc_macro_decls_symbol (std::uint32_t stable_crate_id);
+std::string generate_proc_macro_decls_symbol (std::uint32_t stable_crate_id);
 
 } // namespace Rust
 
diff --git a/gcc/rust/expand/rust-token-tree-desugar.cc b/gcc/rust/expand/rust-token-tree-desugar.cc
index 3b47180..aa20d50 100644
--- a/gcc/rust/expand/rust-token-tree-desugar.cc
+++ b/gcc/rust/expand/rust-token-tree-desugar.cc
@@ -68,5 +68,5 @@ TokenTreeDesugar::visit (Token &tts)
     }
 }
 
-}; // namespace AST
-}; // namespace Rust
+} // namespace AST
+} // namespace Rust
diff --git a/gcc/rust/expand/rust-token-tree-desugar.h b/gcc/rust/expand/rust-token-tree-desugar.h
index ccba53b..da9d732 100644
--- a/gcc/rust/expand/rust-token-tree-desugar.h
+++ b/gcc/rust/expand/rust-token-tree-desugar.h
@@ -49,7 +49,7 @@ private:
   virtual void visit (Token &tts) override;
 };
 
-}; // namespace AST
-}; // namespace Rust
+} // namespace AST
+} // namespace Rust
 
 #endif //! RUST_TOKEN_TREE_DESUGAR_H
diff --git a/gcc/rust/hir/rust-ast-lower-base.cc b/gcc/rust/hir/rust-ast-lower-base.cc
index 2d9a445..b723f59 100644
--- a/gcc/rust/hir/rust-ast-lower-base.cc
+++ b/gcc/rust/hir/rust-ast-lower-base.cc
@@ -47,6 +47,27 @@ ASTLoweringBase::visit (AST::ErrorPropagationExpr &expr)
 }
 
 void
+ASTLoweringBase::visit (AST::TryExpr &expr)
+{
+  rust_fatal_error (expr.get_locus (), "missing desugar for try-blocks");
+  rust_unreachable ();
+}
+
+void
+ASTLoweringBase::visit (AST::ForLoopExpr &expr)
+{
+  rust_fatal_error (expr.get_locus (), "missing desugar for for-loops");
+  rust_unreachable ();
+}
+
+void
+ASTLoweringBase::visit (AST::WhileLetLoopExpr &expr)
+{
+  rust_fatal_error (expr.get_locus (), "missing desugar for while-let loops");
+  rust_unreachable ();
+}
+
+void
 ASTLoweringBase::visit (AST::Token &)
 {}
 void
@@ -115,7 +136,7 @@ void
 ASTLoweringBase::visit (AST::MetaItemLitExpr &)
 {}
 void
-ASTLoweringBase::visit (AST::MetaItemPathLit &)
+ASTLoweringBase::visit (AST::MetaItemPathExpr &)
 {}
 void
 ASTLoweringBase::visit (AST::BorrowExpr &)
@@ -201,6 +222,12 @@ void
 ASTLoweringBase::visit (AST::BlockExpr &)
 {}
 void
+ASTLoweringBase::visit (AST::AnonConst &)
+{}
+void
+ASTLoweringBase::visit (AST::ConstBlock &)
+{}
+void
 ASTLoweringBase::visit (AST::ClosureExprInnerTyped &)
 {}
 void
@@ -245,12 +272,6 @@ void
 ASTLoweringBase::visit (AST::WhileLoopExpr &)
 {}
 void
-ASTLoweringBase::visit (AST::WhileLetLoopExpr &)
-{}
-void
-ASTLoweringBase::visit (AST::ForLoopExpr &)
-{}
-void
 ASTLoweringBase::visit (AST::IfExpr &)
 {}
 void
@@ -474,6 +495,12 @@ void
 ASTLoweringBase::visit (AST::GroupedPattern &)
 {}
 void
+ASTLoweringBase::visit (AST::SlicePatternItemsNoRest &)
+{}
+void
+ASTLoweringBase::visit (AST::SlicePatternItemsHasRest &)
+{}
+void
 ASTLoweringBase::visit (AST::SlicePattern &)
 {}
 void
@@ -551,6 +578,10 @@ void
 ASTLoweringBase::visit (AST::FormatArgs &fmt)
 {}
 
+void
+ASTLoweringBase::visit (AST::OffsetOf &offset_of)
+{}
+
 HIR::Lifetime
 ASTLoweringBase::lower_lifetime (AST::Lifetime &lifetime,
 				 bool default_to_static_lifetime)
@@ -648,12 +679,14 @@ ASTLoweringBase::lower_generic_args (AST::GenericArgs &args)
     {
       switch (arg.get_kind ())
 	{
-	  case AST::GenericArg::Kind::Type: {
+	case AST::GenericArg::Kind::Type:
+	  {
 	    auto type = ASTLoweringType::translate (arg.get_type ());
 	    type_args.emplace_back (std::unique_ptr<HIR::Type> (type));
 	    break;
 	  }
-	  case AST::GenericArg::Kind::Const: {
+	case AST::GenericArg::Kind::Const:
+	  {
 	    auto expr = ASTLoweringExpr::translate (arg.get_expression ());
 	    const_args.emplace_back (
 	      HIR::ConstGenericArg (std::unique_ptr<HIR::Expr> (expr),
@@ -887,7 +920,8 @@ ASTLoweringBase::lower_range_pattern_bound (AST::RangePatternBound &bound)
   std::unique_ptr<HIR::RangePatternBound> hir_bound = nullptr;
   switch (bound.get_bound_type ())
     {
-      case AST::RangePatternBound::RangePatternBoundType::LITERAL: {
+    case AST::RangePatternBound::RangePatternBoundType::LITERAL:
+      {
 	AST::RangePatternBoundLiteral &ref
 	  = static_cast<AST::RangePatternBoundLiteral &> (bound);
 
@@ -898,7 +932,8 @@ ASTLoweringBase::lower_range_pattern_bound (AST::RangePatternBound &bound)
 					     ref.get_has_minus ()));
       }
       break;
-      case AST::RangePatternBound::RangePatternBoundType::PATH: {
+    case AST::RangePatternBound::RangePatternBoundType::PATH:
+      {
 	auto &ref = static_cast<AST::RangePatternBoundPath &> (bound);
 
 	HIR::PathInExpression *path
@@ -908,7 +943,8 @@ ASTLoweringBase::lower_range_pattern_bound (AST::RangePatternBound &bound)
 	  new HIR::RangePatternBoundPath (*path));
       }
       break;
-      case AST::RangePatternBound::RangePatternBoundType::QUALPATH: {
+    case AST::RangePatternBound::RangePatternBoundType::QUALPATH:
+      {
 	auto &ref = static_cast<AST::RangePatternBoundQualPath &> (bound);
 
 	HIR::QualifiedPathInExpression *qualpath
diff --git a/gcc/rust/hir/rust-ast-lower-base.h b/gcc/rust/hir/rust-ast-lower-base.h
index 3116181..e86aacb 100644
--- a/gcc/rust/hir/rust-ast-lower-base.h
+++ b/gcc/rust/hir/rust-ast-lower-base.h
@@ -20,6 +20,8 @@
 #define RUST_AST_LOWER_BASE
 
 #include "rust-ast.h"
+#include "rust-builtin-ast-nodes.h"
+#include "rust-expr.h"
 #include "rust-system.h"
 #include "rust-ast-full.h"
 #include "rust-ast-visitor.h"
@@ -63,6 +65,9 @@ public:
   // Special casing nodes that should never reach the HIR lowering stage
   virtual void visit (AST::MacroInvocation &) override final;
   virtual void visit (AST::ErrorPropagationExpr &) override final;
+  virtual void visit (AST::ForLoopExpr &) override final;
+  virtual void visit (AST::TryExpr &) override final;
+  virtual void visit (AST::WhileLetLoopExpr &) override final;
 
   // visitor impl
   // rust-ast.h
@@ -101,7 +106,7 @@ public:
   virtual void visit (AST::AttrInputLiteral &attr_input) override;
   virtual void visit (AST::AttrInputMacro &attr_input) override;
   virtual void visit (AST::MetaItemLitExpr &meta_item) override;
-  virtual void visit (AST::MetaItemPathLit &meta_item) override;
+  virtual void visit (AST::MetaItemPathExpr &meta_item) override;
   virtual void visit (AST::BorrowExpr &expr) override;
   virtual void visit (AST::DereferenceExpr &expr) override;
   virtual void visit (AST::NegationExpr &expr) override;
@@ -131,6 +136,8 @@ public:
   virtual void visit (AST::FieldAccessExpr &expr) override;
   virtual void visit (AST::ClosureExprInner &expr) override;
   virtual void visit (AST::BlockExpr &expr) override;
+  virtual void visit (AST::AnonConst &expr) override;
+  virtual void visit (AST::ConstBlock &expr) override;
   virtual void visit (AST::ClosureExprInnerTyped &expr) override;
   virtual void visit (AST::ContinueExpr &expr) override;
   virtual void visit (AST::BreakExpr &expr) override;
@@ -145,8 +152,6 @@ public:
   virtual void visit (AST::UnsafeBlockExpr &expr) override;
   virtual void visit (AST::LoopExpr &expr) override;
   virtual void visit (AST::WhileLoopExpr &expr) override;
-  virtual void visit (AST::WhileLetLoopExpr &expr) override;
-  virtual void visit (AST::ForLoopExpr &expr) override;
   virtual void visit (AST::IfExpr &expr) override;
   virtual void visit (AST::IfExprConseqElse &expr) override;
   virtual void visit (AST::IfLetExpr &expr) override;
@@ -231,6 +236,8 @@ public:
   virtual void visit (AST::TuplePatternItemsRanged &tuple_items) override;
   virtual void visit (AST::TuplePattern &pattern) override;
   virtual void visit (AST::GroupedPattern &pattern) override;
+  virtual void visit (AST::SlicePatternItemsNoRest &items) override;
+  virtual void visit (AST::SlicePatternItemsHasRest &items) override;
   virtual void visit (AST::SlicePattern &pattern) override;
   virtual void visit (AST::AltPattern &pattern) override;
 
@@ -259,6 +266,7 @@ public:
   virtual void visit (AST::SelfParam &param) override;
 
   virtual void visit (AST::FormatArgs &fmt) override;
+  virtual void visit (AST::OffsetOf &offset_of) override;
 
 protected:
   ASTLoweringBase ()
diff --git a/gcc/rust/hir/rust-ast-lower-block.h b/gcc/rust/hir/rust-ast-lower-block.h
index 93cd443..f10039b 100644
--- a/gcc/rust/hir/rust-ast-lower-block.h
+++ b/gcc/rust/hir/rust-ast-lower-block.h
@@ -213,8 +213,6 @@ public:
 
   void visit (AST::WhileLoopExpr &expr) override;
 
-  void visit (AST::ForLoopExpr &expr) override;
-
   void visit (AST::MatchExpr &expr) override;
 
 private:
diff --git a/gcc/rust/hir/rust-ast-lower-expr.cc b/gcc/rust/hir/rust-ast-lower-expr.cc
index 07d0c835..4ed51d9 100644
--- a/gcc/rust/hir/rust-ast-lower-expr.cc
+++ b/gcc/rust/hir/rust-ast-lower-expr.cc
@@ -24,7 +24,9 @@
 #include "rust-ast-lower-pattern.h"
 #include "rust-ast-lower-type.h"
 #include "rust-ast.h"
+#include "rust-builtin-ast-nodes.h"
 #include "rust-diagnostics.h"
+#include "rust-hir-map.h"
 #include "rust-system.h"
 #include "tree/rust-hir-expr.h"
 
@@ -127,6 +129,50 @@ ASTLoweringExpr::visit (AST::BlockExpr &expr)
 }
 
 void
+ASTLoweringExpr::visit (AST::AnonConst &expr)
+{
+  auto &mappings = Analysis::Mappings::get ();
+  auto crate_num = mappings.get_current_crate ();
+  auto mapping = Analysis::NodeMapping (crate_num, expr.get_node_id (),
+					mappings.get_next_hir_id (crate_num),
+					UNKNOWN_LOCAL_DEFID);
+
+  if (expr.is_deferred ())
+    {
+      translated = new HIR::AnonConst (std::move (mapping), expr.get_locus ());
+    }
+  else
+    {
+      auto inner_expr = ASTLoweringExpr::translate (expr.get_inner_expr ());
+
+      translated = new HIR::AnonConst (std::move (mapping),
+				       std::unique_ptr<Expr> (inner_expr),
+				       expr.get_locus ());
+    }
+}
+
+void
+ASTLoweringExpr::visit (AST::ConstBlock &expr)
+{
+  auto inner_expr = ASTLoweringExpr::translate (expr.get_const_expr ());
+
+  // we know this will always be an `AnonConst`, or we have an issue. Let's
+  // assert just to be sure.
+  rust_assert (inner_expr->get_expression_type () == Expr::ExprType::AnonConst);
+  auto anon_const = static_cast<AnonConst *> (inner_expr);
+
+  auto &mappings = Analysis::Mappings::get ();
+  auto crate_num = mappings.get_current_crate ();
+  auto mapping = Analysis::NodeMapping (crate_num, expr.get_node_id (),
+					mappings.get_next_hir_id (crate_num),
+					UNKNOWN_LOCAL_DEFID);
+
+  translated
+    = new HIR::ConstBlock (std::move (mapping), std::move (*anon_const),
+			   expr.get_locus (), expr.get_outer_attrs ());
+}
+
+void
 ASTLoweringExpr::visit (AST::UnsafeBlockExpr &expr)
 {
   translated = ASTLoweringBlock::translate (expr, &terminated);
@@ -589,12 +635,6 @@ ASTLoweringExpr::visit (AST::WhileLoopExpr &expr)
 }
 
 void
-ASTLoweringExpr::visit (AST::ForLoopExpr &expr)
-{
-  rust_unreachable ();
-}
-
-void
 ASTLoweringExpr::visit (AST::BreakExpr &expr)
 {
   tl::optional<HIR::Lifetime> break_label = tl::nullopt;
@@ -798,7 +838,7 @@ ASTLoweringExpr::visit (AST::ClosureExprInnerTyped &expr)
 {
   HIR::Type *closure_return_type = nullptr;
   HIR::Expr *closure_expr
-    = ASTLoweringExpr::translate (expr.get_definition_block ());
+    = ASTLoweringExpr::translate (expr.get_definition_expr ());
 
   std::vector<HIR::ClosureParam> closure_params;
   for (auto &param : expr.get_params ())
@@ -841,6 +881,7 @@ translate_operand_out (const AST::InlineAsmOperand &operand)
 					     *out_value.expr.get ())));
   return out;
 }
+
 HIR::InlineAsmOperand
 translate_operand_inout (const AST::InlineAsmOperand &operand)
 {
@@ -851,6 +892,7 @@ translate_operand_inout (const AST::InlineAsmOperand &operand)
 						 *inout_value.expr.get ())));
   return inout;
 }
+
 HIR::InlineAsmOperand
 translate_operand_split_in_out (const AST::InlineAsmOperand &operand)
 {
@@ -863,19 +905,21 @@ translate_operand_split_in_out (const AST::InlineAsmOperand &operand)
       ASTLoweringExpr::translate (*split_in_out_value.out_expr.get ())));
   return split_in_out;
 }
+
 HIR::InlineAsmOperand
 translate_operand_const (const AST::InlineAsmOperand &operand)
 {
   auto const_value = operand.get_const ();
-  struct HIR::AnonConst anon_const (const_value.anon_const.id,
-				    std::unique_ptr<Expr> (
-				      ASTLoweringExpr::translate (
-					*const_value.anon_const.expr.get ())));
-  struct HIR::InlineAsmOperand::Const cnst
-  {
-    anon_const
-  };
-  return cnst;
+
+  auto inner_expr = ASTLoweringExpr::translate (const_value.anon_const);
+
+  // Like `ConstBlock`, we know this should only be an `AnonConst` - let's
+  // assert to make sure and static cast
+  rust_assert (inner_expr->get_expression_type () == Expr::ExprType::AnonConst);
+
+  auto anon_const = static_cast<AnonConst *> (inner_expr);
+
+  return HIR::InlineAsmOperand::Const{*anon_const};
 }
 
 HIR::InlineAsmOperand
@@ -1006,5 +1050,20 @@ ASTLoweringExpr::visit (AST::FormatArgs &fmt)
 		 "FormatArgs lowering is not implemented yet");
 }
 
+void
+ASTLoweringExpr::visit (AST::OffsetOf &offset_of)
+{
+  auto type = std::unique_ptr<Type> (
+    ASTLoweringType::translate (offset_of.get_type ()));
+
+  auto crate_num = mappings.get_current_crate ();
+  Analysis::NodeMapping mapping (crate_num, offset_of.get_node_id (),
+				 mappings.get_next_hir_id (crate_num),
+				 mappings.get_next_localdef_id (crate_num));
+
+  translated = new HIR::OffsetOf (std::move (type), offset_of.get_field (),
+				  mapping, offset_of.get_locus ());
+}
+
 } // namespace HIR
 } // namespace Rust
diff --git a/gcc/rust/hir/rust-ast-lower-expr.h b/gcc/rust/hir/rust-ast-lower-expr.h
index adedeb3..4eed4ec 100644
--- a/gcc/rust/hir/rust-ast-lower-expr.h
+++ b/gcc/rust/hir/rust-ast-lower-expr.h
@@ -21,6 +21,7 @@
 
 #include "rust-ast-lower-base.h"
 #include "rust-ast.h"
+#include "rust-expr.h"
 
 namespace Rust {
 namespace HIR {
@@ -82,6 +83,8 @@ public:
   void visit (AST::IfLetExpr &expr) override;
   void visit (AST::IfLetExprConseqElse &expr) override;
   void visit (AST::BlockExpr &expr) override;
+  void visit (AST::AnonConst &expr) override;
+  void visit (AST::ConstBlock &expr) override;
   void visit (AST::UnsafeBlockExpr &expr) override;
   void visit (AST::PathInExpression &expr) override;
   void visit (AST::QualifiedPathInExpression &expr) override;
@@ -108,7 +111,6 @@ public:
   void visit (AST::FieldAccessExpr &expr) override;
   void visit (AST::LoopExpr &expr) override;
   void visit (AST::WhileLoopExpr &expr) override;
-  void visit (AST::ForLoopExpr &expr) override;
   void visit (AST::BreakExpr &expr) override;
   void visit (AST::ContinueExpr &expr) override;
   void visit (AST::BorrowExpr &expr) override;
@@ -124,8 +126,9 @@ public:
   void visit (AST::InlineAsm &expr) override;
   void visit (AST::LlvmInlineAsm &expr) override;
 
-  // Extra visitor for FormatArgs nodes
+  // Extra visitor for builtin macro nodes
   void visit (AST::FormatArgs &fmt) override;
+  void visit (AST::OffsetOf &offset_of) override;
 
 private:
   ASTLoweringExpr ();
diff --git a/gcc/rust/hir/rust-ast-lower-extern.h b/gcc/rust/hir/rust-ast-lower-extern.h
index 0105e38..3dca1b6 100644
--- a/gcc/rust/hir/rust-ast-lower-extern.h
+++ b/gcc/rust/hir/rust-ast-lower-extern.h
@@ -99,7 +99,7 @@ public:
 	  = static_cast<AST::IdentifierPattern &> (param.get_pattern ());
 	Identifier param_name = param_kind == AST::Pattern::Kind::Identifier
 				  ? param_ident.get_ident ()
-				  : std::string ("_");
+				  : Identifier ("_", param.get_locus ());
 
 	HIR::Type *param_type = ASTLoweringType::translate (param.get_type ());
 
diff --git a/gcc/rust/hir/rust-ast-lower-implitem.cc b/gcc/rust/hir/rust-ast-lower-implitem.cc
index d815a71..5db11cb 100644
--- a/gcc/rust/hir/rust-ast-lower-implitem.cc
+++ b/gcc/rust/hir/rust-ast-lower-implitem.cc
@@ -138,7 +138,8 @@ ASTLowerImplItem::visit (AST::Function &function)
 
   std::unique_ptr<HIR::Type> return_type
     = function.has_return_type () ? std::unique_ptr<HIR::Type> (
-	ASTLoweringType::translate (function.get_return_type ()))
+	ASTLoweringType::translate (function.get_return_type (), false,
+				    true /* impl trait is allowed here*/))
 				  : nullptr;
 
   Defaultness defaultness
@@ -274,7 +275,16 @@ ASTLowerTraitItem::visit (AST::Function &func)
       auto hir_param
 	= HIR::FunctionParam (mapping, std::move (translated_pattern),
 			      std::move (translated_type), param.get_locus ());
-      function_params.push_back (hir_param);
+      function_params.push_back (std::move (hir_param));
+    }
+
+  if (func.has_self_param ())
+    {
+      // insert mappings for self
+      // TODO: Is this correct ? Looks fishy
+      mappings.insert_hir_self_param (&*self_param);
+      mappings.insert_location (self_param->get_mappings ().get_hirid (),
+				self_param->get_locus ());
     }
 
   HIR::TraitFunctionDecl decl (func.get_function_name (),
@@ -300,14 +310,6 @@ ASTLowerTraitItem::visit (AST::Function &func)
     = new HIR::TraitItemFunc (mapping, std::move (decl), std::move (block_expr),
 			      func.get_outer_attrs (), func.get_locus ());
   translated = trait_item;
-  if (func.has_self_param ())
-    {
-      // insert mappings for self
-      // TODO: Is this correct ? Looks fishy
-      mappings.insert_hir_self_param (&*self_param);
-      mappings.insert_location (self_param->get_mappings ().get_hirid (),
-				self_param->get_locus ());
-    }
 
   // add the mappings for the function params at the end
   for (auto &param : trait_item->get_decl ().get_function_params ())
diff --git a/gcc/rust/hir/rust-ast-lower-item.cc b/gcc/rust/hir/rust-ast-lower-item.cc
index f4396b5..4e5a747 100644
--- a/gcc/rust/hir/rust-ast-lower-item.cc
+++ b/gcc/rust/hir/rust-ast-lower-item.cc
@@ -217,7 +217,7 @@ ASTLoweringItem::visit (AST::StructStruct &struct_decl)
 					 field.get_outer_attrs ());
 
       if (struct_field_name_exists (fields, translated_field))
-	break;
+	continue;
 
       fields.push_back (std::move (translated_field));
     }
@@ -367,7 +367,9 @@ ASTLoweringItem::visit (AST::ConstantItem &constant)
   HIR::Visibility vis = translate_visibility (constant.get_visibility ());
 
   HIR::Type *type = ASTLoweringType::translate (constant.get_type (), true);
-  HIR::Expr *expr = ASTLoweringExpr::translate (constant.get_expr ());
+  HIR::Expr *expr = nullptr;
+  if (constant.has_expr ())
+    expr = ASTLoweringExpr::translate (constant.get_expr ());
 
   auto crate_num = mappings.get_current_crate ();
   Analysis::NodeMapping mapping (crate_num, constant.get_node_id (),
@@ -411,7 +413,8 @@ ASTLoweringItem::visit (AST::Function &function)
 
   std::unique_ptr<HIR::Type> return_type
     = function.has_return_type () ? std::unique_ptr<HIR::Type> (
-	ASTLoweringType::translate (function.get_return_type ()))
+	ASTLoweringType::translate (function.get_return_type (), false,
+				    true /* impl trait is allowed here*/))
 				  : nullptr;
 
   std::vector<HIR::FunctionParam> function_params;
@@ -494,7 +497,8 @@ ASTLoweringItem::visit (AST::InherentImpl &impl_block)
 	{
 	  switch (generic_param->get_kind ())
 	    {
-	      case HIR::GenericParam::GenericKind::TYPE: {
+	    case HIR::GenericParam::GenericKind::TYPE:
+	      {
 		const HIR::TypeParam &t
 		  = static_cast<const HIR::TypeParam &> (*generic_param);
 
@@ -651,7 +655,8 @@ ASTLoweringItem::visit (AST::TraitImpl &impl_block)
 	{
 	  switch (generic_param->get_kind ())
 	    {
-	      case HIR::GenericParam::GenericKind::TYPE: {
+	    case HIR::GenericParam::GenericKind::TYPE:
+	      {
 		const HIR::TypeParam &t
 		  = static_cast<const HIR::TypeParam &> (*generic_param);
 
@@ -729,6 +734,25 @@ ASTLoweringItem::visit (AST::MacroRulesDefinition &def)
   lower_macro_definition (def);
 }
 
+void
+ASTLoweringItem::visit (AST::ExternCrate &extern_crate)
+{
+  if (extern_crate.references_self ())
+    return;
+
+  auto &mappings = Analysis::Mappings::get ();
+  CrateNum num
+    = mappings.lookup_crate_name (extern_crate.get_referenced_crate ())
+	.value ();
+  AST::Crate &crate = mappings.get_ast_crate (num);
+
+  auto saved_crate_num = mappings.get_current_crate ();
+  mappings.set_current_crate (num);
+  auto lowered = ASTLowering::Resolve (crate);
+  mappings.insert_hir_crate (std::move (lowered));
+  mappings.set_current_crate (saved_crate_num);
+}
+
 HIR::SimplePath
 ASTLoweringSimplePath::translate (const AST::SimplePath &path)
 {
diff --git a/gcc/rust/hir/rust-ast-lower-item.h b/gcc/rust/hir/rust-ast-lower-item.h
index 4e142ed..dc75057 100644
--- a/gcc/rust/hir/rust-ast-lower-item.h
+++ b/gcc/rust/hir/rust-ast-lower-item.h
@@ -45,6 +45,7 @@ public:
   void visit (AST::TraitImpl &impl_block) override;
   void visit (AST::ExternBlock &extern_block) override;
   void visit (AST::MacroRulesDefinition &rules_def) override;
+  void visit (AST::ExternCrate &extern_crate) override;
 
 private:
   ASTLoweringItem () : translated (nullptr) {}
diff --git a/gcc/rust/hir/rust-ast-lower-pattern.cc b/gcc/rust/hir/rust-ast-lower-pattern.cc
index b7a4c56..8aabcd8 100644
--- a/gcc/rust/hir/rust-ast-lower-pattern.cc
+++ b/gcc/rust/hir/rust-ast-lower-pattern.cc
@@ -23,7 +23,9 @@
 namespace Rust {
 namespace HIR {
 
-ASTLoweringPattern::ASTLoweringPattern () : translated (nullptr) {}
+ASTLoweringPattern::ASTLoweringPattern ()
+  : translated (nullptr), is_let_top_level (false)
+{}
 
 HIR::Pattern *
 ASTLoweringPattern::translate (AST::Pattern &pattern, bool is_let_top_level)
@@ -49,13 +51,18 @@ ASTLoweringPattern::visit (AST::IdentifierPattern &pattern)
 				 mappings.get_next_hir_id (crate_num),
 				 UNKNOWN_LOCAL_DEFID);
 
-  std::unique_ptr<Pattern> to_bind;
+  std::unique_ptr<Pattern> subpattern;
+  if (pattern.has_subpattern ())
+    {
+      subpattern = std::unique_ptr<Pattern> (
+	ASTLoweringPattern::translate (pattern.get_subpattern ()));
+    }
   translated
     = new HIR::IdentifierPattern (mapping, pattern.get_ident (),
 				  pattern.get_locus (), pattern.get_is_ref (),
 				  pattern.get_is_mut () ? Mutability::Mut
 							: Mutability::Imm,
-				  std::move (to_bind));
+				  std::move (subpattern));
 }
 
 void
@@ -74,13 +81,15 @@ ASTLoweringPattern::visit (AST::TupleStructPattern &pattern)
   auto &items = pattern.get_items ();
   switch (items.get_item_type ())
     {
-      case AST::TupleStructItems::RANGE: {
+    case AST::TupleStructItems::RANGE:
+      {
 	// TODO
 	rust_unreachable ();
       }
       break;
 
-      case AST::TupleStructItems::NO_RANGE: {
+    case AST::TupleStructItems::NO_RANGE:
+      {
 	AST::TupleStructItemsNoRange &items_no_range
 	  = static_cast<AST::TupleStructItemsNoRange &> (items);
 
@@ -120,7 +129,8 @@ ASTLoweringPattern::visit (AST::StructPattern &pattern)
       HIR::StructPatternField *f = nullptr;
       switch (field->get_item_type ())
 	{
-	  case AST::StructPatternField::ItemType::TUPLE_PAT: {
+	case AST::StructPatternField::ItemType::TUPLE_PAT:
+	  {
 	    auto &tuple
 	      = static_cast<AST::StructPatternFieldTuplePat &> (*field);
 
@@ -140,7 +150,8 @@ ASTLoweringPattern::visit (AST::StructPattern &pattern)
 	  }
 	  break;
 
-	  case AST::StructPatternField::ItemType::IDENT_PAT: {
+	case AST::StructPatternField::ItemType::IDENT_PAT:
+	  {
 	    AST::StructPatternFieldIdentPat &ident
 	      = static_cast<AST::StructPatternFieldIdentPat &> (*field);
 
@@ -160,7 +171,8 @@ ASTLoweringPattern::visit (AST::StructPattern &pattern)
 	  }
 	  break;
 
-	  case AST::StructPatternField::ItemType::IDENT: {
+	case AST::StructPatternField::ItemType::IDENT:
+	  {
 	    AST::StructPatternFieldIdent &ident
 	      = static_cast<AST::StructPatternFieldIdent &> (*field.get ());
 
@@ -213,7 +225,8 @@ ASTLoweringPattern::visit (AST::TuplePattern &pattern)
   std::unique_ptr<HIR::TuplePatternItems> items;
   switch (pattern.get_items ().get_pattern_type ())
     {
-      case AST::TuplePatternItems::TuplePatternItemType::MULTIPLE: {
+    case AST::TuplePatternItems::TuplePatternItemType::MULTIPLE:
+      {
 	AST::TuplePatternItemsMultiple &ref
 	  = static_cast<AST::TuplePatternItemsMultiple &> (
 	    pattern.get_items ());
@@ -221,7 +234,8 @@ ASTLoweringPattern::visit (AST::TuplePattern &pattern)
       }
       break;
 
-      case AST::TuplePatternItems::TuplePatternItemType::RANGED: {
+    case AST::TuplePatternItems::TuplePatternItemType::RANGED:
+      {
 	AST::TuplePatternItemsRanged &ref
 	  = static_cast<AST::TuplePatternItemsRanged &> (pattern.get_items ());
 	items = lower_tuple_pattern_ranged (ref);
@@ -309,10 +323,27 @@ void
 ASTLoweringPattern::visit (AST::SlicePattern &pattern)
 {
   std::vector<std::unique_ptr<HIR::Pattern>> items;
-  for (auto &p : pattern.get_items ())
+
+  switch (pattern.get_items ().get_pattern_type ())
     {
-      HIR::Pattern *item = ASTLoweringPattern::translate (*p);
-      items.push_back (std::unique_ptr<HIR::Pattern> (item));
+    case AST::SlicePatternItems::SlicePatternItemType::NO_REST:
+      {
+	AST::SlicePatternItemsNoRest &ref
+	  = static_cast<AST::SlicePatternItemsNoRest &> (pattern.get_items ());
+	for (auto &p : ref.get_patterns ())
+	  {
+	    HIR::Pattern *item = ASTLoweringPattern::translate (*p);
+	    items.push_back (std::unique_ptr<HIR::Pattern> (item));
+	  }
+      }
+      break;
+    case AST::SlicePatternItems::SlicePatternItemType::HAS_REST:
+      {
+	rust_error_at (pattern.get_locus (),
+		       "lowering of slice patterns with rest elements are not "
+		       "supported yet");
+      }
+      break;
     }
 
   auto crate_num = mappings.get_current_crate ();
diff --git a/gcc/rust/hir/rust-ast-lower-type.cc b/gcc/rust/hir/rust-ast-lower-type.cc
index a678f18..1841576 100644
--- a/gcc/rust/hir/rust-ast-lower-type.cc
+++ b/gcc/rust/hir/rust-ast-lower-type.cc
@@ -209,10 +209,17 @@ ASTLowerQualifiedPathInType::visit (AST::QualifiedPathInType &path)
 					     path.get_locus ());
 }
 
+ASTLoweringType::ASTLoweringType (bool default_to_static_lifetime,
+				  bool impl_trait_allowed)
+  : ASTLoweringBase (), default_to_static_lifetime (default_to_static_lifetime),
+    impl_trait_allowed (impl_trait_allowed), translated (nullptr)
+{}
+
 HIR::Type *
-ASTLoweringType::translate (AST::Type &type, bool default_to_static_lifetime)
+ASTLoweringType::translate (AST::Type &type, bool default_to_static_lifetime,
+			    bool impl_trait_allowed)
 {
-  ASTLoweringType resolver (default_to_static_lifetime);
+  ASTLoweringType resolver (default_to_static_lifetime, impl_trait_allowed);
   type.accept_vis (resolver);
 
   rust_assert (resolver.translated != nullptr);
@@ -260,7 +267,8 @@ ASTLoweringType::visit (AST::BareFunctionType &fntype)
 
       HIR::Type *param_type
 	= ASTLoweringType::translate (param.get_type (),
-				      default_to_static_lifetime);
+				      default_to_static_lifetime,
+				      impl_trait_allowed);
 
       HIR::MaybeNamedParam p (param.get_name (), kind,
 			      std::unique_ptr<HIR::Type> (param_type),
@@ -272,7 +280,8 @@ ASTLoweringType::visit (AST::BareFunctionType &fntype)
   if (fntype.has_return_type ())
     {
       return_type = ASTLoweringType::translate (fntype.get_return_type (),
-						default_to_static_lifetime);
+						default_to_static_lifetime,
+						impl_trait_allowed);
     }
 
   auto crate_num = mappings.get_current_crate ();
@@ -292,8 +301,8 @@ ASTLoweringType::visit (AST::TupleType &tuple)
   std::vector<std::unique_ptr<HIR::Type>> elems;
   for (auto &e : tuple.get_elems ())
     {
-      HIR::Type *t
-	= ASTLoweringType::translate (*e, default_to_static_lifetime);
+      HIR::Type *t = ASTLoweringType::translate (*e, default_to_static_lifetime,
+						 impl_trait_allowed);
       elems.push_back (std::unique_ptr<HIR::Type> (t));
     }
 
@@ -323,7 +332,8 @@ ASTLoweringType::visit (AST::ArrayType &type)
 {
   HIR::Type *translated_type
     = ASTLoweringType::translate (type.get_elem_type (),
-				  default_to_static_lifetime);
+				  default_to_static_lifetime,
+				  impl_trait_allowed);
   HIR::Expr *array_size = ASTLoweringExpr::translate (type.get_size_expr ());
 
   auto crate_num = mappings.get_current_crate ();
@@ -343,9 +353,9 @@ ASTLoweringType::visit (AST::ReferenceType &type)
   HIR::Lifetime lifetime
     = lower_lifetime (type.get_lifetime (), default_to_static_lifetime);
 
-  HIR::Type *base_type
-    = ASTLoweringType::translate (type.get_base_type (),
-				  default_to_static_lifetime);
+  HIR::Type *base_type = ASTLoweringType::translate (type.get_base_type (),
+						     default_to_static_lifetime,
+						     impl_trait_allowed);
 
   auto crate_num = mappings.get_current_crate ();
   Analysis::NodeMapping mapping (crate_num, type.get_node_id (),
@@ -364,7 +374,8 @@ ASTLoweringType::visit (AST::RawPointerType &type)
 {
   HIR::Type *base_type
     = ASTLoweringType::translate (type.get_type_pointed_to (),
-				  default_to_static_lifetime);
+				  default_to_static_lifetime,
+				  impl_trait_allowed);
 
   auto crate_num = mappings.get_current_crate ();
   Analysis::NodeMapping mapping (crate_num, type.get_node_id (),
@@ -384,9 +395,9 @@ ASTLoweringType::visit (AST::RawPointerType &type)
 void
 ASTLoweringType::visit (AST::SliceType &type)
 {
-  HIR::Type *base_type
-    = ASTLoweringType::translate (type.get_elem_type (),
-				  default_to_static_lifetime);
+  HIR::Type *base_type = ASTLoweringType::translate (type.get_elem_type (),
+						     default_to_static_lifetime,
+						     impl_trait_allowed);
 
   auto crate_num = mappings.get_current_crate ();
   Analysis::NodeMapping mapping (crate_num, type.get_node_id (),
@@ -463,7 +474,8 @@ void
 ASTLoweringType::visit (AST::ParenthesisedType &type)
 {
   auto *inner = ASTLoweringType::translate (*type.get_type_in_parens (),
-					    default_to_static_lifetime);
+					    default_to_static_lifetime,
+					    impl_trait_allowed);
 
   auto crate_num = mappings.get_current_crate ();
   Analysis::NodeMapping mapping (crate_num, type.get_node_id (),
@@ -480,6 +492,9 @@ ASTLoweringType::visit (AST::ParenthesisedType &type)
 void
 ASTLoweringType::visit (AST::ImplTraitType &type)
 {
+  if (!impl_trait_allowed)
+    emit_impl_trait_error (type.get_locus ());
+
   std::vector<std::unique_ptr<HIR::TypeParamBound>> bounds;
   for (auto &bound : type.get_type_param_bounds ())
     {
@@ -499,9 +514,12 @@ ASTLoweringType::visit (AST::ImplTraitType &type)
 void
 ASTLoweringType::visit (AST::ImplTraitTypeOneBound &type)
 {
+  if (!impl_trait_allowed)
+    emit_impl_trait_error (type.get_locus ());
+
   std::vector<std::unique_ptr<HIR::TypeParamBound>> bounds;
 
-  auto b = ASTLoweringTypeBounds::translate (type.get_trait_bound ());
+  auto b = ASTLoweringTypeBounds::translate (*type.get_trait_bound ().get ());
   bounds.push_back (std::unique_ptr<HIR::TypeParamBound> (b));
 
   auto crate_num = mappings.get_current_crate ();
@@ -513,6 +531,15 @@ ASTLoweringType::visit (AST::ImplTraitTypeOneBound &type)
     = new HIR::ImplTraitType (mapping, std::move (bounds), type.get_locus ());
 }
 
+void
+ASTLoweringType::emit_impl_trait_error (location_t locus)
+{
+  rich_location r (line_table, locus);
+  rust_error_at (r, ErrorCode::E0562,
+		 "%<impl Trait%> not allowed outside of function and inherent "
+		 "method return types");
+}
+
 HIR::GenericParam *
 ASTLowerGenericParam::translate (AST::GenericParam &param)
 {
@@ -593,7 +620,8 @@ ASTLowerGenericParam::visit (AST::TypeParam &param)
   translated
     = new HIR::TypeParam (mapping, param.get_type_representation (),
 			  param.get_locus (), std::move (type_param_bounds),
-			  std::move (type), param.get_outer_attrs ());
+			  std::move (type), param.get_outer_attrs (),
+			  param.from_impl_trait ());
 }
 
 HIR::TypeParamBound *
diff --git a/gcc/rust/hir/rust-ast-lower-type.h b/gcc/rust/hir/rust-ast-lower-type.h
index 4efaeee..50f543a 100644
--- a/gcc/rust/hir/rust-ast-lower-type.h
+++ b/gcc/rust/hir/rust-ast-lower-type.h
@@ -66,7 +66,8 @@ class ASTLoweringType : public ASTLoweringBase
 
 public:
   static HIR::Type *translate (AST::Type &type,
-			       bool default_to_static_lifetime = false);
+			       bool default_to_static_lifetime = false,
+			       bool impl_trait_allowed = false);
 
   void visit (AST::BareFunctionType &fntype) override;
   void visit (AST::TupleType &tuple) override;
@@ -81,19 +82,17 @@ public:
   void visit (AST::TraitObjectTypeOneBound &type) override;
   void visit (AST::TraitObjectType &type) override;
   void visit (AST::ParenthesisedType &type) override;
-
   void visit (AST::ImplTraitType &type) override;
   void visit (AST::ImplTraitTypeOneBound &type) override;
 
+  void emit_impl_trait_error (location_t locus);
+
 private:
-  ASTLoweringType (bool default_to_static_lifetime)
-    : ASTLoweringBase (),
-      default_to_static_lifetime (default_to_static_lifetime),
-      translated (nullptr)
-  {}
+  ASTLoweringType (bool default_to_static_lifetime, bool impl_trait_allowed);
 
   /** Used when compiling const and static items. */
   bool default_to_static_lifetime;
+  bool impl_trait_allowed;
 
   HIR::Type *translated;
 };
diff --git a/gcc/rust/hir/rust-ast-lower.cc b/gcc/rust/hir/rust-ast-lower.cc
index 76bd135..e6e327f 100644
--- a/gcc/rust/hir/rust-ast-lower.cc
+++ b/gcc/rust/hir/rust-ast-lower.cc
@@ -423,12 +423,6 @@ ASTLoweringExprWithBlock::visit (AST::WhileLoopExpr &expr)
 }
 
 void
-ASTLoweringExprWithBlock::visit (AST::ForLoopExpr &expr)
-{
-  rust_unreachable ();
-}
-
-void
 ASTLoweringExprWithBlock::visit (AST::MatchExpr &expr)
 {
   HIR::Expr *branch_value
diff --git a/gcc/rust/hir/rust-ast-lower.h b/gcc/rust/hir/rust-ast-lower.h
index cc74082..0787ddf 100644
--- a/gcc/rust/hir/rust-ast-lower.h
+++ b/gcc/rust/hir/rust-ast-lower.h
@@ -28,16 +28,14 @@ namespace HIR {
 
 /* Checks whether the name of a field already exists.  Returns true
    and produces an error if so.  */
-bool
-struct_field_name_exists (std::vector<HIR::StructField> &fields,
-			  HIR::StructField &new_field);
+bool struct_field_name_exists (std::vector<HIR::StructField> &fields,
+			       HIR::StructField &new_field);
 
 /**
  * Lowers a Visibility from the AST into an HIR Visibility, desugaring it in
  * the process
  */
-Visibility
-translate_visibility (const AST::Visibility &vis);
+Visibility translate_visibility (const AST::Visibility &vis);
 
 /**
  * Main base class used for lowering AST to HIR.
diff --git a/gcc/rust/hir/rust-hir-dump.cc b/gcc/rust/hir/rust-hir-dump.cc
index cb32f68..a0cdcb2 100644
--- a/gcc/rust/hir/rust-hir-dump.cc
+++ b/gcc/rust/hir/rust-hir-dump.cc
@@ -92,7 +92,7 @@ Dump::go (HIR::Crate &e)
   end ("Crate");
 }
 
-Dump::Dump (std::ostream &stream) : stream (stream) {}
+Dump::Dump (std::ostream &stream) : beg_of_line (false), stream (stream) {}
 
 /**
  * Writes TEXT with a final newline if ENDLINE is true.
@@ -1297,6 +1297,31 @@ Dump::visit (BlockExpr &e)
 }
 
 void
+Dump::visit (AnonConst &e)
+{
+  begin ("AnonConst");
+  do_expr (e);
+
+  if (e.is_deferred ())
+    put_field ("inner", "_");
+  else
+    visit_field ("inner", e.get_inner_expr ());
+
+  end ("AnonConst");
+}
+
+void
+Dump::visit (ConstBlock &e)
+{
+  begin ("ConstBlock");
+  do_expr (e);
+
+  visit_field ("inner", e.get_const_expr ());
+
+  end ("ConstBlock");
+}
+
+void
 Dump::visit (ContinueExpr &e)
 {
   begin ("ContinueExpr");
@@ -1507,13 +1532,91 @@ Dump::visit (AsyncBlockExpr &e)
 
 void
 Dump::visit (InlineAsm &e)
-{}
+{
+  begin ("InlineAsm");
+  do_expr (e);
+  for (auto &temp : e.get_template_ ())
+    {
+      put_field ("template", temp.string);
+    }
+
+  for (auto &temp_str : e.get_template_strs ())
+    {
+      put_field ("template_str", temp_str.symbol);
+    }
+
+  for (auto &operand : e.get_operands ())
+    {
+      switch (operand.get_register_type ())
+	{
+	case HIR::InlineAsmOperand::RegisterType::In:
+	  {
+	    const auto &in = operand.get_in ();
+	    visit_field ("in expr", *in.expr);
+	    break;
+	  }
+	case HIR::InlineAsmOperand::RegisterType::Out:
+	  {
+	    const auto &out = operand.get_out ();
+	    visit_field ("out expr", *out.expr);
+	    break;
+	  }
+	case HIR::InlineAsmOperand::RegisterType::InOut:
+	  {
+	    const auto &inout = operand.get_in_out ();
+	    visit_field ("inout expr", *inout.expr);
+	    break;
+	  }
+	case HIR::InlineAsmOperand::RegisterType::SplitInOut:
+	  {
+	    const auto &inout = operand.get_split_in_out ();
+	    begin ("Split in out");
+	    visit_field ("in expr", *inout.in_expr);
+	    visit_field ("out expr", *inout.out_expr);
+	    end ("Split in out");
+
+	    break;
+	  }
+	case HIR::InlineAsmOperand::RegisterType::Const:
+	  {
+	    auto &cnst = operand.get_const ();
+	    visit_field ("const expr", cnst.anon_const.get_inner_expr ());
+	    break;
+	  }
+	case HIR::InlineAsmOperand::RegisterType::Sym:
+	  {
+	    auto &sym = operand.get_sym ();
+	    visit_field ("sym expr", *sym.expr);
+	    break;
+	  }
+	case HIR::InlineAsmOperand::RegisterType::Label:
+	  {
+	    auto &label = operand.get_label ();
+	    put_field ("label name", label.label_name);
+	    do_expr (*label.expr);
+	    break;
+	  }
+	}
+    }
+  end ("InlineAsm");
+}
 
 void
 Dump::visit (LlvmInlineAsm &e)
 {}
 
 void
+Dump::visit (OffsetOf &e)
+{
+  begin ("OffsetOf");
+
+  put_field ("type", e.get_type ().as_string ());
+  put_field ("field", e.get_field ());
+
+  end ("OffsetOf");
+}
+
+void
 Dump::visit (TypeParam &e)
 {
   begin ("TypeParam");
@@ -1602,7 +1705,8 @@ Dump::visit (UseTreeGlob &e)
     case UseTreeGlob::PathType::GLOBAL:
       glob = "::*";
       break;
-      case UseTreeGlob::PathType::PATH_PREFIXED: {
+    case UseTreeGlob::PathType::PATH_PREFIXED:
+      {
 	path = e.get_path ().as_string ();
 	glob = "::*";
 	break;
@@ -1630,7 +1734,8 @@ Dump::visit (UseTreeList &e)
     case UseTreeList::PathType::GLOBAL:
       path_type = "::*";
       break;
-      case UseTreeList::PathType::PATH_PREFIXED: {
+    case UseTreeList::PathType::PATH_PREFIXED:
+      {
 	path = e.get_path ().as_string ();
 	path_type = "::*";
 	break;
@@ -1902,7 +2007,8 @@ Dump::visit (ConstantItem &e)
   do_vis_item (e);
   put_field ("identifier", e.get_identifier ().as_string ());
   visit_field ("type", e.get_type ());
-  visit_field ("const_expr", e.get_expr ());
+  if (e.has_expr ())
+    visit_field ("const_expr", e.get_expr ());
   end ("ConstantItem");
 }
 
@@ -2091,10 +2197,10 @@ Dump::visit (IdentifierPattern &e)
   put_field ("is_ref", std::to_string (e.get_is_ref ()));
   put_field ("mut", std::to_string (e.is_mut ()));
 
-  if (e.has_pattern_to_bind ())
-    put_field ("to_bind", e.get_to_bind ().as_string ());
+  if (e.has_subpattern ())
+    visit_field ("subpattern", e.get_subpattern ());
   else
-    put_field ("to_bind", "none");
+    put_field ("subpattern", "none");
 
   end ("IdentifierPattern");
 }
@@ -2172,7 +2278,7 @@ Dump::visit (StructPatternFieldIdentPat &e)
   auto oa = e.get_outer_attrs ();
   do_outer_attrs (oa);
   put_field ("ident", e.get_identifier ().as_string ());
-  put_field ("ident_pattern", e.get_pattern ().as_string ());
+  visit_field ("ident_pattern", e.get_pattern ());
   end ("StructPatternFieldIdentPat");
 }
 
@@ -2290,7 +2396,7 @@ Dump::visit (LetStmt &e)
   auto oa = e.get_outer_attrs ();
   do_outer_attrs (oa);
 
-  put_field ("variable_pattern", e.get_pattern ().as_string ());
+  visit_field ("variable_pattern", e.get_pattern ());
 
   if (e.has_type ())
     visit_field ("type", e.get_type ());
diff --git a/gcc/rust/hir/rust-hir-dump.h b/gcc/rust/hir/rust-hir-dump.h
index 45b1708..3e6ae30 100644
--- a/gcc/rust/hir/rust-hir-dump.h
+++ b/gcc/rust/hir/rust-hir-dump.h
@@ -146,6 +146,8 @@ private:
   virtual void visit (FieldAccessExpr &) override;
   virtual void visit (ClosureExpr &) override;
   virtual void visit (BlockExpr &) override;
+  virtual void visit (AnonConst &) override;
+  virtual void visit (ConstBlock &) override;
   virtual void visit (ContinueExpr &) override;
   virtual void visit (BreakExpr &) override;
   virtual void visit (RangeFromToExpr &) override;
@@ -167,6 +169,7 @@ private:
   virtual void visit (AsyncBlockExpr &) override;
   virtual void visit (InlineAsm &) override;
   virtual void visit (LlvmInlineAsm &) override;
+  virtual void visit (OffsetOf &) override;
 
   virtual void visit (TypeParam &) override;
   virtual void visit (ConstGenericParam &) override;
@@ -252,7 +255,6 @@ private:
 } // namespace Rust
 
 // In the global namespace to make it easier to call from debugger
-void
-debug (Rust::HIR::FullVisitable &v);
+void debug (Rust::HIR::FullVisitable &v);
 
 #endif // !RUST_HIR_DUMP_H
diff --git a/gcc/rust/hir/tree/rust-hir-expr-abstract.h b/gcc/rust/hir/tree/rust-hir-expr-abstract.h
index 5bc5d89..371daa8 100644
--- a/gcc/rust/hir/tree/rust-hir-expr-abstract.h
+++ b/gcc/rust/hir/tree/rust-hir-expr-abstract.h
@@ -43,7 +43,7 @@ public:
     WITHOUT_BLOCK,
   };
 
-  enum ExprType
+  enum class ExprType
   {
     Lit,
     Operator,
@@ -58,6 +58,8 @@ public:
     FieldAccess,
     Closure,
     Block,
+    AnonConst,
+    ConstBlock,
     Continue,
     Break,
     Range,
@@ -72,6 +74,7 @@ public:
     Path,
     InlineAsm,
     LlvmInlineAsm,
+    OffsetOf,
   };
 
   BaseKind get_hir_kind () override final { return Node::BaseKind::EXPR; }
diff --git a/gcc/rust/hir/tree/rust-hir-expr.cc b/gcc/rust/hir/tree/rust-hir-expr.cc
index 266c79c..14786ad 100644
--- a/gcc/rust/hir/tree/rust-hir-expr.cc
+++ b/gcc/rust/hir/tree/rust-hir-expr.cc
@@ -17,6 +17,8 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-hir-expr.h"
+#include "rust-hir-map.h"
+#include "optional.h"
 #include "rust-operators.h"
 #include "rust-hir-stmt.h"
 
@@ -790,6 +792,61 @@ BlockExpr::operator= (BlockExpr const &other)
   return *this;
 }
 
+AnonConst::AnonConst (Analysis::NodeMapping mappings,
+		      std::unique_ptr<Expr> &&expr, location_t locus)
+  : ExprWithBlock (std::move (mappings), {}), locus (locus),
+    kind (Kind::Explicit), expr (std::move (expr))
+{
+  rust_assert (this->expr.value ());
+}
+
+AnonConst::AnonConst (Analysis::NodeMapping mappings, location_t locus)
+  : ExprWithBlock (std::move (mappings), {}), locus (locus),
+    kind (Kind::DeferredInference), expr (tl::nullopt)
+{}
+
+AnonConst::AnonConst (const AnonConst &other)
+  : ExprWithBlock (other), locus (other.locus), kind (other.kind)
+{
+  if (other.expr)
+    expr = other.expr.value ()->clone_expr ();
+}
+
+AnonConst
+AnonConst::operator= (const AnonConst &other)
+{
+  ExprWithBlock::operator= (other);
+
+  locus = other.locus;
+  kind = other.kind;
+
+  if (other.expr)
+    expr = other.expr.value ()->clone_expr ();
+
+  return *this;
+}
+
+ConstBlock::ConstBlock (Analysis::NodeMapping mappings, AnonConst &&expr,
+			location_t locus, AST::AttrVec outer_attrs)
+  : ExprWithBlock (std::move (mappings), std::move (outer_attrs)),
+    expr (std::move (expr)), locus (locus)
+{}
+
+ConstBlock::ConstBlock (const ConstBlock &other)
+  : ExprWithBlock (other), expr (other.expr), locus (other.locus)
+{}
+
+ConstBlock
+ConstBlock::operator= (const ConstBlock &other)
+{
+  ExprWithBlock::operator= (other);
+
+  expr = other.expr;
+  locus = other.locus;
+
+  return *this;
+}
+
 ContinueExpr::ContinueExpr (Analysis::NodeMapping mappings, location_t locus,
 			    tl::optional<Lifetime> label,
 			    AST::AttrVec outer_attribs)
@@ -1277,59 +1334,42 @@ AsyncBlockExpr::operator= (AsyncBlockExpr const &other)
 OperatorExprMeta::OperatorExprMeta (HIR::CompoundAssignmentExpr &expr)
   : node_mappings (expr.get_mappings ()),
     lvalue_mappings (expr.get_expr ().get_mappings ()),
-    locus (expr.get_locus ())
+    rvalue_mappings (expr.get_rhs ().get_mappings ()), locus (expr.get_locus ())
 {}
 
 OperatorExprMeta::OperatorExprMeta (HIR::ArithmeticOrLogicalExpr &expr)
   : node_mappings (expr.get_mappings ()),
     lvalue_mappings (expr.get_expr ().get_mappings ()),
-    locus (expr.get_locus ())
+    rvalue_mappings (expr.get_rhs ().get_mappings ()), locus (expr.get_locus ())
 {}
 
 OperatorExprMeta::OperatorExprMeta (HIR::NegationExpr &expr)
   : node_mappings (expr.get_mappings ()),
     lvalue_mappings (expr.get_expr ().get_mappings ()),
+    rvalue_mappings (Analysis::NodeMapping::get_error ()),
     locus (expr.get_locus ())
 {}
 
 OperatorExprMeta::OperatorExprMeta (HIR::DereferenceExpr &expr)
   : node_mappings (expr.get_mappings ()),
     lvalue_mappings (expr.get_expr ().get_mappings ()),
+    rvalue_mappings (Analysis::NodeMapping::get_error ()),
     locus (expr.get_locus ())
 {}
 
 OperatorExprMeta::OperatorExprMeta (HIR::ArrayIndexExpr &expr)
   : node_mappings (expr.get_mappings ()),
     lvalue_mappings (expr.get_array_expr ().get_mappings ()),
+    rvalue_mappings (expr.get_index_expr ().get_mappings ()),
     locus (expr.get_locus ())
 {}
 
 OperatorExprMeta::OperatorExprMeta (HIR::ComparisonExpr &expr)
   : node_mappings (expr.get_mappings ()),
     lvalue_mappings (expr.get_expr ().get_mappings ()),
-    locus (expr.get_locus ())
+    rvalue_mappings (expr.get_rhs ().get_mappings ()), locus (expr.get_locus ())
 {}
 
-AnonConst::AnonConst (NodeId id, std::unique_ptr<Expr> expr)
-  : id (id), expr (std::move (expr))
-{
-  rust_assert (this->expr != nullptr);
-}
-
-AnonConst::AnonConst (const AnonConst &other)
-{
-  id = other.id;
-  expr = other.expr->clone_expr ();
-}
-
-AnonConst
-AnonConst::operator= (const AnonConst &other)
-{
-  id = other.id;
-  expr = other.expr->clone_expr ();
-  return *this;
-}
-
 InlineAsmOperand::In::In (
   const tl::optional<struct AST::InlineAsmRegOrRegClass> &reg,
   std::unique_ptr<Expr> expr)
@@ -1476,7 +1516,7 @@ InlineAsm::InlineAsm (location_t locus, bool is_global_asm,
 		      std::vector<AST::TupleTemplateStr> template_strs,
 		      std::vector<HIR::InlineAsmOperand> operands,
 		      std::vector<AST::TupleClobber> clobber_abi,
-		      std::set<AST::InlineAsmOption> options,
+		      std::set<AST::InlineAsm::Option> options,
 		      Analysis::NodeMapping mappings,
 		      AST::AttrVec outer_attribs)
   : ExprWithoutBlock (std::move (mappings), std::move (outer_attribs)),
@@ -1486,5 +1526,41 @@ InlineAsm::InlineAsm (location_t locus, bool is_global_asm,
     clobber_abi (std::move (clobber_abi)), options (std::move (options))
 {}
 
+OffsetOf &
+OffsetOf::operator= (const OffsetOf &other)
+{
+  ExprWithoutBlock::operator= (other);
+
+  type = other.type->clone_type ();
+  field = other.field;
+  loc = other.loc;
+
+  return *this;
+}
+
+ExprWithoutBlock *
+OffsetOf::clone_expr_without_block_impl () const
+{
+  return new OffsetOf (*this);
+}
+
+std::string
+OffsetOf::as_string () const
+{
+  return "OffsetOf(" + type->as_string () + ", " + field.as_string () + ")";
+}
+
+void
+OffsetOf::accept_vis (HIRExpressionVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
+OffsetOf::accept_vis (HIRFullVisitor &vis)
+{
+  vis.visit (*this);
+}
+
 } // namespace HIR
 } // namespace Rust
diff --git a/gcc/rust/hir/tree/rust-hir-expr.h b/gcc/rust/hir/tree/rust-hir-expr.h
index 375f474..61e3590 100644
--- a/gcc/rust/hir/tree/rust-hir-expr.h
+++ b/gcc/rust/hir/tree/rust-hir-expr.h
@@ -19,12 +19,15 @@
 #ifndef RUST_HIR_EXPR_H
 #define RUST_HIR_EXPR_H
 
+#include "rust-ast.h"
 #include "rust-hir-expr-abstract.h"
 #include "rust-hir-literal.h"
 #include "rust-common.h"
 #include "rust-hir-bound.h"
 #include "rust-hir-attrs.h"
 #include "rust-expr.h"
+#include "rust-hir-map.h"
+#include "rust-mapping-common.h"
 
 namespace Rust {
 namespace HIR {
@@ -1800,6 +1803,92 @@ protected:
   }
 };
 
+class AnonConst : public ExprWithBlock
+{
+public:
+  enum class Kind
+  {
+    Explicit,
+    DeferredInference
+  };
+
+  AnonConst (Analysis::NodeMapping mappings, std::unique_ptr<Expr> &&expr,
+	     location_t locus = UNKNOWN_LOCATION);
+  AnonConst (Analysis::NodeMapping mappings,
+	     location_t locus = UNKNOWN_LOCATION);
+  AnonConst (const AnonConst &other);
+  AnonConst operator= (const AnonConst &other);
+
+  std::string as_string () const override;
+
+  void accept_vis (HIRFullVisitor &vis) override;
+  void accept_vis (HIRExpressionVisitor &vis) override;
+
+  ExprType get_expression_type () const final override
+  {
+    return ExprType::AnonConst;
+  }
+
+  location_t get_locus () const override { return locus; }
+
+  Expr &get_inner_expr ()
+  {
+    rust_assert (kind == Kind::Explicit);
+    return *expr.value ();
+  }
+
+  const Expr &get_inner_expr () const
+  {
+    rust_assert (kind == Kind::Explicit);
+    return *expr.value ();
+  }
+
+  bool is_deferred () const { return kind == Kind::DeferredInference; }
+
+private:
+  location_t locus;
+  Kind kind;
+  tl::optional<std::unique_ptr<Expr>> expr;
+
+  AnonConst *clone_expr_with_block_impl () const override
+  {
+    return new AnonConst (*this);
+  }
+};
+
+class ConstBlock : public ExprWithBlock
+{
+public:
+  ConstBlock (Analysis::NodeMapping mappings, AnonConst &&expr,
+	      location_t locus = UNKNOWN_LOCATION,
+	      AST::AttrVec outer_attrs = {});
+  ConstBlock (const ConstBlock &other);
+  ConstBlock operator= (const ConstBlock &other);
+
+  void accept_vis (HIRFullVisitor &vis) override;
+  void accept_vis (HIRExpressionVisitor &vis) override;
+
+  std::string as_string () const override;
+
+  ExprType get_expression_type () const final override
+  {
+    return ExprType::ConstBlock;
+  }
+
+  location_t get_locus () const override { return locus; }
+  AnonConst &get_const_expr () { return expr; }
+  const AnonConst &get_const_expr () const { return expr; }
+
+private:
+  AnonConst expr;
+  location_t locus;
+
+  ConstBlock *clone_expr_with_block_impl () const override
+  {
+    return new ConstBlock (*this);
+  }
+};
+
 // HIR node representing continue expression within loops
 class ContinueExpr : public ExprWithoutBlock
 {
@@ -2631,6 +2720,8 @@ public:
   Expr &get_guard_expr () { return *guard_expr; }
 
   location_t get_locus () const { return locus; }
+
+  AST::AttrVec &get_outer_attrs () { return outer_attrs; }
 };
 
 /* A "match case" - a correlated match arm and resulting expression. Not
@@ -2823,6 +2914,22 @@ public:
 
   OperatorExprMeta (HIR::ComparisonExpr &expr);
 
+  OperatorExprMeta (const OperatorExprMeta &other)
+    : node_mappings (other.node_mappings),
+      lvalue_mappings (other.lvalue_mappings),
+      rvalue_mappings (other.rvalue_mappings), locus (other.locus)
+  {}
+
+  OperatorExprMeta &operator= (const OperatorExprMeta &other)
+  {
+    node_mappings = other.node_mappings;
+    lvalue_mappings = other.lvalue_mappings;
+    rvalue_mappings = other.rvalue_mappings;
+    locus = other.locus;
+
+    return *this;
+  }
+
   const Analysis::NodeMapping &get_mappings () const { return node_mappings; }
 
   const Analysis::NodeMapping &get_lvalue_mappings () const
@@ -2830,11 +2937,22 @@ public:
     return lvalue_mappings;
   }
 
+  const Analysis::NodeMapping &get_rvalue_mappings () const
+  {
+    return rvalue_mappings;
+  }
+
+  bool has_rvalue_mappings () const
+  {
+    return rvalue_mappings.get_hirid () != UNKNOWN_HIRID;
+  }
+
   location_t get_locus () const { return locus; }
 
 private:
-  const Analysis::NodeMapping node_mappings;
-  const Analysis::NodeMapping lvalue_mappings;
+  Analysis::NodeMapping node_mappings;
+  Analysis::NodeMapping lvalue_mappings;
+  Analysis::NodeMapping rvalue_mappings;
   location_t locus;
 };
 
@@ -2892,18 +3010,6 @@ class InlineAsmRegClass
   std::string placeholder;
 };
 
-struct AnonConst
-{
-  NodeId id;
-  std::unique_ptr<Expr> expr;
-
-  AnonConst (NodeId id, std::unique_ptr<Expr> expr);
-
-  AnonConst (const AnonConst &other);
-
-  AnonConst operator= (const AnonConst &other);
-};
-
 class InlineAsmOperand
 {
 public:
@@ -2992,8 +3098,9 @@ public:
     Label operator= (const struct Label &other);
   };
 
-private:
   using RegisterType = AST::InlineAsmOperand::RegisterType;
+
+private:
   AST::InlineAsmOperand::RegisterType register_type;
 
   tl::optional<struct In> in;
@@ -3037,13 +3144,24 @@ public:
   RegisterType get_register_type () const { return register_type; }
 
   // Potentially unsafe without get_register_type() check
-  struct In get_in () const { return in.value (); }
-  struct Out get_out () const { return out.value (); }
-  struct InOut get_in_out () const { return in_out.value (); }
-  struct SplitInOut get_split_in_out () const { return split_in_out.value (); }
-  struct Const get_const () const { return cnst.value (); }
-  struct Sym get_sym () const { return sym.value (); }
-  struct Label get_label () const { return label.value (); }
+  const struct In &get_in () const { return in.value (); }
+  const struct Out &get_out () const { return out.value (); }
+  const struct InOut &get_in_out () const { return in_out.value (); }
+  const struct SplitInOut &get_split_in_out () const
+  {
+    return split_in_out.value ();
+  }
+  const struct Const &get_const () const { return cnst.value (); }
+  const struct Sym &get_sym () const { return sym.value (); }
+  const struct Label &get_label () const { return label.value (); }
+
+  struct In &get_in () { return in.value (); }
+  struct Out &get_out () { return out.value (); }
+  struct InOut &get_in_out () { return in_out.value (); }
+  struct SplitInOut &get_split_in_out () { return split_in_out.value (); }
+  struct Const &get_const () { return cnst.value (); }
+  struct Sym &get_sym () { return sym.value (); }
+  struct Label &get_label () { return label.value (); }
 };
 
 // Inline Assembly Node
@@ -3059,7 +3177,7 @@ public:
   std::vector<AST::TupleTemplateStr> template_strs;
   std::vector<HIR::InlineAsmOperand> operands;
   std::vector<AST::TupleClobber> clobber_abi;
-  std::set<AST::InlineAsmOption> options;
+  std::set<AST::InlineAsm::Option> options;
 
   std::vector<location_t> line_spans;
 
@@ -3090,11 +3208,11 @@ public:
     return template_strs;
   }
 
-  std::vector<HIR::InlineAsmOperand> get_operands () { return operands; }
+  std::vector<HIR::InlineAsmOperand> &get_operands () { return operands; }
 
   std::vector<AST::TupleClobber> get_clobber_abi () { return clobber_abi; }
 
-  std::set<AST::InlineAsmOption> get_options () { return options; }
+  std::set<AST::InlineAsm::Option> get_options () { return options; }
 
   bool is_simple_asm ()
   {
@@ -3113,11 +3231,47 @@ public:
 	     std::vector<AST::TupleTemplateStr> template_strs,
 	     std::vector<HIR::InlineAsmOperand> operands,
 	     std::vector<AST::TupleClobber> clobber_abi,
-	     std::set<AST::InlineAsmOption> options,
+	     std::set<AST::InlineAsm::Option> options,
 	     Analysis::NodeMapping mappings,
 	     AST::AttrVec outer_attribs = AST::AttrVec ());
 };
 
+class OffsetOf : public ExprWithoutBlock
+{
+public:
+  OffsetOf (std::unique_ptr<Type> &&type, Identifier field,
+	    Analysis::NodeMapping mappings, location_t loc)
+    : ExprWithoutBlock (mappings), type (std::move (type)), field (field),
+      loc (loc)
+  {}
+
+  OffsetOf (const OffsetOf &other)
+    : ExprWithoutBlock (other), type (other.type->clone_type ()),
+      field (other.field), loc (other.loc)
+  {}
+
+  OffsetOf &operator= (const OffsetOf &other);
+
+  ExprWithoutBlock *clone_expr_without_block_impl () const override;
+  std::string as_string () const override;
+
+  void accept_vis (HIRExpressionVisitor &vis) override;
+  void accept_vis (HIRFullVisitor &vis) override;
+
+  ExprType get_expression_type () const override { return ExprType::OffsetOf; }
+
+  location_t get_locus () const override { return loc; }
+
+  Type &get_type () { return *type; }
+  const Type &get_type () const { return *type; }
+  const Identifier &get_field () const { return field; }
+
+private:
+  std::unique_ptr<Type> type;
+  Identifier field;
+  location_t loc;
+};
+
 struct LlvmOperand
 {
   std::string constraint;
diff --git a/gcc/rust/hir/tree/rust-hir-full-decls.h b/gcc/rust/hir/tree/rust-hir-full-decls.h
index 1e313ec..57b3a4d 100644
--- a/gcc/rust/hir/tree/rust-hir-full-decls.h
+++ b/gcc/rust/hir/tree/rust-hir-full-decls.h
@@ -95,6 +95,8 @@ class FieldAccessExpr;
 struct ClosureParam;
 class ClosureExpr;
 class BlockExpr;
+class AnonConst;
+class ConstBlock;
 class ContinueExpr;
 class BreakExpr;
 class RangeExpr;
@@ -123,10 +125,10 @@ class AwaitExpr;
 class AsyncBlockExpr;
 class InlineAsmReg;
 class InlineAsmRegClass;
-struct AnonConst;
 class InlineAsmOperand;
 class InlineAsm;
 class LlvmInlineAsm;
+class OffsetOf;
 
 // rust-stmt.h
 class EmptyStmt;
diff --git a/gcc/rust/hir/tree/rust-hir-generic-param.h b/gcc/rust/hir/tree/rust-hir-generic-param.h
index 960de56..340b5c6 100644
--- a/gcc/rust/hir/tree/rust-hir-generic-param.h
+++ b/gcc/rust/hir/tree/rust-hir-generic-param.h
@@ -150,7 +150,7 @@ public:
 
   location_t get_locus () const override final { return locus; };
 
-  bool has_default_expression () { return default_expression != nullptr; }
+  bool has_default_expression () const { return default_expression != nullptr; }
 
   std::string get_name () { return name; }
   Type &get_type ()
@@ -160,6 +160,8 @@ public:
   }
   Expr &get_default_expression () { return *default_expression; }
 
+  const Expr &get_default_expression () const { return *default_expression; }
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
diff --git a/gcc/rust/hir/tree/rust-hir-item.cc b/gcc/rust/hir/tree/rust-hir-item.cc
index 160f710..1406e7a 100644
--- a/gcc/rust/hir/tree/rust-hir-item.cc
+++ b/gcc/rust/hir/tree/rust-hir-item.cc
@@ -26,16 +26,18 @@ TypeParam::TypeParam (
   Analysis::NodeMapping mappings, Identifier type_representation,
   location_t locus,
   std::vector<std::unique_ptr<TypeParamBound>> type_param_bounds,
-  tl::optional<std::unique_ptr<Type>> type, AST::AttrVec outer_attrs)
+  tl::optional<std::unique_ptr<Type>> type, AST::AttrVec outer_attrs,
+  bool was_impl_trait)
   : GenericParam (mappings), outer_attrs (std::move (outer_attrs)),
     type_representation (std::move (type_representation)),
     type_param_bounds (std::move (type_param_bounds)), type (std::move (type)),
-    locus (locus)
+    locus (locus), was_impl_trait (was_impl_trait)
 {}
 
 TypeParam::TypeParam (TypeParam const &other)
   : GenericParam (other.mappings), outer_attrs (other.outer_attrs),
-    type_representation (other.type_representation), locus (other.locus)
+    type_representation (other.type_representation), locus (other.locus),
+    was_impl_trait (other.was_impl_trait)
 {
   // guard to prevent null pointer dereference
   if (other.has_type ())
@@ -55,6 +57,7 @@ TypeParam::operator= (TypeParam const &other)
   outer_attrs = other.outer_attrs;
   locus = other.locus;
   mappings = other.mappings;
+  was_impl_trait = other.was_impl_trait;
 
   // guard to prevent null pointer dereference
   if (other.has_type ())
diff --git a/gcc/rust/hir/tree/rust-hir-item.h b/gcc/rust/hir/tree/rust-hir-item.h
index 37f599c..d9df602 100644
--- a/gcc/rust/hir/tree/rust-hir-item.h
+++ b/gcc/rust/hir/tree/rust-hir-item.h
@@ -95,17 +95,11 @@ protected:
 class TypeParam : public GenericParam
 {
   AST::AttrVec outer_attrs;
-
   Identifier type_representation;
-
-  // bool has_type_param_bounds;
-  // TypeParamBounds type_param_bounds;
-  std::vector<std::unique_ptr<TypeParamBound>>
-    type_param_bounds; // inlined form
-
+  std::vector<std::unique_ptr<TypeParamBound>> type_param_bounds;
   tl::optional<std::unique_ptr<Type>> type;
-
   location_t locus;
+  bool was_impl_trait;
 
 public:
   // Returns whether the type of the type param has been specified.
@@ -121,9 +115,9 @@ public:
   TypeParam (Analysis::NodeMapping mappings, Identifier type_representation,
 	     location_t locus = UNDEF_LOCATION,
 	     std::vector<std::unique_ptr<TypeParamBound>> type_param_bounds
-	     = std::vector<std::unique_ptr<TypeParamBound>> (),
+	     = {},
 	     tl::optional<std::unique_ptr<Type>> type = tl::nullopt,
-	     AST::AttrVec outer_attrs = std::vector<AST::Attribute> ());
+	     AST::AttrVec outer_attrs = {}, bool was_impl_trait = false);
 
   // Copy constructor uses clone
   TypeParam (TypeParam const &other);
@@ -154,6 +148,8 @@ public:
 
   std::vector<std::unique_ptr<TypeParamBound>> &get_type_param_bounds ();
 
+  bool from_impl_trait () const { return was_impl_trait; }
+
 protected:
   // Clone function implementation as (not pure) virtual method
   TypeParam *clone_generic_param_impl () const override
@@ -213,6 +209,8 @@ public:
 
   std::string as_string () const override;
 
+  location_t get_locus () const { return locus; }
+
   void accept_vis (HIRFullVisitor &vis) override;
 
   Lifetime &get_lifetime () { return lifetime; }
@@ -406,6 +404,8 @@ public:
 
   const Lifetime &get_lifetime () const { return lifetime.value (); }
 
+  Lifetime &get_lifetime () { return lifetime.value (); }
+
   std::string as_string () const;
 
   location_t get_locus () const { return locus; }
@@ -1801,6 +1801,8 @@ public:
     return *type;
   }
 
+  bool has_expr () const { return const_expr != nullptr; }
+
   Expr &get_expr () { return *const_expr; }
 
   Identifier get_identifier () const { return identifier; }
diff --git a/gcc/rust/hir/tree/rust-hir-path.h b/gcc/rust/hir/tree/rust-hir-path.h
index 3ce2662..4f296d8 100644
--- a/gcc/rust/hir/tree/rust-hir-path.h
+++ b/gcc/rust/hir/tree/rust-hir-path.h
@@ -41,11 +41,15 @@ public:
     : segment_name (std::move (segment_name))
   {}
 
-  /* TODO: insert check in constructor for this? Or is this a semantic error
-   * best handled then? */
+  PathIdentSegment (const PathIdentSegment &other)
+    : segment_name (other.segment_name)
+  {}
 
-  /* TODO: does this require visitor? pretty sure this isn't polymorphic, but
-   * not entirely sure */
+  PathIdentSegment &operator= (PathIdentSegment const &other)
+  {
+    segment_name = other.segment_name;
+    return *this;
+  }
 
   // Creates an error PathIdentSegment.
   static PathIdentSegment create_error () { return PathIdentSegment (""); }
@@ -128,6 +132,8 @@ public:
 
   std::unique_ptr<Expr> &get_expression () { return expression; }
 
+  location_t get_locus () const { return locus; }
+
 private:
   std::unique_ptr<Expr> expression;
   location_t locus;
@@ -146,7 +152,7 @@ public:
   bool has_generic_args () const
   {
     return !(lifetime_args.empty () && type_args.empty ()
-	     && binding_args.empty ());
+	     && binding_args.empty () && const_args.empty ());
   }
 
   GenericArgs (std::vector<Lifetime> lifetime_args,
diff --git a/gcc/rust/hir/tree/rust-hir-pattern.h b/gcc/rust/hir/tree/rust-hir-pattern.h
index 5cc5c95..9c636ca 100644
--- a/gcc/rust/hir/tree/rust-hir-pattern.h
+++ b/gcc/rust/hir/tree/rust-hir-pattern.h
@@ -80,7 +80,7 @@ class IdentifierPattern : public Pattern
   Identifier variable_ident;
   bool is_ref;
   Mutability mut;
-  std::unique_ptr<Pattern> to_bind;
+  std::unique_ptr<Pattern> subpattern;
   location_t locus;
   Analysis::NodeMapping mappings;
 
@@ -88,15 +88,15 @@ public:
   std::string as_string () const override;
 
   // Returns whether the IdentifierPattern has a pattern to bind.
-  bool has_pattern_to_bind () const { return to_bind != nullptr; }
+  bool has_subpattern () const { return subpattern != nullptr; }
 
   // Constructor
   IdentifierPattern (Analysis::NodeMapping mappings, Identifier ident,
 		     location_t locus, bool is_ref = false,
 		     Mutability mut = Mutability::Imm,
-		     std::unique_ptr<Pattern> to_bind = nullptr)
+		     std::unique_ptr<Pattern> subpattern = nullptr)
     : variable_ident (std::move (ident)), is_ref (is_ref), mut (mut),
-      to_bind (std::move (to_bind)), locus (locus), mappings (mappings)
+      subpattern (std::move (subpattern)), locus (locus), mappings (mappings)
   {}
 
   // Copy constructor with clone
@@ -105,8 +105,8 @@ public:
       mut (other.mut), locus (other.locus), mappings (other.mappings)
   {
     // fix to get prevent null pointer dereference
-    if (other.to_bind != nullptr)
-      to_bind = other.to_bind->clone_pattern ();
+    if (other.subpattern != nullptr)
+      subpattern = other.subpattern->clone_pattern ();
   }
 
   // Overload assignment operator to use clone
@@ -119,8 +119,8 @@ public:
     mappings = other.mappings;
 
     // fix to get prevent null pointer dereference
-    if (other.to_bind != nullptr)
-      to_bind = other.to_bind->clone_pattern ();
+    if (other.subpattern != nullptr)
+      subpattern = other.subpattern->clone_pattern ();
 
     return *this;
   }
@@ -133,7 +133,7 @@ public:
 
   bool is_mut () const { return mut == Mutability::Mut; }
   bool get_is_ref () const { return is_ref; }
-  Pattern &get_to_bind () { return *to_bind; }
+  Pattern &get_subpattern () { return *subpattern; }
 
   void accept_vis (HIRFullVisitor &vis) override;
   void accept_vis (HIRPatternVisitor &vis) override;
diff --git a/gcc/rust/hir/tree/rust-hir-visibility.h b/gcc/rust/hir/tree/rust-hir-visibility.h
index a750d88..9dd6ff2 100644
--- a/gcc/rust/hir/tree/rust-hir-visibility.h
+++ b/gcc/rust/hir/tree/rust-hir-visibility.h
@@ -73,6 +73,8 @@ public:
   }
 
   std::string as_string () const;
+
+  location_t get_locus () const { return locus; }
 };
 } // namespace HIR
 } // namespace Rust
diff --git a/gcc/rust/hir/tree/rust-hir-visitor.cc b/gcc/rust/hir/tree/rust-hir-visitor.cc
new file mode 100644
index 0000000..58c1e1a
--- /dev/null
+++ b/gcc/rust/hir/tree/rust-hir-visitor.cc
@@ -0,0 +1,1187 @@
+// Copyright (C) 2021-2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-expr.h"
+#include "rust-hir-full-decls.h"
+#include "rust-hir-visitor.h"
+#include "rust-hir-full.h"
+#include "rust-system.h"
+
+namespace Rust {
+namespace HIR {
+
+void
+DefaultHIRVisitor::walk (Lifetime &)
+{}
+
+void
+DefaultHIRVisitor::walk (LifetimeParam &lifetime_param)
+{
+  visit_outer_attrs (lifetime_param);
+  lifetime_param.get_lifetime ().accept_vis (*this);
+  for (Lifetime &lifetime_bound : lifetime_param.get_lifetime_bounds ())
+    lifetime_bound.accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_generic_args (GenericArgs &generic_args)
+{
+  for (auto &lifetime : generic_args.get_lifetime_args ())
+    lifetime.accept_vis (*this);
+  for (auto &type : generic_args.get_type_args ())
+    type->accept_vis (*this);
+  for (auto &binding : generic_args.get_binding_args ())
+    binding.get_type ().accept_vis (*this);
+  for (auto &const_arg : generic_args.get_const_args ())
+    const_arg.get_expression ()->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (PathInExpression &path_in_expr)
+{
+  visit_outer_attrs (path_in_expr);
+  if (!path_in_expr.is_lang_item ())
+    for (auto &segment : path_in_expr.get_segments ())
+      visit_path_expr_segment (segment);
+}
+
+void
+DefaultHIRVisitor::walk (TypePathSegment &)
+{}
+
+void
+DefaultHIRVisitor::walk (TypePathSegmentFunction &segment_function)
+{
+  TypePathFunction &function_path = segment_function.get_function_path ();
+  if (function_path.has_inputs ())
+    for (auto &param : function_path.get_params ())
+      param->accept_vis (*this);
+  if (function_path.has_return_type ())
+    function_path.get_return_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TypePathSegmentGeneric &segment_generic)
+{
+  if (segment_generic.has_generic_args ())
+    visit_generic_args (segment_generic.get_generic_args ());
+}
+
+void
+DefaultHIRVisitor::walk (TypePath &type_path)
+{
+  for (auto &segment : type_path.get_segments ())
+    segment->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_qualified_path_type (QualifiedPathType &path)
+{
+  path.get_type ().accept_vis (*this);
+  if (path.has_as_clause ())
+    path.get_trait ().accept_vis (*this);
+}
+
+// TODO: Implement visit_path_expr_segment
+void
+DefaultHIRVisitor::visit_path_expr_segment (PathExprSegment &segment)
+{
+  if (segment.has_generic_args ())
+    visit_generic_args (segment.get_generic_args ());
+}
+
+void
+DefaultHIRVisitor::walk (QualifiedPathInExpression &path_in_expr)
+{
+  visit_outer_attrs (path_in_expr);
+  visit_qualified_path_type (path_in_expr.get_path_type ());
+  for (auto &segment : path_in_expr.get_segments ())
+    visit_path_expr_segment (segment);
+}
+
+void
+DefaultHIRVisitor::walk (QualifiedPathInType &path_in_type)
+{
+  visit_qualified_path_type (path_in_type.get_path_type ());
+  path_in_type.get_associated_segment ().accept_vis (*this);
+  for (auto &segment : path_in_type.get_segments ())
+    segment->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (LiteralExpr &expr)
+{
+  visit_outer_attrs (expr);
+}
+
+void
+DefaultHIRVisitor::walk (BorrowExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (DereferenceExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ErrorPropagationExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (NegationExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ArithmeticOrLogicalExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_lhs ().accept_vis (*this);
+  expr.get_rhs ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ComparisonExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_lhs ().accept_vis (*this);
+  expr.get_rhs ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (LazyBooleanExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_lhs ().accept_vis (*this);
+  expr.get_rhs ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TypeCastExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_expr ().accept_vis (*this);
+  expr.get_type_to_convert_to ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (AssignmentExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_lhs ().accept_vis (*this);
+  expr.get_rhs ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (CompoundAssignmentExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_lhs ().accept_vis (*this);
+  expr.get_rhs ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (GroupedExpr &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  expr.get_expr_in_parens ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ArrayElemsValues &elems)
+{
+  for (auto &elem : elems.get_values ())
+    elem->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ArrayElemsCopied &elems)
+{
+  elems.get_elem_to_copy ().accept_vis (*this);
+  elems.get_num_copies_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ArrayExpr &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  expr.get_internal_elements ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ArrayIndexExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_array_expr ().accept_vis (*this);
+  expr.get_index_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TupleExpr &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  for (auto &elem : expr.get_tuple_elems ())
+    elem->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TupleIndexExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_tuple_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructExprStruct &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  expr.get_struct_name ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructExprFieldIdentifier &)
+{}
+
+void
+DefaultHIRVisitor::walk (StructExprFieldIdentifierValue &field)
+{
+  field.get_value ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructExprFieldIndexValue &field)
+{
+  field.get_value ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructExprStructFields &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  expr.get_struct_name ().accept_vis (*this);
+  if (expr.has_struct_base ())
+    {
+      StructBase &base = expr.get_struct_base ();
+      base.get_base ().accept_vis (*this);
+    }
+  for (auto &field : expr.get_fields ())
+    field->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructExprStructBase &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  expr.get_struct_name ().accept_vis (*this);
+  StructBase &base = expr.get_struct_base ();
+  base.get_base ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (CallExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_fnexpr ().accept_vis (*this);
+  for (auto &arg : expr.get_arguments ())
+    arg->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (MethodCallExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_receiver ().accept_vis (*this);
+  visit_path_expr_segment (expr.get_method_name ());
+  for (auto &arg : expr.get_arguments ())
+    arg->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (FieldAccessExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_receiver_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_closure_param (ClosureParam &param)
+{
+  visit_outer_attrs (param);
+  param.get_pattern ().accept_vis (*this);
+  if (param.has_type_given ())
+    {
+      param.get_type ().accept_vis (*this);
+    }
+}
+
+void
+DefaultHIRVisitor::walk (ClosureExpr &expr)
+{
+  visit_outer_attrs (expr);
+  for (auto &param : expr.get_params ())
+    visit_closure_param (param);
+  if (expr.has_return_type ())
+    expr.get_return_type ().accept_vis (*this);
+  expr.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (BlockExpr &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  for (auto &stmt : expr.get_statements ())
+    stmt->accept_vis (*this);
+  if (expr.has_expr ())
+    expr.get_final_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (AnonConst &expr)
+{
+  if (!expr.is_deferred ())
+    expr.get_inner_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ConstBlock &expr)
+{
+  expr.get_const_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ContinueExpr &expr)
+{
+  visit_outer_attrs (expr);
+  if (expr.has_label ())
+    expr.get_label ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (BreakExpr &expr)
+{
+  visit_outer_attrs (expr);
+  if (expr.has_label ())
+    expr.get_label ().accept_vis (*this);
+
+  if (expr.has_break_expr ())
+    expr.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (RangeFromToExpr &expr)
+{
+  expr.get_from_expr ().accept_vis (*this);
+  expr.get_to_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (RangeFromExpr &expr)
+{
+  expr.get_from_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (RangeToExpr &expr)
+{
+  expr.get_to_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (RangeFullExpr &)
+{}
+
+void
+DefaultHIRVisitor::walk (RangeFromToInclExpr &expr)
+{
+  expr.get_from_expr ().accept_vis (*this);
+  expr.get_to_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (RangeToInclExpr &expr)
+{
+  expr.get_to_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ReturnExpr &expr)
+{
+  visit_outer_attrs (expr);
+  if (expr.has_return_expr ())
+    expr.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (UnsafeBlockExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_block_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_loop_label (LoopLabel &label)
+{
+  label.get_lifetime ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (LoopExpr &expr)
+{
+  visit_outer_attrs (expr);
+  if (expr.has_loop_label ())
+    visit_loop_label (expr.get_loop_label ());
+  expr.get_loop_block ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (WhileLoopExpr &expr)
+{
+  visit_outer_attrs (expr);
+  if (expr.has_loop_label ())
+    visit_loop_label (expr.get_loop_label ());
+  expr.get_predicate_expr ().accept_vis (*this);
+  expr.get_loop_block ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (WhileLetLoopExpr &expr)
+{
+  visit_outer_attrs (expr);
+  for (auto &pattern : expr.get_patterns ())
+    pattern->accept_vis (*this);
+  if (expr.has_loop_label ())
+    visit_loop_label (expr.get_loop_label ());
+  expr.get_cond ().accept_vis (*this);
+  expr.get_loop_block ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (IfExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_if_condition ().accept_vis (*this);
+  expr.get_if_block ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (IfExprConseqElse &expr)
+{
+  expr.IfExpr::accept_vis (*this);
+  expr.get_else_block ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_match_arm (MatchArm &arm)
+{
+  // visit_outer_attrs (arm);
+  for (auto &pattern : arm.get_patterns ())
+    pattern->accept_vis (*this);
+  if (arm.has_match_arm_guard ())
+    arm.get_guard_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_match_case (MatchCase &arm)
+{
+  visit_match_arm (arm.get_arm ());
+  arm.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (MatchExpr &expr)
+{
+  visit_outer_attrs (expr);
+  visit_inner_attrs (expr);
+  expr.get_scrutinee_expr ().accept_vis (*this);
+  for (auto &arm : expr.get_match_cases ())
+    visit_match_case (arm);
+}
+
+void
+DefaultHIRVisitor::walk (AwaitExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_awaited_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (AsyncBlockExpr &expr)
+{
+  visit_outer_attrs (expr);
+  expr.get_block_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (InlineAsm &expr)
+{
+  visit_outer_attrs (expr);
+  auto &operands = expr.get_operands ();
+  using RegisterType = AST::InlineAsmOperand::RegisterType;
+  for (auto &operand : operands)
+    {
+      switch (operand.get_register_type ())
+	{
+	case RegisterType::In:
+	  {
+	    operand.get_in ().expr->accept_vis (*this);
+	    break;
+	  }
+	case RegisterType::Out:
+	  {
+	    operand.get_out ().expr->accept_vis (*this);
+	    break;
+	  }
+	case RegisterType::InOut:
+	  {
+	    operand.get_in_out ().expr->accept_vis (*this);
+	    break;
+	  }
+	case RegisterType::SplitInOut:
+	  {
+	    operand.get_split_in_out ().in_expr->accept_vis (*this);
+	    operand.get_split_in_out ().out_expr->accept_vis (*this);
+	    break;
+	  }
+	case RegisterType::Const:
+	  {
+	    operand.get_const ().anon_const.get_inner_expr ().accept_vis (
+	      *this);
+	    break;
+	  }
+	case RegisterType::Sym:
+	  {
+	    operand.get_sym ().expr->accept_vis (*this);
+	    break;
+	  }
+	case RegisterType::Label:
+	  {
+	    operand.get_label ().expr->accept_vis (*this);
+	    break;
+	  }
+	}
+    }
+}
+
+void
+DefaultHIRVisitor::walk (LlvmInlineAsm &expr)
+{
+  for (auto &output : expr.outputs)
+    output.expr->accept_vis (*this);
+  for (auto &input : expr.inputs)
+    input.expr->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (OffsetOf &expr)
+{
+  expr.get_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TypeParam &param)
+{
+  visit_outer_attrs (param);
+  for (auto &bounds : param.get_type_param_bounds ())
+    bounds->accept_vis (*this);
+  if (param.has_type ())
+    param.get_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ConstGenericParam &const_param)
+{
+  visit_outer_attrs (const_param);
+  const_param.get_type ().accept_vis (*this);
+  if (const_param.has_default_expression ())
+    const_param.get_default_expression ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (LifetimeWhereClauseItem &item)
+{
+  item.get_lifetime ().accept_vis (*this);
+  for (auto &bound : item.get_lifetime_bounds ())
+    bound.accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TypeBoundWhereClauseItem &item)
+{
+  for (auto &lifetime : item.get_for_lifetimes ())
+    lifetime.accept_vis (*this);
+  item.get_bound_type ().accept_vis (*this);
+  for (auto &param : item.get_type_param_bounds ())
+    param->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (Module &module)
+{
+  visit_outer_attrs (module);
+  visit_inner_attrs (module);
+  for (auto &item : module.get_items ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ExternCrate &crate)
+{
+  visit_outer_attrs (crate);
+}
+
+void
+DefaultHIRVisitor::walk (UseTreeGlob &)
+{}
+
+void
+DefaultHIRVisitor::walk (UseTreeList &)
+{}
+
+void
+DefaultHIRVisitor::walk (UseTreeRebind &)
+{}
+
+void
+DefaultHIRVisitor::walk (UseDeclaration &)
+{}
+
+void
+DefaultHIRVisitor::visit_function_param (FunctionParam &param)
+{
+  param.get_param_name ().accept_vis (*this);
+  param.get_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (Function &function)
+{
+  visit_outer_attrs (function);
+  for (auto &generic : function.get_generic_params ())
+    generic->accept_vis (*this);
+  for (auto &param : function.get_function_params ())
+    visit_function_param (param);
+  if (function.has_return_type ())
+    function.get_return_type ().accept_vis (*this);
+  if (function.has_where_clause ())
+    visit_where_clause (function.get_where_clause ());
+  function.get_definition ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TypeAlias &type_alias)
+{
+  visit_outer_attrs (type_alias);
+  for (auto &generic : type_alias.get_generic_params ())
+    generic->accept_vis (*this);
+  if (type_alias.has_where_clause ())
+    visit_where_clause (type_alias.get_where_clause ());
+  type_alias.get_type_aliased ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_struct_field (StructField &field)
+{
+  field.get_field_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructStruct &struct_item)
+{
+  visit_outer_attrs (struct_item);
+  for (auto &generic : struct_item.get_generic_params ())
+    generic->accept_vis (*this);
+  if (struct_item.has_where_clause ())
+    visit_where_clause (struct_item.get_where_clause ());
+  for (auto &field : struct_item.get_fields ())
+    visit_struct_field (field);
+}
+
+void
+DefaultHIRVisitor::walk (TupleStruct &tuple_struct)
+{
+  visit_outer_attrs (tuple_struct);
+  for (auto &generic : tuple_struct.get_generic_params ())
+    generic->accept_vis (*this);
+  if (tuple_struct.has_where_clause ())
+    visit_where_clause (tuple_struct.get_where_clause ());
+  for (auto &field : tuple_struct.get_fields ())
+    field.get_field_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (EnumItem &item)
+{
+  visit_outer_attrs (item);
+}
+
+void
+DefaultHIRVisitor::walk (EnumItemTuple &item_tuple)
+{
+  item_tuple.EnumItem::accept_vis (*this);
+  for (auto &field : item_tuple.get_tuple_fields ())
+    field.get_field_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (EnumItemStruct &item_struct)
+{
+  item_struct.EnumItem::accept_vis (*this);
+  for (auto &field : item_struct.get_struct_fields ())
+    field.get_field_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (EnumItemDiscriminant &item)
+{
+  item.EnumItem::accept_vis (*this);
+  item.get_discriminant_expression ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (Enum &enum_item)
+{
+  visit_outer_attrs (enum_item);
+  for (auto &generic : enum_item.get_generic_params ())
+    generic->accept_vis (*this);
+  if (enum_item.has_where_clause ())
+    visit_where_clause (enum_item.get_where_clause ());
+  for (auto &item : enum_item.get_variants ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (Union &union_item)
+{
+  visit_outer_attrs (union_item);
+  for (auto &generic : union_item.get_generic_params ())
+    generic->accept_vis (*this);
+  if (union_item.has_where_clause ())
+    visit_where_clause (union_item.get_where_clause ());
+  for (auto &variant : union_item.get_variants ())
+    visit_struct_field (variant);
+}
+
+void
+DefaultHIRVisitor::walk (ConstantItem &const_item)
+{
+  visit_outer_attrs (const_item);
+  const_item.get_type ().accept_vis (*this);
+  const_item.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StaticItem &static_item)
+{
+  visit_outer_attrs (static_item);
+  static_item.get_type ().accept_vis (*this);
+  static_item.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_self_param (SelfParam &self_param)
+{
+  if (self_param.has_lifetime ())
+    {
+      Lifetime lifetime = self_param.get_lifetime ();
+      lifetime.accept_vis (*this);
+    }
+  if (self_param.has_type ())
+    self_param.get_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TraitItemFunc &item)
+{
+  visit_outer_attrs (item);
+  TraitFunctionDecl &decl = item.get_decl ();
+  for (auto &generic : decl.get_generic_params ())
+    generic->accept_vis (*this);
+  if (decl.get_self ().has_value ())
+    visit_self_param (decl.get_self ().value ());
+  for (auto &param : decl.get_function_params ())
+    visit_function_param (param);
+  if (decl.has_return_type ())
+    decl.get_return_type ().accept_vis (*this);
+  if (decl.has_where_clause ())
+    visit_where_clause (decl.get_where_clause ());
+  if (item.has_definition ())
+    item.get_block_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TraitItemConst &item)
+{
+  visit_outer_attrs (item);
+  item.get_type ().accept_vis (*this);
+  if (item.has_expr ())
+    item.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TraitItemType &item)
+{
+  visit_outer_attrs (item);
+  for (auto &bound : item.get_type_param_bounds ())
+    bound->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_where_clause (const WhereClause &where_clause)
+{
+  for (auto &item : where_clause.get_items ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_where_clause (WhereClause &where_clause)
+{
+  for (auto &item : where_clause.get_items ())
+    {
+      item->accept_vis (*this);
+    }
+}
+
+void
+DefaultHIRVisitor::walk (WhereClauseItem &node)
+{}
+
+void
+DefaultHIRVisitor::walk (Trait &trait)
+{
+  visit_outer_attrs (trait);
+  for (auto &generic : trait.get_generic_params ())
+    generic->accept_vis (*this);
+  if (trait.has_where_clause ())
+    visit_where_clause (trait.get_where_clause ());
+  for (auto &bound : trait.get_type_param_bounds ())
+    bound->accept_vis (*this);
+  for (auto &item : trait.get_trait_items ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ImplBlock &impl)
+{
+  visit_outer_attrs (impl);
+  for (auto &generic : impl.get_generic_params ())
+    generic->accept_vis (*this);
+  if (impl.has_trait_ref ())
+    impl.get_trait_ref ().accept_vis (*this);
+  impl.get_type ().accept_vis (*this);
+  if (impl.has_where_clause ())
+    visit_where_clause (impl.get_where_clause ());
+  visit_inner_attrs (impl);
+  for (auto &item : impl.get_impl_items ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ExternalStaticItem &item)
+{
+  visit_outer_attrs (item);
+  item.get_item_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::visit_named_function_param (NamedFunctionParam &param)
+{
+  param.get_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ExternalFunctionItem &item)
+{
+  visit_outer_attrs (item);
+  for (auto &generic : item.get_generic_params ())
+    generic->accept_vis (*this);
+  for (auto &param : item.get_function_params ())
+    visit_named_function_param (param);
+  if (item.has_return_type ())
+    item.get_return_type ().accept_vis (*this);
+  if (item.has_where_clause ())
+    visit_where_clause (item.get_where_clause ());
+}
+
+void
+DefaultHIRVisitor::walk (ExternalTypeItem &item)
+{
+  visit_outer_attrs (item);
+}
+
+void
+DefaultHIRVisitor::walk (ExternBlock &block)
+{
+  visit_outer_attrs (block);
+  visit_inner_attrs (block);
+  for (auto &item : block.get_extern_items ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (LiteralPattern &)
+{}
+
+void
+DefaultHIRVisitor::walk (IdentifierPattern &pattern)
+{
+  if (pattern.has_subpattern ())
+    pattern.get_subpattern ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (WildcardPattern &)
+{}
+
+void
+DefaultHIRVisitor::walk (RangePatternBoundLiteral &)
+{}
+
+void
+DefaultHIRVisitor::walk (RangePatternBoundPath &bound)
+{
+  bound.get_path ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (RangePatternBoundQualPath &bound)
+{
+  bound.get_qualified_path ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (RangePattern &pattern)
+{
+  pattern.get_lower_bound ().accept_vis (*this);
+  pattern.get_upper_bound ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ReferencePattern &pattern)
+{
+  pattern.get_referenced_pattern ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructPatternFieldTuplePat &field)
+{
+  visit_outer_attrs (field);
+  field.get_tuple_pattern ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructPatternFieldIdentPat &field)
+{
+  visit_outer_attrs (field);
+  field.get_pattern ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (StructPatternFieldIdent &field)
+{
+  visit_outer_attrs (field);
+}
+
+void
+DefaultHIRVisitor::walk (StructPattern &pattern)
+{
+  pattern.get_path ().accept_vis (*this);
+  StructPatternElements &elements = pattern.get_struct_pattern_elems ();
+  for (auto &field : elements.get_struct_pattern_fields ())
+    field->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TupleStructItemsNoRange &tuple_items)
+{
+  for (auto &item : tuple_items.get_patterns ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TupleStructItemsRange &tuple_items)
+{
+  for (auto &lower : tuple_items.get_lower_patterns ())
+    lower->accept_vis (*this);
+  for (auto &upper : tuple_items.get_upper_patterns ())
+    upper->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TupleStructPattern &pattern)
+{
+  pattern.get_path ().accept_vis (*this);
+  pattern.get_items ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TuplePatternItemsMultiple &tuple_items)
+{
+  for (auto &pattern : tuple_items.get_patterns ())
+    pattern->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TuplePatternItemsRanged &tuple_items)
+{
+  for (auto &lower : tuple_items.get_lower_patterns ())
+    lower->accept_vis (*this);
+  for (auto &upper : tuple_items.get_upper_patterns ())
+    upper->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TuplePattern &pattern)
+{
+  pattern.get_items ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (SlicePattern &pattern)
+{
+  for (auto &item : pattern.get_items ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (AltPattern &pattern)
+{
+  for (auto &item : pattern.get_alts ())
+    item->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (EmptyStmt &stmt)
+{}
+
+void
+DefaultHIRVisitor::walk (LetStmt &stmt)
+{
+  visit_outer_attrs (stmt);
+  stmt.get_pattern ().accept_vis (*this);
+  if (stmt.has_type ())
+    stmt.get_type ().accept_vis (*this);
+  if (stmt.has_init_expr ())
+    stmt.get_init_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ExprStmt &stmt)
+{
+  stmt.get_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TraitBound &bound)
+{
+  for (auto &lifetime : bound.get_for_lifetimes ())
+    lifetime.accept_vis (*this);
+  bound.get_path ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ImplTraitType &type)
+{
+  for (auto &bound : type.get_type_param_bounds ())
+    bound->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TraitObjectType &type)
+{
+  for (auto &bound : type.get_type_param_bounds ())
+    bound->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ParenthesisedType &type)
+{
+  type.get_type_in_parens ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (TupleType &type)
+{
+  for (auto &elem : type.get_elems ())
+    elem->accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (NeverType &type)
+{}
+
+void
+DefaultHIRVisitor::walk (RawPointerType &type)
+{
+  type.get_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ReferenceType &type)
+{
+  if (type.has_lifetime ())
+    type.get_lifetime ().accept_vis (*this);
+  type.get_base_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (ArrayType &type)
+{
+  type.get_element_type ().accept_vis (*this);
+  type.get_size_expr ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (SliceType &type)
+{
+  type.get_element_type ().accept_vis (*this);
+}
+
+void
+DefaultHIRVisitor::walk (InferredType &type)
+{}
+
+void
+DefaultHIRVisitor::walk (BareFunctionType &type)
+{
+  for (auto &lifetime : type.get_for_lifetimes ())
+    lifetime.accept_vis (*this);
+  for (auto &param : type.get_function_params ())
+    param.get_type ().accept_vis (*this);
+  if (type.has_return_type ())
+    type.get_return_type ().accept_vis (*this);
+}
+
+} // namespace HIR
+} // namespace Rust
diff --git a/gcc/rust/hir/tree/rust-hir-visitor.h b/gcc/rust/hir/tree/rust-hir-visitor.h
index 283cc34..7996260 100644
--- a/gcc/rust/hir/tree/rust-hir-visitor.h
+++ b/gcc/rust/hir/tree/rust-hir-visitor.h
@@ -20,6 +20,7 @@
 #define RUST_HIR_VISITOR_H
 
 #include "rust-hir-full-decls.h"
+#include "rust-ast.h"
 
 namespace Rust {
 namespace HIR {
@@ -64,6 +65,8 @@ public:
   virtual void visit (MethodCallExpr &expr) = 0;
   virtual void visit (FieldAccessExpr &expr) = 0;
   virtual void visit (BlockExpr &expr) = 0;
+  virtual void visit (AnonConst &expr) = 0;
+  virtual void visit (ConstBlock &expr) = 0;
   virtual void visit (ClosureExpr &expr) = 0;
   virtual void visit (ContinueExpr &expr) = 0;
   virtual void visit (BreakExpr &expr) = 0;
@@ -85,6 +88,7 @@ public:
   virtual void visit (AsyncBlockExpr &expr) = 0;
   virtual void visit (InlineAsm &expr) = 0;
   virtual void visit (LlvmInlineAsm &expr) = 0;
+  virtual void visit (OffsetOf &expr) = 0;
   virtual void visit (TypeParam &param) = 0;
   virtual void visit (ConstGenericParam &param) = 0;
   virtual void visit (LifetimeWhereClauseItem &item) = 0;
@@ -153,6 +157,312 @@ public:
   virtual void visit (BareFunctionType &type) = 0;
 };
 
+class DefaultHIRVisitor : public HIRFullVisitor
+{
+public:
+  virtual void visit_where_clause (WhereClause &);
+  virtual void visit_where_clause (const WhereClause &);
+  virtual void visit_named_function_param (NamedFunctionParam &param);
+  virtual void visit_function_param (FunctionParam &param);
+  virtual void visit_self_param (SelfParam &param);
+  virtual void visit_match_arm (MatchArm &arm);
+  virtual void visit_match_case (MatchCase &);
+  virtual void visit_struct_field (StructField &field);
+  virtual void visit_generic_args (GenericArgs &args);
+  virtual void visit_qualified_path_type (QualifiedPathType &);
+  virtual void visit_path_expr_segment (PathExprSegment &segment);
+  virtual void visit_closure_param (ClosureParam &param);
+  virtual void visit_loop_label (LoopLabel &);
+
+  virtual void visit_attribute (AST::Attribute &attr)
+  {
+    visit_attribute (static_cast<const AST::Attribute &> (attr));
+  }
+  virtual void visit_attribute (const AST::Attribute &attr) {}
+  template <typename T> void visit_outer_attrs (T &node)
+  {
+    for (auto &attr : node.get_outer_attrs ())
+      visit_attribute (attr);
+  }
+  template <typename T> void visit_inner_attrs (T &node)
+  {
+    for (auto &attr : node.get_inner_attrs ())
+      visit_attribute (attr);
+  }
+
+  virtual void visit (WhereClauseItem &node) { walk (node); }
+
+  virtual void visit (Lifetime &node) override { walk (node); }
+  virtual void visit (LifetimeParam &node) override { walk (node); }
+  virtual void visit (PathInExpression &node) override { walk (node); }
+  virtual void visit (TypePathSegment &node) override { walk (node); }
+  virtual void visit (TypePathSegmentGeneric &node) override { walk (node); }
+  virtual void visit (TypePathSegmentFunction &node) override { walk (node); }
+  virtual void visit (TypePath &node) override { walk (node); }
+  virtual void visit (QualifiedPathInExpression &node) override { walk (node); }
+  virtual void visit (QualifiedPathInType &node) override { walk (node); }
+  virtual void visit (LiteralExpr &node) override { walk (node); }
+  virtual void visit (BorrowExpr &node) override { walk (node); }
+  virtual void visit (DereferenceExpr &node) override { walk (node); }
+  virtual void visit (ErrorPropagationExpr &node) override { walk (node); }
+  virtual void visit (NegationExpr &node) override { walk (node); }
+  virtual void visit (ArithmeticOrLogicalExpr &node) override { walk (node); }
+  virtual void visit (ComparisonExpr &node) override { walk (node); }
+  virtual void visit (LazyBooleanExpr &node) override { walk (node); }
+  virtual void visit (TypeCastExpr &node) override { walk (node); }
+  virtual void visit (AssignmentExpr &node) override { walk (node); }
+  virtual void visit (CompoundAssignmentExpr &node) override { walk (node); }
+  virtual void visit (GroupedExpr &node) override { walk (node); }
+  virtual void visit (ArrayElemsValues &node) override { walk (node); }
+  virtual void visit (ArrayElemsCopied &node) override { walk (node); }
+  virtual void visit (ArrayExpr &node) override { walk (node); }
+  virtual void visit (ArrayIndexExpr &node) override { walk (node); }
+  virtual void visit (TupleExpr &node) override { walk (node); }
+  virtual void visit (TupleIndexExpr &node) override { walk (node); }
+  virtual void visit (StructExprStruct &node) override { walk (node); }
+  virtual void visit (StructExprFieldIdentifier &node) override { walk (node); }
+  virtual void visit (StructExprFieldIdentifierValue &node) override
+  {
+    walk (node);
+  }
+  virtual void visit (StructExprFieldIndexValue &node) override { walk (node); }
+  virtual void visit (StructExprStructFields &node) override { walk (node); }
+  virtual void visit (StructExprStructBase &node) override { walk (node); }
+  virtual void visit (CallExpr &node) override { walk (node); }
+  virtual void visit (MethodCallExpr &node) override { walk (node); }
+  virtual void visit (FieldAccessExpr &node) override { walk (node); }
+  virtual void visit (ClosureExpr &node) override { walk (node); }
+  virtual void visit (BlockExpr &node) override { walk (node); }
+  virtual void visit (AnonConst &node) override { walk (node); }
+  virtual void visit (ConstBlock &node) override { walk (node); }
+  virtual void visit (ContinueExpr &node) override { walk (node); }
+  virtual void visit (BreakExpr &node) override { walk (node); }
+  virtual void visit (RangeFromToExpr &node) override { walk (node); }
+  virtual void visit (RangeFromExpr &node) override { walk (node); }
+  virtual void visit (RangeToExpr &node) override { walk (node); }
+  virtual void visit (RangeFullExpr &node) override { walk (node); }
+  virtual void visit (RangeFromToInclExpr &node) override { walk (node); }
+  virtual void visit (RangeToInclExpr &node) override { walk (node); }
+  virtual void visit (ReturnExpr &node) override { walk (node); }
+  virtual void visit (UnsafeBlockExpr &node) override { walk (node); }
+  virtual void visit (LoopExpr &node) override { walk (node); }
+  virtual void visit (WhileLoopExpr &node) override { walk (node); }
+  virtual void visit (WhileLetLoopExpr &node) override { walk (node); }
+  virtual void visit (IfExpr &node) override { walk (node); }
+  virtual void visit (IfExprConseqElse &node) override { walk (node); }
+  virtual void visit (MatchExpr &node) override { walk (node); }
+  virtual void visit (AwaitExpr &node) override { walk (node); }
+  virtual void visit (AsyncBlockExpr &node) override { walk (node); }
+  virtual void visit (InlineAsm &node) override { walk (node); }
+  virtual void visit (LlvmInlineAsm &node) override { walk (node); }
+  virtual void visit (OffsetOf &node) override { walk (node); }
+  virtual void visit (TypeParam &node) override { walk (node); }
+  virtual void visit (ConstGenericParam &node) override { walk (node); }
+  virtual void visit (LifetimeWhereClauseItem &node) override { walk (node); }
+  virtual void visit (TypeBoundWhereClauseItem &node) override { walk (node); }
+  virtual void visit (Module &node) override { walk (node); }
+  virtual void visit (ExternCrate &node) override { walk (node); }
+  virtual void visit (UseTreeGlob &node) override { walk (node); }
+  virtual void visit (UseTreeList &node) override { walk (node); }
+  virtual void visit (UseTreeRebind &node) override { walk (node); }
+  virtual void visit (UseDeclaration &node) override { walk (node); }
+  virtual void visit (Function &node) override { walk (node); }
+  virtual void visit (TypeAlias &node) override { walk (node); }
+  virtual void visit (StructStruct &node) override { walk (node); }
+  virtual void visit (TupleStruct &node) override { walk (node); }
+  virtual void visit (EnumItem &node) override { walk (node); }
+  virtual void visit (EnumItemTuple &node) override { walk (node); }
+  virtual void visit (EnumItemStruct &node) override { walk (node); }
+  virtual void visit (EnumItemDiscriminant &node) override { walk (node); }
+  virtual void visit (Enum &node) override { walk (node); }
+  virtual void visit (Union &node) override { walk (node); }
+  virtual void visit (ConstantItem &node) override { walk (node); }
+  virtual void visit (StaticItem &node) override { walk (node); }
+  virtual void visit (TraitItemFunc &node) override { walk (node); }
+  virtual void visit (TraitItemConst &node) override { walk (node); }
+  virtual void visit (TraitItemType &node) override { walk (node); }
+  virtual void visit (Trait &node) override { walk (node); }
+  virtual void visit (ImplBlock &node) override { walk (node); }
+  virtual void visit (ExternalStaticItem &node) override { walk (node); }
+  virtual void visit (ExternalFunctionItem &node) override { walk (node); }
+  virtual void visit (ExternalTypeItem &node) override { walk (node); }
+  virtual void visit (ExternBlock &node) override { walk (node); }
+  virtual void visit (LiteralPattern &node) override { walk (node); }
+  virtual void visit (IdentifierPattern &node) override { walk (node); }
+  virtual void visit (WildcardPattern &node) override { walk (node); }
+  virtual void visit (RangePatternBoundLiteral &node) override { walk (node); }
+  virtual void visit (RangePatternBoundPath &node) override { walk (node); }
+  virtual void visit (RangePatternBoundQualPath &node) override { walk (node); }
+  virtual void visit (RangePattern &node) override { walk (node); }
+  virtual void visit (ReferencePattern &node) override { walk (node); }
+  virtual void visit (StructPatternFieldTuplePat &node) override
+  {
+    walk (node);
+  }
+  virtual void visit (StructPatternFieldIdentPat &node) override
+  {
+    walk (node);
+  }
+  virtual void visit (StructPatternFieldIdent &node) override { walk (node); }
+  virtual void visit (StructPattern &node) override { walk (node); }
+  virtual void visit (TupleStructItemsNoRange &node) override { walk (node); }
+  virtual void visit (TupleStructItemsRange &node) override { walk (node); }
+  virtual void visit (TupleStructPattern &node) override { walk (node); }
+  virtual void visit (TuplePatternItemsMultiple &node) override { walk (node); }
+  virtual void visit (TuplePatternItemsRanged &node) override { walk (node); }
+  virtual void visit (TuplePattern &node) override { walk (node); }
+  virtual void visit (SlicePattern &node) override { walk (node); }
+  virtual void visit (AltPattern &node) override { walk (node); }
+  virtual void visit (EmptyStmt &node) override { walk (node); }
+  virtual void visit (LetStmt &node) override { walk (node); }
+  virtual void visit (ExprStmt &node) override { walk (node); }
+  virtual void visit (TraitBound &node) override { walk (node); }
+  virtual void visit (ImplTraitType &node) override { walk (node); }
+  virtual void visit (TraitObjectType &node) override { walk (node); }
+  virtual void visit (ParenthesisedType &node) override { walk (node); }
+  virtual void visit (TupleType &node) override { walk (node); }
+  virtual void visit (NeverType &node) override { walk (node); }
+  virtual void visit (RawPointerType &node) override { walk (node); }
+  virtual void visit (ReferenceType &node) override { walk (node); }
+  virtual void visit (ArrayType &node) override { walk (node); }
+  virtual void visit (SliceType &node) override { walk (node); }
+  virtual void visit (InferredType &node) override { walk (node); }
+  virtual void visit (BareFunctionType &node) override { walk (node); }
+
+protected:
+  virtual void walk (WhereClauseItem &) final;
+
+  virtual void walk (Lifetime &) final;
+  virtual void walk (LifetimeParam &) final;
+  virtual void walk (PathInExpression &) final;
+  virtual void walk (TypePathSegment &) final;
+  virtual void walk (TypePathSegmentGeneric &) final;
+  virtual void walk (TypePathSegmentFunction &) final;
+  virtual void walk (TypePath &) final;
+  virtual void walk (QualifiedPathInExpression &) final;
+  virtual void walk (QualifiedPathInType &) final;
+
+  virtual void walk (LiteralExpr &) final;
+  virtual void walk (BorrowExpr &) final;
+  virtual void walk (DereferenceExpr &) final;
+  virtual void walk (ErrorPropagationExpr &) final;
+  virtual void walk (NegationExpr &) final;
+  virtual void walk (ArithmeticOrLogicalExpr &) final;
+  virtual void walk (ComparisonExpr &) final;
+  virtual void walk (LazyBooleanExpr &) final;
+  virtual void walk (TypeCastExpr &) final;
+  virtual void walk (AssignmentExpr &) final;
+  virtual void walk (CompoundAssignmentExpr &) final;
+  virtual void walk (GroupedExpr &) final;
+
+  virtual void walk (ArrayElemsValues &) final;
+  virtual void walk (ArrayElemsCopied &) final;
+  virtual void walk (ArrayExpr &) final;
+  virtual void walk (ArrayIndexExpr &) final;
+  virtual void walk (TupleExpr &) final;
+  virtual void walk (TupleIndexExpr &) final;
+  virtual void walk (StructExprStruct &) final;
+  virtual void walk (StructExprFieldIdentifier &) final;
+  virtual void walk (StructExprFieldIdentifierValue &) final;
+  virtual void walk (StructExprFieldIndexValue &) final;
+  virtual void walk (StructExprStructFields &) final;
+  virtual void walk (StructExprStructBase &) final;
+  virtual void walk (CallExpr &) final;
+  virtual void walk (MethodCallExpr &) final;
+  virtual void walk (FieldAccessExpr &) final;
+  virtual void walk (ClosureExpr &) final;
+  virtual void walk (BlockExpr &) final;
+  virtual void walk (AnonConst &) final;
+  virtual void walk (ConstBlock &) final;
+  virtual void walk (ContinueExpr &) final;
+  virtual void walk (BreakExpr &) final;
+  virtual void walk (RangeFromToExpr &) final;
+  virtual void walk (RangeFromExpr &) final;
+  virtual void walk (RangeToExpr &) final;
+  virtual void walk (RangeFullExpr &) final;
+  virtual void walk (RangeFromToInclExpr &) final;
+  virtual void walk (RangeToInclExpr &) final;
+  virtual void walk (ReturnExpr &) final;
+  virtual void walk (UnsafeBlockExpr &) final;
+  virtual void walk (LoopExpr &) final;
+  virtual void walk (WhileLoopExpr &) final;
+  virtual void walk (WhileLetLoopExpr &) final;
+  virtual void walk (IfExpr &) final;
+  virtual void walk (IfExprConseqElse &) final;
+  virtual void walk (MatchExpr &) final;
+  virtual void walk (AwaitExpr &) final;
+  virtual void walk (AsyncBlockExpr &) final;
+  virtual void walk (InlineAsm &) final;
+  virtual void walk (LlvmInlineAsm &) final;
+  virtual void walk (OffsetOf &) final;
+  virtual void walk (TypeParam &) final;
+  virtual void walk (ConstGenericParam &) final;
+  virtual void walk (LifetimeWhereClauseItem &) final;
+  virtual void walk (TypeBoundWhereClauseItem &) final;
+  virtual void walk (Module &) final;
+  virtual void walk (ExternCrate &) final;
+  virtual void walk (UseTreeGlob &) final;
+  virtual void walk (UseTreeList &) final;
+  virtual void walk (UseTreeRebind &) final;
+  virtual void walk (UseDeclaration &) final;
+  virtual void walk (Function &) final;
+  virtual void walk (TypeAlias &) final;
+  virtual void walk (StructStruct &) final;
+  virtual void walk (TupleStruct &) final;
+  virtual void walk (EnumItem &) final;
+  virtual void walk (EnumItemTuple &) final;
+  virtual void walk (EnumItemStruct &) final;
+  virtual void walk (EnumItemDiscriminant &) final;
+  virtual void walk (Enum &) final;
+  virtual void walk (Union &) final;
+  virtual void walk (ConstantItem &) final;
+  virtual void walk (StaticItem &) final;
+  virtual void walk (TraitItemFunc &) final;
+  virtual void walk (TraitItemConst &) final;
+  virtual void walk (TraitItemType &) final;
+  virtual void walk (Trait &) final;
+  virtual void walk (ImplBlock &) final;
+  virtual void walk (ExternalStaticItem &) final;
+  virtual void walk (ExternalFunctionItem &) final;
+  virtual void walk (ExternalTypeItem &) final;
+  virtual void walk (ExternBlock &) final;
+  virtual void walk (LiteralPattern &) final;
+  virtual void walk (IdentifierPattern &) final;
+  virtual void walk (WildcardPattern &) final;
+  virtual void walk (RangePatternBoundLiteral &) final;
+  virtual void walk (RangePatternBoundPath &) final;
+  virtual void walk (RangePatternBoundQualPath &) final;
+  virtual void walk (RangePattern &) final;
+  virtual void walk (ReferencePattern &) final;
+  virtual void walk (StructPatternFieldTuplePat &) final;
+  virtual void walk (StructPatternFieldIdentPat &) final;
+  virtual void walk (StructPatternFieldIdent &) final;
+  virtual void walk (StructPattern &) final;
+  virtual void walk (TupleStructItemsNoRange &) final;
+  virtual void walk (TupleStructItemsRange &) final;
+  virtual void walk (TupleStructPattern &) final;
+  virtual void walk (TuplePatternItemsMultiple &) final;
+  virtual void walk (TuplePatternItemsRanged &) final;
+  virtual void walk (TuplePattern &) final;
+  virtual void walk (SlicePattern &) final;
+  virtual void walk (AltPattern &) final;
+  virtual void walk (EmptyStmt &) final;
+  virtual void walk (LetStmt &) final;
+  virtual void walk (ExprStmt &) final;
+  virtual void walk (TraitBound &) final;
+  virtual void walk (ImplTraitType &) final;
+  virtual void walk (TraitObjectType &) final;
+  virtual void walk (ParenthesisedType &) final;
+  virtual void walk (TupleType &) final;
+  virtual void walk (NeverType &) final;
+  virtual void walk (RawPointerType &) final;
+  virtual void walk (ReferenceType &) final;
+  virtual void walk (ArrayType &) final;
+  virtual void walk (SliceType &) final;
+  virtual void walk (InferredType &) final;
+  virtual void walk (BareFunctionType &) final;
+};
+
 class HIRFullVisitorBase : public HIRFullVisitor
 {
 public:
@@ -201,6 +511,8 @@ public:
   virtual void visit (FieldAccessExpr &) override {}
   virtual void visit (ClosureExpr &) override {}
   virtual void visit (BlockExpr &) override {}
+  virtual void visit (AnonConst &) override {}
+  virtual void visit (ConstBlock &) override {}
   virtual void visit (ContinueExpr &) override {}
   virtual void visit (BreakExpr &) override {}
   virtual void visit (RangeFromToExpr &) override {}
@@ -222,6 +534,7 @@ public:
   virtual void visit (AsyncBlockExpr &) override {}
   virtual void visit (InlineAsm &) override {}
   virtual void visit (LlvmInlineAsm &) override {}
+  virtual void visit (OffsetOf &) override {}
 
   virtual void visit (TypeParam &) override {}
   virtual void visit (ConstGenericParam &) override {}
@@ -427,6 +740,8 @@ public:
   virtual void visit (MethodCallExpr &expr) = 0;
   virtual void visit (FieldAccessExpr &expr) = 0;
   virtual void visit (BlockExpr &expr) = 0;
+  virtual void visit (AnonConst &expr) = 0;
+  virtual void visit (ConstBlock &expr) = 0;
   virtual void visit (ContinueExpr &expr) = 0;
   virtual void visit (BreakExpr &expr) = 0;
   virtual void visit (RangeFromToExpr &expr) = 0;
@@ -443,6 +758,7 @@ public:
   virtual void visit (IfExpr &expr) = 0;
   virtual void visit (IfExprConseqElse &expr) = 0;
   virtual void visit (InlineAsm &expr) = 0;
+  virtual void visit (OffsetOf &expr) = 0;
   virtual void visit (LlvmInlineAsm &expr) = 0;
   virtual void visit (MatchExpr &expr) = 0;
   virtual void visit (AwaitExpr &expr) = 0;
diff --git a/gcc/rust/hir/tree/rust-hir.cc b/gcc/rust/hir/tree/rust-hir.cc
index 093d8d5..a802e8c 100644
--- a/gcc/rust/hir/tree/rust-hir.cc
+++ b/gcc/rust/hir/tree/rust-hir.cc
@@ -577,7 +577,8 @@ UseTreeGlob::as_string () const
       return "*";
     case GLOBAL:
       return "::*";
-      case PATH_PREFIXED: {
+    case PATH_PREFIXED:
+      {
 	std::string path_str = path.as_string ();
 	return path_str + "::*";
       }
@@ -600,7 +601,8 @@ UseTreeList::as_string () const
     case GLOBAL:
       path_str = "::{";
       break;
-      case PATH_PREFIXED: {
+    case PATH_PREFIXED:
+      {
 	path_str = path.as_string () + "::{";
 	break;
       }
@@ -1048,6 +1050,36 @@ BlockExpr::as_string () const
 }
 
 std::string
+AnonConst::as_string () const
+{
+  std::string istr = indent_spaces (enter);
+  std::string str = istr + "AnonConst:\n" + istr;
+
+  if (expr.has_value ())
+    str += get_inner_expr ().as_string ();
+  else
+    str += "_";
+
+  str += "\n" + indent_spaces (out);
+
+  return str;
+}
+
+std::string
+ConstBlock::as_string () const
+{
+  std::string istr = indent_spaces (enter);
+
+  std::string str = istr + "ConstBlock:\n" + istr;
+
+  str += get_const_expr ().as_string ();
+
+  str += "\n" + indent_spaces (out);
+
+  return str;
+}
+
+std::string
 TypeAlias::as_string () const
 {
   std::string str = VisItem::as_string ();
@@ -2579,9 +2611,9 @@ IdentifierPattern::as_string () const
 
   str += variable_ident.as_string ();
 
-  if (has_pattern_to_bind ())
+  if (has_subpattern ())
     {
-      str += " @ " + to_bind->as_string ();
+      str += " @ " + subpattern->as_string ();
     }
 
   return str;
@@ -4055,6 +4087,18 @@ BlockExpr::accept_vis (HIRFullVisitor &vis)
 }
 
 void
+AnonConst::accept_vis (HIRFullVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
+ConstBlock::accept_vis (HIRFullVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
 ContinueExpr::accept_vis (HIRFullVisitor &vis)
 {
   vis.visit (*this);
@@ -5027,6 +5071,18 @@ BlockExpr::accept_vis (HIRExpressionVisitor &vis)
 }
 
 void
+AnonConst::accept_vis (HIRExpressionVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
+ConstBlock::accept_vis (HIRExpressionVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
 Function::accept_vis (HIRStmtVisitor &vis)
 {
   vis.visit (*this);
diff --git a/gcc/rust/lang.opt b/gcc/rust/lang.opt
index 9cdbce2..d9824f1 100644
--- a/gcc/rust/lang.opt
+++ b/gcc/rust/lang.opt
@@ -205,7 +205,7 @@ EnumValue
 Enum(frust_compile_until) String(end) Value(13)
 
 frust-name-resolution-2.0
-Rust Var(flag_name_resolution_2_0)
+Rust Var(flag_name_resolution_2_0) Init(1)
 Use the temporary and experimental name resolution pipeline instead of the stable one
 
 frust-borrowcheck
@@ -229,4 +229,8 @@ frust-overflow-checks
 Rust Var(flag_overflow_checks) Init(1)
 Enable the overflow checks in code generation
 
+frust-assume-builtin-offset-of
+Rust Var(flag_assume_builtin_offset_of)
+Define a built-in offset_of macro in the compiler and assume it is present
+
 ; This comment is to ensure we retain the blank line above.
diff --git a/gcc/rust/lex/rust-lex.cc b/gcc/rust/lex/rust-lex.cc
index b143e70..76ff15c 100644
--- a/gcc/rust/lex/rust-lex.cc
+++ b/gcc/rust/lex/rust-lex.cc
@@ -1317,7 +1317,8 @@ Lexer::parse_escape (char opening_char)
 
   switch (current_char.value)
     {
-      case 'x': {
+    case 'x':
+      {
 	auto hex_escape_pair = parse_partial_hex_escape ();
 	long hexLong = hex_escape_pair.first;
 	additional_length_offset += hex_escape_pair.second;
@@ -1400,7 +1401,8 @@ Lexer::parse_utf8_escape ()
 
   switch (current_char.value)
     {
-      case 'x': {
+    case 'x':
+      {
 	auto hex_escape_pair = parse_partial_hex_escape ();
 	long hexLong = hex_escape_pair.first;
 	additional_length_offset += hex_escape_pair.second;
@@ -1438,7 +1440,8 @@ Lexer::parse_utf8_escape ()
     case '"':
       output_char = '"';
       break;
-      case 'u': {
+    case 'u':
+      {
 	auto unicode_escape_pair = parse_partial_unicode_escape ();
 	output_char = unicode_escape_pair.first;
 	additional_length_offset += unicode_escape_pair.second;
@@ -1894,6 +1897,11 @@ Lexer::parse_raw_byte_string (location_t loc)
 	      break;
 	    }
 	}
+      else if (current_char.is_eof ())
+	{
+	  rust_error_at (string_begin_locus, "unended raw byte string literal");
+	  return Token::make (END_OF_FILE, get_current_location ());
+	}
       else if (current_char.value > 127)
 	{
 	  rust_error_at (get_current_location (),
@@ -1901,11 +1909,6 @@ Lexer::parse_raw_byte_string (location_t loc)
 			 current_char.as_string ().c_str ());
 	  current_char = 0;
 	}
-      else if (current_char.is_eof ())
-	{
-	  rust_error_at (string_begin_locus, "unended raw byte string literal");
-	  return Token::make (END_OF_FILE, get_current_location ());
-	}
 
       length++;
       current_column++;
diff --git a/gcc/rust/lex/rust-lex.h b/gcc/rust/lex/rust-lex.h
index 10293e0..383ffac 100644
--- a/gcc/rust/lex/rust-lex.h
+++ b/gcc/rust/lex/rust-lex.h
@@ -263,8 +263,7 @@ private:
 #if CHECKING_P
 
 namespace selftest {
-void
-rust_input_source_test ();
+void rust_input_source_test ();
 
 } // namespace selftest
 
diff --git a/gcc/rust/lex/rust-token.cc b/gcc/rust/lex/rust-token.cc
index 8493889..c396e10 100644
--- a/gcc/rust/lex/rust-token.cc
+++ b/gcc/rust/lex/rust-token.cc
@@ -20,6 +20,7 @@
 #include "rust-token.h"
 #include "rust-diagnostics.h"
 #include "rust-unicode.h"
+#include "rust-ast.h"
 
 namespace Rust {
 // Hackily defined way to get token description for enum value using x-macros
@@ -88,7 +89,8 @@ token_id_keyword_string (TokenId id)
   switch (id)
     {
 #define RS_TOKEN_KEYWORD_2015(id, str_ptr)                                     \
-    case id: {                                                                 \
+  case id:                                                                     \
+    {                                                                          \
       static const std::string str (str_ptr);                                  \
       return str;                                                              \
     }                                                                          \
@@ -234,6 +236,13 @@ escape_special_chars (const std::string &source, Context ctx)
 
 } // namespace
 
+TokenPtr
+Token::make_identifier (const Identifier &ident)
+{
+  std::string str = ident;
+  return make_identifier (ident.get_locus (), std::move (str));
+}
+
 std::string
 Token::as_string () const
 {
diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h
index c683ecd..2021aec 100644
--- a/gcc/rust/lex/rust-token.h
+++ b/gcc/rust/lex/rust-token.h
@@ -24,6 +24,10 @@
 #include "rust-unicode.h"
 
 namespace Rust {
+
+// used by Rust::Token::make_identifier
+class Identifier;
+
 // "Primitive core types" in Rust - the different int and float types, as well
 // as some others
 enum PrimitiveCoreType
@@ -221,25 +225,20 @@ typedef std::shared_ptr<Token> TokenPtr;
 typedef std::shared_ptr<const Token> const_TokenPtr;
 
 // Hackily defined way to get token description for enum value using x-macros
-const char *
-get_token_description (TokenId id);
+const char *get_token_description (TokenId id);
 /* Hackily defined way to get token description as a string for enum value using
  * x-macros */
-const char *
-token_id_to_str (TokenId id);
+const char *token_id_to_str (TokenId id);
 /* checks if a token is a keyword */
-bool
-token_id_is_keyword (TokenId id);
+bool token_id_is_keyword (TokenId id);
 /* gets the string associated with a keyword */
-const std::string &
-token_id_keyword_string (TokenId id);
+const std::string &token_id_keyword_string (TokenId id);
 // Get type hint description as a string.
-const char *
-get_type_hint_string (PrimitiveCoreType type);
+const char *get_type_hint_string (PrimitiveCoreType type);
 
 /* Normalize string if a token is a identifier */
-std::string
-nfc_normalize_token_string (location_t loc, TokenId id, const std::string &str);
+std::string nfc_normalize_token_string (location_t loc, TokenId id,
+					const std::string &str);
 
 // Represents a single token. Create using factory static methods.
 class Token
@@ -329,6 +328,8 @@ public:
     return TokenPtr (new Token (IDENTIFIER, locus, std::move (str)));
   }
 
+  static TokenPtr make_identifier (const Identifier &ident);
+
   // Makes and returns a new TokenPtr of type INT_LITERAL.
   static TokenPtr make_int (location_t locus, std::string &&str,
 			    PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
diff --git a/gcc/rust/metadata/rust-export-metadata.cc b/gcc/rust/metadata/rust-export-metadata.cc
index 771bec6..1829a85 100644
--- a/gcc/rust/metadata/rust-export-metadata.cc
+++ b/gcc/rust/metadata/rust-export-metadata.cc
@@ -263,8 +263,7 @@ PublicInterface::write_to_path (const std::string &path) const
   FILE *nfd = fopen (path.c_str (), "wb");
   if (nfd == NULL)
     {
-      rust_error_at (UNDEF_LOCATION,
-		     "failed to open file %qs for writing: %s",
+      rust_error_at (UNDEF_LOCATION, "failed to open file %qs for writing: %s",
 		     path.c_str (), xstrerror (errno));
       return;
     }
diff --git a/gcc/rust/metadata/rust-import-archive.cc b/gcc/rust/metadata/rust-import-archive.cc
index cf24607..f64de4e 100644
--- a/gcc/rust/metadata/rust-import-archive.cc
+++ b/gcc/rust/metadata/rust-import-archive.cc
@@ -683,7 +683,7 @@ public:
 
   const Header &operator* () const { return this->header_; }
 
-  const Header *operator-> () const { return &this->header_; }
+  const Header *operator->() const { return &this->header_; }
 
   Archive_iterator &operator++ ()
   {
diff --git a/gcc/rust/metadata/rust-imports.h b/gcc/rust/metadata/rust-imports.h
index a497c67..65a2af1 100644
--- a/gcc/rust/metadata/rust-imports.h
+++ b/gcc/rust/metadata/rust-imports.h
@@ -11,8 +11,7 @@
 
 namespace Rust {
 
-extern void
-add_search_path (const std::string &path);
+extern void add_search_path (const std::string &path);
 
 class Import
 {
diff --git a/gcc/rust/parse/rust-cfg-parser.h b/gcc/rust/parse/rust-cfg-parser.h
index 0d64016..61db240 100644
--- a/gcc/rust/parse/rust-cfg-parser.h
+++ b/gcc/rust/parse/rust-cfg-parser.h
@@ -36,15 +36,14 @@ namespace Rust {
  *
  * @return false if the given input was invalid, true otherwise
  */
-bool
-parse_cfg_option (std::string &input, std::string &key, std::string &value);
+bool parse_cfg_option (std::string &input, std::string &key,
+		       std::string &value);
 } // namespace Rust
 
 #if CHECKING_P
 
 namespace selftest {
-extern void
-rust_cfg_parser_test (void);
+extern void rust_cfg_parser_test (void);
 } // namespace selftest
 
 #endif // CHECKING_P
diff --git a/gcc/rust/ast/rust-macro.cc b/gcc/rust/parse/rust-parse-impl-lexer.cc
index 2703438..fec91e8 100644
--- a/gcc/rust/ast/rust-macro.cc
+++ b/gcc/rust/parse/rust-parse-impl-lexer.cc
@@ -1,4 +1,4 @@
-// Copyright (C) 2020-2025 Free Software Foundation, Inc.
+// Copyright (C) 2025 Free Software Foundation, Inc.
 
 // This file is part of GCC.
 
@@ -16,10 +16,10 @@
 // along with GCC; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
-#include "rust-macro.h"
+#include "rust-parse-impl.h"
 
 namespace Rust {
-namespace AST {
 
-} // namespace AST
+template class Parser<Lexer>;
+
 } // namespace Rust
diff --git a/gcc/rust/parse/rust-parse-impl-macro.cc b/gcc/rust/parse/rust-parse-impl-macro.cc
new file mode 100644
index 0000000..e632887
--- /dev/null
+++ b/gcc/rust/parse/rust-parse-impl-macro.cc
@@ -0,0 +1,26 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-parse-impl.h"
+#include "rust-macro-invoc-lexer.h"
+
+namespace Rust {
+
+template class Parser<MacroInvocLexer>;
+
+} // namespace Rust
diff --git a/gcc/rust/parse/rust-parse-impl-proc-macro.cc b/gcc/rust/parse/rust-parse-impl-proc-macro.cc
new file mode 100644
index 0000000..edc484f
--- /dev/null
+++ b/gcc/rust/parse/rust-parse-impl-proc-macro.cc
@@ -0,0 +1,34 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-parse-impl.h"
+#include "rust-proc-macro-invoc-lexer.h"
+
+namespace Rust {
+
+template std::unique_ptr<AST::Item>
+Parser<ProcMacroInvocLexer>::parse_item (bool);
+
+template std::unique_ptr<AST::Stmt>
+  Parser<ProcMacroInvocLexer>::parse_stmt (ParseRestrictions);
+
+// instantiate entire class (or just more functions) if necessary
+
+// template class Parser<ProcMacroInvocLexer>;
+
+} // namespace Rust
diff --git a/gcc/rust/parse/rust-parse-impl.h b/gcc/rust/parse/rust-parse-impl.h
index 9dda231..14bccbd 100644
--- a/gcc/rust/parse/rust-parse-impl.h
+++ b/gcc/rust/parse/rust-parse-impl.h
@@ -19,13 +19,17 @@
 /* Template implementation for Rust::Parser. Previously in rust-parse.cc (before
  * Parser was template). Separated from rust-parse.h for readability. */
 
-/* DO NOT INCLUDE ANYWHERE - this is automatically included with rust-parse.h
+/* DO NOT INCLUDE ANYWHERE - this is automatically included
+ *   by rust-parse-impl-*.cc
  * This is also the reason why there are no include guards. */
 
+#include "expected.h"
+#include "rust-ast.h"
 #include "rust-common.h"
 #include "rust-expr.h"
 #include "rust-item.h"
 #include "rust-common.h"
+#include "rust-parse.h"
 #include "rust-token.h"
 #define INCLUDE_ALGORITHM
 #include "rust-diagnostics.h"
@@ -227,19 +231,22 @@ Parser<ManagedTokenSource>::skip_generics_right_angle ()
       // this is good - skip token
       lexer.skip_token ();
       return true;
-      case RIGHT_SHIFT: {
+    case RIGHT_SHIFT:
+      {
 	// new implementation that should be better
 	lexer.split_current_token (RIGHT_ANGLE, RIGHT_ANGLE);
 	lexer.skip_token ();
 	return true;
       }
-      case GREATER_OR_EQUAL: {
+    case GREATER_OR_EQUAL:
+      {
 	// new implementation that should be better
 	lexer.split_current_token (RIGHT_ANGLE, EQUAL);
 	lexer.skip_token ();
 	return true;
       }
-      case RIGHT_SHIFT_EQ: {
+    case RIGHT_SHIFT_EQ:
+      {
 	// new implementation that should be better
 	lexer.split_current_token (RIGHT_ANGLE, GREATER_OR_EQUAL);
 	lexer.skip_token ();
@@ -651,10 +658,7 @@ Parser<ManagedTokenSource>::parse_simple_path ()
   // Parse all other simple path segments
   while (lexer.peek_token ()->get_id () == SCOPE_RESOLUTION)
     {
-      // Skip scope resolution operator
-      lexer.skip_token ();
-
-      AST::SimplePathSegment new_segment = parse_simple_path_segment ();
+      AST::SimplePathSegment new_segment = parse_simple_path_segment (1);
 
       // Return path as currently constructed if segment in error state.
       if (new_segment.is_error ())
@@ -682,35 +686,36 @@ Parser<ManagedTokenSource>::parse_simple_path ()
 }
 
 /* Parses a single SimplePathSegment (does not handle the scope resolution
- * operators) */
+ * operators)
+ * Starts parsing at an offset of base_peek */
 template <typename ManagedTokenSource>
 AST::SimplePathSegment
-Parser<ManagedTokenSource>::parse_simple_path_segment ()
+Parser<ManagedTokenSource>::parse_simple_path_segment (int base_peek)
 {
   using namespace Values;
-  const_TokenPtr t = lexer.peek_token ();
+  const_TokenPtr t = lexer.peek_token (base_peek);
   switch (t->get_id ())
     {
     case IDENTIFIER:
-      lexer.skip_token ();
+      lexer.skip_token (base_peek);
 
       return AST::SimplePathSegment (t->get_str (), t->get_locus ());
     case SUPER:
-      lexer.skip_token ();
+      lexer.skip_token (base_peek);
 
       return AST::SimplePathSegment (Keywords::SUPER, t->get_locus ());
     case SELF:
-      lexer.skip_token ();
+      lexer.skip_token (base_peek);
 
       return AST::SimplePathSegment (Keywords::SELF, t->get_locus ());
     case CRATE:
-      lexer.skip_token ();
+      lexer.skip_token (base_peek);
 
       return AST::SimplePathSegment (Keywords::CRATE, t->get_locus ());
     case DOLLAR_SIGN:
-      if (lexer.peek_token (1)->get_id () == CRATE)
+      if (lexer.peek_token (base_peek + 1)->get_id () == CRATE)
 	{
-	  lexer.skip_token (1);
+	  lexer.skip_token (base_peek + 1);
 
 	  return AST::SimplePathSegment ("$crate", t->get_locus ());
 	}
@@ -790,7 +795,8 @@ Parser<ManagedTokenSource>::parse_attr_input ()
     {
     case LEFT_PAREN:
     case LEFT_SQUARE:
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	// must be a delimited token tree, so parse that
 	std::unique_ptr<AST::AttrInput> input_tree (
 	  new AST::DelimTokenTree (parse_delim_token_tree ()));
@@ -799,7 +805,8 @@ Parser<ManagedTokenSource>::parse_attr_input ()
 
 	return input_tree;
       }
-      case EQUAL: {
+    case EQUAL:
+      {
 	// = LiteralExpr
 	lexer.skip_token ();
 
@@ -877,7 +884,10 @@ Parser<ManagedTokenSource>::parse_attr_input ()
 	return attr_input_lit;
       }
       break;
+    case RIGHT_PAREN:
     case RIGHT_SQUARE:
+    case RIGHT_CURLY:
+    case END_OF_FILE:
       // means AttrInput is missing, which is allowed
       return nullptr;
     default:
@@ -2117,7 +2127,8 @@ Parser<ManagedTokenSource>::parse_macro_match ()
     {
     case LEFT_PAREN:
     case LEFT_SQUARE:
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	// must be macro matcher as delimited
 	AST::MacroMatcher matcher = parse_macro_matcher ();
 	if (matcher.is_error ())
@@ -2131,7 +2142,8 @@ Parser<ManagedTokenSource>::parse_macro_match ()
 	return std::unique_ptr<AST::MacroMatcher> (
 	  new AST::MacroMatcher (std::move (matcher)));
       }
-      case DOLLAR_SIGN: {
+    case DOLLAR_SIGN:
+      {
 	// have to do more lookahead to determine if fragment or repetition
 	const_TokenPtr t2 = lexer.peek_token (1);
 	switch (t2->get_id ())
@@ -2402,7 +2414,8 @@ Parser<ManagedTokenSource>::parse_visibility ()
       skip_token (RIGHT_PAREN);
 
       return AST::Visibility::create_super (path_loc, vis_loc);
-      case IN: {
+    case IN:
+      {
 	lexer.skip_token ();
 
 	// parse the "in" path as well
@@ -2466,7 +2479,8 @@ Parser<ManagedTokenSource>::parse_module (AST::Visibility vis,
 	new AST::Module (std::move (name), std::move (vis),
 			 std::move (outer_attrs), locus, safety,
 			 lexer.get_filename (), inline_module_stack));
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	lexer.skip_token ();
 
 	// parse inner attributes
@@ -2730,7 +2744,8 @@ Parser<ManagedTokenSource>::parse_use_tree ()
 	    return std::unique_ptr<AST::UseTreeGlob> (
 	      new AST::UseTreeGlob (AST::UseTreeGlob::NO_PATH,
 				    AST::SimplePath::create_empty (), locus));
-	  case LEFT_CURLY: {
+	case LEFT_CURLY:
+	  {
 	    // nested tree UseTree type
 	    lexer.skip_token ();
 
@@ -2794,58 +2809,12 @@ Parser<ManagedTokenSource>::parse_use_tree ()
     }
   else
     {
-      /* Due to aforementioned implementation issues, the trailing :: token is
-       * consumed by the path, so it can not be used as a disambiguator.
-       * NOPE, not true anymore - TODO what are the consequences of this? */
-
       const_TokenPtr t = lexer.peek_token ();
+
       switch (t->get_id ())
 	{
-	case ASTERISK:
-	  // glob UseTree type
-	  lexer.skip_token ();
-
-	  return std::unique_ptr<AST::UseTreeGlob> (
-	    new AST::UseTreeGlob (AST::UseTreeGlob::PATH_PREFIXED,
-				  std::move (path), locus));
-	  case LEFT_CURLY: {
-	    // nested tree UseTree type
-	    lexer.skip_token ();
-
-	    std::vector<std::unique_ptr<AST::UseTree>> use_trees;
-
-	    // TODO: think of better control structure
-	    const_TokenPtr t = lexer.peek_token ();
-	    while (t->get_id () != RIGHT_CURLY)
-	      {
-		std::unique_ptr<AST::UseTree> use_tree = parse_use_tree ();
-		if (use_tree == nullptr)
-		  {
-		    break;
-		  }
-
-		use_trees.push_back (std::move (use_tree));
-
-		if (lexer.peek_token ()->get_id () != COMMA)
-		  break;
-
-		lexer.skip_token ();
-		t = lexer.peek_token ();
-	      }
-
-	    // skip end curly delimiter
-	    if (!skip_token (RIGHT_CURLY))
-	      {
-		// skip after somewhere?
-		return nullptr;
-	      }
-
-	    return std::unique_ptr<AST::UseTreeList> (
-	      new AST::UseTreeList (AST::UseTreeList::PATH_PREFIXED,
-				    std::move (path), std::move (use_trees),
-				    locus));
-	  }
-	  case AS: {
+	case AS:
+	  {
 	    // rebind UseTree type
 	    lexer.skip_token ();
 
@@ -2884,16 +2853,72 @@ Parser<ManagedTokenSource>::parse_use_tree ()
 
 	  // don't skip semicolon - handled in parse_use_tree
 	  // lexer.skip_token();
-
-	  return std::unique_ptr<AST::UseTreeRebind> (
-	    new AST::UseTreeRebind (AST::UseTreeRebind::NONE, std::move (path),
-				    locus));
 	case COMMA:
 	case RIGHT_CURLY:
 	  // this may occur in recursive calls - assume it is ok and ignore it
 	  return std::unique_ptr<AST::UseTreeRebind> (
 	    new AST::UseTreeRebind (AST::UseTreeRebind::NONE, std::move (path),
 				    locus));
+	case SCOPE_RESOLUTION:
+	  // keep going
+	  break;
+	default:
+	  add_error (Error (t->get_locus (),
+			    "unexpected token %qs in use tree with valid path",
+			    t->get_token_description ()));
+	  return nullptr;
+	}
+
+      skip_token ();
+      t = lexer.peek_token ();
+
+      switch (t->get_id ())
+	{
+	case ASTERISK:
+	  // glob UseTree type
+	  lexer.skip_token ();
+
+	  return std::unique_ptr<AST::UseTreeGlob> (
+	    new AST::UseTreeGlob (AST::UseTreeGlob::PATH_PREFIXED,
+				  std::move (path), locus));
+	case LEFT_CURLY:
+	  {
+	    // nested tree UseTree type
+	    lexer.skip_token ();
+
+	    std::vector<std::unique_ptr<AST::UseTree>> use_trees;
+
+	    // TODO: think of better control structure
+	    const_TokenPtr t = lexer.peek_token ();
+	    while (t->get_id () != RIGHT_CURLY)
+	      {
+		std::unique_ptr<AST::UseTree> use_tree = parse_use_tree ();
+		if (use_tree == nullptr)
+		  {
+		    break;
+		  }
+
+		use_trees.push_back (std::move (use_tree));
+
+		if (lexer.peek_token ()->get_id () != COMMA)
+		  break;
+
+		lexer.skip_token ();
+		t = lexer.peek_token ();
+	      }
+
+	    // skip end curly delimiter
+	    if (!skip_token (RIGHT_CURLY))
+	      {
+		// skip after somewhere?
+		return nullptr;
+	      }
+
+	    return std::unique_ptr<AST::UseTreeList> (
+	      new AST::UseTreeList (AST::UseTreeList::PATH_PREFIXED,
+				    std::move (path), std::move (use_trees),
+				    locus));
+	  }
 	default:
 	  add_error (Error (t->get_locus (),
 			    "unexpected token %qs in use tree with valid path",
@@ -3100,7 +3125,8 @@ Parser<ManagedTokenSource>::parse_generic_param (EndTokenPred is_end_token)
 
   switch (token->get_id ())
     {
-      case LIFETIME: {
+    case LIFETIME:
+      {
 	auto lifetime = parse_lifetime (false);
 	if (!lifetime)
 	  {
@@ -3126,7 +3152,8 @@ Parser<ManagedTokenSource>::parse_generic_param (EndTokenPred is_end_token)
 	  std::move (outer_attrs), token->get_locus ()));
 	break;
       }
-      case IDENTIFIER: {
+    case IDENTIFIER:
+      {
 	auto type_ident = token->get_str ();
 	lexer.skip_token ();
 
@@ -3161,7 +3188,8 @@ Parser<ManagedTokenSource>::parse_generic_param (EndTokenPred is_end_token)
 			      std::move (outer_attrs)));
 	break;
       }
-      case CONST: {
+    case CONST:
+      {
 	lexer.skip_token ();
 
 	auto name_token = expect_token (IDENTIFIER);
@@ -3345,7 +3373,8 @@ Parser<ManagedTokenSource>::parse_lifetime_params (EndTokenPred is_end_token)
 
 /* Parses lifetime generic parameters (objects). Will also consume any
  * trailing comma. No extra checks for end token.
- * TODO: is this best solution? implements most of the same algorithm. */
+ * TODO: is this best solution? implements most of the same algorithm.
+ * TODO: seems to be unused, remove? */
 template <typename ManagedTokenSource>
 std::vector<AST::LifetimeParam>
 Parser<ManagedTokenSource>::parse_lifetime_params_objs ()
@@ -3363,7 +3392,7 @@ Parser<ManagedTokenSource>::parse_lifetime_params_objs ()
 	  break;
 	}
 
-      lifetime_params.push_back (std::move (lifetime_param));
+      lifetime_params.push_back (std::move (lifetime_param.value ()));
 
       if (lexer.peek_token ()->get_id () != COMMA)
 	break;
@@ -4356,7 +4385,8 @@ Parser<ManagedTokenSource>::parse_struct (AST::Visibility vis,
   const_TokenPtr t = lexer.peek_token ();
   switch (t->get_id ())
     {
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	// struct with body
 
 	// skip curly bracket
@@ -4759,7 +4789,8 @@ Parser<ManagedTokenSource>::parse_enum_item ()
   const_TokenPtr t = lexer.peek_token ();
   switch (t->get_id ())
     {
-      case LEFT_PAREN: {
+    case LEFT_PAREN:
+      {
 	// tuple enum item
 	lexer.skip_token ();
 
@@ -4780,7 +4811,8 @@ Parser<ManagedTokenSource>::parse_enum_item ()
 	  std::move (item_name), std::move (vis), std::move (tuple_fields),
 	  std::move (outer_attrs), item_name_tok->get_locus ()));
       }
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	// struct enum item
 	lexer.skip_token ();
 
@@ -4797,7 +4829,8 @@ Parser<ManagedTokenSource>::parse_enum_item ()
 	  std::move (item_name), std::move (vis), std::move (struct_fields),
 	  std::move (outer_attrs), item_name_tok->get_locus ()));
       }
-      case EQUAL: {
+    case EQUAL:
+      {
 	// discriminant enum item
 	lexer.skip_token ();
 
@@ -5450,7 +5483,8 @@ Parser<ManagedTokenSource>::parse_inherent_impl_item ()
     case SUPER:
     case SELF:
     case CRATE:
-      case PUB: {
+    case PUB:
+      {
 	// visibility, so not a macro invocation semi - must be constant,
 	// function, or method
 	AST::Visibility vis = parse_visibility ();
@@ -5581,7 +5615,8 @@ Parser<ManagedTokenSource>::parse_inherent_impl_function_or_method (
   // param
   auto initial_param = parse_self_param ();
 
-  if (!initial_param.has_value () && initial_param.error () != NOT_SELF)
+  if (!initial_param.has_value ()
+      && initial_param.error () != ParseSelfError::NOT_SELF)
     return nullptr;
 
   /* FIXME: ensure that self param doesn't accidently consume tokens for a
@@ -5780,7 +5815,8 @@ Parser<ManagedTokenSource>::parse_trait_impl_function_or_method (
   // param
   auto initial_param = parse_self_param ();
 
-  if (!initial_param.has_value () && initial_param.error () != NOT_SELF)
+  if (!initial_param.has_value ()
+      && initial_param.error () != ParseSelfError::NOT_SELF)
     return nullptr;
 
   // FIXME: ensure that self param doesn't accidently consume tokens for a
@@ -5964,7 +6000,8 @@ Parser<ManagedTokenSource>::parse_external_item ()
     {
     case IDENTIFIER:
       return parse_macro_invocation_semi (outer_attrs);
-      case STATIC_KW: {
+    case STATIC_KW:
+      {
 	// parse extern static item
 	lexer.skip_token ();
 
@@ -6261,7 +6298,8 @@ Parser<ManagedTokenSource>::parse_generic_arg ()
 
   switch (tok->get_id ())
     {
-      case IDENTIFIER: {
+    case IDENTIFIER:
+      {
 	// This is a bit of a weird situation: With an identifier token, we
 	// could either have a valid type or a macro (FIXME: anything else?). So
 	// we need one bit of lookahead to differentiate if this is really
@@ -6309,9 +6347,10 @@ Parser<ManagedTokenSource>::parse_generic_arg ()
     case FALSE_LITERAL:
       expr = parse_literal_expr ();
       break;
-      // FIXME: Because of this, error reporting is garbage for const generic
-      // parameter's default values
-      default: {
+    // FIXME: Because of this, error reporting is garbage for const generic
+    // parameter's default values
+    default:
+      {
 	auto type = parse_type ();
 	// FIXME: Find a better way to do this?
 	if (type)
@@ -6510,7 +6549,8 @@ Parser<ManagedTokenSource>::parse_type_path_segment ()
   switch (t->get_id ())
     {
     case LEFT_SHIFT:
-      case LEFT_ANGLE: {
+    case LEFT_ANGLE:
+      {
 	// parse generic args
 	AST::GenericArgs generic_args = parse_path_generic_args ();
 
@@ -6519,7 +6559,8 @@ Parser<ManagedTokenSource>::parse_type_path_segment ()
 					   has_separating_scope_resolution,
 					   std::move (generic_args), locus));
       }
-      case LEFT_PAREN: {
+    case LEFT_PAREN:
+      {
 	// parse type path function
 	AST::TypePathFunction type_path_function
 	  = parse_type_path_function (locus);
@@ -7068,16 +7109,14 @@ Parser<ManagedTokenSource>::parse_expr_stmt (AST::AttrVec outer_attrs,
     case SELF:
     case SELF_ALIAS:
     case DOLLAR_SIGN:
-      case SCOPE_RESOLUTION: {
+    case SCOPE_RESOLUTION:
+      {
 	AST::PathInExpression path = parse_path_in_expression ();
 	std::unique_ptr<AST::Expr> null_denotation;
 
 	if (lexer.peek_token ()->get_id () == EXCLAM)
 	  {
-	    // Bind a reference to avoid -Wredundant-move on post-P1825R0
-	    // compilers. Change to non-reference type and remove the moves
-	    // below once C++20 is required to build gcc.
-	    std::unique_ptr<AST::MacroInvocation> &&invoc
+	    std::unique_ptr<AST::MacroInvocation> invoc
 	      = parse_macro_invocation_partial (std::move (path),
 						std::move (outer_attrs));
 
@@ -7085,7 +7124,7 @@ Parser<ManagedTokenSource>::parse_expr_stmt (AST::AttrVec outer_attrs,
 	      {
 		invoc->add_semicolon ();
 		// Macro invocation with semicolon.
-		return std::move (invoc);
+		return invoc;
 	      }
 
 	    TokenId after_macro = lexer.peek_token ()->get_id ();
@@ -7093,14 +7132,14 @@ Parser<ManagedTokenSource>::parse_expr_stmt (AST::AttrVec outer_attrs,
 	    if (restrictions.allow_close_after_expr_stmt
 		&& (after_macro == RIGHT_PAREN || after_macro == RIGHT_CURLY
 		    || after_macro == RIGHT_SQUARE))
-	      return std::move (invoc);
+	      return invoc;
 
 	    if (invoc->get_invoc_data ().get_delim_tok_tree ().get_delim_type ()
 		  == AST::CURLY
 		&& after_macro != DOT && after_macro != QUESTION_MARK)
 	      {
 		rust_debug ("braced macro statement");
-		return std::move (invoc);
+		return invoc;
 	      }
 
 	    null_denotation = std::move (invoc);
@@ -7238,6 +7277,51 @@ Parser<ManagedTokenSource>::parse_block_expr (
 			std::move (label), locus, end_locus));
 }
 
+/* Parse an anonymous const expression. This can be a regular const expression
+ * or an underscore for deferred const inference */
+template <typename ManagedTokenSource>
+tl::expected<AST::AnonConst, AnonConstError>
+Parser<ManagedTokenSource>::parse_anon_const ()
+{
+  auto current = lexer.peek_token ();
+  auto locus = current->get_locus ();
+
+  // Special case deferred inference constants
+  if (maybe_skip_token (UNDERSCORE))
+    return AST::AnonConst (locus);
+
+  auto expr = parse_expr ();
+
+  if (!expr)
+    return tl::make_unexpected (AnonConstError::InvalidSizeExpr);
+
+  return AST::AnonConst (std::move (expr), locus);
+}
+
+/* Parse a "const block", a block preceded by the `const` keyword whose
+ * statements can be const evaluated and used in constant contexts */
+template <typename ManagedTokenSource>
+std::unique_ptr<AST::ConstBlock>
+Parser<ManagedTokenSource>::parse_const_block_expr (AST::AttrVec outer_attrs,
+						    location_t locus)
+{
+  auto block = parse_block_expr ();
+
+  if (!block)
+    {
+      add_error (Error (locus, "failed to parse inner block in const block"));
+      skip_after_end_block ();
+
+      return nullptr;
+    }
+
+  auto block_locus = block->get_locus ();
+
+  return std::make_unique<AST::ConstBlock> (AST::AnonConst (std::move (block),
+							    block_locus),
+					    locus, std::move (outer_attrs));
+}
+
 /* Parses a "grouped" expression (expression in parentheses), used to control
  * precedence. */
 template <typename ManagedTokenSource>
@@ -7518,6 +7602,34 @@ Parser<ManagedTokenSource>::parse_return_expr (AST::AttrVec outer_attrs,
 			 locus));
 }
 
+// Parses a try expression.
+template <typename ManagedTokenSource>
+std::unique_ptr<AST::TryExpr>
+Parser<ManagedTokenSource>::parse_try_expr (AST::AttrVec outer_attrs,
+					    location_t pratt_parsed_loc)
+{
+  location_t locus = pratt_parsed_loc;
+  if (locus == UNKNOWN_LOCATION)
+    {
+      locus = lexer.peek_token ()->get_locus ();
+      skip_token (TRY);
+    }
+
+  std::unique_ptr<AST::BlockExpr> block_expr = parse_block_expr ();
+
+  if (!block_expr)
+    {
+      Error error (lexer.peek_token ()->get_locus (),
+		   "failed to parse try block expression");
+      add_error (std::move (error));
+
+      return nullptr;
+    }
+
+  return std::unique_ptr<AST::TryExpr> (
+    new AST::TryExpr (std::move (block_expr), std::move (outer_attrs), locus));
+}
+
 /* Parses a break expression (including any label to break to AND any return
  * expression). */
 template <typename ManagedTokenSource>
@@ -7675,7 +7787,8 @@ Parser<ManagedTokenSource>::parse_if_expr (AST::AttrVec outer_attrs,
       const_TokenPtr t = lexer.peek_token ();
       switch (t->get_id ())
 	{
-	  case LEFT_CURLY: {
+	case LEFT_CURLY:
+	  {
 	    // double selection - else
 	    // parse else block expr (required)
 	    std::unique_ptr<AST::BlockExpr> else_body = parse_block_expr ();
@@ -7696,7 +7809,8 @@ Parser<ManagedTokenSource>::parse_if_expr (AST::AttrVec outer_attrs,
 					 std::move (else_body),
 					 std::move (outer_attrs), locus));
 	  }
-	  case IF: {
+	case IF:
+	  {
 	    // multiple selection - else if or else if let
 	    // branch on whether next token is 'let' or not
 	    if (lexer.peek_token (1)->get_id () == LET)
@@ -7857,7 +7971,8 @@ Parser<ManagedTokenSource>::parse_if_let_expr (AST::AttrVec outer_attrs,
       const_TokenPtr t = lexer.peek_token ();
       switch (t->get_id ())
 	{
-	  case LEFT_CURLY: {
+	case LEFT_CURLY:
+	  {
 	    // double selection - else
 	    // parse else block expr (required)
 	    std::unique_ptr<AST::BlockExpr> else_body = parse_block_expr ();
@@ -7879,7 +7994,8 @@ Parser<ManagedTokenSource>::parse_if_let_expr (AST::AttrVec outer_attrs,
 					    std::move (else_body),
 					    std::move (outer_attrs), locus));
 	  }
-	  case IF: {
+	case IF:
+	  {
 	    // multiple selection - else if or else if let
 	    // branch on whether next token is 'let' or not
 	    if (lexer.peek_token (1)->get_id () == LET)
@@ -8931,7 +9047,8 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
       // slice type or array type - requires further disambiguation
       return parse_slice_or_array_type ();
     case LEFT_SHIFT:
-      case LEFT_ANGLE: {
+    case LEFT_ANGLE:
+      {
 	// qualified path in type
 	AST::QualifiedPathInType path = parse_qualified_path_in_type ();
 	if (path.is_error ())
@@ -8960,7 +9077,8 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
     case LOGICAL_AND:
       // reference type
       return parse_reference_type ();
-      case LIFETIME: {
+    case LIFETIME:
+      {
 	/* probably a lifetime bound, so probably type param bounds in
 	 * TraitObjectType */
 	std::vector<std::unique_ptr<AST::TypeParamBound>> bounds
@@ -8976,7 +9094,8 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
     case SELF_ALIAS:
     case CRATE:
     case DOLLAR_SIGN:
-      case SCOPE_RESOLUTION: {
+    case SCOPE_RESOLUTION:
+      {
 	// macro invocation or type path - requires further disambiguation.
 	/* for parsing path component of each rule, perhaps parse it as a
 	 * typepath and attempt conversion to simplepath if a trailing '!' is
@@ -9006,7 +9125,8 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
 	t = lexer.peek_token ();
 	switch (t->get_id ())
 	  {
-	    case EXCLAM: {
+	  case EXCLAM:
+	    {
 	      // macro invocation
 	      // convert to simple path
 	      AST::SimplePath macro_path = path.as_simple_path ();
@@ -9032,7 +9152,8 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
 				     std::move (tok_tree)),
 		{}, locus);
 	    }
-	    case PLUS: {
+	  case PLUS:
+	    {
 	      // type param bounds
 	      std::vector<std::unique_ptr<AST::TypeParamBound>> bounds;
 
@@ -9117,14 +9238,8 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
 	  t = lexer.peek_token ();
 	  if (t->get_id () != PLUS)
 	    {
-	      // convert trait bound to value object
-	      AST::TraitBound value_bound (*initial_bound);
-
-	      // DEBUG: removed as unique ptr, so should auto-delete
-	      // delete initial_bound;
-
 	      return std::unique_ptr<AST::ImplTraitTypeOneBound> (
-		new AST::ImplTraitTypeOneBound (std::move (value_bound),
+		new AST::ImplTraitTypeOneBound (std::move (initial_bound),
 						locus));
 	    }
 
@@ -9152,7 +9267,8 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
 	    new AST::ImplTraitType (std::move (bounds), locus));
 	}
     case DYN:
-      case QUESTION_MARK: {
+    case QUESTION_MARK:
+      {
 	// either TraitObjectType or TraitObjectTypeOneBound
 	bool has_dyn = false;
 	if (t->get_id () == DYN)
@@ -9405,7 +9521,8 @@ Parser<ManagedTokenSource>::parse_for_prefixed_type ()
     case SELF:
     case SELF_ALIAS:
     case CRATE:
-      case DOLLAR_SIGN: {
+    case DOLLAR_SIGN:
+      {
 	// path, so trait type
 
 	// parse type path to finish parsing trait bound
@@ -9751,13 +9868,15 @@ Parser<ManagedTokenSource>::parse_slice_or_array_type ()
 
       return std::unique_ptr<AST::SliceType> (
 	new AST::SliceType (std::move (inner_type), locus));
-      case SEMICOLON: {
+    case SEMICOLON:
+      {
 	// array type
 	lexer.skip_token ();
 
 	// parse required array size expression
-	std::unique_ptr<AST::Expr> size = parse_expr ();
-	if (size == nullptr)
+	auto size = parse_anon_const ();
+
+	if (!size)
 	  {
 	    Error error (lexer.peek_token ()->get_locus (),
 			 "failed to parse size expression in array type");
@@ -9772,7 +9891,8 @@ Parser<ManagedTokenSource>::parse_slice_or_array_type ()
 	  }
 
 	return std::unique_ptr<AST::ArrayType> (
-	  new AST::ArrayType (std::move (inner_type), std::move (size), locus));
+	  new AST::ArrayType (std::move (inner_type), std::move (*size),
+			      locus));
       }
     default:
       // error
@@ -9802,7 +9922,8 @@ Parser<ManagedTokenSource>::parse_type_no_bounds ()
       // slice type or array type - requires further disambiguation
       return parse_slice_or_array_type ();
     case LEFT_SHIFT:
-      case LEFT_ANGLE: {
+    case LEFT_ANGLE:
+      {
 	// qualified path in type
 	AST::QualifiedPathInType path = parse_qualified_path_in_type ();
 	if (path.is_error ())
@@ -9843,7 +9964,8 @@ Parser<ManagedTokenSource>::parse_type_no_bounds ()
     case SELF_ALIAS:
     case CRATE:
     case DOLLAR_SIGN:
-      case SCOPE_RESOLUTION: {
+    case SCOPE_RESOLUTION:
+      {
 	// macro invocation or type path - requires further disambiguation.
 	/* for parsing path component of each rule, perhaps parse it as a
 	 * typepath and attempt conversion to simplepath if a trailing '!' is
@@ -9871,7 +9993,8 @@ Parser<ManagedTokenSource>::parse_type_no_bounds ()
 	t = lexer.peek_token ();
 	switch (t->get_id ())
 	  {
-	    case EXCLAM: {
+	  case EXCLAM:
+	    {
 	      // macro invocation
 	      // convert to simple path
 	      AST::SimplePath macro_path = path.as_simple_path ();
@@ -9955,14 +10078,12 @@ Parser<ManagedTokenSource>::parse_type_no_bounds ()
 	      return nullptr;
 	    }
 
-	  // convert trait bound to value object
-	  AST::TraitBound value_bound (*initial_bound);
-
 	  return std::unique_ptr<AST::ImplTraitTypeOneBound> (
-	    new AST::ImplTraitTypeOneBound (std::move (value_bound), locus));
+	    new AST::ImplTraitTypeOneBound (std::move (initial_bound), locus));
 	}
     case DYN:
-      case QUESTION_MARK: {
+    case QUESTION_MARK:
+      {
 	// either TraitObjectTypeOneBound
 	bool has_dyn = false;
 	if (t->get_id () == DYN)
@@ -10303,7 +10424,8 @@ Parser<ManagedTokenSource>::parse_range_pattern_bound ()
     case SELF_ALIAS:
     case CRATE:
     case SCOPE_RESOLUTION:
-      case DOLLAR_SIGN: {
+    case DOLLAR_SIGN:
+      {
 	// path in expression
 	AST::PathInExpression path = parse_path_in_expression ();
 	if (path.is_error ())
@@ -10319,7 +10441,8 @@ Parser<ManagedTokenSource>::parse_range_pattern_bound ()
 	  new AST::RangePatternBoundPath (std::move (path)));
       }
     case LEFT_SHIFT:
-      case LEFT_ANGLE: {
+    case LEFT_ANGLE:
+      {
 	// qualified path in expression
 	AST::QualifiedPathInExpression path
 	  = parse_qualified_path_in_expression ();
@@ -10464,7 +10587,8 @@ Parser<ManagedTokenSource>::parse_pattern_no_alt ()
       // slice pattern
       return parse_slice_pattern ();
     case LEFT_SHIFT:
-      case LEFT_ANGLE: {
+    case LEFT_ANGLE:
+      {
 	// qualified path in expression or qualified range pattern bound
 	AST::QualifiedPathInExpression path
 	  = parse_qualified_path_in_expression ();
@@ -10500,7 +10624,8 @@ Parser<ManagedTokenSource>::parse_pattern_no_alt ()
     case SELF_ALIAS:
     case CRATE:
     case SCOPE_RESOLUTION:
-      case DOLLAR_SIGN: {
+    case DOLLAR_SIGN:
+      {
 	// path in expression or range pattern bound
 	AST::PathInExpression path = parse_path_in_expression ();
 
@@ -10509,7 +10634,8 @@ Parser<ManagedTokenSource>::parse_pattern_no_alt ()
 	  {
 	  case DOT_DOT_EQ:
 	  case DOT_DOT:
-	    case ELLIPSIS: {
+	  case ELLIPSIS:
+	    {
 	      // qualified range pattern bound, so parse rest of range pattern
 	      AST::RangeKind kind = AST::tokenid_to_rangekind (next->get_id ());
 	      lexer.skip_token ();
@@ -10527,7 +10653,8 @@ Parser<ManagedTokenSource>::parse_pattern_no_alt ()
 	  case EXCLAM:
 	    return parse_macro_invocation_partial (std::move (path),
 						   AST::AttrVec ());
-	    case LEFT_PAREN: {
+	  case LEFT_PAREN:
+	    {
 	      // tuple struct
 	      lexer.skip_token ();
 
@@ -10552,7 +10679,8 @@ Parser<ManagedTokenSource>::parse_pattern_no_alt ()
 		new AST::TupleStructPattern (std::move (path),
 					     std::move (items)));
 	    }
-	    case LEFT_CURLY: {
+	  case LEFT_CURLY:
+	    {
 	      // struct
 	      lexer.skip_token ();
 
@@ -10722,7 +10850,8 @@ Parser<ManagedTokenSource>::parse_grouped_or_tuple_pattern ()
 
       return std::unique_ptr<AST::GroupedPattern> (
 	new AST::GroupedPattern (std::move (initial_pattern), paren_locus));
-      case COMMA: {
+    case COMMA:
+      {
 	// tuple pattern
 	lexer.skip_token ();
 
@@ -10835,27 +10964,47 @@ Parser<ManagedTokenSource>::parse_slice_pattern ()
 {
   location_t square_locus = lexer.peek_token ()->get_locus ();
   std::vector<std::unique_ptr<AST::Pattern>> patterns;
+  tl::optional<std::vector<std::unique_ptr<AST::Pattern>>> upper_patterns
+    = tl::nullopt;
+
+  // lambda function to determine which vector to push new patterns into
+  auto get_pattern_ref
+    = [&] () -> std::vector<std::unique_ptr<AST::Pattern>> & {
+    return upper_patterns.has_value () ? upper_patterns.value () : patterns;
+  };
+
   skip_token (LEFT_SQUARE);
 
   if (lexer.peek_token ()->get_id () == RIGHT_SQUARE)
     {
       skip_token (RIGHT_SQUARE);
+      std::unique_ptr<AST::SlicePatternItemsNoRest> items (
+	new AST::SlicePatternItemsNoRest (std::move (patterns)));
       return std::unique_ptr<AST::SlicePattern> (
-	new AST::SlicePattern (std::move (patterns), square_locus));
+	new AST::SlicePattern (std::move (items), square_locus));
     }
 
   // parse initial pattern (required)
-  std::unique_ptr<AST::Pattern> initial_pattern = parse_pattern ();
-  if (initial_pattern == nullptr)
+  if (lexer.peek_token ()->get_id () == DOT_DOT)
     {
-      Error error (lexer.peek_token ()->get_locus (),
-		   "failed to parse initial pattern in slice pattern");
-      add_error (std::move (error));
-
-      return nullptr;
+      lexer.skip_token ();
+      upper_patterns = std::vector<std::unique_ptr<AST::Pattern>> ();
     }
+  else
+    {
+      // Not a rest pattern `..`, parse normally
+      std::unique_ptr<AST::Pattern> initial_pattern = parse_pattern ();
+      if (initial_pattern == nullptr)
+	{
+	  Error error (lexer.peek_token ()->get_locus (),
+		       "failed to parse initial pattern in slice pattern");
+	  add_error (std::move (error));
 
-  patterns.push_back (std::move (initial_pattern));
+	  return nullptr;
+	}
+
+      patterns.push_back (std::move (initial_pattern));
+    }
 
   const_TokenPtr t = lexer.peek_token ();
   while (t->get_id () == COMMA)
@@ -10866,6 +11015,23 @@ Parser<ManagedTokenSource>::parse_slice_pattern ()
       if (lexer.peek_token ()->get_id () == RIGHT_SQUARE)
 	break;
 
+      if (lexer.peek_token ()->get_id () == DOT_DOT)
+	{
+	  if (upper_patterns.has_value ())
+	    {
+	      // DOT_DOT has been parsed before
+	      Error error (lexer.peek_token ()->get_locus (), "%s",
+			   "`..` can only be used once per slice pattern");
+	      add_error (std::move (error));
+
+	      return nullptr;
+	    }
+	  upper_patterns = std::vector<std::unique_ptr<AST::Pattern>> ();
+	  lexer.skip_token ();
+	  t = lexer.peek_token ();
+	  continue;
+	}
+
       // parse pattern (required)
       std::unique_ptr<AST::Pattern> pattern = parse_pattern ();
       if (pattern == nullptr)
@@ -10876,7 +11042,7 @@ Parser<ManagedTokenSource>::parse_slice_pattern ()
 
 	  return nullptr;
 	}
-      patterns.push_back (std::move (pattern));
+      get_pattern_ref ().push_back (std::move (pattern));
 
       t = lexer.peek_token ();
     }
@@ -10886,8 +11052,21 @@ Parser<ManagedTokenSource>::parse_slice_pattern ()
       return nullptr;
     }
 
+  if (upper_patterns.has_value ())
+    {
+      // Slice pattern with rest
+      std::unique_ptr<AST::SlicePatternItemsHasRest> items (
+	new AST::SlicePatternItemsHasRest (
+	  std::move (patterns), std::move (upper_patterns.value ())));
+      return std::unique_ptr<AST::SlicePattern> (
+	new AST::SlicePattern (std::move (items), square_locus));
+    }
+
+  // Rest-less slice pattern
+  std::unique_ptr<AST::SlicePatternItemsNoRest> items (
+    new AST::SlicePatternItemsNoRest (std::move (patterns)));
   return std::unique_ptr<AST::SlicePattern> (
-    new AST::SlicePattern (std::move (patterns), square_locus));
+    new AST::SlicePattern (std::move (items), square_locus));
 }
 
 /* Parses an identifier pattern (pattern that binds a value matched to a
@@ -10979,7 +11158,8 @@ Parser<ManagedTokenSource>::parse_ident_leading_pattern ()
     {
     case EXCLAM:
       return parse_macro_invocation_partial (std::move (path), AST::AttrVec ());
-      case LEFT_PAREN: {
+    case LEFT_PAREN:
+      {
 	// tuple struct
 	lexer.skip_token ();
 
@@ -11012,7 +11192,8 @@ Parser<ManagedTokenSource>::parse_ident_leading_pattern ()
 	return std::unique_ptr<AST::TupleStructPattern> (
 	  new AST::TupleStructPattern (std::move (path), std::move (items)));
       }
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	// struct
 	lexer.skip_token ();
 
@@ -11033,7 +11214,8 @@ Parser<ManagedTokenSource>::parse_ident_leading_pattern ()
       }
     case DOT_DOT_EQ:
     case DOT_DOT:
-      case ELLIPSIS: {
+    case ELLIPSIS:
+      {
 	// range
 	AST::RangeKind kind
 	  = AST::tokenid_to_rangekind (lexer.peek_token ()->get_id ());
@@ -11050,7 +11232,8 @@ Parser<ManagedTokenSource>::parse_ident_leading_pattern ()
 				 std::move (upper_bound), kind,
 				 t->get_locus ()));
       }
-      case PATTERN_BIND: {
+    case PATTERN_BIND:
+      {
 	// only allow on single-segment paths
 	if (path.is_single_segment ())
 	  {
@@ -11188,7 +11371,8 @@ Parser<ManagedTokenSource>::parse_tuple_struct_items ()
     case RIGHT_PAREN:
       return std::unique_ptr<AST::TupleStructItemsNoRange> (
 	new AST::TupleStructItemsNoRange (std::move (lower_patterns)));
-      case DOT_DOT: {
+    case DOT_DOT:
+      {
 	// has an upper range that must be parsed separately
 	lexer.skip_token ();
 
@@ -11308,7 +11492,8 @@ Parser<ManagedTokenSource>::parse_struct_pattern_field_partial (
   const_TokenPtr t = lexer.peek_token ();
   switch (t->get_id ())
     {
-      case INT_LITERAL: {
+    case INT_LITERAL:
+      {
 	// tuple index
 	std::string index_str = t->get_str ();
 	int index = atoi (index_str.c_str ());
@@ -11342,7 +11527,8 @@ Parser<ManagedTokenSource>::parse_struct_pattern_field_partial (
       // branch on next token
       switch (lexer.peek_token (1)->get_id ())
 	{
-	  case COLON: {
+	case COLON:
+	  {
 	    // identifier-pattern
 	    Identifier ident{t};
 	    lexer.skip_token ();
@@ -11367,7 +11553,8 @@ Parser<ManagedTokenSource>::parse_struct_pattern_field_partial (
 						   t->get_locus ()));
 	  }
 	case COMMA:
-	  case RIGHT_CURLY: {
+	case RIGHT_CURLY:
+	  {
 	    // identifier only
 	    Identifier ident = {t};
 	    lexer.skip_token ();
@@ -11386,7 +11573,8 @@ Parser<ManagedTokenSource>::parse_struct_pattern_field_partial (
 	  return nullptr;
 	}
     case REF:
-      case MUT: {
+    case MUT:
+      {
 	// only identifier
 	bool has_ref = false;
 	if (t->get_id () == REF)
@@ -11458,7 +11646,8 @@ Parser<ManagedTokenSource>::parse_stmt_or_expr ()
   t = lexer.peek_token ();
   switch (t->get_id ())
     {
-      case LET: {
+    case LET:
+      {
 	// let statement
 	std::unique_ptr<AST::LetStmt> stmt (
 	  parse_let_stmt (std::move (outer_attrs)));
@@ -11476,42 +11665,48 @@ Parser<ManagedTokenSource>::parse_stmt_or_expr ()
     case STATIC_KW:
     case AUTO:
     case TRAIT:
-      case IMPL: {
+    case IMPL:
+      {
 	std::unique_ptr<AST::VisItem> item (
 	  parse_vis_item (std::move (outer_attrs)));
 	return ExprOrStmt (std::move (item));
       }
-      /* TODO: implement union keyword but not really because of
-       * context-dependence crappy hack way to parse a union written below to
-       * separate it from the good code. */
-      // case UNION:
-      case UNSAFE: { // maybe - unsafe traits are a thing
+    /* TODO: implement union keyword but not really because of
+     * context-dependence crappy hack way to parse a union written below to
+     * separate it from the good code. */
+    // case UNION:
+    case UNSAFE:
+      { // maybe - unsafe traits are a thing
 	/* if any of these (should be all possible VisItem prefixes), parse a
 	 * VisItem - can't parse item because would require reparsing outer
 	 * attributes */
 	const_TokenPtr t2 = lexer.peek_token (1);
 	switch (t2->get_id ())
 	  {
-	    case LEFT_CURLY: {
+	  case LEFT_CURLY:
+	    {
 	      // unsafe block: parse as expression
 	      expr = parse_expr (std::move (outer_attrs), restrictions);
 	      break;
 	    }
 	  case AUTO:
-	    case TRAIT: {
+	  case TRAIT:
+	    {
 	      // unsafe trait
 	      std::unique_ptr<AST::VisItem> item (
 		parse_vis_item (std::move (outer_attrs)));
 	      return ExprOrStmt (std::move (item));
 	    }
 	  case EXTERN_KW:
-	    case FN_KW: {
+	  case FN_KW:
+	    {
 	      // unsafe function
 	      std::unique_ptr<AST::VisItem> item (
 		parse_vis_item (std::move (outer_attrs)));
 	      return ExprOrStmt (std::move (item));
 	    }
-	    case IMPL: {
+	  case IMPL:
+	    {
 	      // unsafe trait impl
 	      std::unique_ptr<AST::VisItem> item (
 		parse_vis_item (std::move (outer_attrs)));
@@ -11556,7 +11751,8 @@ Parser<ManagedTokenSource>::parse_stmt_or_expr ()
     case SELF_ALIAS:
     case CRATE:
     case SCOPE_RESOLUTION:
-      case DOLLAR_SIGN: {
+    case DOLLAR_SIGN:
+      {
 	AST::PathInExpression path = parse_path_in_expression ();
 	std::unique_ptr<AST::Expr> null_denotation;
 
@@ -11680,7 +11876,8 @@ Parser<ManagedTokenSource>::parse_struct_expr_field ()
 						std::move (outer_attrs),
 						t->get_locus ()));
 	}
-      case INT_LITERAL: {
+    case INT_LITERAL:
+      {
 	// parse tuple index field
 	int index = atoi (t->get_str ().c_str ());
 	lexer.skip_token ();
@@ -11725,11 +11922,12 @@ Parser<ManagedTokenSource>::parse_struct_expr_field ()
 }
 
 // "Unexpected token" panic mode - flags gcc error at unexpected token
+// TODO: seems to be unused, remove?
 template <typename ManagedTokenSource>
 void
 Parser<ManagedTokenSource>::unexpected_token (const_TokenPtr t)
 {
-  Error error (t->get_locus (), "unexpected token %qs\n",
+  Error error (t->get_locus (), "unexpected token %qs",
 	       t->get_token_description ());
   add_error (std::move (error));
 }
@@ -11920,7 +12118,7 @@ Parser<ManagedTokenSource>::skip_after_end_attribute ()
 {
   const_TokenPtr t = lexer.peek_token ();
 
-  while (t->get_id () != RIGHT_SQUARE)
+  while (t->get_id () != RIGHT_SQUARE && t->get_id () != END_OF_FILE)
     {
       lexer.skip_token ();
       t = lexer.peek_token ();
@@ -12050,7 +12248,8 @@ Parser<ManagedTokenSource>::null_denotation (const_TokenPtr tok,
     case SELF_ALIAS:
     case DOLLAR_SIGN:
     case CRATE:
-      case SUPER: {
+    case SUPER:
+      {
 	// DEBUG
 	rust_debug ("beginning null denotation identifier handling");
 
@@ -12061,7 +12260,8 @@ Parser<ManagedTokenSource>::null_denotation (const_TokenPtr tok,
 	return null_denotation_path (std::move (path), std::move (outer_attrs),
 				     restrictions);
       }
-      case SCOPE_RESOLUTION: {
+    case SCOPE_RESOLUTION:
+      {
 	// TODO: fix: this is for global paths, i.e. std::string::whatever
 	Error error (tok->get_locus (),
 		     "found null denotation scope resolution operator, and "
@@ -12103,7 +12303,8 @@ Parser<ManagedTokenSource>::null_denotation_path (
       // macro
       return parse_macro_invocation_partial (std::move (path),
 					     std::move (outer_attrs));
-      case LEFT_CURLY: {
+    case LEFT_CURLY:
+      {
 	bool not_a_block = lexer.peek_token (1)->get_id () == IDENTIFIER
 			   && (lexer.peek_token (2)->get_id () == COMMA
 			       || (lexer.peek_token (2)->get_id () == COLON
@@ -12179,7 +12380,8 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
     {
     // FIXME: Handle in null_denotation_path?
     case LEFT_SHIFT:
-      case LEFT_ANGLE: {
+    case LEFT_ANGLE:
+      {
 	// qualified path
 	// HACK: add outer attrs to path
 	AST::QualifiedPathInExpression path
@@ -12237,23 +12439,24 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
       return parse_grouped_or_tuple_expr (std::move (outer_attrs),
 					  tok->get_locus ());
 
-      /*case PLUS: { // unary plus operator
-	  // invoke parse_expr recursively with appropriate priority, etc. for
-      below AST::Expr* expr = parse_expr(LBP_UNARY_PLUS);
+    /*case PLUS: { // unary plus operator
+	// invoke parse_expr recursively with appropriate priority, etc. for
+    below AST::Expr* expr = parse_expr(LBP_UNARY_PLUS);
 
-	  if (expr == nullptr)
-	      return nullptr;
-	  // can only apply to integer and float expressions
-	  if (expr->get_type() != integer_type_node || expr->get_type() !=
-      float_type_node) { rust_error_at(tok->get_locus(), "operand of unary
-      plus must be int or float but it is %s", print_type(expr->get_type()));
-      return nullptr;
-	  }
+	if (expr == nullptr)
+	    return nullptr;
+	// can only apply to integer and float expressions
+	if (expr->get_type() != integer_type_node || expr->get_type() !=
+    float_type_node) { rust_error_at(tok->get_locus(), "operand of unary
+    plus must be int or float but it is %s", print_type(expr->get_type()));
+    return nullptr;
+	}
 
-	  return Tree(expr, tok->get_locus());
-      }*/
-      // Rust has no unary plus operator
-      case MINUS: { // unary minus
+	return Tree(expr, tok->get_locus());
+    }*/
+    // Rust has no unary plus operator
+    case MINUS:
+      { // unary minus
 	ParseRestrictions entered_from_unary;
 	entered_from_unary.entered_from_unary = true;
 	if (!restrictions.can_be_struct_expr)
@@ -12280,7 +12483,8 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
 	  new AST::NegationExpr (std::move (expr), NegationOperator::NEGATE,
 				 std::move (outer_attrs), tok->get_locus ()));
       }
-      case EXCLAM: { // logical or bitwise not
+    case EXCLAM:
+      { // logical or bitwise not
 	ParseRestrictions entered_from_unary;
 	entered_from_unary.entered_from_unary = true;
 	if (!restrictions.can_be_struct_expr)
@@ -12305,7 +12509,8 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
 	  new AST::NegationExpr (std::move (expr), NegationOperator::NOT,
 				 std::move (outer_attrs), tok->get_locus ()));
       }
-      case ASTERISK: {
+    case ASTERISK:
+      {
 	/* pointer dereference only - HACK: as struct expressions should
 	 * always be value expressions, cannot be dereferenced */
 	ParseRestrictions entered_from_unary;
@@ -12318,7 +12523,8 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
 	  new AST::DereferenceExpr (std::move (expr), std::move (outer_attrs),
 				    tok->get_locus ()));
       }
-      case AMP: {
+    case AMP:
+      {
 	// (single) "borrow" expression - shared (mutable) or immutable
 	std::unique_ptr<AST::Expr> expr = nullptr;
 	Mutability mutability = Mutability::Imm;
@@ -12375,7 +12581,8 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
 	  new AST::BorrowExpr (std::move (expr), mutability, raw_borrow, false,
 			       std::move (outer_attrs), tok->get_locus ()));
       }
-      case LOGICAL_AND: {
+    case LOGICAL_AND:
+      {
 	// (double) "borrow" expression - shared (mutable) or immutable
 	std::unique_ptr<AST::Expr> expr = nullptr;
 	Mutability mutability = Mutability::Imm;
@@ -12414,6 +12621,9 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
     case RETURN_KW:
       // FIXME: is this really a null denotation expression?
       return parse_return_expr (std::move (outer_attrs), tok->get_locus ());
+    case TRY:
+      // FIXME: is this really a null denotation expression?
+      return parse_try_expr (std::move (outer_attrs), tok->get_locus ());
     case BREAK:
       // FIXME: is this really a null denotation expression?
       return parse_break_expr (std::move (outer_attrs), tok->get_locus ());
@@ -12469,6 +12679,9 @@ Parser<ManagedTokenSource>::null_denotation_not_path (
 	       "use of %qs is not allowed on the right-side of an assignment",
 	       tok->get_token_description ()));
       return nullptr;
+    case CONST:
+      return parse_const_block_expr (std::move (outer_attrs),
+				     tok->get_locus ());
     default:
       if (!restrictions.expr_can_be_null)
 	add_error (Error (tok->get_locus (),
@@ -12491,8 +12704,9 @@ Parser<ManagedTokenSource>::left_denotation (const_TokenPtr tok,
   // Token passed in has already been skipped, so peek gives "next" token
   switch (tok->get_id ())
     {
-      // FIXME: allow for outer attributes to be applied
-      case QUESTION_MARK: {
+    // FIXME: allow for outer attributes to be applied
+    case QUESTION_MARK:
+      {
 	location_t left_locus = left->get_locus ();
 	// error propagation expression - unary postfix
 	return std::unique_ptr<AST::ErrorPropagationExpr> (
@@ -12746,7 +12960,8 @@ Parser<ManagedTokenSource>::left_denotation (const_TokenPtr tok,
 	       "function - this should probably be handled elsewhere"));
 
       return nullptr;
-      case DOT: {
+    case DOT:
+      {
 	/* field expression or method call - relies on parentheses after next
 	 * identifier or await if token after is "await" (unary postfix) or
 	 * tuple index if token after is a decimal int literal */
@@ -14062,7 +14277,8 @@ Parser<ManagedTokenSource>::parse_struct_expr_struct_partial (
        * algorithm should work too. As such, AST type not happening. */
     case IDENTIFIER:
     case HASH:
-      case INT_LITERAL: {
+    case INT_LITERAL:
+      {
 	// struct with struct expr fields
 
 	// parse struct expr fields
@@ -14359,7 +14575,8 @@ Parser<ManagedTokenSource>::parse_closure_expr_pratt (const_TokenPtr tok,
     case OR:
       // no parameters, don't skip token
       break;
-      case PIPE: {
+    case PIPE:
+      {
 	// actually may have parameters
 	// don't skip token
 	const_TokenPtr t = lexer.peek_token ();
diff --git a/gcc/rust/parse/rust-parse.cc b/gcc/rust/parse/rust-parse.cc
index 43d15aa..860fd11 100644
--- a/gcc/rust/parse/rust-parse.cc
+++ b/gcc/rust/parse/rust-parse.cc
@@ -42,8 +42,7 @@ extract_module_path (const AST::AttrVec &inner_attrs,
     {
       rust_error_at (
 	path_attr.get_locus (),
-	// Split the format string so that -Wformat-diag does not complain...
-	"path attributes must contain a filename: '%s'", "#![path = \"file\"]");
+	"path attributes must contain a filename: %<#[path = \"file\"]%>");
       return name;
     }
 
@@ -67,8 +66,7 @@ extract_module_path (const AST::AttrVec &inner_attrs,
     {
       rust_error_at (
 	path_attr.get_locus (),
-	// Split the format string so that -Wformat-diag does not complain...
-	"path attributes must contain a filename: '%s'", "#[path = \"file\"]");
+	"path attributes must contain a filename: %<#[path = \"file\"]%>");
       return name;
     }
 
@@ -80,6 +78,15 @@ extract_module_path (const AST::AttrVec &inner_attrs,
   // a character that is not an equal sign or whitespace
   auto filename_begin = path_value.find_first_not_of ("=\t ");
 
+  // If the path consists of only whitespace, then we have an error
+  if (filename_begin == std::string::npos)
+    {
+      rust_error_at (
+	path_attr.get_locus (),
+	"path attributes must contain a filename: %<#[path = \"file\"]%>");
+      return name;
+    }
+
   auto path = path_value.substr (filename_begin);
 
   // On windows, the path might mix '/' and '\' separators. Replace the
@@ -144,10 +151,9 @@ peculiar_fragment_match_compatible_fragment (
     = contains (fragment_follow_set[last_spec.get_kind ()], spec.get_kind ());
 
   if (!is_valid)
-    rust_error_at (
-      match_locus,
-      "fragment specifier %qs is not allowed after %qs fragments",
-      spec.as_string ().c_str (), last_spec.as_string ().c_str ());
+    rust_error_at (match_locus,
+		   "fragment specifier %qs is not allowed after %qs fragments",
+		   spec.as_string ().c_str (), last_spec.as_string ().c_str ());
 
   return is_valid;
 }
@@ -244,7 +250,8 @@ peculiar_fragment_match_compatible (const AST::MacroMatchFragment &last_match,
   // the error.
   switch (match.get_macro_match_type ())
     {
-      case AST::MacroMatch::Tok: {
+    case AST::MacroMatch::Tok:
+      {
 	auto tok = static_cast<const AST::Token *> (&match);
 	if (contains (allowed_toks, tok->get_id ()))
 	  return true;
@@ -254,7 +261,8 @@ peculiar_fragment_match_compatible (const AST::MacroMatchFragment &last_match,
 	break;
       }
       break;
-      case AST::MacroMatch::Repetition: {
+    case AST::MacroMatch::Repetition:
+      {
 	auto repetition
 	  = static_cast<const AST::MacroMatchRepetition *> (&match);
 	auto &matches = repetition->get_matches ();
@@ -263,7 +271,8 @@ peculiar_fragment_match_compatible (const AST::MacroMatchFragment &last_match,
 	  return peculiar_fragment_match_compatible (last_match, *first_frag);
 	break;
       }
-      case AST::MacroMatch::Matcher: {
+    case AST::MacroMatch::Matcher:
+      {
 	auto matcher = static_cast<const AST::MacroMatcher *> (&match);
 	auto first_token = matcher->get_delim_type ();
 	TokenId delim_id;
@@ -289,7 +298,8 @@ peculiar_fragment_match_compatible (const AST::MacroMatchFragment &last_match,
 	error_locus = matcher->get_match_locus ();
 	break;
       }
-      case AST::MacroMatch::Fragment: {
+    case AST::MacroMatch::Fragment:
+      {
 	auto last_spec = last_match.get_frag_spec ();
 	auto fragment = static_cast<const AST::MacroMatchFragment *> (&match);
 	if (last_spec.has_follow_set_fragment_restrictions ())
@@ -328,10 +338,11 @@ is_match_compatible (const AST::MacroMatch &last_match,
 
   switch (last_match.get_macro_match_type ())
     {
-      // This is our main stop condition: When we are finally looking at the
-      // last match (or its actual last component), and it is a fragment, it
-      // may contain some follow up restrictions.
-      case AST::MacroMatch::Fragment: {
+    // This is our main stop condition: When we are finally looking at the
+    // last match (or its actual last component), and it is a fragment, it
+    // may contain some follow up restrictions.
+    case AST::MacroMatch::Fragment:
+      {
 	auto fragment
 	  = static_cast<const AST::MacroMatchFragment *> (&last_match);
 	if (fragment->get_frag_spec ().has_follow_set_restrictions ())
@@ -339,7 +350,8 @@ is_match_compatible (const AST::MacroMatch &last_match,
 	else
 	  return true;
       }
-      case AST::MacroMatch::Repetition: {
+    case AST::MacroMatch::Repetition:
+      {
 	// A repetition on the left hand side means we want to make sure the
 	// last match of the repetition is compatible with the new match
 	auto repetition
diff --git a/gcc/rust/parse/rust-parse.h b/gcc/rust/parse/rust-parse.h
index 827d91d..8253885 100644
--- a/gcc/rust/parse/rust-parse.h
+++ b/gcc/rust/parse/rust-parse.h
@@ -17,6 +17,7 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef RUST_PARSE_H
 #define RUST_PARSE_H
 
+#include "rust-ast.h"
 #include "rust-item.h"
 #include "rust-lex.h"
 #include "rust-ast-full.h"
@@ -33,12 +34,19 @@ class ParseLifetimeParamError
 class ParseLifetimeError
 {
 };
+
+enum class AnonConstError
+{
+  InvalidSizeExpr,
+};
+
 enum class ParseLoopLabelError
 {
   NOT_LOOP_LABEL,
   MISSING_COLON,
 };
-enum ParseSelfError
+
+enum class ParseSelfError
 {
   SELF_PTR,
   PARSING,
@@ -165,6 +173,12 @@ public:
 		    tl::optional<AST::LoopLabel> = tl::nullopt,
 		    location_t pratt_parsed_loc = UNKNOWN_LOCATION);
 
+  tl::expected<AST::AnonConst, AnonConstError> parse_anon_const ();
+
+  std::unique_ptr<AST::ConstBlock>
+  parse_const_block_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
+			  location_t loc = UNKNOWN_LOCATION);
+
   bool is_macro_rules_def (const_TokenPtr t);
   std::unique_ptr<AST::Item> parse_item (bool called_from_statement);
   std::unique_ptr<AST::Pattern> parse_pattern ();
@@ -222,7 +236,7 @@ private:
 
   // Path-related
   AST::SimplePath parse_simple_path ();
-  AST::SimplePathSegment parse_simple_path_segment ();
+  AST::SimplePathSegment parse_simple_path_segment (int base_peek = 0);
   AST::TypePath parse_type_path ();
   std::unique_ptr<AST::TypePathSegment> parse_type_path_segment ();
   AST::PathIdentSegment parse_path_ident_segment ();
@@ -644,6 +658,9 @@ private:
   std::unique_ptr<AST::ReturnExpr>
   parse_return_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
 		     location_t pratt_parsed_loc = UNKNOWN_LOCATION);
+  std::unique_ptr<AST::TryExpr>
+  parse_try_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
+		  location_t pratt_parsed_loc = UNKNOWN_LOCATION);
   std::unique_ptr<AST::BreakExpr>
   parse_break_expr (AST::AttrVec outer_attrs = AST::AttrVec (),
 		    location_t pratt_parsed_loc = UNKNOWN_LOCATION);
@@ -762,11 +779,15 @@ private:
     }
     ~InlineModuleStackScope () { parser.inline_module_stack.pop_back (); }
   };
+
+  // don't want to make things *only* AttributeParser uses public
+  // TODO: fold more of AttributeParser into Parser?
+  friend class ::Rust::AST::AttributeParser;
 };
 
-std::string
-extract_module_path (const AST::AttrVec &inner_attrs,
-		     const AST::AttrVec &outer_attrs, const std::string &name);
+std::string extract_module_path (const AST::AttrVec &inner_attrs,
+				 const AST::AttrVec &outer_attrs,
+				 const std::string &name);
 
 /**
  * Check if a MacroMatch is allowed to follow the last parsed MacroMatch.
@@ -776,12 +797,8 @@ extract_module_path (const AST::AttrVec &inner_attrs,
  *
  * @return true if the follow-up is valid, false otherwise
  */
-bool
-is_match_compatible (const AST::MacroMatch &last_match,
-		     const AST::MacroMatch &current_match);
+bool is_match_compatible (const AST::MacroMatch &last_match,
+			  const AST::MacroMatch &current_match);
 } // namespace Rust
 
-// as now template, include implementations of all methods
-#include "rust-parse-impl.h"
-
 #endif // RUST_PARSE_H
diff --git a/gcc/rust/resolve/rust-ast-resolve-base.cc b/gcc/rust/resolve/rust-ast-resolve-base.cc
index b781ce33..3c7b425 100644
--- a/gcc/rust/resolve/rust-ast-resolve-base.cc
+++ b/gcc/rust/resolve/rust-ast-resolve-base.cc
@@ -116,7 +116,7 @@ ResolverBase::visit (AST::MetaItemLitExpr &)
 {}
 
 void
-ResolverBase::visit (AST::MetaItemPathLit &)
+ResolverBase::visit (AST::MetaItemPathExpr &)
 {}
 
 void
@@ -232,6 +232,14 @@ ResolverBase::visit (AST::BlockExpr &)
 {}
 
 void
+ResolverBase::visit (AST::AnonConst &)
+{}
+
+void
+ResolverBase::visit (AST::ConstBlock &)
+{}
+
+void
 ResolverBase::visit (AST::ClosureExprInnerTyped &)
 {}
 
@@ -276,6 +284,10 @@ ResolverBase::visit (AST::ReturnExpr &)
 {}
 
 void
+ResolverBase::visit (AST::TryExpr &)
+{}
+
+void
 ResolverBase::visit (AST::UnsafeBlockExpr &)
 {}
 
@@ -572,6 +584,14 @@ ResolverBase::visit (AST::GroupedPattern &)
 {}
 
 void
+ResolverBase::visit (AST::SlicePatternItemsNoRest &)
+{}
+
+void
+ResolverBase::visit (AST::SlicePatternItemsHasRest &)
+{}
+
+void
 ResolverBase::visit (AST::SlicePattern &)
 {}
 
@@ -663,5 +683,9 @@ void
 ResolverBase::visit (AST::FormatArgs &fmt)
 {}
 
+void
+ResolverBase::visit (AST::OffsetOf &offset_of)
+{}
+
 } // namespace Resolver
 } // namespace Rust
diff --git a/gcc/rust/resolve/rust-ast-resolve-base.h b/gcc/rust/resolve/rust-ast-resolve-base.h
index 5bb9e4f..89c5c35 100644
--- a/gcc/rust/resolve/rust-ast-resolve-base.h
+++ b/gcc/rust/resolve/rust-ast-resolve-base.h
@@ -21,6 +21,8 @@
 
 #include "rust-ast-visitor.h"
 #include "rust-ast.h"
+#include "rust-builtin-ast-nodes.h"
+#include "rust-expr.h"
 #include "rust-name-resolver.h"
 #include "rust-diagnostics.h"
 #include "rust-location.h"
@@ -56,7 +58,7 @@ public:
   void visit (AST::AttrInputLiteral &);
   void visit (AST::AttrInputMacro &);
   void visit (AST::MetaItemLitExpr &);
-  void visit (AST::MetaItemPathLit &);
+  void visit (AST::MetaItemPathExpr &);
   void visit (AST::BorrowExpr &);
   void visit (AST::DereferenceExpr &);
   void visit (AST::ErrorPropagationExpr &);
@@ -85,6 +87,8 @@ public:
   void visit (AST::FieldAccessExpr &);
   void visit (AST::ClosureExprInner &);
   void visit (AST::BlockExpr &);
+  void visit (AST::AnonConst &);
+  void visit (AST::ConstBlock &);
   void visit (AST::ClosureExprInnerTyped &);
   void visit (AST::ContinueExpr &);
   void visit (AST::BreakExpr &);
@@ -96,6 +100,7 @@ public:
   void visit (AST::RangeToInclExpr &);
   void visit (AST::BoxExpr &);
   void visit (AST::ReturnExpr &);
+  void visit (AST::TryExpr &);
   void visit (AST::UnsafeBlockExpr &);
   void visit (AST::LoopExpr &);
   void visit (AST::WhileLoopExpr &);
@@ -181,6 +186,8 @@ public:
   void visit (AST::TuplePatternItemsRanged &);
   void visit (AST::TuplePattern &);
   void visit (AST::GroupedPattern &);
+  void visit (AST::SlicePatternItemsNoRest &);
+  void visit (AST::SlicePatternItemsHasRest &);
   void visit (AST::SlicePattern &);
   void visit (AST::AltPattern &);
 
@@ -207,6 +214,7 @@ public:
   void visit (AST::SelfParam &param);
 
   void visit (AST::FormatArgs &fmt);
+  void visit (AST::OffsetOf &offset_of);
 
 protected:
   ResolverBase ()
diff --git a/gcc/rust/resolve/rust-ast-resolve-expr.cc b/gcc/rust/resolve/rust-ast-resolve-expr.cc
index 6242235..a410193 100644
--- a/gcc/rust/resolve/rust-ast-resolve-expr.cc
+++ b/gcc/rust/resolve/rust-ast-resolve-expr.cc
@@ -129,8 +129,8 @@ ResolveExpr::visit (AST::IdentifierExpr &expr)
 	 resolve.  Emit a funny ICE.  We set the finalizer to our custom one,
 	 and use the lower-level emit_diagnostic () instead of the more common
 	 internal_error_no_backtrace () in order to pass our locus.  */
-      diagnostic_text_finalizer (global_dc) = funny_ice_text_finalizer;
-      emit_diagnostic (DK_ICE_NOBT, expr.get_locus (), -1,
+      diagnostics::text_finalizer (global_dc) = funny_ice_text_finalizer;
+      emit_diagnostic (diagnostics::kind::ice_nobt, expr.get_locus (), -1,
 		       "are you trying to break %s? how dare you?",
 		       expr.as_string ().c_str ());
     }
@@ -315,6 +315,18 @@ ResolveExpr::visit (AST::BlockExpr &expr)
 }
 
 void
+ResolveExpr::visit (AST::AnonConst &expr)
+{
+  ResolveExpr::go (expr.get_inner_expr (), prefix, canonical_prefix);
+}
+
+void
+ResolveExpr::visit (AST::ConstBlock &expr)
+{
+  ResolveExpr::go (expr.get_const_expr (), prefix, canonical_prefix);
+}
+
+void
 translate_operand (AST::InlineAsm &expr, const CanonicalPath &prefix,
 		   const CanonicalPath &canonical_prefix)
 {
@@ -324,38 +336,46 @@ translate_operand (AST::InlineAsm &expr, const CanonicalPath &prefix,
     {
       switch (operand.get_register_type ())
 	{
-	  case RegisterType::In: {
+	case RegisterType::In:
+	  {
 	    auto in = operand.get_in ();
 	    ResolveExpr::go (*in.expr, prefix, canonical_prefix);
 	    break;
 	  }
-	  case RegisterType::Out: {
+	case RegisterType::Out:
+	  {
 	    auto out = operand.get_out ();
 	    ResolveExpr::go (*out.expr, prefix, canonical_prefix);
 	    break;
 	  }
-	  case RegisterType::InOut: {
+	case RegisterType::InOut:
+	  {
 	    auto in_out = operand.get_in_out ();
 	    ResolveExpr::go (*in_out.expr, prefix, canonical_prefix);
 	    break;
 	  }
-	  case RegisterType::SplitInOut: {
+	case RegisterType::SplitInOut:
+	  {
 	    auto split_in_out = operand.get_split_in_out ();
 	    ResolveExpr::go (*split_in_out.in_expr, prefix, canonical_prefix);
 	    ResolveExpr::go (*split_in_out.out_expr, prefix, canonical_prefix);
 	    break;
 	  }
-	  case RegisterType::Const: {
+	case RegisterType::Const:
+	  {
 	    auto anon_const = operand.get_const ().anon_const;
-	    ResolveExpr::go (*anon_const.expr, prefix, canonical_prefix);
+	    ResolveExpr::go (anon_const.get_inner_expr (), prefix,
+			     canonical_prefix);
 	    break;
 	  }
-	  case RegisterType::Sym: {
+	case RegisterType::Sym:
+	  {
 	    auto sym = operand.get_sym ();
 	    ResolveExpr::go (*sym.expr, prefix, canonical_prefix);
 	    break;
 	  }
-	  case RegisterType::Label: {
+	case RegisterType::Label:
+	  {
 	    auto label = operand.get_label ();
 	    ResolveExpr::go (*label.expr, prefix, canonical_prefix);
 	    break;
@@ -766,7 +786,7 @@ ResolveExpr::visit (AST::ClosureExprInnerTyped &expr)
 
   resolver->push_closure_context (expr.get_node_id ());
 
-  ResolveExpr::go (expr.get_definition_block (), prefix, canonical_prefix);
+  ResolveExpr::go (expr.get_definition_expr (), prefix, canonical_prefix);
 
   resolver->pop_closure_context ();
 
diff --git a/gcc/rust/resolve/rust-ast-resolve-expr.h b/gcc/rust/resolve/rust-ast-resolve-expr.h
index b296d66..aad1605 100644
--- a/gcc/rust/resolve/rust-ast-resolve-expr.h
+++ b/gcc/rust/resolve/rust-ast-resolve-expr.h
@@ -56,6 +56,8 @@ public:
   void visit (AST::IfLetExpr &expr) override;
   void visit (AST::IfLetExprConseqElse &expr) override;
   void visit (AST::BlockExpr &expr) override;
+  void visit (AST::AnonConst &expr) override;
+  void visit (AST::ConstBlock &expr) override;
   void visit (AST::InlineAsm &expr) override;
   void visit (AST::LlvmInlineAsm &expr) override;
   void visit (AST::UnsafeBlockExpr &expr) override;
diff --git a/gcc/rust/resolve/rust-ast-resolve-item.cc b/gcc/rust/resolve/rust-ast-resolve-item.cc
index 30f6d43..1d5ebed 100644
--- a/gcc/rust/resolve/rust-ast-resolve-item.cc
+++ b/gcc/rust/resolve/rust-ast-resolve-item.cc
@@ -453,7 +453,8 @@ ResolveItem::visit (AST::ConstantItem &constant)
   resolve_visibility (constant.get_visibility ());
 
   ResolveType::go (constant.get_type ());
-  ResolveExpr::go (constant.get_expr (), path, cpath);
+  if (constant.has_expr ())
+    ResolveExpr::go (constant.get_expr (), path, cpath);
 }
 
 void
@@ -608,10 +609,7 @@ ResolveItem::visit (AST::InherentImpl &impl_block)
     }
   else
     {
-      std::string seg_buf = "<impl " + self_cpath.get () + ">";
-      CanonicalPath seg
-	= CanonicalPath::new_seg (impl_block.get_node_id (), seg_buf);
-      cpath = canonical_prefix.append (seg);
+      cpath = canonical_prefix.append (impl_type_seg);
     }
 
   // done setup paths
@@ -732,13 +730,7 @@ ResolveItem::visit (AST::TraitImpl &impl_block)
     }
   else
     {
-      std::string projection_str = canonical_projection.get ();
-      std::string seg_buf
-	= "<impl " + projection_str.substr (1, projection_str.size () - 2)
-	  + ">";
-      CanonicalPath seg
-	= CanonicalPath::new_seg (impl_block.get_node_id (), seg_buf);
-      cpath = canonical_prefix.append (seg);
+      cpath = canonical_prefix.append (canonical_projection);
     }
 
   // DONE setup canonical-path
@@ -838,29 +830,32 @@ ResolveItem::resolve_extern_item (AST::ExternalItem &item)
   ResolveExternItem::go (item, prefix, canonical_prefix);
 }
 
-static void
-flatten_glob (const AST::UseTreeGlob &glob, std::vector<Import> &imports);
-static void
-flatten_rebind (const AST::UseTreeRebind &glob, std::vector<Import> &imports);
-static void
-flatten_list (const AST::UseTreeList &glob, std::vector<Import> &imports);
+static void flatten_glob (const AST::UseTreeGlob &glob,
+			  std::vector<Import> &imports);
+static void flatten_rebind (const AST::UseTreeRebind &glob,
+			    std::vector<Import> &imports);
+static void flatten_list (const AST::UseTreeList &glob,
+			  std::vector<Import> &imports);
 
 static void
 flatten (const AST::UseTree *tree, std::vector<Import> &imports)
 {
   switch (tree->get_kind ())
     {
-      case AST::UseTree::Glob: {
+    case AST::UseTree::Glob:
+      {
 	auto glob = static_cast<const AST::UseTreeGlob *> (tree);
 	flatten_glob (*glob, imports);
 	break;
       }
-      case AST::UseTree::Rebind: {
+    case AST::UseTree::Rebind:
+      {
 	auto rebind = static_cast<const AST::UseTreeRebind *> (tree);
 	flatten_rebind (*rebind, imports);
 	break;
       }
-      case AST::UseTree::List: {
+    case AST::UseTree::List:
+      {
 	auto list = static_cast<const AST::UseTreeList *> (tree);
 	flatten_list (*list, imports);
 	break;
diff --git a/gcc/rust/resolve/rust-ast-resolve-item.h b/gcc/rust/resolve/rust-ast-resolve-item.h
index 776dd53..d31f910 100644
--- a/gcc/rust/resolve/rust-ast-resolve-item.h
+++ b/gcc/rust/resolve/rust-ast-resolve-item.h
@@ -153,8 +153,7 @@ private:
 #if CHECKING_P
 
 namespace selftest {
-extern void
-rust_simple_path_resolve_test (void);
+extern void rust_simple_path_resolve_test (void);
 } // namespace selftest
 
 #endif // CHECKING_P
diff --git a/gcc/rust/resolve/rust-ast-resolve-pattern.cc b/gcc/rust/resolve/rust-ast-resolve-pattern.cc
index ee84be8..3b80f9f 100644
--- a/gcc/rust/resolve/rust-ast-resolve-pattern.cc
+++ b/gcc/rust/resolve/rust-ast-resolve-pattern.cc
@@ -62,6 +62,11 @@ PatternDeclaration::go (AST::Pattern &pattern, Rib::ItemType type,
 void
 PatternDeclaration::visit (AST::IdentifierPattern &pattern)
 {
+  if (pattern.has_subpattern ())
+    {
+      pattern.get_subpattern ().accept_vis (*this);
+    }
+
   Mutability mut = pattern.get_is_mut () ? Mutability::Mut : Mutability::Imm;
   add_new_binding (pattern.get_ident (), pattern.get_node_id (),
 		   BindingTypeInfo (mut, pattern.get_is_ref (),
@@ -94,13 +99,15 @@ PatternDeclaration::visit (AST::TupleStructPattern &pattern)
   AST::TupleStructItems &items = pattern.get_items ();
   switch (items.get_item_type ())
     {
-      case AST::TupleStructItems::RANGE: {
+    case AST::TupleStructItems::RANGE:
+      {
 	// TODO
 	rust_unreachable ();
       }
       break;
 
-      case AST::TupleStructItems::NO_RANGE: {
+    case AST::TupleStructItems::NO_RANGE:
+      {
 	auto &items_no_range
 	  = static_cast<AST::TupleStructItemsNoRange &> (items);
 
@@ -123,7 +130,8 @@ PatternDeclaration::visit (AST::StructPattern &pattern)
     {
       switch (field->get_item_type ())
 	{
-	  case AST::StructPatternField::ItemType::TUPLE_PAT: {
+	case AST::StructPatternField::ItemType::TUPLE_PAT:
+	  {
 	    AST::StructPatternFieldTuplePat &tuple
 	      = static_cast<AST::StructPatternFieldTuplePat &> (*field);
 
@@ -131,7 +139,8 @@ PatternDeclaration::visit (AST::StructPattern &pattern)
 	  }
 	  break;
 
-	  case AST::StructPatternField::ItemType::IDENT_PAT: {
+	case AST::StructPatternField::ItemType::IDENT_PAT:
+	  {
 	    AST::StructPatternFieldIdentPat &ident
 	      = static_cast<AST::StructPatternFieldIdentPat &> (*field);
 
@@ -139,7 +148,8 @@ PatternDeclaration::visit (AST::StructPattern &pattern)
 	  }
 	  break;
 
-	  case AST::StructPatternField::ItemType::IDENT: {
+	case AST::StructPatternField::ItemType::IDENT:
+	  {
 	    auto &ident = static_cast<AST::StructPatternFieldIdent &> (*field);
 
 	    Mutability mut
@@ -160,7 +170,8 @@ PatternDeclaration::visit (AST::TuplePattern &pattern)
   auto &items = pattern.get_items ();
   switch (items.get_pattern_type ())
     {
-      case AST::TuplePatternItems::TuplePatternItemType::MULTIPLE: {
+    case AST::TuplePatternItems::TuplePatternItemType::MULTIPLE:
+      {
 	auto &ref = static_cast<AST::TuplePatternItemsMultiple &> (
 	  pattern.get_items ());
 
@@ -169,7 +180,8 @@ PatternDeclaration::visit (AST::TuplePattern &pattern)
       }
       break;
 
-      case AST::TuplePatternItems::TuplePatternItemType::RANGED: {
+    case AST::TuplePatternItems::TuplePatternItemType::RANGED:
+      {
 	auto &ref
 	  = static_cast<AST::TuplePatternItemsRanged &> (pattern.get_items ());
 
@@ -348,14 +360,16 @@ resolve_range_pattern_bound (AST::RangePatternBound &bound)
       // Nothing to resolve for a literal.
       break;
 
-      case AST::RangePatternBound::RangePatternBoundType::PATH: {
+    case AST::RangePatternBound::RangePatternBoundType::PATH:
+      {
 	auto &ref = static_cast<AST::RangePatternBoundPath &> (bound);
 
 	ResolvePath::go (ref.get_path ());
       }
       break;
 
-      case AST::RangePatternBound::RangePatternBoundType::QUALPATH: {
+    case AST::RangePatternBound::RangePatternBoundType::QUALPATH:
+      {
 	auto &ref = static_cast<AST::RangePatternBoundQualPath &> (bound);
 
 	ResolvePath::go (ref.get_qualified_path ());
@@ -374,9 +388,30 @@ PatternDeclaration::visit (AST::RangePattern &pattern)
 void
 PatternDeclaration::visit (AST::SlicePattern &pattern)
 {
-  for (auto &p : pattern.get_items ())
+  auto &items = pattern.get_items ();
+  switch (items.get_pattern_type ())
     {
-      p->accept_vis (*this);
+    case AST::SlicePatternItems::SlicePatternItemType::NO_REST:
+      {
+	auto &ref
+	  = static_cast<AST::SlicePatternItemsNoRest &> (pattern.get_items ());
+
+	for (auto &p : ref.get_patterns ())
+	  p->accept_vis (*this);
+      }
+      break;
+
+    case AST::SlicePatternItems::SlicePatternItemType::HAS_REST:
+      {
+	auto &ref
+	  = static_cast<AST::SlicePatternItemsHasRest &> (pattern.get_items ());
+
+	for (auto &p : ref.get_lower_patterns ())
+	  p->accept_vis (*this);
+	for (auto &p : ref.get_upper_patterns ())
+	  p->accept_vis (*this);
+      }
+      break;
     }
 }
 
diff --git a/gcc/rust/resolve/rust-ast-resolve-stmt.h b/gcc/rust/resolve/rust-ast-resolve-stmt.h
index d413a7c..d714511 100644
--- a/gcc/rust/resolve/rust-ast-resolve-stmt.h
+++ b/gcc/rust/resolve/rust-ast-resolve-stmt.h
@@ -67,7 +67,8 @@ public:
       });
 
     ResolveType::go (constant.get_type ());
-    ResolveExpr::go (constant.get_expr (), prefix, canonical_prefix);
+    if (constant.has_expr ())
+      ResolveExpr::go (constant.get_expr (), prefix, canonical_prefix);
   }
 
   void visit (AST::LetStmt &stmt) override
diff --git a/gcc/rust/resolve/rust-ast-resolve-type.cc b/gcc/rust/resolve/rust-ast-resolve-type.cc
index 8fd69c3..a040228 100644
--- a/gcc/rust/resolve/rust-ast-resolve-type.cc
+++ b/gcc/rust/resolve/rust-ast-resolve-type.cc
@@ -140,7 +140,7 @@ ResolveType::visit (AST::ImplTraitType &type)
 void
 ResolveType::visit (AST::ImplTraitTypeOneBound &type)
 {
-  ResolveTypeBound::go (type.get_trait_bound ());
+  ResolveTypeBound::go (*type.get_trait_bound ().get ());
 }
 
 // resolve relative type-paths
@@ -210,7 +210,8 @@ ResolveRelativeTypePath::go (AST::TypePath &path, NodeId &resolved_node_id)
 
       switch (segment->get_type ())
 	{
-	  case AST::TypePathSegment::SegmentType::GENERIC: {
+	case AST::TypePathSegment::SegmentType::GENERIC:
+	  {
 	    AST::TypePathSegmentGeneric *s
 	      = static_cast<AST::TypePathSegmentGeneric *> (segment.get ());
 	    if (s->has_generic_args ())
@@ -509,7 +510,8 @@ ResolveTypeToCanonicalPath::visit (AST::TypePath &path)
       auto &final_seg = path.get_segments ().back ();
       switch (final_seg->get_type ())
 	{
-	  case AST::TypePathSegment::SegmentType::GENERIC: {
+	case AST::TypePathSegment::SegmentType::GENERIC:
+	  {
 	    AST::TypePathSegmentGeneric *s
 	      = static_cast<AST::TypePathSegmentGeneric *> (final_seg.get ());
 
@@ -651,7 +653,8 @@ ResolveTypeToCanonicalPath::visit (AST::TraitObjectType &type)
 
       switch (additional_bound->get_bound_type ())
 	{
-	  case AST::TypeParamBound::TRAIT: {
+	case AST::TypeParamBound::TRAIT:
+	  {
 	    auto bound_path = CanonicalPath::create_empty ();
 
 	    auto &bound_type_path
diff --git a/gcc/rust/resolve/rust-ast-resolve.cc b/gcc/rust/resolve/rust-ast-resolve.cc
index 3e3c992..2208f70 100644
--- a/gcc/rust/resolve/rust-ast-resolve.cc
+++ b/gcc/rust/resolve/rust-ast-resolve.cc
@@ -24,8 +24,7 @@
 #include "rust-ast-resolve-expr.h"
 #include "rust-ast-resolve-struct-expr-field.h"
 
-extern bool
-saw_errors (void);
+extern bool saw_errors (void);
 
 namespace Rust {
 namespace Resolver {
diff --git a/gcc/rust/resolve/rust-default-resolver.cc b/gcc/rust/resolve/rust-default-resolver.cc
index 480034c..f1c0e5c 100644
--- a/gcc/rust/resolve/rust-default-resolver.cc
+++ b/gcc/rust/resolve/rust-default-resolver.cc
@@ -25,6 +25,21 @@ namespace Rust {
 namespace Resolver2_0 {
 
 void
+DefaultResolver::visit (AST::Crate &crate)
+{
+  auto inner_fn = [this, &crate] () { AST::DefaultASTVisitor::visit (crate); };
+
+  auto &mappings = Analysis::Mappings::get ();
+
+  auto crate_num = mappings.lookup_crate_num (crate.get_node_id ());
+  rust_assert (crate_num.has_value ());
+  auto crate_name = mappings.get_crate_name (*crate_num);
+  rust_assert (crate_name.has_value ());
+
+  ctx.canonical_ctx.scope_crate (crate.get_node_id (), *crate_name, inner_fn);
+}
+
+void
 DefaultResolver::visit (AST::BlockExpr &expr)
 {
   // extracting the lambda from the `scoped` call otherwise the code looks like
@@ -38,19 +53,32 @@ DefaultResolver::visit (AST::BlockExpr &expr)
 void
 DefaultResolver::visit (AST::Module &module)
 {
-  auto item_fn = [this, &module] () { AST::DefaultASTVisitor::visit (module); };
+  auto item_fn_1
+    = [this, &module] () { AST::DefaultASTVisitor::visit (module); };
+
+  auto item_fn_2 = [this, &module, &item_fn_1] () {
+    ctx.canonical_ctx.scope (module.get_node_id (), module.get_name (),
+			     std::move (item_fn_1));
+  };
 
-  ctx.scoped (Rib::Kind::Module, module.get_node_id (), item_fn,
+  ctx.scoped (Rib::Kind::Module, module.get_node_id (), item_fn_2,
 	      module.get_name ());
 }
 
 void
 DefaultResolver::visit (AST::Function &function)
 {
-  auto def_fn
+  auto def_fn_1
     = [this, &function] () { AST::DefaultASTVisitor::visit (function); };
 
-  ctx.scoped (Rib::Kind::Function, function.get_node_id (), def_fn);
+  auto def_fn_2 = [this, &function, &def_fn_1] () {
+    ctx.canonical_ctx.scope (function.get_node_id (),
+			     function.get_function_name (),
+			     std::move (def_fn_1));
+  };
+
+  ctx.scoped (Rib::Kind::Function, function.get_node_id (), def_fn_2,
+	      function.get_function_name ());
 }
 
 void
@@ -61,73 +89,252 @@ DefaultResolver::visit (AST::ForLoopExpr &expr)
 }
 
 void
-DefaultResolver::visit (AST::Trait &trait)
+DefaultResolver::visit_if_let_patterns (AST::IfLetExpr &expr)
+{
+  for (auto &pattern : expr.get_patterns ())
+    visit (pattern);
+}
+
+void
+DefaultResolver::visit (AST::IfLetExpr &expr)
 {
-  auto inner_fn = [this, &trait] () { AST::DefaultASTVisitor::visit (trait); };
+  auto inner_vis = [this, &expr] () {
+    visit_if_let_patterns (expr);
+    visit (expr.get_if_block ());
+  };
+
+  visit_outer_attrs (expr);
+
+  visit (expr.get_value_expr ());
+
+  ctx.scoped (Rib::Kind::Normal, expr.get_node_id (), inner_vis);
+}
+
+void
+DefaultResolver::visit (AST::IfLetExprConseqElse &expr)
+{
+  DefaultResolver::visit (static_cast<AST::IfLetExpr &> (expr));
+  visit (expr.get_else_block ());
+}
 
-  ctx.scoped (Rib::Kind::TraitOrImpl, trait.get_node_id (), inner_fn,
+void
+DefaultResolver::visit (AST::Trait &trait)
+{
+  visit_outer_attrs (trait);
+  visit (trait.get_visibility ());
+  visit_inner_attrs (trait);
+
+  auto inner_fn_1 = [this, &trait] () {
+    for (auto &item : trait.get_trait_items ())
+      visit (item);
+  };
+
+  auto inner_fn_2 = [this, &trait, &inner_fn_1] () {
+    visit (trait.get_implicit_self ());
+    for (auto &generic : trait.get_generic_params ())
+      visit (generic);
+    if (trait.has_where_clause ())
+      visit (trait.get_where_clause ());
+    for (auto &bound : trait.get_type_param_bounds ())
+      visit (bound);
+
+    ctx.scoped (Rib::Kind::TraitOrImpl, trait.get_node_id (), inner_fn_1);
+  };
+
+  auto inner_fn_3 = [this, &trait, &inner_fn_2] () {
+    ctx.canonical_ctx.scope (trait.get_node_id (), trait.get_identifier (),
+			     std::move (inner_fn_2));
+  };
+
+  ctx.scoped (Rib::Kind::Generics, trait.get_node_id (), inner_fn_3,
 	      trait.get_identifier () /* FIXME: Is that valid?*/);
 }
 
 void
 DefaultResolver::visit (AST::InherentImpl &impl)
 {
-  auto inner_fn = [this, &impl] () { AST::DefaultASTVisitor::visit (impl); };
-
-  ctx.scoped (Rib::Kind::TraitOrImpl, impl.get_node_id (), inner_fn);
+  visit_outer_attrs (impl);
+  visit (impl.get_visibility ());
+  visit_inner_attrs (impl);
+
+  auto inner_fn_1 = [this, &impl] () {
+    for (auto &item : impl.get_impl_items ())
+      visit (item);
+  };
+
+  auto inner_fn_2 = [this, &impl, &inner_fn_1] () {
+    maybe_insert_big_self (impl);
+    for (auto &generic : impl.get_generic_params ())
+      visit (generic);
+    if (impl.has_where_clause ())
+      visit (impl.get_where_clause ());
+    visit_impl_type (impl.get_type ());
+
+    ctx.scoped (Rib::Kind::TraitOrImpl, impl.get_node_id (), inner_fn_1);
+  };
+
+  auto inner_fn_3 = [this, &impl, &inner_fn_2] () {
+    ctx.canonical_ctx.scope_impl (impl, std::move (inner_fn_2));
+  };
+
+  ctx.scoped (Rib::Kind::Generics, impl.get_node_id (), inner_fn_3);
 }
 
 void
 DefaultResolver::visit (AST::TraitImpl &impl)
 {
-  auto inner_fn = [this, &impl] () { AST::DefaultASTVisitor::visit (impl); };
-
-  ctx.scoped (Rib::Kind::TraitOrImpl, impl.get_node_id (), inner_fn);
+  visit_outer_attrs (impl);
+  visit (impl.get_visibility ());
+  visit_inner_attrs (impl);
+
+  auto inner_fn_1 = [this, &impl] () {
+    for (auto &item : impl.get_impl_items ())
+      visit (item);
+  };
+
+  auto inner_fn_2 = [this, &impl, &inner_fn_1] () {
+    maybe_insert_big_self (impl);
+    for (auto &generic : impl.get_generic_params ())
+      visit (generic);
+    if (impl.has_where_clause ())
+      visit (impl.get_where_clause ());
+    visit_impl_type (impl.get_type ());
+    visit (impl.get_trait_path ());
+
+    ctx.scoped (Rib::Kind::TraitOrImpl, impl.get_node_id (), inner_fn_1);
+  };
+
+  auto inner_fn_3 = [this, &impl, &inner_fn_2] () {
+    ctx.canonical_ctx.scope_impl (impl, std::move (inner_fn_2));
+  };
+
+  ctx.scoped (Rib::Kind::Generics, impl.get_node_id (), inner_fn_3);
 }
 
 void
 DefaultResolver::visit (AST::StructStruct &type)
 {
-  auto inner_fn = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+  auto inner_fn_1 = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+
+  auto inner_fn_2 = [this, &type, &inner_fn_1] () {
+    ctx.canonical_ctx.scope (type.get_node_id (), type.get_struct_name (),
+			     std::move (inner_fn_1));
+  };
 
   ctx.scoped (Rib::Kind::Item /* FIXME: Correct? */, type.get_node_id (),
-	      inner_fn, type.get_struct_name ());
+	      inner_fn_2, type.get_struct_name ());
 }
 
 void
 DefaultResolver::visit (AST::TupleStruct &type)
 {
-  auto inner_fn = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+  auto inner_fn_1 = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+
+  auto inner_fn_2 = [this, &type, &inner_fn_1] () {
+    ctx.canonical_ctx.scope (type.get_node_id (), type.get_struct_name (),
+			     std::move (inner_fn_1));
+  };
 
   ctx.scoped (Rib::Kind::Item /* FIXME: Correct? */, type.get_node_id (),
-	      inner_fn, type.get_struct_name ());
+	      inner_fn_2, type.get_struct_name ());
+}
+
+void
+DefaultResolver::visit (AST::EnumItem &item)
+{
+  auto inner_fn = [this, &item] () { AST::DefaultASTVisitor::visit (item); };
+
+  ctx.canonical_ctx.scope (item.get_node_id (), item.get_identifier (),
+			   inner_fn);
+}
+
+void
+DefaultResolver::visit (AST::EnumItemTuple &item)
+{
+  auto inner_fn = [this, &item] () { AST::DefaultASTVisitor::visit (item); };
+
+  ctx.canonical_ctx.scope (item.get_node_id (), item.get_identifier (),
+			   inner_fn);
+}
+
+void
+DefaultResolver::visit (AST::EnumItemStruct &item)
+{
+  auto inner_fn = [this, &item] () { AST::DefaultASTVisitor::visit (item); };
+
+  ctx.canonical_ctx.scope (item.get_node_id (), item.get_identifier (),
+			   inner_fn);
+}
+
+void
+DefaultResolver::visit (AST::EnumItemDiscriminant &item)
+{
+  auto inner_fn = [this, &item] () { AST::DefaultASTVisitor::visit (item); };
+
+  ctx.canonical_ctx.scope (item.get_node_id (), item.get_identifier (),
+			   inner_fn);
 }
 
 void
 DefaultResolver::visit (AST::Enum &type)
 {
-  auto variant_fn = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+  auto inner_fn_1 = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+
+  auto inner_fn_2 = [this, &type, &inner_fn_1] () {
+    ctx.canonical_ctx.scope (type.get_node_id (), type.get_identifier (),
+			     std::move (inner_fn_1));
+  };
 
   ctx.scoped (Rib::Kind::Item /* FIXME: Correct? */, type.get_node_id (),
-	      variant_fn, type.get_identifier ());
+	      inner_fn_2, type.get_identifier ());
 }
 
 void
 DefaultResolver::visit (AST::Union &type)
 {
-  auto inner_fn = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+  auto inner_fn_1 = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+
+  auto inner_fn_2 = [this, &type, &inner_fn_1] () {
+    ctx.canonical_ctx.scope (type.get_node_id (), type.get_identifier (),
+			     std::move (inner_fn_1));
+  };
 
   ctx.scoped (Rib::Kind::Item /* FIXME: Correct? */, type.get_node_id (),
-	      inner_fn, type.get_identifier ());
+	      inner_fn_2, type.get_identifier ());
 }
 
 void
 DefaultResolver::visit (AST::TypeAlias &type)
 {
-  auto inner_fn = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+  auto inner_fn_1 = [this, &type] () { AST::DefaultASTVisitor::visit (type); };
+
+  auto inner_fn_2 = [this, &type, &inner_fn_1] () {
+    ctx.canonical_ctx.scope (type.get_node_id (), type.get_new_type_name (),
+			     std::move (inner_fn_1));
+  };
 
   ctx.scoped (Rib::Kind::Item /* FIXME: Correct? */, type.get_node_id (),
-	      inner_fn, type.get_new_type_name ());
+	      inner_fn_2, type.get_new_type_name ());
+}
+
+void
+DefaultResolver::visit_closure_params (AST::ClosureExpr &expr)
+{
+  for (auto &param : expr.get_params ())
+    visit (param);
+}
+
+void
+DefaultResolver::visit (AST::ClosureExpr &expr)
+{
+  auto expr_fn = [this, &expr] () {
+    visit_closure_params (expr);
+    visit (expr.get_definition_expr ());
+  };
+
+  visit_outer_attrs (expr);
+
+  ctx.scoped (Rib::Kind::Normal, expr.get_node_id (), expr_fn);
 }
 
 void
@@ -136,7 +343,7 @@ DefaultResolver::visit (AST::ClosureExprInner &expr)
   if (expr.is_marked_for_strip ())
     return;
 
-  AST::DefaultASTVisitor::visit (expr);
+  visit (static_cast<AST::ClosureExpr &> (expr));
 }
 
 void
@@ -145,7 +352,8 @@ DefaultResolver::visit (AST::ClosureExprInnerTyped &expr)
   if (expr.is_marked_for_strip ())
     return;
 
-  AST::DefaultASTVisitor::visit (expr);
+  visit (static_cast<AST::ClosureExpr &> (expr));
+  visit (expr.get_return_type ());
 }
 
 void
@@ -160,23 +368,29 @@ DefaultResolver::visit (AST::MatchExpr &expr)
 void
 DefaultResolver::visit (AST::ConstantItem &item)
 {
-  if (item.has_expr ())
-    {
-      auto expr_vis
-	= [this, &item] () { AST::DefaultASTVisitor::visit (item); };
+  auto expr_vis_1 = [this, &item] () { AST::DefaultASTVisitor::visit (item); };
 
-      // FIXME: Why do we need a Rib here?
-      ctx.scoped (Rib::Kind::ConstantItem, item.get_node_id (), expr_vis);
-    }
+  auto expr_vis_2 = [this, &item, &expr_vis_1] () {
+    ctx.canonical_ctx.scope (item.get_node_id (), item.get_identifier (),
+			     std::move (expr_vis_1));
+  };
+
+  // FIXME: Why do we need a Rib here?
+  ctx.scoped (Rib::Kind::ConstantItem, item.get_node_id (), expr_vis_2);
 }
 
 void
 DefaultResolver::visit (AST::StaticItem &item)
 {
-  auto expr_vis = [this, &item] () { AST::DefaultASTVisitor::visit (item); };
+  auto expr_vis_1 = [this, &item] () { AST::DefaultASTVisitor::visit (item); };
+
+  auto expr_vis_2 = [this, &item, &expr_vis_1] () {
+    ctx.canonical_ctx.scope (item.get_node_id (), item.get_identifier (),
+			     std::move (expr_vis_1));
+  };
 
   // FIXME: Why do we need a Rib here?
-  ctx.scoped (Rib::Kind::ConstantItem, item.get_node_id (), expr_vis);
+  ctx.scoped (Rib::Kind::ConstantItem, item.get_node_id (), expr_vis_2);
 }
 
 void
@@ -187,5 +401,46 @@ DefaultResolver::visit (AST::TypeParam &param)
   ctx.scoped (Rib::Kind::ForwardTypeParamBan, param.get_node_id (), expr_vis);
 }
 
+void
+DefaultResolver::visit_extern_crate (AST::ExternCrate &extern_crate,
+				     AST::Crate &crate, CrateNum num)
+{
+  visit (crate);
+}
+
+void
+DefaultResolver::visit (AST::ExternCrate &crate)
+{
+  auto &mappings = Analysis::Mappings::get ();
+  auto num_opt = mappings.lookup_crate_name (crate.get_referenced_crate ());
+
+  if (!num_opt)
+    {
+      rust_error_at (crate.get_locus (), "unknown crate %qs",
+		     crate.get_referenced_crate ().c_str ());
+      return;
+    }
+
+  CrateNum num = *num_opt;
+
+  AST::Crate &referenced_crate = mappings.get_ast_crate (num);
+
+  auto sub_visitor_1
+    = [&, this] () { visit_extern_crate (crate, referenced_crate, num); };
+
+  auto sub_visitor_2 = [&] () {
+    ctx.canonical_ctx.scope_crate (referenced_crate.get_node_id (),
+				   crate.get_referenced_crate (),
+				   std::move (sub_visitor_1));
+  };
+
+  if (crate.has_as_clause ())
+    ctx.scoped (Rib::Kind::Module, referenced_crate.get_node_id (),
+		sub_visitor_2, crate.get_as_clause ());
+  else
+    ctx.scoped (Rib::Kind::Module, referenced_crate.get_node_id (),
+		sub_visitor_2, crate.get_referenced_crate ());
+}
+
 } // namespace Resolver2_0
 } // namespace Rust
diff --git a/gcc/rust/resolve/rust-default-resolver.h b/gcc/rust/resolve/rust-default-resolver.h
index 2a987ef..cf0df68 100644
--- a/gcc/rust/resolve/rust-default-resolver.h
+++ b/gcc/rust/resolve/rust-default-resolver.h
@@ -39,6 +39,7 @@ public:
 
   virtual ~DefaultResolver () {}
 
+  void visit (AST::Crate &) override;
   // First, our lexical scope expressions - these visit their sub nodes, always
   // these nodes create new scopes and ribs - they are often used to declare new
   // variables, such as a for loop's iterator, or a function's arguments
@@ -46,20 +47,35 @@ public:
   void visit (AST::Module &) override;
   void visit (AST::Function &) override;
   void visit (AST::ForLoopExpr &expr) override;
+  virtual void visit_if_let_patterns (AST::IfLetExpr &expr);
+  void visit (AST::IfLetExpr &expr) override;
+  void visit (AST::IfLetExprConseqElse &expr) override;
   void visit (AST::Trait &) override;
+  // used to handle Self insertion in TopLevel
+  virtual void maybe_insert_big_self (AST::Impl &) {}
+  virtual void visit_impl_type (AST::Type &type) { visit (type); }
   void visit (AST::InherentImpl &) override;
   void visit (AST::TraitImpl &) override;
 
   void visit (AST::TypeParam &) override;
 
+  virtual void visit_extern_crate (AST::ExternCrate &, AST::Crate &, CrateNum);
+  void visit (AST::ExternCrate &) override;
+
   // type dec nodes, which visit their fields or variants by default
   void visit (AST::StructStruct &) override;
   void visit (AST::TupleStruct &) override;
+  void visit (AST::EnumItem &) override;
+  void visit (AST::EnumItemTuple &) override;
+  void visit (AST::EnumItemStruct &) override;
+  void visit (AST::EnumItemDiscriminant &) override;
   void visit (AST::Enum &) override;
   void visit (AST::Union &) override;
   void visit (AST::TypeAlias &) override;
 
   // Visitors that visit their expression node(s)
+  virtual void visit_closure_params (AST::ClosureExpr &);
+  virtual void visit (AST::ClosureExpr &);
   void visit (AST::ClosureExprInner &) override;
   void visit (AST::ClosureExprInnerTyped &) override;
   void visit (AST::MatchExpr &) override;
diff --git a/gcc/rust/resolve/rust-early-name-resolver-2.0.cc b/gcc/rust/resolve/rust-early-name-resolver-2.0.cc
index 3390f09..4fd1dd2 100644
--- a/gcc/rust/resolve/rust-early-name-resolver-2.0.cc
+++ b/gcc/rust/resolve/rust-early-name-resolver-2.0.cc
@@ -17,8 +17,11 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-early-name-resolver-2.0.h"
-#include "rust-ast-full.h"
+#include "optional.h"
+#include "options.h"
 #include "rust-diagnostics.h"
+#include "rust-hir-map.h"
+#include "rust-item.h"
 #include "rust-toplevel-name-resolver-2.0.h"
 #include "rust-attributes.h"
 #include "rust-finalize-imports-2.0.h"
@@ -62,8 +65,9 @@ Early::go (AST::Crate &crate)
   // We now proceed with resolving macros, which can be nested in almost any
   // items
   textual_scope.push ();
-  for (auto &item : crate.items)
-    item->accept_vis (*this);
+
+  visit (crate);
+
   textual_scope.pop ();
 }
 
@@ -74,8 +78,9 @@ Early::resolve_glob_import (NodeId use_dec_id, TopLevel::ImportKind &&glob)
   if (!resolved.has_value ())
     return false;
 
-  auto result
-    = Analysis::Mappings::get ().lookup_ast_module (resolved->get_node_id ());
+  auto result = Analysis::Mappings::get ().lookup_glob_container (
+    resolved->get_node_id ());
+
   if (!result)
     return false;
 
@@ -249,7 +254,12 @@ Early::visit (AST::Module &module)
 void
 Early::visit (AST::MacroInvocation &invoc)
 {
-  auto path = invoc.get_invoc_data ().get_path ();
+  auto &path = invoc.get_invoc_data ().get_path ();
+
+  // We special case the `offset_of!()` macro if the flag is here, otherwise
+  // we accept whatever `offset_of!()` definition we resolved to.
+  auto resolve_offset_of
+    = flag_assume_builtin_offset_of && (path.as_string () == "offset_of");
 
   if (invoc.get_kind () == AST::MacroInvocation::InvocKind::Builtin)
     for (auto &pending_invoc : invoc.get_pending_eager_invocations ())
@@ -272,14 +282,16 @@ Early::visit (AST::MacroInvocation &invoc)
   // we won't have changed `definition` from `nullopt` if there are more
   // than one segments in our path
   if (!definition.has_value ())
-    definition = ctx.resolve_path (path.get_segments (), Namespace::Macros);
+    definition = ctx.resolve_path (path, Namespace::Macros);
 
-  // if the definition still does not have a value, then it's an error
+  // if the definition still does not have a value, then it's an error - unless
+  // we should automatically resolve offset_of!() calls
   if (!definition.has_value ())
     {
-      collect_error (Error (invoc.get_locus (), ErrorCode::E0433,
-			    "could not resolve macro invocation %qs",
-			    path.as_string ().c_str ()));
+      if (!resolve_offset_of)
+	collect_error (Error (invoc.get_locus (), ErrorCode::E0433,
+			      "could not resolve macro invocation %qs",
+			      path.as_string ().c_str ()));
       return;
     }
 
@@ -314,8 +326,8 @@ Early::visit_attributes (std::vector<AST::Attribute> &attrs)
 	  auto traits = attr.get_traits_to_derive ();
 	  for (auto &trait : traits)
 	    {
-	      auto definition = ctx.resolve_path (trait.get ().get_segments (),
-						  Namespace::Macros);
+	      auto definition
+		= ctx.resolve_path (trait.get (), Namespace::Macros);
 	      if (!definition.has_value ())
 		{
 		  // FIXME: Change to proper error message
@@ -337,8 +349,8 @@ Early::visit_attributes (std::vector<AST::Attribute> &attrs)
 		 ->lookup_builtin (name)
 		 .is_error ()) // Do not resolve builtins
 	{
-	  auto definition = ctx.resolve_path (attr.get_path ().get_segments (),
-					      Namespace::Macros);
+	  auto definition
+	    = ctx.resolve_path (attr.get_path (), Namespace::Macros);
 	  if (!definition.has_value ())
 	    {
 	      // FIXME: Change to proper error message
@@ -350,7 +362,8 @@ Early::visit_attributes (std::vector<AST::Attribute> &attrs)
 	  auto pm_def = mappings.lookup_attribute_proc_macro_def (
 	    definition->get_node_id ());
 
-	  rust_assert (pm_def.has_value ());
+	  if (!pm_def.has_value ())
+	    return;
 
 	  mappings.insert_attribute_proc_macro_invocation (attr.get_path (),
 							   pm_def.value ());
@@ -392,12 +405,12 @@ void
 Early::finalize_glob_import (NameResolutionContext &ctx,
 			     const Early::ImportPair &mapping)
 {
-  auto module = Analysis::Mappings::get ().lookup_ast_module (
-    mapping.data.module ().get_node_id ());
-  rust_assert (module);
+  auto container = Analysis::Mappings::get ().lookup_glob_container (
+    mapping.data.container ().get_node_id ());
+
+  rust_assert (container);
 
-  GlobbingVisitor glob_visitor (ctx);
-  glob_visitor.go (module.value ());
+  GlobbingVisitor (ctx).go (container.value ());
 }
 
 void
@@ -419,7 +432,8 @@ Early::finalize_rebind_import (const Early::ImportPair &mapping)
       declared_name = rebind.get_identifier ().as_string ();
       locus = rebind.get_identifier ().get_locus ();
       break;
-      case AST::UseTreeRebind::NewBindType::NONE: {
+    case AST::UseTreeRebind::NewBindType::NONE:
+      {
 	const auto &segments = path.get_segments ();
 	// We don't want to insert `self` with `use module::self`
 	if (path.get_final_segment ().is_lower_self_seg ())
diff --git a/gcc/rust/resolve/rust-early-name-resolver-2.0.h b/gcc/rust/resolve/rust-early-name-resolver-2.0.h
index e78bec0..960de0e 100644
--- a/gcc/rust/resolve/rust-early-name-resolver-2.0.h
+++ b/gcc/rust/resolve/rust-early-name-resolver-2.0.h
@@ -83,15 +83,15 @@ public:
       return ImportData (Kind::Rebind, std::move (definitions));
     }
 
-    static ImportData Glob (Rib::Definition module)
+    static ImportData Glob (Rib::Definition container)
     {
-      return ImportData (Kind::Glob, module);
+      return ImportData (Kind::Glob, container);
     }
 
-    Rib::Definition module () const
+    Rib::Definition container () const
     {
       rust_assert (kind == Kind::Glob);
-      return glob_module;
+      return glob_container;
     }
 
     std::vector<std::pair<Rib::Definition, Namespace>> definitions () const
@@ -107,8 +107,8 @@ public:
       : kind (kind), resolved_definitions (std::move (definitions))
     {}
 
-    ImportData (Kind kind, Rib::Definition module)
-      : kind (kind), glob_module (module)
+    ImportData (Kind kind, Rib::Definition container)
+      : kind (kind), glob_container (container)
     {}
 
     // TODO: Should this be a union?
@@ -117,7 +117,7 @@ public:
     std::vector<std::pair<Rib::Definition, Namespace>> resolved_definitions;
 
     // For Glob
-    Rib::Definition glob_module;
+    Rib::Definition glob_container;
   };
 
   struct ImportPair
diff --git a/gcc/rust/resolve/rust-early-name-resolver.cc b/gcc/rust/resolve/rust-early-name-resolver.cc
index fc9a26c..7b365ef 100644
--- a/gcc/rust/resolve/rust-early-name-resolver.cc
+++ b/gcc/rust/resolve/rust-early-name-resolver.cc
@@ -205,7 +205,7 @@ EarlyNameResolver::visit (AST::MetaItemLitExpr &)
 {}
 
 void
-EarlyNameResolver::visit (AST::MetaItemPathLit &)
+EarlyNameResolver::visit (AST::MetaItemPathExpr &)
 {}
 
 void
diff --git a/gcc/rust/resolve/rust-early-name-resolver.h b/gcc/rust/resolve/rust-early-name-resolver.h
index 26fc84d..d3c5225 100644
--- a/gcc/rust/resolve/rust-early-name-resolver.h
+++ b/gcc/rust/resolve/rust-early-name-resolver.h
@@ -142,7 +142,7 @@ private:
   virtual void visit (AST::AttrInputLiteral &attr_input);
   virtual void visit (AST::AttrInputMacro &attr_input);
   virtual void visit (AST::MetaItemLitExpr &meta_item);
-  virtual void visit (AST::MetaItemPathLit &meta_item);
+  virtual void visit (AST::MetaItemPathExpr &meta_item);
   virtual void visit (AST::StructExprStruct &expr);
   virtual void visit (AST::StructExprFieldIdentifier &field);
   virtual void visit (AST::StructExprStructBase &expr);
diff --git a/gcc/rust/resolve/rust-finalize-imports-2.0.cc b/gcc/rust/resolve/rust-finalize-imports-2.0.cc
index b0e8651..317acb0 100644
--- a/gcc/rust/resolve/rust-finalize-imports-2.0.cc
+++ b/gcc/rust/resolve/rust-finalize-imports-2.0.cc
@@ -21,19 +21,44 @@
 #include "rust-hir-map.h"
 #include "rust-name-resolution-context.h"
 #include "rust-rib.h"
+#include "rust-system.h"
 #include "rust-toplevel-name-resolver-2.0.h"
 
 namespace Rust {
 namespace Resolver2_0 {
 
 void
-GlobbingVisitor::go (AST::Module *module)
+GlobbingVisitor::go (AST::Item *container)
 {
-  for (auto &i : module->get_items ())
+  switch (container->get_item_kind ())
+    {
+    case AST::Item::Kind::Module:
+      visit_module_container (static_cast<AST::Module &> (*container));
+      break;
+    case AST::Item::Kind::Enum:
+      visit_enum_container (static_cast<AST::Enum &> (*container));
+      break;
+    default:
+      rust_unreachable ();
+    }
+}
+
+void
+GlobbingVisitor::visit_module_container (AST::Module &module)
+{
+  for (auto &i : module.get_items ())
     visit (i);
 }
 
 void
+GlobbingVisitor::visit_enum_container (AST::Enum &item)
+{
+  for (auto &variant : item.get_variants ())
+    ctx.insert_globbed (variant->get_identifier (), variant->get_node_id (),
+			Namespace::Types);
+}
+
+void
 GlobbingVisitor::visit (AST::Module &module)
 {
   if (module.get_visibility ().is_public ())
diff --git a/gcc/rust/resolve/rust-finalize-imports-2.0.h b/gcc/rust/resolve/rust-finalize-imports-2.0.h
index d587a5e..4ae1d6d 100644
--- a/gcc/rust/resolve/rust-finalize-imports-2.0.h
+++ b/gcc/rust/resolve/rust-finalize-imports-2.0.h
@@ -18,6 +18,7 @@
 
 #include "rust-ast.h"
 #include "rust-expr.h"
+#include "rust-item.h"
 #include "rust-name-resolution-context.h"
 #include "rust-toplevel-name-resolver-2.0.h"
 #include "rust-early-name-resolver-2.0.h"
@@ -32,7 +33,11 @@ class GlobbingVisitor : public AST::DefaultASTVisitor
 public:
   GlobbingVisitor (NameResolutionContext &ctx) : ctx (ctx) {}
 
-  void go (AST::Module *module);
+  void go (AST::Item *container);
+
+  void visit_module_container (AST::Module &module);
+  void visit_enum_container (AST::Enum &item);
+
   void visit (AST::Module &module) override;
   void visit (AST::MacroRulesDefinition &macro) override;
   void visit (AST::Function &function) override;
diff --git a/gcc/rust/resolve/rust-forever-stack.h b/gcc/rust/resolve/rust-forever-stack.h
index 81468e5..75dd873 100644
--- a/gcc/rust/resolve/rust-forever-stack.h
+++ b/gcc/rust/resolve/rust-forever-stack.h
@@ -543,6 +543,13 @@ private:
   Node root;
 };
 
+enum class ResolutionMode
+{
+  Normal,
+  FromRoot,
+  FromExtern, // extern prelude
+};
+
 template <Namespace N> class ForeverStack
 {
 public:
@@ -672,14 +679,11 @@ public:
    */
   template <typename S>
   tl::optional<Rib::Definition> resolve_path (
-    const std::vector<S> &segments, bool has_opening_scope_resolution,
+    const std::vector<S> &segments, ResolutionMode mode,
     std::function<void (const S &, NodeId)> insert_segment_resolution,
     std::vector<Error> &collect_errors);
 
   // FIXME: Documentation
-  tl::optional<Resolver::CanonicalPath> to_canonical_path (NodeId id) const;
-
-  // FIXME: Documentation
   tl::optional<Rib &> to_rib (NodeId rib_id);
   tl::optional<const Rib &> to_rib (NodeId rib_id) const;
 
@@ -739,6 +743,9 @@ private:
     tl::optional<Node &> parent; // `None` only if the node is a root
   };
 
+  // private overload which allows specifying a starting point
+  tl::optional<Rib::Definition> get (Node &start, const Identifier &name);
+
   /* Should we keep going upon seeing a Rib? */
   enum class KeepGoing
   {
diff --git a/gcc/rust/resolve/rust-forever-stack.hxx b/gcc/rust/resolve/rust-forever-stack.hxx
index 069111ee..1ed87b3 100644
--- a/gcc/rust/resolve/rust-forever-stack.hxx
+++ b/gcc/rust/resolve/rust-forever-stack.hxx
@@ -291,12 +291,16 @@ ForeverStack<N>::update_cursor (Node &new_cursor)
 
 template <Namespace N>
 tl::optional<Rib::Definition>
-ForeverStack<N>::get (const Identifier &name)
+ForeverStack<N>::get (Node &start, const Identifier &name)
 {
   tl::optional<Rib::Definition> resolved_definition = tl::nullopt;
 
   // TODO: Can we improve the API? have `reverse_iter` return an optional?
-  reverse_iter ([&resolved_definition, &name] (Node &current) {
+  reverse_iter (start, [&resolved_definition, &name] (Node &current) {
+    // we can't reference associated types/functions like this
+    if (current.rib.kind == Rib::Kind::TraitOrImpl)
+      return KeepGoing::Yes;
+
     auto candidate = current.rib.get (name.as_string ());
 
     return candidate.map_or (
@@ -320,6 +324,13 @@ ForeverStack<N>::get (const Identifier &name)
 
 template <Namespace N>
 tl::optional<Rib::Definition>
+ForeverStack<N>::get (const Identifier &name)
+{
+  return get (cursor (), name);
+}
+
+template <Namespace N>
+tl::optional<Rib::Definition>
 ForeverStack<N>::get_lang_prelude (const Identifier &name)
 {
   return lang_prelude.rib.get (name.as_string ());
@@ -542,6 +553,14 @@ ForeverStack<N>::resolve_segments (
       bool searched_prelude = false;
       while (true)
 	{
+	  if (is_start (iterator, segments)
+	      && current_node->rib.kind == Rib::Kind::TraitOrImpl)
+	    {
+	      // we can't reference associated types/functions like this
+	      current_node = &current_node->parent.value ();
+	      continue;
+	    }
+
 	  // may set the value of child
 	  for (auto &kv : current_node->children)
 	    {
@@ -625,88 +644,158 @@ template <Namespace N>
 template <typename S>
 tl::optional<Rib::Definition>
 ForeverStack<N>::resolve_path (
-  const std::vector<S> &segments, bool has_opening_scope_resolution,
+  const std::vector<S> &segments, ResolutionMode mode,
   std::function<void (const S &, NodeId)> insert_segment_resolution,
   std::vector<Error> &collect_errors)
 {
-  // TODO: What to do if segments.empty() ?
+  rust_assert (!segments.empty ());
 
-  // handle paths with opening scopes
-  std::function<void (void)> cleanup_current = [] () {};
-  if (has_opening_scope_resolution)
+  std::reference_wrapper<Node> starting_point = cursor ();
+  switch (mode)
     {
-      Node *last_current = &cursor_reference.get ();
-      if (get_rust_edition () == Edition::E2015)
-	cursor_reference = root;
-      else
-	cursor_reference = extern_prelude;
-      cleanup_current
-	= [this, last_current] () { cursor_reference = *last_current; };
+    case ResolutionMode::Normal:
+      break; // default
+    case ResolutionMode::FromRoot:
+      starting_point = root;
+      break;
+    case ResolutionMode::FromExtern:
+      starting_point = extern_prelude;
+      break;
+    default:
+      rust_unreachable ();
     }
 
   // if there's only one segment, we just use `get`
   if (segments.size () == 1)
     {
-      auto &seg = segments.front ();
-      if (auto lang_item = unwrap_segment_get_lang_item (seg))
+      auto &outer_seg = segments.front ();
+      if (auto lang_item = unwrap_segment_get_lang_item (outer_seg))
 	{
 	  NodeId seg_id = Analysis::Mappings::get ().get_lang_item_node (
 	    lang_item.value ());
 
-	  insert_segment_resolution (seg, seg_id);
-	  cleanup_current ();
+	  insert_segment_resolution (outer_seg, seg_id);
 	  // TODO: does NonShadowable matter?
 	  return Rib::Definition::NonShadowable (seg_id);
 	}
 
+      auto &seg = unwrap_type_segment (outer_seg);
+
       tl::optional<Rib::Definition> res
-	= get (unwrap_type_segment (segments.back ()).as_string ());
+	= get (starting_point.get (), seg.as_string ());
 
       if (!res)
-	res = get_lang_prelude (
-	  unwrap_type_segment (segments.back ()).as_string ());
+	res = get_lang_prelude (seg.as_string ());
+
+      if (N == Namespace::Types && !res)
+	{
+	  if (seg.is_crate_path_seg ())
+	    {
+	      insert_segment_resolution (outer_seg, root.id);
+	      // TODO: does NonShadowable matter?
+	      return Rib::Definition::NonShadowable (root.id);
+	    }
+	  else if (seg.is_lower_self_seg ())
+	    {
+	      NodeId id = find_closest_module (starting_point.get ()).id;
+	      insert_segment_resolution (outer_seg, id);
+	      // TODO: does NonShadowable matter?
+	      return Rib::Definition::NonShadowable (id);
+	    }
+	  else if (seg.is_super_path_seg ())
+	    {
+	      Node &closest_module
+		= find_closest_module (starting_point.get ());
+	      if (closest_module.is_root ())
+		{
+		  rust_error_at (seg.get_locus (), ErrorCode::E0433,
+				 "too many leading %<super%> keywords");
+		  return tl::nullopt;
+		}
+
+	      NodeId id
+		= find_closest_module (closest_module.parent.value ()).id;
+	      insert_segment_resolution (outer_seg, id);
+	      // TODO: does NonShadowable matter?
+	      return Rib::Definition::NonShadowable (id);
+	    }
+	  else
+	    {
+	      // HACK: check for a module after we check the language prelude
+	      for (auto &kv :
+		   find_closest_module (starting_point.get ()).children)
+		{
+		  auto &link = kv.first;
+
+		  if (link.path.map_or (
+			[&seg] (Identifier path) {
+			  auto &path_str = path.as_string ();
+			  return path_str == seg.as_string ();
+			},
+			false))
+		    {
+		      insert_segment_resolution (outer_seg, kv.second.id);
+		      return Rib::Definition::NonShadowable (kv.second.id);
+		    }
+		}
+	    }
+	}
 
       if (res && !res->is_ambiguous ())
-	insert_segment_resolution (segments.back (), res->get_node_id ());
-      cleanup_current ();
+	insert_segment_resolution (outer_seg, res->get_node_id ());
       return res;
     }
 
-  std::reference_wrapper<Node> starting_point = cursor ();
+  return find_starting_point (segments, starting_point,
+			      insert_segment_resolution, collect_errors)
+    .and_then (
+      [this, &segments, &starting_point, &insert_segment_resolution,
+       &collect_errors] (typename std::vector<S>::const_iterator iterator) {
+	return resolve_segments (starting_point.get (), segments, iterator,
+				 insert_segment_resolution, collect_errors);
+      })
+    .and_then ([this, &segments, &insert_segment_resolution] (
+		 Node &final_node) -> tl::optional<Rib::Definition> {
+      // leave resolution within impl blocks to type checker
+      if (final_node.rib.kind == Rib::Kind::TraitOrImpl)
+	return tl::nullopt;
+
+      auto &seg = unwrap_type_segment (segments.back ());
+      std::string seg_name = seg.as_string ();
+
+      // assuming this can't be a lang item segment
+      tl::optional<Rib::Definition> res
+	= resolve_final_segment (final_node, seg_name,
+				 seg.is_lower_self_seg ());
+      // Ok we didn't find it in the rib, Lets try the prelude...
+      if (!res)
+	res = get_lang_prelude (seg_name);
 
-  auto res
-    = find_starting_point (segments, starting_point, insert_segment_resolution,
-			   collect_errors)
-	.and_then (
-	  [this, &segments, &starting_point, &insert_segment_resolution,
-	   &collect_errors] (typename std::vector<S>::const_iterator iterator) {
-	    return resolve_segments (starting_point.get (), segments, iterator,
-				     insert_segment_resolution, collect_errors);
-	  })
-	.and_then ([this, &segments, &insert_segment_resolution] (
-		     Node &final_node) -> tl::optional<Rib::Definition> {
-	  // leave resolution within impl blocks to type checker
-	  if (final_node.rib.kind == Rib::Kind::TraitOrImpl)
-	    return tl::nullopt;
-
-	  auto &seg = unwrap_type_segment (segments.back ());
-	  std::string seg_name = seg.as_string ();
-
-	  // assuming this can't be a lang item segment
-	  tl::optional<Rib::Definition> res
-	    = resolve_final_segment (final_node, seg_name,
-				     seg.is_lower_self_seg ());
-	  // Ok we didn't find it in the rib, Lets try the prelude...
-	  if (!res)
-	    res = get_lang_prelude (seg_name);
-
-	  if (res && !res->is_ambiguous ())
-	    insert_segment_resolution (segments.back (), res->get_node_id ());
-
-	  return res;
-	});
-  cleanup_current ();
-  return res;
+      if (N == Namespace::Types && !res)
+	{
+	  // HACK: check for a module after we check the language prelude
+	  for (auto &kv : final_node.children)
+	    {
+	      auto &link = kv.first;
+
+	      if (link.path.map_or (
+		    [&seg_name] (Identifier path) {
+		      auto &path_str = path.as_string ();
+		      return path_str == seg_name;
+		    },
+		    false))
+		{
+		  insert_segment_resolution (segments.back (), kv.second.id);
+		  return Rib::Definition::NonShadowable (kv.second.id);
+		}
+	    }
+	}
+
+      if (res && !res->is_ambiguous ())
+	insert_segment_resolution (segments.back (), res->get_node_id ());
+
+      return res;
+    });
 }
 
 template <Namespace N>
@@ -771,67 +860,6 @@ ForeverStack<N>::dfs (const ForeverStack<N>::Node &starting_point,
 }
 
 template <Namespace N>
-tl::optional<Resolver::CanonicalPath>
-ForeverStack<N>::to_canonical_path (NodeId id) const
-{
-  // find the id in the current forever stack, starting from the root,
-  // performing either a BFS or DFS once the Node containing the ID is found, go
-  // back up to the root (parent().parent().parent()...) accumulate link
-  // segments reverse them that's your canonical path
-
-  return dfs (root, id).map ([this, id] (ConstDfsResult tuple) {
-    auto containing_node = tuple.first;
-    auto name = tuple.second;
-
-    auto segments = std::vector<Resolver::CanonicalPath> ();
-
-    reverse_iter (containing_node, [&segments] (const Node &current) {
-      if (current.is_root ())
-	return KeepGoing::No;
-
-      auto children = current.parent.value ().children;
-      const Link *outer_link = nullptr;
-
-      for (auto &kv : children)
-	{
-	  auto &link = kv.first;
-	  auto &child = kv.second;
-
-	  if (current.id == child.id)
-	    {
-	      outer_link = &link;
-	      break;
-	    }
-	}
-
-      rust_assert (outer_link);
-
-      outer_link->path.map ([&segments, outer_link] (Identifier path) {
-	segments.emplace (segments.begin (),
-			  Resolver::CanonicalPath::new_seg (outer_link->id,
-							    path.as_string ()));
-      });
-
-      return KeepGoing::Yes;
-    });
-
-    auto &mappings = Analysis::Mappings::get ();
-    CrateNum crate_num = mappings.lookup_crate_num (root.id).value ();
-    auto path = Resolver::CanonicalPath::new_seg (
-      root.id, mappings.get_crate_name (crate_num).value ());
-    path.set_crate_num (crate_num);
-
-    for (const auto &segment : segments)
-      path = path.append (segment);
-
-    // Finally, append the name
-    path = path.append (Resolver::CanonicalPath::new_seg (id, name));
-
-    return path;
-  });
-}
-
-template <Namespace N>
 tl::optional<Rib &>
 ForeverStack<N>::dfs_rib (ForeverStack<N>::Node &starting_point, NodeId to_find)
 {
diff --git a/gcc/rust/resolve/rust-ice-finalizer.cc b/gcc/rust/resolve/rust-ice-finalizer.cc
index bd4763f..79a06c3 100644
--- a/gcc/rust/resolve/rust-ice-finalizer.cc
+++ b/gcc/rust/resolve/rust-ice-finalizer.cc
@@ -22,12 +22,12 @@ namespace Rust {
 namespace Resolver {
 
 void ATTRIBUTE_NORETURN
-funny_ice_text_finalizer (diagnostic_text_output_format &text_output,
-			  const diagnostic_info *diagnostic,
-			  diagnostic_t diag_kind)
+funny_ice_text_finalizer (diagnostics::text_sink &text_output,
+			  const diagnostics::diagnostic_info *diagnostic,
+			  enum diagnostics::kind diag_kind)
 {
-  gcc_assert (diag_kind == DK_ICE_NOBT);
-  default_diagnostic_text_finalizer (text_output, diagnostic, diag_kind);
+  gcc_assert (diag_kind == diagnostics::kind::ice_nobt);
+  diagnostics::default_text_finalizer (text_output, diagnostic, diag_kind);
   fnotice (stderr, "You have broken GCC Rust. This is a feature.\n");
   exit (ICE_EXIT_CODE);
 }
diff --git a/gcc/rust/resolve/rust-ice-finalizer.h b/gcc/rust/resolve/rust-ice-finalizer.h
index 85ab88f..5120176 100644
--- a/gcc/rust/resolve/rust-ice-finalizer.h
+++ b/gcc/rust/resolve/rust-ice-finalizer.h
@@ -55,9 +55,9 @@ namespace Resolver {
    the default bug reporting instructions, as there is no bug to report.  */
 
 void ATTRIBUTE_NORETURN
-funny_ice_text_finalizer (diagnostic_text_output_format &text_output,
-			  const diagnostic_info *diagnostic,
-			  diagnostic_t diag_kind);
+funny_ice_text_finalizer (diagnostics::text_sink &text_output,
+			  const diagnostics::diagnostic_info *diagnostic,
+			  enum diagnostics::kind diag_kind);
 
 } // namespace Resolver
 } // namespace Rust
diff --git a/gcc/rust/resolve/rust-late-name-resolver-2.0.cc b/gcc/rust/resolve/rust-late-name-resolver-2.0.cc
index 6ec0422..e39ca15 100644
--- a/gcc/rust/resolve/rust-late-name-resolver-2.0.cc
+++ b/gcc/rust/resolve/rust-late-name-resolver-2.0.cc
@@ -19,6 +19,7 @@
 #include "optional.h"
 #include "rust-ast-full.h"
 #include "rust-diagnostics.h"
+#include "rust-expr.h"
 #include "rust-hir-map.h"
 #include "rust-late-name-resolver-2.0.h"
 #include "rust-default-resolver.h"
@@ -33,7 +34,9 @@
 namespace Rust {
 namespace Resolver2_0 {
 
-Late::Late (NameResolutionContext &ctx) : DefaultResolver (ctx) {}
+Late::Late (NameResolutionContext &ctx)
+  : DefaultResolver (ctx), funny_error (false), block_big_self (false)
+{}
 
 static NodeId
 next_node_id ()
@@ -114,8 +117,7 @@ Late::go (AST::Crate &crate)
 {
   setup_builtin_types ();
 
-  for (auto &item : crate.items)
-    item->accept_vis (*this);
+  visit (crate);
 }
 
 void
@@ -140,24 +142,21 @@ Late::visit (AST::ForLoopExpr &expr)
   ctx.bindings.exit ();
 
   visit (expr.get_iterator_expr ());
-  visit (expr.get_loop_label ());
+
+  if (expr.has_loop_label ())
+    visit (expr.get_loop_label ());
+
   visit (expr.get_loop_block ());
 }
 
 void
-Late::visit (AST::IfLetExpr &expr)
+Late::visit_if_let_patterns (AST::IfLetExpr &expr)
 {
-  visit_outer_attrs (expr);
+  ctx.bindings.enter (BindingSource::IfLet);
 
-  ctx.bindings.enter (BindingSource::Let);
-
-  for (auto &pattern : expr.get_patterns ())
-    visit (pattern);
+  DefaultResolver::visit_if_let_patterns (expr);
 
   ctx.bindings.exit ();
-
-  visit (expr.get_value_expr ());
-  visit (expr.get_if_block ());
 }
 
 void
@@ -214,45 +213,80 @@ Late::visit (AST::LetStmt &let)
 }
 
 void
-Late::visit (AST::IdentifierPattern &identifier)
+Late::visit (AST::WhileLetLoopExpr &while_let)
+{
+  DefaultASTVisitor::visit_outer_attrs (while_let);
+
+  if (while_let.has_loop_label ())
+    visit (while_let.get_loop_label ());
+
+  // visit expression before pattern
+  // this makes variable shadowing work properly
+  visit (while_let.get_scrutinee_expr ());
+
+  ctx.bindings.enter (BindingSource::WhileLet);
+
+  for (auto &pattern : while_let.get_patterns ())
+    visit (pattern);
+
+  ctx.bindings.exit ();
+
+  visit (while_let.get_loop_block ());
+}
+
+static void
+visit_identifier_as_pattern (NameResolutionContext &ctx,
+			     const Identifier &ident, location_t locus,
+			     NodeId node_id, bool is_ref, bool is_mut)
 {
   // do we insert in labels or in values
   // but values does not allow shadowing... since functions cannot shadow
   // do we insert functions in labels as well?
 
-  if (ctx.bindings.peek ().is_and_bound (identifier.get_ident ()))
+  if (ctx.bindings.peek ().is_and_bound (ident))
     {
       if (ctx.bindings.peek ().get_source () == BindingSource::Param)
 	rust_error_at (
-	  identifier.get_locus (), ErrorCode::E0415,
+	  locus, ErrorCode::E0415,
 	  "identifier %qs is bound more than once in the same parameter list",
-	  identifier.as_string ().c_str ());
+	  ident.as_string ().c_str ());
       else
 	rust_error_at (
-	  identifier.get_locus (), ErrorCode::E0416,
+	  locus, ErrorCode::E0416,
 	  "identifier %qs is bound more than once in the same pattern",
-	  identifier.as_string ().c_str ());
+	  ident.as_string ().c_str ());
       return;
     }
 
-  ctx.bindings.peek ().insert_ident (identifier.get_ident ());
+  ctx.bindings.peek ().insert_ident (ident.as_string (), locus, is_ref, is_mut);
 
-  if (ctx.bindings.peek ().is_or_bound (identifier.get_ident ()))
+  if (ctx.bindings.peek ().is_or_bound (ident))
     {
-      // FIXME: map usage instead
-      std::ignore = ctx.values.insert_shadowable (identifier.get_ident (),
-						  identifier.get_node_id ());
+      auto res = ctx.values.get (ident);
+      rust_assert (res.has_value () && !res->is_ambiguous ());
+      ctx.map_usage (Usage (node_id), Definition (res->get_node_id ()));
     }
   else
     {
       // We do want to ignore duplicated data because some situations rely on
       // it.
-      std::ignore = ctx.values.insert_shadowable (identifier.get_ident (),
-						  identifier.get_node_id ());
+      std::ignore = ctx.values.insert_shadowable (ident, node_id);
     }
 }
 
 void
+Late::visit (AST::IdentifierPattern &identifier)
+{
+  DefaultResolver::visit (identifier);
+
+  visit_identifier_as_pattern (ctx, identifier.get_ident (),
+			       identifier.get_locus (),
+			       identifier.get_node_id (),
+			       identifier.get_is_ref (),
+			       identifier.get_is_mut ());
+}
+
+void
 Late::visit (AST::AltPattern &pattern)
 {
   ctx.bindings.peek ().push (Binding::Kind::Or);
@@ -279,9 +313,9 @@ Late::visit_function_params (AST::Function &function)
 void
 Late::visit (AST::StructPatternFieldIdent &field)
 {
-  // We do want to ignore duplicated data because some situations rely on it.
-  std::ignore = ctx.values.insert_shadowable (field.get_identifier (),
-					      field.get_node_id ());
+  visit_identifier_as_pattern (ctx, field.get_identifier (), field.get_locus (),
+			       field.get_node_id (), field.is_ref (),
+			       field.is_mut ());
 }
 
 void
@@ -375,8 +409,9 @@ Late::visit (AST::IdentifierExpr &expr)
     }
   else if (funny_error)
     {
-      diagnostic_text_finalizer (global_dc) = Resolver::funny_ice_text_finalizer;
-      emit_diagnostic (DK_ICE_NOBT, expr.get_locus (), -1,
+      diagnostics::text_finalizer (global_dc)
+	= Resolver::funny_ice_text_finalizer;
+      emit_diagnostic (diagnostics::kind::ice_nobt, expr.get_locus (), -1,
 		       "are you trying to break %s? how dare you?",
 		       expr.as_string ().c_str ());
     }
@@ -477,6 +512,16 @@ Late::visit (AST::PathInExpression &expr)
 }
 
 void
+Late::visit_impl_type (AST::Type &type)
+{
+  // TODO: does this have to handle reentrancy?
+  rust_assert (!block_big_self);
+  block_big_self = true;
+  visit (type);
+  block_big_self = false;
+}
+
+void
 Late::visit (AST::TypePath &type)
 {
   // should we add type path resolution in `ForeverStack` directly? Since it's
@@ -486,6 +531,16 @@ Late::visit (AST::TypePath &type)
 
   DefaultResolver::visit (type);
 
+  // prevent "impl Self {}" and similar
+  if (type.get_segments ().size () == 1
+      && !type.get_segments ().front ()->is_lang_item ()
+      && type.get_segments ().front ()->is_big_self_seg () && block_big_self)
+    {
+      rust_error_at (type.get_locus (),
+		     "%<Self%> is not valid in the self type of an impl block");
+      return;
+    }
+
   // this *should* mostly work
   // TODO: make sure typepath-like path resolution (?) is working
   auto resolved = ctx.resolve_path (type, Namespace::Types);
@@ -493,15 +548,16 @@ Late::visit (AST::TypePath &type)
   if (!resolved.has_value ())
     {
       if (!ctx.lookup (type.get_segments ().front ()->get_node_id ()))
-	rust_error_at (type.get_locus (), "could not resolve type path %qs",
-		       type.as_string ().c_str ());
+	rust_error_at (type.get_locus (), ErrorCode::E0412,
+		       "could not resolve type path %qs",
+		       type.make_debug_string ().c_str ());
       return;
     }
 
   if (resolved->is_ambiguous ())
     {
       rust_error_at (type.get_locus (), ErrorCode::E0659, "%qs is ambiguous",
-		     type.as_string ().c_str ());
+		     type.make_debug_string ().c_str ());
       return;
     }
 
@@ -518,6 +574,62 @@ Late::visit (AST::TypePath &type)
 }
 
 void
+Late::visit (AST::Visibility &vis)
+{
+  if (!vis.has_path ())
+    return;
+
+  AST::SimplePath &path = vis.get_path ();
+
+  rust_assert (path.get_segments ().size ());
+  auto &first_seg = path.get_segments ()[0];
+
+  auto mode = ResolutionMode::Normal;
+
+  if (path.has_opening_scope_resolution ())
+    {
+      if (get_rust_edition () == Edition::E2015)
+	mode = ResolutionMode::FromRoot;
+      else
+	mode = ResolutionMode::FromExtern;
+    }
+  else if (!first_seg.is_crate_path_seg () && !first_seg.is_super_path_seg ()
+	   && !first_seg.is_lower_self_seg ())
+    {
+      if (get_rust_edition () == Edition::E2015)
+	{
+	  mode = ResolutionMode::FromRoot;
+	}
+      else
+	{
+	  rust_error_at (path.get_locus (),
+			 "relative paths are not supported in visibilities in "
+			 "2018 edition or later");
+	  return;
+	}
+    }
+
+  auto res = ctx.resolve_path (path.get_segments (), mode, Namespace::Types);
+
+  if (!res.has_value ())
+    {
+      rust_error_at (path.get_locus (), ErrorCode::E0433,
+		     "could not resolve path %qs", path.as_string ().c_str ());
+      return;
+    }
+
+  // TODO: is this possible?
+  if (res->is_ambiguous ())
+    {
+      rust_error_at (path.get_locus (), ErrorCode::E0659, "%qs is ambiguous",
+		     path.as_string ().c_str ());
+      return;
+    }
+
+  ctx.map_usage (Usage (path.get_node_id ()), Definition (res->get_node_id ()));
+}
+
+void
 Late::visit (AST::Trait &trait)
 {
   // kind of weird how this is done
@@ -531,13 +643,6 @@ Late::visit (AST::Trait &trait)
 }
 
 void
-Late::visit (AST::StructStruct &s)
-{
-  auto s_vis = [this, &s] () { AST::DefaultASTVisitor::visit (s); };
-  ctx.scoped (Rib::Kind::Item, s.get_node_id (), s_vis);
-}
-
-void
 Late::visit (AST::StructExprStruct &s)
 {
   visit_outer_attrs (s);
@@ -613,51 +718,27 @@ Late::visit (AST::GenericArg &arg)
   DefaultResolver::visit (arg);
 }
 
-template <class Closure>
-static void
-add_captures (Closure &closure, NameResolutionContext &ctx)
-{
-  auto vals = ctx.values.peek ().get_values ();
-  for (auto &val : vals)
-    {
-      ctx.mappings.add_capture (closure.get_node_id (),
-				val.second.get_node_id ());
-    }
-}
-
 void
-Late::visit (AST::ClosureExprInner &closure)
+Late::visit_closure_params (AST::ClosureExpr &closure)
 {
-  add_captures (closure, ctx);
-
-  visit_outer_attrs (closure);
-
   ctx.bindings.enter (BindingSource::Param);
 
-  for (auto &param : closure.get_params ())
-    visit (param);
+  DefaultResolver::visit_closure_params (closure);
 
   ctx.bindings.exit ();
-
-  visit (closure.get_definition_expr ());
 }
 
 void
-Late::visit (AST::ClosureExprInnerTyped &closure)
+Late::visit (AST::ClosureExpr &expr)
 {
-  add_captures (closure, ctx);
-
-  visit_outer_attrs (closure);
-
-  ctx.bindings.enter (BindingSource::Param);
-
-  for (auto &param : closure.get_params ())
-    visit (param);
-
-  ctx.bindings.exit ();
+  // add captures
+  auto vals = ctx.values.peek ().get_values ();
+  for (auto &val : vals)
+    {
+      ctx.mappings.add_capture (expr.get_node_id (), val.second.get_node_id ());
+    }
 
-  visit (closure.get_return_type ());
-  visit (closure.get_definition_block ());
+  DefaultResolver::visit (expr);
 }
 
 } // namespace Resolver2_0
diff --git a/gcc/rust/resolve/rust-late-name-resolver-2.0.h b/gcc/rust/resolve/rust-late-name-resolver-2.0.h
index 171d9bf..95540e3 100644
--- a/gcc/rust/resolve/rust-late-name-resolver-2.0.h
+++ b/gcc/rust/resolve/rust-late-name-resolver-2.0.h
@@ -42,6 +42,7 @@ public:
 
   // some more label declarations
   void visit (AST::LetStmt &) override;
+  void visit (AST::WhileLetLoopExpr &) override;
   // TODO: Do we need this?
   // void visit (AST::Method &) override;
   void visit (AST::IdentifierPattern &) override;
@@ -50,7 +51,7 @@ public:
   void visit (AST::SelfParam &) override;
   void visit (AST::MatchArm &) override;
   void visit (AST::ForLoopExpr &) override;
-  void visit (AST::IfLetExpr &) override;
+  void visit_if_let_patterns (AST::IfLetExpr &) override;
 
   // resolutions
   void visit (AST::IdentifierExpr &) override;
@@ -59,16 +60,17 @@ public:
   void visit (AST::ContinueExpr &) override;
   void visit (AST::LoopLabel &) override;
   void visit (AST::PathInExpression &) override;
+  void visit_impl_type (AST::Type &) override;
   void visit (AST::TypePath &) override;
+  void visit (AST::Visibility &) override;
   void visit (AST::Trait &) override;
   void visit (AST::StructExprStruct &) override;
   void visit (AST::StructExprStructBase &) override;
   void visit (AST::StructExprStructFields &) override;
-  void visit (AST::StructStruct &) override;
   void visit (AST::GenericArgs &) override;
   void visit (AST::GenericArg &);
-  void visit (AST::ClosureExprInner &) override;
-  void visit (AST::ClosureExprInnerTyped &) override;
+  void visit_closure_params (AST::ClosureExpr &) override;
+  void visit (AST::ClosureExpr &) override;
 
 private:
   void resolve_label (AST::Lifetime &lifetime);
@@ -77,6 +79,9 @@ private:
   void setup_builtin_types ();
 
   bool funny_error;
+
+  /* used to prevent "impl Self {}", "impl (Self, i32) {}", etc */
+  bool block_big_self;
 };
 
 // TODO: Add missing mappings and data structures
diff --git a/gcc/rust/resolve/rust-name-resolution-context.cc b/gcc/rust/resolve/rust-name-resolution-context.cc
index f098e48..1b84f1d 100644
--- a/gcc/rust/resolve/rust-name-resolution-context.cc
+++ b/gcc/rust/resolve/rust-name-resolution-context.cc
@@ -33,7 +33,8 @@ BindingLayer::bind_test (Identifier ident, Binding::Kind kind)
 {
   for (auto &bind : bindings)
     {
-      if (bind.set.find (ident) != bind.set.cend () && bind.kind == kind)
+      if (bind.idents.find (ident.as_string ()) != bind.idents.cend ()
+	  && bind.kind == kind)
 	{
 	  return true;
 	}
@@ -60,20 +61,66 @@ BindingLayer::is_or_bound (Identifier ident)
 }
 
 void
-BindingLayer::insert_ident (Identifier ident)
+BindingLayer::insert_ident (std::string ident, location_t locus, bool is_ref,
+			    bool is_mut)
 {
-  bindings.back ().set.insert (ident);
+  bindings.back ().idents.emplace (
+    std::move (ident), std::make_pair (locus, IdentifierMode (is_ref, is_mut)));
 }
 
 void
 BindingLayer::merge ()
 {
-  auto last_binding = bindings.back ();
+  auto last_binding = std::move (bindings.back ());
   bindings.pop_back ();
-  for (auto &value : last_binding.set)
+
+  if (bindings.back ().has_expected_bindings)
+    {
+      for (auto &value : bindings.back ().idents)
+	{
+	  auto ident = value.first;
+	  if (last_binding.idents.find (ident) == last_binding.idents.end ())
+	    {
+	      location_t locus = value.second.first;
+	      rust_error_at (locus, ErrorCode::E0408,
+			     "variable %qs is not bound in all patterns",
+			     ident.c_str ());
+	    }
+	}
+    }
+
+  for (auto &value : last_binding.idents)
     {
-      bindings.back ().set.insert (value);
+      auto res = bindings.back ().idents.emplace (value);
+      if (res.second)
+	{
+	  if (bindings.back ().has_expected_bindings)
+	    {
+	      auto &ident = value.first;
+	      location_t locus = value.second.first;
+	      rust_error_at (locus, ErrorCode::E0408,
+			     "variable %qs is not bound in all patterns",
+			     ident.c_str ());
+	    }
+	}
+      else
+	{
+	  auto this_mode = value.second.second;
+	  auto other_mode = res.first->second.second;
+	  if (this_mode != other_mode)
+	    {
+	      auto &ident = value.first;
+	      location_t locus = value.second.first;
+	      rust_error_at (locus, ErrorCode::E0409,
+			     "variable %qs is bound inconsistently across "
+			     "pattern alternatives",
+			     ident.c_str ());
+	    }
+	}
     }
+
+  if (bindings.back ().kind == Binding::Kind::Or)
+    bindings.back ().has_expected_bindings = true;
 }
 
 BindingSource
@@ -82,8 +129,63 @@ BindingLayer::get_source () const
   return source;
 }
 
+Resolver::CanonicalPath
+CanonicalPathRecordCrateRoot::as_path (const NameResolutionContext &)
+{
+  auto ret = Resolver::CanonicalPath::new_seg (node_id, seg);
+  ret.set_crate_num (crate_num);
+  return ret;
+}
+
+Resolver::CanonicalPath
+CanonicalPathRecordNormal::as_path (const NameResolutionContext &ctx)
+{
+  auto parent_path = get_parent ().as_path (ctx);
+  return parent_path.append (Resolver::CanonicalPath::new_seg (node_id, seg));
+}
+
+Resolver::CanonicalPath
+CanonicalPathRecordLookup::as_path (const NameResolutionContext &ctx)
+{
+  if (!cache)
+    {
+      auto res = ctx.lookup (lookup_id).and_then (
+	[&ctx] (NodeId id) { return ctx.canonical_ctx.get_record_opt (id); });
+
+      if (!res)
+	{
+	  // HACK: use a dummy value
+	  // this should bring us roughly to parity with nr1.0
+	  // since nr1.0 doesn't seem to handle canonical paths for generics
+	  //   quite right anyways
+	  return Resolver::CanonicalPath::new_seg (UNKNOWN_NODEID, "XXX");
+	}
+
+      cache = res.value ();
+    }
+  return cache->as_path (ctx);
+}
+
+Resolver::CanonicalPath
+CanonicalPathRecordImpl::as_path (const NameResolutionContext &ctx)
+{
+  auto parent_path = get_parent ().as_path (ctx);
+  return parent_path.append (
+    Resolver::CanonicalPath::inherent_impl_seg (impl_id,
+						type_record.as_path (ctx)));
+}
+
+Resolver::CanonicalPath
+CanonicalPathRecordTraitImpl::as_path (const NameResolutionContext &ctx)
+{
+  auto parent_path = get_parent ().as_path (ctx);
+  return parent_path.append (
+    Resolver::CanonicalPath::trait_impl_projection_seg (
+      impl_id, trait_path_record.as_path (ctx), type_record.as_path (ctx)));
+}
+
 NameResolutionContext::NameResolutionContext ()
-  : mappings (Analysis::Mappings::get ())
+  : mappings (Analysis::Mappings::get ()), canonical_ctx (*this)
 {}
 
 tl::expected<NodeId, DuplicateNameError>
diff --git a/gcc/rust/resolve/rust-name-resolution-context.h b/gcc/rust/resolve/rust-name-resolution-context.h
index 19ba750..558b3ca 100644
--- a/gcc/rust/resolve/rust-name-resolution-context.h
+++ b/gcc/rust/resolve/rust-name-resolution-context.h
@@ -24,6 +24,7 @@
 #include "rust-hir-map.h"
 #include "rust-rib.h"
 #include "rust-stacked-contexts.h"
+#include "rust-item.h"
 
 namespace Rust {
 namespace Resolver2_0 {
@@ -157,6 +158,22 @@ public:
   NodeId id;
 };
 
+struct IdentifierMode
+{
+  bool is_ref;
+  bool is_mut;
+
+  IdentifierMode (bool is_ref, bool is_mut) : is_ref (is_ref), is_mut (is_mut)
+  {}
+
+  bool operator== (const IdentifierMode &other)
+  {
+    return other.is_ref == is_ref && other.is_mut == is_mut;
+  }
+
+  bool operator!= (const IdentifierMode &other) { return !(*this == other); }
+};
+
 struct Binding
 {
   enum class Kind
@@ -165,9 +182,12 @@ struct Binding
     Or,
   } kind;
 
-  std::unordered_set<Identifier> set;
+  // used to check the correctness of or-bindings
+  bool has_expected_bindings;
 
-  Binding (Binding::Kind kind) : kind (kind) {}
+  std::unordered_map<std::string, std::pair<location_t, IdentifierMode>> idents;
+
+  Binding (Binding::Kind kind) : kind (kind), has_expected_bindings (false) {}
 };
 
 /**
@@ -177,6 +197,8 @@ enum class BindingSource
 {
   Match,
   Let,
+  IfLet,
+  WhileLet,
   For,
   /* Closure param or function param */
   Param
@@ -206,13 +228,256 @@ public:
    */
   bool is_or_bound (Identifier ident);
 
-  void insert_ident (Identifier ident);
+  void insert_ident (std::string ident, location_t locus, bool is_ref,
+		     bool is_mut);
 
   void merge ();
 
   BindingSource get_source () const;
 };
 
+class NameResolutionContext;
+/*
+ * Used to handle canonical paths
+ * Similar to ForeverStack, but namespace independent and more specialized
+ */
+class CanonicalPathRecord
+{
+public:
+  virtual Resolver::CanonicalPath as_path (const NameResolutionContext &) = 0;
+
+  virtual bool is_root () const = 0;
+
+  virtual ~CanonicalPathRecord () = default;
+};
+
+class CanonicalPathRecordWithParent : public CanonicalPathRecord
+{
+public:
+  CanonicalPathRecordWithParent (CanonicalPathRecord &parent) : parent (&parent)
+  {}
+
+  CanonicalPathRecord &get_parent () { return *parent; }
+
+  bool is_root () const override final { return false; }
+
+private:
+  CanonicalPathRecord *parent;
+};
+
+class CanonicalPathRecordCrateRoot : public CanonicalPathRecord
+{
+public:
+  CanonicalPathRecordCrateRoot (NodeId node_id, std::string seg)
+    : node_id (node_id), seg (std::move (seg))
+  {
+    rust_assert (Analysis::Mappings::get ().node_is_crate (node_id));
+    crate_num = Analysis::Mappings::get ().lookup_crate_num (node_id).value ();
+  }
+
+  Resolver::CanonicalPath as_path (const NameResolutionContext &) override;
+
+  bool is_root () const override final { return true; }
+
+private:
+  NodeId node_id;
+  CrateNum crate_num;
+  std::string seg;
+};
+
+class CanonicalPathRecordNormal : public CanonicalPathRecordWithParent
+{
+public:
+  CanonicalPathRecordNormal (CanonicalPathRecord &parent, NodeId node_id,
+			     std::string seg)
+    : CanonicalPathRecordWithParent (parent), node_id (node_id),
+      seg (std::move (seg))
+  {
+    rust_assert (!Analysis::Mappings::get ().node_is_crate (node_id));
+  }
+
+  Resolver::CanonicalPath as_path (const NameResolutionContext &) override;
+
+private:
+  NodeId node_id;
+  std::string seg;
+};
+
+class CanonicalPathRecordLookup : public CanonicalPathRecord
+{
+public:
+  CanonicalPathRecordLookup (NodeId lookup_id)
+    : lookup_id (lookup_id), cache (nullptr)
+  {}
+
+  Resolver::CanonicalPath as_path (const NameResolutionContext &) override;
+
+  bool is_root () const override final { return true; }
+
+private:
+  NodeId lookup_id;
+  CanonicalPathRecord *cache;
+};
+
+class CanonicalPathRecordImpl : public CanonicalPathRecordWithParent
+{
+public:
+  CanonicalPathRecordImpl (CanonicalPathRecord &parent, NodeId impl_id,
+			   NodeId type_id)
+    : CanonicalPathRecordWithParent (parent), impl_id (impl_id),
+      type_record (type_id)
+  {}
+
+  Resolver::CanonicalPath as_path (const NameResolutionContext &) override;
+
+private:
+  NodeId impl_id;
+  CanonicalPathRecordLookup type_record;
+};
+
+class CanonicalPathRecordTraitImpl : public CanonicalPathRecordWithParent
+{
+public:
+  CanonicalPathRecordTraitImpl (CanonicalPathRecord &parent, NodeId impl_id,
+				NodeId type_id, NodeId trait_path_id)
+    : CanonicalPathRecordWithParent (parent), impl_id (impl_id),
+      type_record (type_id), trait_path_record (trait_path_id)
+  {}
+
+  Resolver::CanonicalPath as_path (const NameResolutionContext &) override;
+
+private:
+  NodeId impl_id;
+  CanonicalPathRecordLookup type_record;
+  CanonicalPathRecordLookup trait_path_record;
+};
+
+class CanonicalPathCtx
+{
+public:
+  CanonicalPathCtx (const NameResolutionContext &ctx)
+    : current_record (nullptr), nr_ctx (&ctx)
+  {}
+
+  Resolver::CanonicalPath get_path (NodeId id) const
+  {
+    return get_record (id).as_path (*nr_ctx);
+  }
+
+  CanonicalPathRecord &get_record (NodeId id) const
+  {
+    auto it = records.find (id);
+    rust_assert (it != records.end ());
+    return *it->second;
+  }
+
+  tl::optional<CanonicalPathRecord *> get_record_opt (NodeId id) const
+  {
+    auto it = records.find (id);
+    if (it == records.end ())
+      return tl::nullopt;
+    else
+      return it->second.get ();
+  }
+
+  void insert_record (NodeId id, const Identifier &ident)
+  {
+    insert_record (id, ident.as_string ());
+  }
+
+  void insert_record (NodeId id, std::string seg)
+  {
+    rust_assert (current_record != nullptr);
+
+    auto it = records.find (id);
+    if (it == records.end ())
+      {
+	auto record = new CanonicalPathRecordNormal (*current_record, id,
+						     std::move (seg));
+	bool ok
+	  = records.emplace (id, std::unique_ptr<CanonicalPathRecord> (record))
+	      .second;
+	rust_assert (ok);
+      }
+  }
+
+  template <typename F> void scope (NodeId id, const Identifier &ident, F &&f)
+  {
+    scope (id, ident.as_string (), std::forward<F> (f));
+  }
+
+  template <typename F> void scope (NodeId id, std::string seg, F &&f)
+  {
+    rust_assert (current_record != nullptr);
+
+    scope_inner (id, std::forward<F> (f), [this, id, &seg] () {
+      return new CanonicalPathRecordNormal (*current_record, id,
+					    std::move (seg));
+    });
+  }
+
+  template <typename F> void scope_impl (AST::InherentImpl &impl, F &&f)
+  {
+    rust_assert (current_record != nullptr);
+
+    NodeId id = impl.get_node_id ();
+    scope_inner (id, std::forward<F> (f), [this, id, &impl] () {
+      return new CanonicalPathRecordImpl (*current_record, id,
+					  impl.get_type ().get_node_id ());
+    });
+  }
+
+  template <typename F> void scope_impl (AST::TraitImpl &impl, F &&f)
+  {
+    rust_assert (current_record != nullptr);
+
+    NodeId id = impl.get_node_id ();
+    scope_inner (id, std::forward<F> (f), [this, id, &impl] () {
+      return new CanonicalPathRecordTraitImpl (
+	*current_record, id, impl.get_type ().get_node_id (),
+	impl.get_trait_path ().get_node_id ());
+    });
+  }
+
+  template <typename F>
+  void scope_crate (NodeId node_id, std::string crate_name, F &&f)
+  {
+    scope_inner (node_id, std::forward<F> (f), [node_id, &crate_name] () {
+      return new CanonicalPathRecordCrateRoot (node_id, std::move (crate_name));
+    });
+  }
+
+private:
+  template <typename FCreate, typename FCallback>
+  void scope_inner (NodeId id, FCallback &&f_callback, FCreate &&f_create)
+  {
+    auto it = records.find (id);
+    if (it == records.end ())
+      {
+	CanonicalPathRecord *record = std::forward<FCreate> (f_create) ();
+	it = records.emplace (id, std::unique_ptr<CanonicalPathRecord> (record))
+	       .first;
+      }
+
+    rust_assert (it->second->is_root ()
+		 || &static_cast<CanonicalPathRecordWithParent &> (*it->second)
+			.get_parent ()
+		      == current_record);
+
+    CanonicalPathRecord *stash = it->second.get ();
+    std::swap (stash, current_record);
+
+    std::forward<FCallback> (f_callback) ();
+
+    std::swap (stash, current_record);
+  }
+
+  std::unordered_map<NodeId, std::unique_ptr<CanonicalPathRecord>> records;
+  CanonicalPathRecord *current_record;
+
+  const NameResolutionContext *nr_ctx;
+};
+
 // Now our resolver, which keeps track of all the `ForeverStack`s we could want
 class NameResolutionContext
 {
@@ -271,16 +536,22 @@ public:
   Analysis::Mappings &mappings;
   StackedContexts<BindingLayer> bindings;
 
+  CanonicalPathCtx canonical_ctx;
+
   // TODO: Rename
   // TODO: Use newtype pattern for Usage and Definition
   void map_usage (Usage usage, Definition definition);
 
   tl::optional<NodeId> lookup (NodeId usage) const;
 
+  Resolver::CanonicalPath to_canonical_path (NodeId id) const
+  {
+    return canonical_ctx.get_path (id);
+  }
+
   template <typename S>
   tl::optional<Rib::Definition>
-  resolve_path (const std::vector<S> &segments,
-		bool has_opening_scope_resolution,
+  resolve_path (const std::vector<S> &segments, ResolutionMode mode,
 		std::vector<Error> &collect_errors, Namespace ns)
   {
     std::function<void (const S &, NodeId)> insert_segment_resolution
@@ -292,17 +563,17 @@ public:
     switch (ns)
       {
       case Namespace::Values:
-	return values.resolve_path (segments, has_opening_scope_resolution,
-				    insert_segment_resolution, collect_errors);
+	return values.resolve_path (segments, mode, insert_segment_resolution,
+				    collect_errors);
       case Namespace::Types:
-	return types.resolve_path (segments, has_opening_scope_resolution,
-				   insert_segment_resolution, collect_errors);
+	return types.resolve_path (segments, mode, insert_segment_resolution,
+				   collect_errors);
       case Namespace::Macros:
-	return macros.resolve_path (segments, has_opening_scope_resolution,
-				    insert_segment_resolution, collect_errors);
+	return macros.resolve_path (segments, mode, insert_segment_resolution,
+				    collect_errors);
       case Namespace::Labels:
-	return labels.resolve_path (segments, has_opening_scope_resolution,
-				    insert_segment_resolution, collect_errors);
+	return labels.resolve_path (segments, mode, insert_segment_resolution,
+				    collect_errors);
       default:
 	rust_unreachable ();
       }
@@ -310,8 +581,7 @@ public:
 
   template <typename S, typename... Args>
   tl::optional<Rib::Definition>
-  resolve_path (const std::vector<S> &segments,
-		bool has_opening_scope_resolution,
+  resolve_path (const std::vector<S> &segments, ResolutionMode mode,
 		tl::optional<std::vector<Error> &> collect_errors,
 		Namespace ns_first, Args... ns_args)
   {
@@ -320,8 +590,7 @@ public:
     for (auto ns : namespaces)
       {
 	std::vector<Error> collect_errors_inner;
-	if (auto ret = resolve_path (segments, has_opening_scope_resolution,
-				     collect_errors_inner, ns))
+	if (auto ret = resolve_path (segments, mode, collect_errors_inner, ns))
 	  return ret;
 	if (!collect_errors_inner.empty ())
 	  {
@@ -343,52 +612,68 @@ public:
     return tl::nullopt;
   }
 
-  template <typename... Args>
+  template <typename S, typename... Args>
   tl::optional<Rib::Definition>
-  resolve_path (const AST::SimplePath &path,
+  resolve_path (const std::vector<S> &path_segments,
+		bool has_opening_scope_resolution,
 		tl::optional<std::vector<Error> &> collect_errors,
 		Namespace ns_first, Args... ns_args)
   {
-    return resolve_path (path.get_segments (),
-			 path.has_opening_scope_resolution (), collect_errors,
-			 ns_first, ns_args...);
+    auto mode = ResolutionMode::Normal;
+    if (has_opening_scope_resolution)
+      {
+	if (get_rust_edition () == Edition::E2015)
+	  mode = ResolutionMode::FromRoot;
+	else
+	  mode = ResolutionMode::FromExtern;
+      }
+    return resolve_path (path_segments, mode, collect_errors, ns_first,
+			 ns_args...);
   }
 
-  template <typename... Args>
+  template <typename S, typename... Args>
   tl::optional<Rib::Definition>
-  resolve_path (const AST::PathInExpression &path,
-		tl::optional<std::vector<Error> &> collect_errors,
-		Namespace ns_first, Args... ns_args)
+  resolve_path (const std::vector<S> &path_segments,
+		bool has_opening_scope_resolution, Namespace ns_first,
+		Args... ns_args)
   {
-    return resolve_path (path.get_segments (), path.opening_scope_resolution (),
-			 collect_errors, ns_first, ns_args...);
+    return resolve_path (path_segments, has_opening_scope_resolution,
+			 tl::nullopt, ns_first, ns_args...);
   }
 
-  template <typename... Args>
+  template <typename S, typename... Args>
   tl::optional<Rib::Definition>
-  resolve_path (const AST::TypePath &path,
-		tl::optional<std::vector<Error> &> collect_errors,
+  resolve_path (const std::vector<S> &path_segments, ResolutionMode mode,
 		Namespace ns_first, Args... ns_args)
   {
+    return resolve_path (path_segments, mode, tl::nullopt, ns_first,
+			 ns_args...);
+  }
+
+  template <typename... Args>
+  tl::optional<Rib::Definition> resolve_path (const AST::SimplePath &path,
+					      Args &&...args)
+  {
     return resolve_path (path.get_segments (),
-			 path.has_opening_scope_resolution_op (),
-			 collect_errors, ns_first, ns_args...);
+			 path.has_opening_scope_resolution (),
+			 std::forward<Args> (args)...);
   }
 
-  template <typename P, typename... Args>
-  tl::optional<Rib::Definition> resolve_path (const P &path, Namespace ns_first,
-					      Args... ns_args)
+  template <typename... Args>
+  tl::optional<Rib::Definition> resolve_path (const AST::PathInExpression &path,
+					      Args &&...args)
   {
-    return resolve_path (path, tl::nullopt, ns_first, ns_args...);
+    return resolve_path (path.get_segments (), path.opening_scope_resolution (),
+			 std::forward<Args> (args)...);
   }
 
-  template <typename P, typename... Args>
-  tl::optional<Rib::Definition>
-  resolve_path (const P &path_segments, bool has_opening_scope_resolution,
-		Namespace ns_first, Args... ns_args)
+  template <typename... Args>
+  tl::optional<Rib::Definition> resolve_path (const AST::TypePath &path,
+					      Args &&...args)
   {
-    return resolve_path (path_segments, has_opening_scope_resolution,
-			 tl::nullopt, ns_first, ns_args...);
+    return resolve_path (path.get_segments (),
+			 path.has_opening_scope_resolution_op (),
+			 std::forward<Args> (args)...);
   }
 
 private:
diff --git a/gcc/rust/resolve/rust-rib.h b/gcc/rust/resolve/rust-rib.h
index c498328..140c991 100644
--- a/gcc/rust/resolve/rust-rib.h
+++ b/gcc/rust/resolve/rust-rib.h
@@ -188,6 +188,8 @@ public:
      * restriction that you cannot `use` items from the Prelude
      */
     Prelude,
+    /* Generic rib, used to store generics */
+    Generics,
   } kind;
 
   static std::string kind_to_string (Rib::Kind kind)
@@ -214,9 +216,13 @@ public:
 	return "Forward type param ban";
       case Rib::Kind::ConstParamType:
 	return "Const Param Type";
-      default:
-	rust_unreachable ();
+      case Kind::Prelude:
+	return "Prelude";
+      case Kind::Generics:
+	return "Generics";
       }
+
+    rust_unreachable ();
   }
 
   Rib (Kind kind);
diff --git a/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.cc b/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.cc
index 2f036fe..0930f96 100644
--- a/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.cc
+++ b/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.cc
@@ -95,41 +95,17 @@ TopLevel::go (AST::Crate &crate)
   // times in a row in a fixed-point fashion, so it would make the code
   // responsible for this ugly and perfom a lot of error checking.
 
-  for (auto &item : crate.items)
-    item->accept_vis (*this);
+  visit (crate);
 }
 
 void
 TopLevel::visit (AST::Module &module)
 {
-  insert_or_error_out (module.get_name (), module, Namespace::Types);
-
-  // Parse the module's items if they haven't been expanded and the file
-  // should be parsed (i.e isn't hidden behind an untrue or impossible cfg
-  // directive
-  // TODO: make sure this is right
-  // TODO: avoid loading items if cfg attributes are present?
-  //       might not be needed if this runs after early resolution?
-  // This was copied from the old early resolver method
-  // 'accumulate_escaped_macros'
-  if (module.get_kind () == AST::Module::UNLOADED)
-    {
-      module.load_items ();
-
-      // If the module was previously unloaded, then we don't want to visit it
-      // this time around as the CfgStrip hasn't run on its inner items yet.
-      // Skip it for now, mark the visitor as dirty and try again
-
-      dirty = true;
-
-      return;
-    }
-
   DefaultResolver::visit (module);
 
-  if (Analysis::Mappings::get ().lookup_ast_module (module.get_node_id ())
+  if (Analysis::Mappings::get ().lookup_glob_container (module.get_node_id ())
       == tl::nullopt)
-    Analysis::Mappings::get ().insert_ast_module (&module);
+    Analysis::Mappings::get ().insert_glob_container (&module);
 }
 
 void
@@ -141,33 +117,10 @@ TopLevel::visit (AST::Trait &trait)
 }
 
 void
-TopLevel::visit (AST::InherentImpl &impl)
-{
-  auto inner_fn = [this, &impl] () {
-    insert_or_error_out (Identifier ("Self", impl.get_type ().get_locus ()),
-			 impl.get_type (), Namespace::Types);
-
-    // We do want to visit with the default visitor instead of default resolver
-    // because we don't want to insert the scope twice.
-    AST::DefaultASTVisitor::visit (impl);
-  };
-
-  ctx.scoped (Rib::Kind::TraitOrImpl, impl.get_node_id (), inner_fn);
-}
-
-void
-TopLevel::visit (AST::TraitImpl &impl)
+TopLevel::maybe_insert_big_self (AST::Impl &impl)
 {
-  auto inner_fn = [this, &impl] () {
-    insert_or_error_out (Identifier ("Self", impl.get_type ().get_locus ()),
-			 impl.get_type (), Namespace::Types);
-
-    // We do want to visit using the default visitor instead of default resolver
-    // because we don't want to insert the scope twice.
-    AST::DefaultASTVisitor::visit (impl);
-  };
-
-  ctx.scoped (Rib::Kind::TraitOrImpl, impl.get_node_id (), inner_fn);
+  insert_or_error_out (Identifier ("Self", impl.get_type ().get_locus ()),
+		       impl.get_type (), Namespace::Types);
 }
 
 void
@@ -197,19 +150,10 @@ insert_macros (std::vector<PROC_MACRO> &macros, NameResolutionContext &ctx)
 }
 
 void
-TopLevel::visit (AST::ExternCrate &crate)
+TopLevel::visit_extern_crate (AST::ExternCrate &extern_crate, AST::Crate &crate,
+			      CrateNum num)
 {
   auto &mappings = Analysis::Mappings::get ();
-  auto num_opt = mappings.lookup_crate_name (crate.get_referenced_crate ());
-
-  if (!num_opt)
-    {
-      rust_error_at (crate.get_locus (), "unknown crate %qs",
-		     crate.get_referenced_crate ().c_str ());
-      return;
-    }
-
-  CrateNum num = *num_opt;
 
   auto attribute_macros = mappings.lookup_attribute_proc_macros (num);
 
@@ -217,34 +161,27 @@ TopLevel::visit (AST::ExternCrate &crate)
 
   auto derive_macros = mappings.lookup_derive_proc_macros (num);
 
-  auto sub_visitor = [&] () {
-    // TODO: Find a way to keep this part clean without the double dispatch.
-    if (derive_macros.has_value ())
-      {
-	insert_macros (derive_macros.value (), ctx);
-	for (auto &macro : derive_macros.value ())
-	  mappings.insert_derive_proc_macro_def (macro);
-      }
-    if (attribute_macros.has_value ())
-      {
-	insert_macros (attribute_macros.value (), ctx);
-	for (auto &macro : attribute_macros.value ())
-	  mappings.insert_attribute_proc_macro_def (macro);
-      }
-    if (bang_macros.has_value ())
-      {
-	insert_macros (bang_macros.value (), ctx);
-	for (auto &macro : bang_macros.value ())
-	  mappings.insert_bang_proc_macro_def (macro);
-      }
-  };
+  // TODO: Find a way to keep this part clean without the double dispatch.
+  if (derive_macros.has_value ())
+    {
+      insert_macros (derive_macros.value (), ctx);
+      for (auto &macro : derive_macros.value ())
+	mappings.insert_derive_proc_macro_def (macro);
+    }
+  if (attribute_macros.has_value ())
+    {
+      insert_macros (attribute_macros.value (), ctx);
+      for (auto &macro : attribute_macros.value ())
+	mappings.insert_attribute_proc_macro_def (macro);
+    }
+  if (bang_macros.has_value ())
+    {
+      insert_macros (bang_macros.value (), ctx);
+      for (auto &macro : bang_macros.value ())
+	mappings.insert_bang_proc_macro_def (macro);
+    }
 
-  if (crate.has_as_clause ())
-    ctx.scoped (Rib::Kind::Module, crate.get_node_id (), sub_visitor,
-		crate.get_as_clause ());
-  else
-    ctx.scoped (Rib::Kind::Module, crate.get_node_id (), sub_visitor,
-		crate.get_referenced_crate ());
+  visit (crate);
 }
 
 static bool
@@ -321,14 +258,7 @@ TopLevel::visit (AST::ExternalStaticItem &static_item)
 void
 TopLevel::visit (AST::StructStruct &struct_item)
 {
-  auto generic_vis = [this, &struct_item] () {
-    for (auto &g : struct_item.get_generic_params ())
-      {
-	g->accept_vis (*this);
-      }
-  };
-
-  ctx.scoped (Rib::Kind::Item, struct_item.get_node_id (), generic_vis);
+  DefaultResolver::visit (struct_item);
 
   insert_or_error_out (struct_item.get_struct_name (), struct_item,
 		       Namespace::Types);
@@ -374,24 +304,32 @@ void
 TopLevel::visit (AST::EnumItem &variant)
 {
   insert_enum_variant_or_error_out (variant.get_identifier (), variant);
+
+  DefaultResolver::visit (variant);
 }
 
 void
 TopLevel::visit (AST::EnumItemTuple &variant)
 {
   insert_enum_variant_or_error_out (variant.get_identifier (), variant);
+
+  DefaultResolver::visit (variant);
 }
 
 void
 TopLevel::visit (AST::EnumItemStruct &variant)
 {
   insert_enum_variant_or_error_out (variant.get_identifier (), variant);
+
+  DefaultResolver::visit (variant);
 }
 
 void
 TopLevel::visit (AST::EnumItemDiscriminant &variant)
 {
   insert_or_error_out (variant.get_identifier (), variant, Namespace::Types);
+
+  DefaultResolver::visit (variant);
 }
 
 void
@@ -401,6 +339,13 @@ TopLevel::visit (AST::Enum &enum_item)
 		       Namespace::Types);
 
   DefaultResolver::visit (enum_item);
+
+  // Since enums can be containers for imports, we need to insert them like we
+  // do for modules
+  if (Analysis::Mappings::get ().lookup_glob_container (
+	enum_item.get_node_id ())
+      == tl::nullopt)
+    Analysis::Mappings::get ().insert_glob_container (&enum_item);
 }
 
 void
@@ -430,21 +375,18 @@ TopLevel::visit (AST::TypeAlias &type_item)
   DefaultResolver::visit (type_item);
 }
 
-static void
-flatten_rebind (
+static void flatten_rebind (
   const AST::UseTreeRebind &glob,
   std::vector<std::pair<AST::SimplePath, AST::UseTreeRebind>> &rebind_paths);
 
-static void
-flatten_list (
+static void flatten_list (
   const AST::UseTreeList &glob, std::vector<AST::SimplePath> &paths,
   std::vector<AST::SimplePath> &glob_paths,
   std::vector<std::pair<AST::SimplePath, AST::UseTreeRebind>> &rebind_paths,
   NameResolutionContext &ctx);
-static void
-flatten_glob (const AST::UseTreeGlob &glob,
-	      std::vector<AST::SimplePath> &glob_paths,
-	      NameResolutionContext &ctx);
+static void flatten_glob (const AST::UseTreeGlob &glob,
+			  std::vector<AST::SimplePath> &glob_paths,
+			  NameResolutionContext &ctx);
 
 static void
 flatten (
@@ -455,17 +397,20 @@ flatten (
 {
   switch (tree->get_kind ())
     {
-      case AST::UseTree::Rebind: {
+    case AST::UseTree::Rebind:
+      {
 	auto rebind = static_cast<const AST::UseTreeRebind *> (tree);
 	flatten_rebind (*rebind, rebind_paths);
 	break;
       }
-      case AST::UseTree::List: {
+    case AST::UseTree::List:
+      {
 	auto list = static_cast<const AST::UseTreeList *> (tree);
 	flatten_list (*list, paths, glob_paths, rebind_paths, ctx);
 	break;
       }
-      case AST::UseTree::Glob: {
+    case AST::UseTree::Glob:
+      {
 	auto glob = static_cast<const AST::UseTreeGlob *> (tree);
 	flatten_glob (*glob, glob_paths, ctx);
 	break;
diff --git a/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.h b/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.h
index 3ff37ed..8d3da92 100644
--- a/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.h
+++ b/gcc/rust/resolve/rust-toplevel-name-resolver-2.0.h
@@ -160,8 +160,7 @@ private:
 
   void visit (AST::Module &module) override;
   void visit (AST::Trait &trait) override;
-  void visit (AST::InherentImpl &impl) override;
-  void visit (AST::TraitImpl &impl) override;
+  void maybe_insert_big_self (AST::Impl &impl) override;
   void visit (AST::TraitItemType &trait_item) override;
   void visit (AST::MacroRulesDefinition &macro) override;
   void visit (AST::Function &function) override;
@@ -177,7 +176,7 @@ private:
   void visit (AST::Union &union_item) override;
   void visit (AST::ConstantItem &const_item) override;
   void visit (AST::TypeAlias &type_item) override;
-  void visit (AST::ExternCrate &crate) override;
+  void visit_extern_crate (AST::ExternCrate &, AST::Crate &, CrateNum) override;
   void visit (AST::TypeParam &type_param) override;
   void visit (AST::ConstGenericParam &const_param) override;
 
diff --git a/gcc/rust/rust-attribs.cc b/gcc/rust/rust-attribs.cc
index 74cb2af..a98c1fa 100644
--- a/gcc/rust/rust-attribs.cc
+++ b/gcc/rust/rust-attribs.cc
@@ -38,35 +38,27 @@ along with GCC; see the file COPYING3.  If not see
  * future.
  */
 
-extern const attribute_spec grs_langhook_common_attribute_table[];
+extern const struct scoped_attribute_specs grs_langhook_gnu_attribute_table;
+extern const struct scoped_attribute_specs grs_langhook_common_attribute_table;
 
 /* Internal attribute handlers for built-in functions.  */
-static tree
-handle_noreturn_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_leaf_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_const_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_malloc_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_pure_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_novops_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_nonnull_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_nothrow_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_type_generic_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_transaction_pure_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_returns_twice_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_fnspec_attribute (tree *, tree, tree, int, bool *);
-static tree
-handle_omp_declare_simd_attribute (tree *, tree, tree, int, bool *);
+static tree handle_noreturn_attribute (tree *, tree, tree, int, bool *);
+static tree handle_leaf_attribute (tree *, tree, tree, int, bool *);
+static tree handle_const_attribute (tree *, tree, tree, int, bool *);
+static tree handle_malloc_attribute (tree *, tree, tree, int, bool *);
+static tree handle_pure_attribute (tree *, tree, tree, int, bool *);
+static tree handle_novops_attribute (tree *, tree, tree, int, bool *);
+static tree handle_nonnull_attribute (tree *, tree, tree, int, bool *);
+static tree handle_nothrow_attribute (tree *, tree, tree, int, bool *);
+static tree handle_type_generic_attribute (tree *, tree, tree, int, bool *);
+static tree handle_transaction_pure_attribute (tree *, tree, tree, int, bool *);
+static tree handle_returns_twice_attribute (tree *, tree, tree, int, bool *);
+static tree handle_fnspec_attribute (tree *, tree, tree, int, bool *);
+static tree handle_omp_declare_simd_attribute (tree *, tree, tree, int, bool *);
+
+/* Rust attribute handlers for user defined attributes.  */
+static tree handle_cold_attribute (tree *, tree, tree, int, bool *);
+static tree handle_hot_attribute (tree *, tree, tree, int, bool *);
 
 /* Helper to define attribute exclusions.  */
 #define ATTR_EXCL(name, function, type, variable)                              \
@@ -74,6 +66,10 @@ handle_omp_declare_simd_attribute (tree *, tree, tree, int, bool *);
     name, function, type, variable                                             \
   }
 
+// clang-format off
+// Disabling clang-format because it insists in having several ATTR_EXCL() on a
+// single line.
+
 static const struct attribute_spec::exclusions attr_noreturn_exclusions[] = {
   //  ATTR_EXCL ("alloc_size", true, true, true),
   ATTR_EXCL ("const", true, true, true),
@@ -89,11 +85,22 @@ static const struct attribute_spec::exclusions attr_returns_twice_exclusions[]
     ATTR_EXCL (NULL, false, false, false),
 };
 
+extern const struct attribute_spec::exclusions attr_cold_hot_exclusions[] = {
+
+  ATTR_EXCL ("cold", true, true, true),
+  ATTR_EXCL ("hot", true, true, true),
+  ATTR_EXCL (NULL, false, false, false)
+};
+
 static const struct attribute_spec::exclusions attr_const_pure_exclusions[] = {
   // ATTR_EXCL ("alloc_size", true, true, true),
   ATTR_EXCL ("const", true, true, true),
   ATTR_EXCL ("noreturn", true, true, true),
-  ATTR_EXCL ("pure", true, true, true), ATTR_EXCL (NULL, false, false, false)};
+  ATTR_EXCL ("pure", true, true, true),
+  ATTR_EXCL (NULL, false, false, false)
+};
+
+// clang-format on
 
 /* Helper to define an attribute.  */
 #define ATTR_SPEC(name, min_len, max_len, decl_req, type_req, fn_type_req,     \
@@ -105,7 +112,7 @@ static const struct attribute_spec::exclusions attr_const_pure_exclusions[] = {
 
 /* Table of machine-independent attributes.
    For internal use (marking of built-ins) only.  */
-const attribute_spec grs_langhook_common_attribute_table[] = {
+static const attribute_spec grs_langhook_common_attributes[] = {
   ATTR_SPEC ("noreturn", 0, 0, true, false, false, false,
 	     handle_noreturn_attribute, attr_noreturn_exclusions),
   ATTR_SPEC ("leaf", 0, 0, true, false, false, false, handle_leaf_attribute,
@@ -132,9 +139,21 @@ const attribute_spec grs_langhook_common_attribute_table[] = {
 	     NULL),
   ATTR_SPEC ("omp declare simd", 0, -1, true, false, false, false,
 	     handle_omp_declare_simd_attribute, NULL),
-  ATTR_SPEC (NULL, 0, 0, false, false, false, false, NULL, NULL),
 };
 
+const scoped_attribute_specs grs_langhook_common_attribute_table
+  = {"gnu", {grs_langhook_common_attributes}};
+
+static const attribute_spec grs_langhook_gnu_attributes[] = {
+  ATTR_SPEC ("cold", 0, 0, true, false, false, false, handle_cold_attribute,
+	     attr_cold_hot_exclusions),
+  ATTR_SPEC ("hot", 0, 0, true, false, false, false, handle_hot_attribute,
+	     attr_cold_hot_exclusions),
+};
+
+const scoped_attribute_specs grs_langhook_gnu_attribute_table
+  = {"gnu", {grs_langhook_gnu_attributes}};
+
 /* Built-in attribute handlers.
    These functions take the arguments:
    (tree *node, tree name, tree args, int flags, bool *no_add_attrs)  */
@@ -204,7 +223,7 @@ handle_const_attribute (tree *node, tree, tree, int, bool *)
 /* Handle a "malloc" attribute; arguments as in
    struct attribute_spec.handler.  */
 
-tree
+static tree
 handle_malloc_attribute (tree *node, tree, tree, int, bool *)
 {
   gcc_assert (TREE_CODE (*node) == FUNCTION_DECL
@@ -217,9 +236,14 @@ handle_malloc_attribute (tree *node, tree, tree, int, bool *)
    struct attribute_spec.handler.  */
 
 static tree
-handle_pure_attribute (tree *node, tree, tree, int, bool *)
+handle_pure_attribute (tree *node, tree name, tree, int, bool *no_add_attrs)
 {
-  gcc_assert (TREE_CODE (*node) == FUNCTION_DECL);
+  if (TREE_CODE (*node) != FUNCTION_DECL)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
+
   DECL_PURE_P (*node) = 1;
   return NULL_TREE;
 }
@@ -228,9 +252,14 @@ handle_pure_attribute (tree *node, tree, tree, int, bool *)
    struct attribute_spec.handler.  */
 
 static tree
-handle_novops_attribute (tree *node, tree, tree, int, bool *)
+handle_novops_attribute (tree *node, tree name, tree, int, bool *no_add_attrs)
 {
-  gcc_assert (TREE_CODE (*node) == FUNCTION_DECL);
+  if (TREE_CODE (*node) != FUNCTION_DECL)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
+
   DECL_IS_NOVOPS (*node) = 1;
   return NULL_TREE;
 }
@@ -301,9 +330,14 @@ handle_nonnull_attribute (tree *node, tree, tree args, int, bool *)
    struct attribute_spec.handler.  */
 
 static tree
-handle_nothrow_attribute (tree *node, tree, tree, int, bool *)
+handle_nothrow_attribute (tree *node, tree name, tree, int, bool *no_add_attrs)
 {
-  gcc_assert (TREE_CODE (*node) == FUNCTION_DECL);
+  if (TREE_CODE (*node) != FUNCTION_DECL)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
+
   TREE_NOTHROW (*node) = 1;
   return NULL_TREE;
 }
@@ -339,9 +373,14 @@ handle_transaction_pure_attribute (tree *node, tree, tree, int, bool *)
    struct attribute_spec.handler.  */
 
 static tree
-handle_returns_twice_attribute (tree *node, tree, tree, int, bool *)
+handle_returns_twice_attribute (tree *node, tree name, tree, int,
+				bool *no_add_attrs)
 {
-  gcc_assert (TREE_CODE (*node) == FUNCTION_DECL);
+  if (TREE_CODE (*node) != FUNCTION_DECL)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
 
   DECL_IS_RETURNS_TWICE (*node) = 1;
 
@@ -351,7 +390,7 @@ handle_returns_twice_attribute (tree *node, tree, tree, int, bool *)
 /* Handle a "fn spec" attribute; arguments as in
    struct attribute_spec.handler.  */
 
-tree
+static tree
 handle_fnspec_attribute (tree *, tree, tree args, int, bool *)
 {
   gcc_assert (args && TREE_CODE (TREE_VALUE (args)) == STRING_CST
@@ -362,9 +401,46 @@ handle_fnspec_attribute (tree *, tree, tree args, int, bool *)
 /* Handle an "omp declare simd" attribute; arguments as in
    struct attribute_spec.handler.  */
 
-tree
-handle_omp_declare_simd_attribute (tree *node, tree, tree, int, bool *)
+static tree
+handle_omp_declare_simd_attribute (tree *node, tree name, tree, int,
+				   bool *no_add_attrs)
 {
-  gcc_assert (TREE_CODE (*node) == FUNCTION_DECL);
+  if (TREE_CODE (*node) != FUNCTION_DECL)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
+
+  return NULL_TREE;
+}
+
+/* Language specific attribute handlers.
+   These functions take the arguments:
+   (tree *node, tree name, tree args, int flags, bool *no_add_attrs)  */
+
+/* Handle a "cold" and attribute; arguments as in
+   struct attribute_spec.handler.  */
+
+static tree
+handle_cold_attribute (tree *node, tree name, tree, int, bool *no_add_attrs)
+{
+  if (TREE_CODE (*node) != FUNCTION_DECL)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
+
+  return NULL_TREE;
+}
+
+static tree
+handle_hot_attribute (tree *node, tree name, tree, int, bool *no_add_attrs)
+{
+  if (TREE_CODE (*node) != FUNCTION_DECL)
+    {
+      warning (OPT_Wattributes, "%qE attribute ignored", name);
+      *no_add_attrs = true;
+    }
+
   return NULL_TREE;
 }
diff --git a/gcc/rust/rust-backend.h b/gcc/rust/rust-backend.h
index 414799e..95ca7a9 100644
--- a/gcc/rust/rust-backend.h
+++ b/gcc/rust/rust-backend.h
@@ -27,6 +27,8 @@
 #include "rust-linemap.h"
 #include "rust-diagnostics.h"
 #include "util/rust-operators.h"
+#include "util/rust-ggc.h"
+#include "util/optional.h"
 #include "tree.h"
 #include "rust-gcc.h"
 
@@ -42,69 +44,60 @@ class Bvariable;
 
 namespace Backend {
 
-void
-init ();
+namespace GGC {
+
+using Rust::GGC::Ident;
+
+} // namespace GGC
+
+void init ();
 
 // Name/type/location.  Used for function parameters, struct fields,
 // interface methods.
 struct typed_identifier
 {
-  std::string name;
+  GGC::Ident name;
   tree type;
   location_t location;
 
-  typed_identifier () : name (), type (NULL_TREE), location (UNKNOWN_LOCATION)
-  {}
-
-  typed_identifier (const std::string &a_name, tree a_type,
-		    location_t a_location)
+  typed_identifier (GGC::Ident a_name, tree a_type, location_t a_location)
     : name (a_name), type (a_type), location (a_location)
   {}
 };
 
 // debug
 void debug (tree);
-void
-debug (Bvariable *);
+void debug (Bvariable *);
 
-tree
-get_identifier_node (const std::string &str);
+tree get_identifier_node (const std::string &str);
 
 // Types.
 
 // Get the wchar type
-tree
-wchar_type ();
+tree wchar_type ();
 
 // Get the Host pointer size in bits
-int
-get_pointer_size ();
+int get_pointer_size ();
 
 // Get the raw str type const char*
-tree
-raw_str_type ();
+tree raw_str_type ();
 
 // Get an unnamed integer type with the given signedness and number
 // of bits.
-tree
-integer_type (bool is_unsigned, int bits);
+tree integer_type (bool is_unsigned, int bits);
 
 // Get an unnamed floating point type with the given number of bits
 // (32 or 64).
-tree
-float_type (int bits);
+tree float_type (int bits);
 
 // Get a pointer type.
-tree
-pointer_type (tree to_type);
+tree pointer_type (tree to_type);
 
 // Get a reference type.
-tree
-reference_type (tree to_type);
+tree reference_type (tree to_type);
 
 // make type immutable
-tree
-immutable_type (tree base);
+tree immutable_type (tree base);
 
 // Get a function type.  The receiver, parameter, and results are
 // generated from the types in the Function_type.  The Function_type
@@ -115,41 +108,36 @@ immutable_type (tree base);
 // one result, RESULT_STRUCT is a struct type to hold the results,
 // and RESULTS may be ignored; if there are zero or one results,
 // RESULT_STRUCT is NULL.
-tree
-function_type (const typed_identifier &receiver,
-	       const std::vector<typed_identifier> &parameters,
-	       const std::vector<typed_identifier> &results, tree result_struct,
-	       location_t location);
-
-tree
-function_type_variadic (const typed_identifier &receiver,
-			const std::vector<typed_identifier> &parameters,
-			const std::vector<typed_identifier> &results,
-			tree result_struct, location_t location);
-
-tree
-function_ptr_type (tree result, const std::vector<tree> &praameters,
-		   location_t location);
+tree function_type (const typed_identifier &receiver,
+		    const std::vector<typed_identifier> &parameters,
+		    const std::vector<typed_identifier> &results,
+		    tree result_struct, location_t location);
+
+tree function_type_variadic (const typed_identifier &receiver,
+			     const std::vector<typed_identifier> &parameters,
+			     const std::vector<typed_identifier> &results,
+			     tree result_struct, location_t location);
+
+tree function_ptr_type (tree result, const std::vector<tree> &praameters,
+			location_t location);
 
 // Get a struct type.
-tree
-struct_type (const std::vector<typed_identifier> &fields, bool layout = true);
+tree struct_type (const std::vector<typed_identifier> &fields,
+		  bool layout = true);
 
 // Get a union type.
-tree
-union_type (const std::vector<typed_identifier> &fields, bool layout = true);
+tree union_type (const std::vector<typed_identifier> &fields,
+		 bool layout = true);
 
 // Get an array type.
-tree
-array_type (tree element_type, tree length);
+tree array_type (tree element_type, tree length);
 
 // Return a named version of a type.  The location is the location
 // of the type definition.  This will not be called for a type
 // created via placeholder_pointer_type, placeholder_struct_type, or
 // placeholder_array_type..  (It may be called for a pointer,
 // struct, or array type in a case like "type P *byte; type Q P".)
-tree
-named_type (const std::string &name, tree, location_t);
+tree named_type (GGC::Ident name, tree, location_t);
 
 // Return the size of a type.
 int64_t type_size (tree);
@@ -164,8 +152,7 @@ int64_t type_field_alignment (tree);
 // Return the offset of field INDEX in a struct type.  INDEX is the
 // entry in the FIELDS std::vector parameter of struct_type or
 // set_placeholder_struct_type.
-int64_t
-type_field_offset (tree, size_t index);
+int64_t type_field_offset (tree, size_t index);
 
 // Expressions.
 
@@ -175,155 +162,135 @@ type_field_offset (tree, size_t index);
 tree zero_expression (tree);
 
 // Create a reference to a variable.
-tree
-var_expression (Bvariable *var, location_t);
+tree var_expression (Bvariable *var, location_t);
 
 // Return an expression for the floating point value VAL in BTYPE.
-tree
-float_constant_expression (tree btype, mpfr_t val);
+tree float_constant_expression (tree btype, mpfr_t val);
 
 // Return an expression for the string value VAL.
-tree
-string_constant_expression (const std::string &val);
+tree string_constant_expression (const std::string &val);
 
 // Get a char literal
-tree
-char_constant_expression (char c);
+tree char_constant_expression (char c);
 
 // Get a char literal
-tree
-wchar_constant_expression (wchar_t c);
+tree wchar_constant_expression (wchar_t c);
+
+// Get a size literal
+tree size_constant_expression (size_t val);
 
 // Return an expression for the boolean value VAL.
-tree
-boolean_constant_expression (bool val);
+tree boolean_constant_expression (bool val);
 
 // Return an expression that converts EXPR to TYPE.
-tree
-convert_expression (tree type, tree expr, location_t);
+tree convert_expression (tree type, tree expr, location_t);
 
 // Return an expression for the field at INDEX in BSTRUCT.
-tree
-struct_field_expression (tree bstruct, size_t index, location_t);
+tree struct_field_expression (tree bstruct, size_t index, location_t);
 
 // Create an expression that executes BSTAT before BEXPR.
-tree
-compound_expression (tree bstat, tree bexpr, location_t);
+tree compound_expression (tree bstat, tree bexpr, location_t);
 
 // Return an expression that executes THEN_EXPR if CONDITION is true, or
 // ELSE_EXPR otherwise and returns the result as type BTYPE, within the
 // specified function FUNCTION.  ELSE_EXPR may be NULL.  BTYPE may be NULL.
-tree
-conditional_expression (tree function, tree btype, tree condition,
-			tree then_expr, tree else_expr, location_t);
+tree conditional_expression (tree function, tree btype, tree condition,
+			     tree then_expr, tree else_expr, location_t);
 
 // Return an expression for the negation operation OP EXPR.
 // Supported values of OP are enumerated in NegationOperator.
-tree
-negation_expression (NegationOperator op, tree expr, location_t);
+tree negation_expression (NegationOperator op, tree expr, location_t);
 
 // Return an expression for the operation LEFT OP RIGHT.
 // Supported values of OP are enumerated in ArithmeticOrLogicalOperator.
-tree
-arithmetic_or_logical_expression (ArithmeticOrLogicalOperator op, tree left,
-				  tree right, location_t loc);
+tree arithmetic_or_logical_expression (ArithmeticOrLogicalOperator op,
+				       tree left, tree right, location_t loc);
 
 // Return an expression for the operation LEFT OP RIGHT.
 // Supported values of OP are enumerated in ArithmeticOrLogicalOperator.
 // This function adds overflow checking and returns a list of statements to
 // add to the current function context. The `receiver` variable refers to the
 // variable which will contain the result of that operation.
-tree
-arithmetic_or_logical_expression_checked (ArithmeticOrLogicalOperator op,
-					  tree left, tree right, location_t loc,
-					  Bvariable *receiver);
+tree arithmetic_or_logical_expression_checked (ArithmeticOrLogicalOperator op,
+					       tree left, tree right,
+					       location_t loc,
+					       Bvariable *receiver);
 
 // Return an expression for the operation LEFT OP RIGHT.
 // Supported values of OP are enumerated in ComparisonOperator.
-tree
-comparison_expression (ComparisonOperator op, tree left, tree right,
-		       location_t loc);
+tree comparison_expression (ComparisonOperator op, tree left, tree right,
+			    location_t loc);
 
 // Return an expression for the operation LEFT OP RIGHT.
 // Supported values of OP are enumerated in LazyBooleanOperator.
-tree
-lazy_boolean_expression (LazyBooleanOperator op, tree left, tree right,
-			 location_t);
+tree lazy_boolean_expression (LazyBooleanOperator op, tree left, tree right,
+			      location_t);
 
 // Return an expression that constructs BTYPE with VALS.  BTYPE must be the
 // backend representation a of struct.  VALS must be in the same order as the
 // corresponding fields in BTYPE.
-tree
-constructor_expression (tree btype, bool is_variant,
-			const std::vector<tree> &vals, int, location_t);
+tree constructor_expression (tree btype, bool is_variant,
+			     const std::vector<tree> &vals, int, location_t);
 
 // Return an expression that constructs an array of BTYPE with INDEXES and
 // VALS.  INDEXES and VALS must have the same amount of elements. Each index
 // in INDEXES must be in the same order as the corresponding value in VALS.
-tree
-array_constructor_expression (tree btype,
-			      const std::vector<unsigned long> &indexes,
-			      const std::vector<tree> &vals, location_t);
+tree array_constructor_expression (tree btype,
+				   const std::vector<unsigned long> &indexes,
+				   const std::vector<tree> &vals, location_t);
 
-tree
-array_initializer (tree, tree, tree, tree, tree, tree *, location_t);
+tree array_initializer (tree, tree, tree, tree, tree, tree *, location_t);
 
 // Return an expression for ARRAY[INDEX] as an l-value.  ARRAY is a valid
 // fixed-length array, not a slice.
-tree
-array_index_expression (tree array, tree index, location_t);
+tree array_index_expression (tree array, tree index, location_t);
+
+// Return an expresison for SLICE[INDEX] as an l-value.  SLICE is represented
+// with a DST.
+tree slice_index_expression (tree slice, tree index, location_t);
 
 // Create an expression for a call to FN with ARGS, taking place within
 // caller CALLER.
-tree
-call_expression (tree fn, const std::vector<tree> &args, tree static_chain,
-		 location_t);
+tree call_expression (tree fn, const std::vector<tree> &args, tree static_chain,
+		      location_t);
 
 // Statements.
 
 // Create a variable initialization statement in the specified
 // function.  This initializes a local variable at the point in the
 // program flow where it is declared.
-tree
-init_statement (tree, Bvariable *var, tree init);
+tree init_statement (tree, Bvariable *var, tree init);
 
 // Create an assignment statement within the specified function.
-tree
-assignment_statement (tree lhs, tree rhs, location_t);
+tree assignment_statement (tree lhs, tree rhs, location_t);
 
 // Create return statement for an decl for a value (can be NULL_TREE) at a
 // location
-tree
-return_statement (tree fndecl, tree val, location_t);
+tree return_statement (tree fndecl, tree val, location_t);
 
 // Create an if statement within a function.  ELSE_BLOCK may be NULL.
-tree
-if_statement (tree, tree condition, tree then_block, tree else_block,
-	      location_t);
+tree if_statement (tree, tree condition, tree then_block, tree else_block,
+		   location_t);
 
 // infinite loop expressions
-tree
-loop_expression (tree body, location_t);
+tree loop_expression (tree body, location_t);
 
 // exit expressions
-tree
-exit_expression (tree condition, location_t);
+tree exit_expression (tree condition, location_t);
 
 // Create a single statement from two statements.
 tree compound_statement (tree, tree);
 
 // Create a single statement from a list of statements.
-tree
-statement_list (const std::vector<tree> &);
+tree statement_list (const std::vector<tree> &);
 
 // Create a statement that attempts to execute BSTAT and calls EXCEPT_STMT if
 // an exception occurs. EXCEPT_STMT may be NULL.  FINALLY_STMT may be NULL and
 // if not NULL, it will always be executed.  This is used for handling defers
 // in Go functions.  In C++, the resulting code is of this form:
 //   try { BSTAT; } catch { EXCEPT_STMT; } finally { FINALLY_STMT; }
-tree
-exception_handler_statement (tree bstat, tree except_stmt, tree finally_stmt,
-			     location_t);
+tree exception_handler_statement (tree bstat, tree except_stmt,
+				  tree finally_stmt, location_t);
 
 // Blocks.
 
@@ -337,16 +304,14 @@ exception_handler_statement (tree bstat, tree except_stmt, tree finally_stmt,
 // the initial curly brace.  END_LOCATION is the location of the end
 // of the block, more or less the location of the final curly brace.
 // The statements will be added after the block is created.
-tree
-block (tree function, tree enclosing, const std::vector<Bvariable *> &vars,
-       location_t start_location, location_t end_location);
+tree block (tree function, tree enclosing, const std::vector<Bvariable *> &vars,
+	    location_t start_location, location_t end_location);
 
 // Add the statements to a block.  The block is created first.  Then
 // the statements are created.  Then the statements are added to the
 // block.  This will called exactly once per block.  The vector may
 // be empty if there are no statements.
-void
-block_add_statements (tree, const std::vector<tree> &);
+void block_add_statements (tree, const std::vector<tree> &);
 
 // Variables.
 
@@ -360,10 +325,9 @@ block_add_statements (tree, const std::vector<tree> &);
 // be put into a unique section if possible; this is intended to
 // permit the linker to garbage collect the variable if it is not
 // referenced.  LOCATION is where the variable was defined.
-Bvariable *
-global_variable (const std::string &name, const std::string &asm_name,
-		 tree btype, bool is_external, bool is_hidden,
-		 bool in_unique_section, location_t location);
+Bvariable *global_variable (GGC::Ident name, GGC::Ident asm_name, tree btype,
+			    bool is_external, bool is_hidden,
+			    bool in_unique_section, location_t location);
 
 // A global variable will 1) be initialized to zero, or 2) be
 // initialized to a constant value, or 3) be initialized in the init
@@ -371,8 +335,7 @@ global_variable (const std::string &name, const std::string &asm_name,
 // global_variable_set_init to set the initial value.  If this is
 // not called, the backend should initialize a global variable to 0.
 // The init function may then assign a value to it.
-void
-global_variable_set_init (Bvariable *, tree);
+void global_variable_set_init (Bvariable *, tree);
 
 // Create a local variable.  The frontend will create the local
 // variables first, and then create the block which contains them.
@@ -386,21 +349,18 @@ global_variable_set_init (Bvariable *, tree);
 // the function, as otherwise the variable would be on the heap).
 // LOCATION is where the variable is defined.  For each local variable
 // the frontend will call init_statement to set the initial value.
-Bvariable *
-local_variable (tree function, const std::string &name, tree type,
-		Bvariable *decl_var, location_t location);
+Bvariable *local_variable (tree function, GGC::Ident name, tree type,
+			   Bvariable *decl_var, location_t location);
 
 // Create a function parameter.  This is an incoming parameter, not
 // a result parameter (result parameters are treated as local
 // variables).  The arguments are as for local_variable.
-Bvariable *
-parameter_variable (tree function, const std::string &name, tree type,
-		    location_t location);
+Bvariable *parameter_variable (tree function, GGC::Ident name, tree type,
+			       location_t location);
 
 // Create a static chain parameter.  This is the closure parameter.
-Bvariable *
-static_chain_variable (tree function, const std::string &name, tree type,
-		       location_t location);
+Bvariable *static_chain_variable (tree function, GGC::Ident name, tree type,
+				  location_t location);
 
 // Create a temporary variable.  A temporary variable has no name,
 // just a type.  We pass in FUNCTION and BLOCK in case they are
@@ -413,18 +373,16 @@ static_chain_variable (tree function, const std::string &name, tree type,
 // variable, and may not be very useful.  This function should
 // return a variable which can be referenced later and should set
 // *PSTATEMENT to a statement which initializes the variable.
-Bvariable *
-temporary_variable (tree fndecl, tree bind_tree, tree type, tree init,
-		    bool address_is_taken, location_t location,
-		    tree *pstatement);
+Bvariable *temporary_variable (tree fndecl, tree bind_tree, tree type,
+			       tree init, bool address_is_taken,
+			       location_t location, tree *pstatement);
 
 // Labels.
 
-// Create a new label.  NAME will be empty if this is a label
+// Create a new label.  NAME will be tl::nullopt if this is a label
 // created by the frontend for a loop construct.  The location is
 // where the label is defined.
-tree
-label (tree, const std::string &name, location_t);
+tree label (tree, tl::optional<GGC::Ident> name, location_t);
 
 // Create a statement which defines a label.  This statement will be
 // put into the codestream at the point where the label should be
@@ -439,6 +397,12 @@ tree goto_statement (tree, location_t);
 // recover.
 tree label_address (tree, location_t);
 
+// Lookup a field from a type given its name.
+// Build the `component` tree with `Backend::get_identifier_node`.
+//
+// Forked from the C frontend.
+tree lookup_field (const_tree, tree);
+
 // Functions.
 
 // Bit flags to pass to the function method.
@@ -460,43 +424,38 @@ static const unsigned int function_does_not_return = 1 << 2;
 static const unsigned int function_in_unique_section = 1 << 3;
 
 // Declare or define a function of FNTYPE.
-// NAME is the Go name of the function.  ASM_NAME, if not the empty
-// string, is the name that should be used in the symbol table; this
+// NAME is the Go name of the function.  ASM_NAME, if not tl::nullopt,
+// is the name that should be used in the symbol table; this
 // will be non-empty if a magic extern comment is used.  FLAGS is
 // bit flags described above.
-tree
-function (tree fntype, const std::string &name, const std::string &asm_name,
-	  unsigned int flags, location_t);
+tree function (tree fntype, GGC::Ident name, tl::optional<GGC::Ident> asm_name,
+	       unsigned int flags, location_t);
 
 // Create a statement that runs all deferred calls for FUNCTION.  This should
 // be a statement that looks like this in C++:
 //   finish:
 //     try { DEFER_RETURN; } catch { CHECK_DEFER; goto finish; }
-tree
-function_defer_statement (tree function, tree undefer, tree check_defer,
-			  location_t);
+tree function_defer_statement (tree function, tree undefer, tree check_defer,
+			       location_t);
 
 // Record PARAM_VARS as the variables to use for the parameters of FUNCTION.
 // This will only be called for a function definition.  Returns true on
 // success, false on failure.
-bool
-function_set_parameters (tree function,
-			 const std::vector<Bvariable *> &param_vars);
+bool function_set_parameters (tree function,
+			      const std::vector<Bvariable *> &param_vars);
 
 // Utility.
 
 // Write the definitions for all TYPE_DECLS, CONSTANT_DECLS,
 // FUNCTION_DECLS, and VARIABLE_DECLS declared globally.
-void
-write_global_definitions (const std::vector<tree> &type_decls,
-			  const std::vector<tree> &constant_decls,
-			  const std::vector<tree> &function_decls,
-			  const std::vector<Bvariable *> &variable_decls);
+void write_global_definitions (const std::vector<tree> &type_decls,
+			       const std::vector<tree> &constant_decls,
+			       const std::vector<tree> &function_decls,
+			       const std::vector<Bvariable *> &variable_decls);
 
 // TODO: make static
 
-tree
-fill_in_fields (tree, const std::vector<typed_identifier> &, bool);
+tree fill_in_fields (tree, const std::vector<typed_identifier> &, bool);
 
 tree fill_in_array (tree, tree, tree);
 
diff --git a/gcc/rust/rust-diagnostics.cc b/gcc/rust/rust-diagnostics.cc
index 0c0ef6e..5965bb4 100644
--- a/gcc/rust/rust-diagnostics.cc
+++ b/gcc/rust/rust-diagnostics.cc
@@ -22,7 +22,7 @@
 #include "rust-diagnostics.h"
 
 #include "options.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/metadata.h"
 
 static std::string
 mformat_value ()
@@ -48,27 +48,33 @@ expand_format (const char *fmt)
       c++;
       switch (*c)
 	{
-	  case '\0': {
+	case '\0':
+	  {
 	    // malformed format string
 	    rust_unreachable ();
 	  }
-	  case '%': {
+	case '%':
+	  {
 	    ss << "%";
 	    break;
 	  }
-	  case 'm': {
+	case 'm':
+	  {
 	    ss << mformat_value ();
 	    break;
 	  }
-	  case '<': {
+	case '<':
+	  {
 	    ss << rust_open_quote ();
 	    break;
 	  }
-	  case '>': {
+	case '>':
+	  {
 	    ss << rust_close_quote ();
 	    break;
 	  }
-	  case 'q': {
+	case 'q':
+	  {
 	    ss << rust_open_quote ();
 	    c++;
 	    if (*c == 'm')
@@ -82,7 +88,8 @@ expand_format (const char *fmt)
 	    ss << rust_close_quote ();
 	    break;
 	  }
-	  default: {
+	default:
+	  {
 	    ss << "%" << *c;
 	  }
 	}
@@ -104,8 +111,8 @@ expand_format (const char *fmt)
 // calling function must need to have attribute gnu_printf as well, even
 // though there is already an attribute declaration for it.
 
-static std::string
-expand_message (const char *fmt, va_list ap) RUST_ATTRIBUTE_GCC_DIAG (1, 0);
+static std::string expand_message (const char *fmt, va_list ap)
+  RUST_ATTRIBUTE_GCC_DIAG (1, 0);
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wsuggest-attribute=format"
@@ -192,7 +199,7 @@ rust_error_at (const location_t location, const char *fmt, ...)
   va_end (ap);
 }
 
-class rust_error_code_rule : public diagnostic_metadata::rule
+class rust_error_code_rule : public diagnostics::metadata::rule
 {
 public:
   rust_error_code_rule (const ErrorCode code) : m_code (code) {}
@@ -237,7 +244,7 @@ rust_be_error_at (const location_t location, const ErrorCode code,
 		  const std::string &errmsg)
 {
   rich_location gcc_loc (line_table, location);
-  diagnostic_metadata m;
+  diagnostics::metadata m;
   rust_error_code_rule rule (code);
   m.add_rule (rule);
   error_meta (&gcc_loc, m, "%s", errmsg.c_str ());
@@ -260,7 +267,7 @@ rust_be_error_at (const rich_location &location, const ErrorCode code,
 {
   /* TODO: 'error_at' would like a non-'const' 'rich_location *'.  */
   rich_location &gcc_loc = const_cast<rich_location &> (location);
-  diagnostic_metadata m;
+  diagnostics::metadata m;
   rust_error_code_rule rule (code);
   m.add_rule (rule);
   error_meta (&gcc_loc, m, "%s", errmsg.c_str ());
@@ -281,7 +288,7 @@ void
 rust_be_error_at (rich_location *richloc, const ErrorCode code,
 		  const std::string &errmsg)
 {
-  diagnostic_metadata m;
+  diagnostics::metadata m;
   rust_error_code_rule rule (code);
   m.add_rule (rule);
   error_meta (richloc, m, "%s", errmsg.c_str ());
@@ -420,24 +427,24 @@ namespace Rust {
  */
 
 // simple location
-static Error
-va_constructor (Error::Kind kind, location_t locus, const char *fmt,
-		va_list args) RUST_ATTRIBUTE_GCC_DIAG (3, 0);
+static Error va_constructor (Error::Kind kind, location_t locus,
+			     const char *fmt, va_list args)
+  RUST_ATTRIBUTE_GCC_DIAG (3, 0);
 
 // simple location + error code
-static Error
-va_constructor (Error::Kind kind, location_t locus, const ErrorCode code,
-		const char *fmt, va_list args) RUST_ATTRIBUTE_GCC_DIAG (4, 0);
+static Error va_constructor (Error::Kind kind, location_t locus,
+			     const ErrorCode code, const char *fmt,
+			     va_list args) RUST_ATTRIBUTE_GCC_DIAG (4, 0);
 
 // rich location
-static Error
-va_constructor (Error::Kind kind, rich_location *r_locus, const char *fmt,
-		va_list args) RUST_ATTRIBUTE_GCC_DIAG (3, 0);
+static Error va_constructor (Error::Kind kind, rich_location *r_locus,
+			     const char *fmt, va_list args)
+  RUST_ATTRIBUTE_GCC_DIAG (3, 0);
 
 // rich location + error code
-static Error
-va_constructor (Error::Kind kind, rich_location *r_locus, const ErrorCode code,
-		const char *fmt, va_list args) RUST_ATTRIBUTE_GCC_DIAG (4, 0);
+static Error va_constructor (Error::Kind kind, rich_location *r_locus,
+			     const ErrorCode code, const char *fmt,
+			     va_list args) RUST_ATTRIBUTE_GCC_DIAG (4, 0);
 
 // simple location
 static Error
diff --git a/gcc/rust/rust-diagnostics.h b/gcc/rust/rust-diagnostics.h
index a13dc6a2..09d5e24 100644
--- a/gcc/rust/rust-diagnostics.h
+++ b/gcc/rust/rust-diagnostics.h
@@ -31,7 +31,7 @@
 #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
 #define RUST_ATTRIBUTE_GCC_DIAG(m, n)                                          \
   __attribute__ ((__format__ (__gcc_tdiag__, m, n)))                           \
-    __attribute__ ((__nonnull__ (m)))
+  __attribute__ ((__nonnull__ (m)))
 #else
 #define RUST_ATTRIBUTE_GCC_DIAG(m, n)
 #endif
@@ -119,10 +119,8 @@ rust_error_at(rich_location *richloc, const ErrorCode, const char *fmt, ...)
 // These interfaces provide a way for the front end to ask for
 // the open/close quote characters it should use when formatting
 // diagnostics (warnings, errors).
-extern const char *
-rust_open_quote ();
-extern const char *
-rust_close_quote ();
+extern const char *rust_open_quote ();
+extern const char *rust_close_quote ();
 
 // These interfaces are used by utilities above to pass warnings and
 // errors (once format specifiers have been expanded) to the back end,
@@ -185,7 +183,7 @@ struct Error
   Error (Kind kind, location_t locus, std::string message)
     : kind (kind), locus (locus), message (std::move (message))
   {
-    message.shrink_to_fit ();
+    this->message.shrink_to_fit ();
   }
   // simple location + error code
   Error (Kind kind, location_t locus, ErrorCode code, std::string message)
@@ -193,13 +191,13 @@ struct Error
       message (std::move (message))
   {
     is_errorcode = true;
-    message.shrink_to_fit ();
+    this->message.shrink_to_fit ();
   }
   // rich location
   Error (Kind kind, rich_location *richlocus, std::string message)
     : kind (kind), richlocus (richlocus), message (std::move (message))
   {
-    message.shrink_to_fit ();
+    this->message.shrink_to_fit ();
   }
   // rich location + error code
   Error (Kind kind, rich_location *richlocus, ErrorCode code,
@@ -208,7 +206,7 @@ struct Error
       message (std::move (message))
   {
     is_errorcode = true;
-    message.shrink_to_fit ();
+    this->message.shrink_to_fit ();
   }
   // simple location
   Error (location_t locus, std::string message)
@@ -306,8 +304,7 @@ struct Error
 
 #define rust_sorry_at(location, ...) sorry_at (location, __VA_ARGS__)
 
-void
-rust_debug_loc (const location_t location, const char *fmt,
-		...) ATTRIBUTE_PRINTF_2;
+void rust_debug_loc (const location_t location, const char *fmt,
+		     ...) ATTRIBUTE_PRINTF_2;
 
 #endif // !defined(RUST_DIAGNOSTICS_H)
diff --git a/gcc/rust/rust-gcc.cc b/gcc/rust/rust-gcc.cc
index e5319d3..398dea1 100644
--- a/gcc/rust/rust-gcc.cc
+++ b/gcc/rust/rust-gcc.cc
@@ -91,12 +91,6 @@ Bvariable::error_variable ()
 
 // A helper function to create a GCC identifier from a C++ string.
 
-static inline tree
-get_identifier_from_string (const std::string &str)
-{
-  return get_identifier_with_length (str.data (), str.length ());
-}
-
 namespace Backend {
 
 // Define the built-in functions that are exposed to GCCRust.
@@ -609,7 +603,7 @@ fill_in_fields (tree fill, const std::vector<typed_identifier> &fields,
   tree *pp = &field_trees;
   for (const auto &p : fields)
     {
-      tree name_tree = get_identifier_from_string (p.name);
+      tree name_tree = p.name.as_tree ();
       tree type_tree = p.type;
       if (error_operand_p (type_tree))
 	return error_mark_node;
@@ -675,7 +669,7 @@ fill_in_array (tree fill, tree element_type, tree length_tree)
 // Return a named version of a type.
 
 tree
-named_type (const std::string &name, tree type, location_t location)
+named_type (GGC::Ident name, tree type, location_t location)
 {
   if (error_operand_p (type))
     return error_mark_node;
@@ -688,15 +682,14 @@ named_type (const std::string &name, tree type, location_t location)
 	  || TREE_CODE (type) == COMPLEX_TYPE
 	  || TREE_CODE (type) == BOOLEAN_TYPE))
     {
-      tree decl = build_decl (BUILTINS_LOCATION, TYPE_DECL,
-			      get_identifier_from_string (name), type);
+      tree decl
+	= build_decl (BUILTINS_LOCATION, TYPE_DECL, name.as_tree (), type);
       TYPE_NAME (type) = decl;
       return type;
     }
 
   tree copy = build_variant_type_copy (type);
-  tree decl
-    = build_decl (location, TYPE_DECL, get_identifier_from_string (name), copy);
+  tree decl = build_decl (location, TYPE_DECL, name.as_tree (), copy);
   DECL_ORIGINAL_TYPE (decl) = type;
   TYPE_NAME (copy) = decl;
   return copy;
@@ -825,6 +818,12 @@ char_constant_expression (char c)
   return build_int_cst (char_type_node, c);
 }
 
+tree
+size_constant_expression (size_t val)
+{
+  return size_int (val);
+}
+
 // Make a constant boolean expression.
 
 tree
@@ -1505,6 +1504,34 @@ array_index_expression (tree array_tree, tree index_tree, location_t location)
   return ret;
 }
 
+// Return an expression representing SLICE[INDEX]
+
+tree
+slice_index_expression (tree slice_tree, tree index_tree, location_t location)
+{
+  if (error_operand_p (slice_tree) || error_operand_p (index_tree))
+    return error_mark_node;
+
+  // A slice is created in TyTyResolvecompile::create_slice_type_record
+  // For example:
+  //   &[i32] is turned directly into a struct { i32* data, usize len };
+  //   [i32] is also turned into struct { i32* data, usize len }
+
+  // it should have RS_DST_FLAG set to 1
+  rust_assert (RS_DST_FLAG_P (TREE_TYPE (slice_tree)));
+
+  tree data_field = struct_field_expression (slice_tree, 0, location);
+  tree data_field_deref = build_fold_indirect_ref_loc (location, data_field);
+
+  tree element_type = TREE_TYPE (data_field_deref);
+  tree data_pointer = TREE_OPERAND (data_field_deref, 0);
+  rust_assert (POINTER_TYPE_P (TREE_TYPE (data_pointer)));
+  tree data_offset_expr
+    = Rust::pointer_offset_expression (data_pointer, index_tree, location);
+
+  return build1_loc (location, INDIRECT_REF, element_type, data_offset_expr);
+}
+
 // Create an expression for a call to FN_EXPR with FN_ARGS.
 tree
 call_expression (tree fn, const std::vector<tree> &fn_args, tree chain_expr,
@@ -1876,7 +1903,8 @@ non_zero_size_type (tree type)
 	}
       return rust_non_zero_struct;
 
-      case ARRAY_TYPE: {
+    case ARRAY_TYPE:
+      {
 	tree element_type = non_zero_size_type (TREE_TYPE (type));
 	return build_array_type_nelts (element_type, 1);
       }
@@ -1923,9 +1951,9 @@ convert_tree (tree type_tree, tree expr_tree, location_t location)
 // Make a global variable.
 
 Bvariable *
-global_variable (const std::string &var_name, const std::string &asm_name,
-		 tree type_tree, bool is_external, bool is_hidden,
-		 bool in_unique_section, location_t location)
+global_variable (GGC::Ident var_name, GGC::Ident asm_name, tree type_tree,
+		 bool is_external, bool is_hidden, bool in_unique_section,
+		 location_t location)
 {
   if (error_operand_p (type_tree))
     return Bvariable::error_variable ();
@@ -1935,8 +1963,7 @@ global_variable (const std::string &var_name, const std::string &asm_name,
   if ((is_external || !is_hidden) && int_size_in_bytes (type_tree) == 0)
     type_tree = non_zero_size_type (type_tree);
 
-  tree decl = build_decl (location, VAR_DECL,
-			  get_identifier_from_string (var_name), type_tree);
+  tree decl = build_decl (location, VAR_DECL, var_name.as_tree (), type_tree);
   if (is_external)
     DECL_EXTERNAL (decl) = 1;
   else
@@ -1944,11 +1971,11 @@ global_variable (const std::string &var_name, const std::string &asm_name,
   if (!is_hidden)
     {
       TREE_PUBLIC (decl) = 1;
-      SET_DECL_ASSEMBLER_NAME (decl, get_identifier_from_string (asm_name));
+      SET_DECL_ASSEMBLER_NAME (decl, asm_name.as_tree ());
     }
   else
     {
-      SET_DECL_ASSEMBLER_NAME (decl, get_identifier_from_string (asm_name));
+      SET_DECL_ASSEMBLER_NAME (decl, asm_name.as_tree ());
     }
 
   TREE_USED (decl) = 1;
@@ -1988,13 +2015,12 @@ global_variable_set_init (Bvariable *var, tree expr_tree)
 // Make a local variable.
 
 Bvariable *
-local_variable (tree function, const std::string &name, tree type_tree,
+local_variable (tree function, GGC::Ident name, tree type_tree,
 		Bvariable *decl_var, location_t location)
 {
   if (error_operand_p (type_tree))
     return Bvariable::error_variable ();
-  tree decl = build_decl (location, VAR_DECL, get_identifier_from_string (name),
-			  type_tree);
+  tree decl = build_decl (location, VAR_DECL, name.as_tree (), type_tree);
   DECL_CONTEXT (decl) = function;
 
   if (decl_var != NULL)
@@ -2009,13 +2035,12 @@ local_variable (tree function, const std::string &name, tree type_tree,
 // Make a function parameter variable.
 
 Bvariable *
-parameter_variable (tree function, const std::string &name, tree type_tree,
+parameter_variable (tree function, GGC::Ident name, tree type_tree,
 		    location_t location)
 {
   if (error_operand_p (type_tree))
     return Bvariable::error_variable ();
-  tree decl = build_decl (location, PARM_DECL,
-			  get_identifier_from_string (name), type_tree);
+  tree decl = build_decl (location, PARM_DECL, name.as_tree (), type_tree);
   DECL_CONTEXT (decl) = function;
   DECL_ARG_TYPE (decl) = type_tree;
 
@@ -2026,13 +2051,12 @@ parameter_variable (tree function, const std::string &name, tree type_tree,
 // Make a static chain variable.
 
 Bvariable *
-static_chain_variable (tree fndecl, const std::string &name, tree type_tree,
+static_chain_variable (tree fndecl, GGC::Ident name, tree type_tree,
 		       location_t location)
 {
   if (error_operand_p (type_tree))
     return Bvariable::error_variable ();
-  tree decl = build_decl (location, PARM_DECL,
-			  get_identifier_from_string (name), type_tree);
+  tree decl = build_decl (location, PARM_DECL, name.as_tree (), type_tree);
   DECL_CONTEXT (decl) = fndecl;
   DECL_ARG_TYPE (decl) = type_tree;
   TREE_USED (decl) = 1;
@@ -2123,10 +2147,10 @@ temporary_variable (tree fndecl, tree bind_tree, tree type_tree, tree init_tree,
 // Make a label.
 
 tree
-label (tree func_tree, const std::string &name, location_t location)
+label (tree func_tree, tl::optional<GGC::Ident> name, location_t location)
 {
   tree decl;
-  if (name.empty ())
+  if (!name.has_value ())
     {
       if (DECL_STRUCT_FUNCTION (func_tree) == NULL)
 	push_struct_function (func_tree);
@@ -2139,7 +2163,7 @@ label (tree func_tree, const std::string &name, location_t location)
     }
   else
     {
-      tree id = get_identifier_from_string (name);
+      tree id = name->as_tree ();
       decl = build_decl (location, LABEL_DECL, id, void_type_node);
       DECL_CONTEXT (decl) = func_tree;
     }
@@ -2178,7 +2202,7 @@ label_address (tree label, location_t location)
 // Declare or define a new function.
 
 tree
-function (tree functype, const std::string &name, const std::string &asm_name,
+function (tree functype, GGC::Ident name, tl::optional<GGC::Ident> asm_name,
 	  unsigned int flags, location_t location)
 {
   if (error_operand_p (functype))
@@ -2186,13 +2210,13 @@ function (tree functype, const std::string &name, const std::string &asm_name,
 
   gcc_assert (FUNCTION_POINTER_TYPE_P (functype));
   functype = TREE_TYPE (functype);
-  tree id = get_identifier_from_string (name);
+  tree id = name.as_tree ();
   if (error_operand_p (id))
     return error_mark_node;
 
   tree decl = build_decl (location, FUNCTION_DECL, id, functype);
-  if (!asm_name.empty ())
-    SET_DECL_ASSEMBLER_NAME (decl, get_identifier_from_string (asm_name));
+  if (asm_name.has_value ())
+    SET_DECL_ASSEMBLER_NAME (decl, asm_name->as_tree ());
 
   if ((flags & function_is_declaration) != 0)
     DECL_EXTERNAL (decl) = 1;
@@ -2235,7 +2259,7 @@ function_defer_statement (tree function, tree undefer_tree, tree defer_tree,
     push_cfun (DECL_STRUCT_FUNCTION (function));
 
   tree stmt_list = NULL;
-  tree label = Backend::label (function, "", location);
+  tree label = Backend::label (function, tl::nullopt, location);
   tree label_def = label_definition_statement (label);
   append_to_statement_list (label_def, &stmt_list);
 
@@ -2342,4 +2366,30 @@ write_global_definitions (const std::vector<tree> &type_decls,
   delete[] defs;
 }
 
+tree
+lookup_field (const_tree type, tree component)
+{
+  tree field;
+
+  for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
+    {
+      if (DECL_NAME (field) == NULL_TREE
+	  && RECORD_OR_UNION_TYPE_P (TREE_TYPE (field)))
+	{
+	  tree anon = lookup_field (TREE_TYPE (field), component);
+
+	  if (anon)
+	    return tree_cons (NULL_TREE, field, anon);
+	}
+
+      if (DECL_NAME (field) == component)
+	break;
+    }
+
+  if (field == NULL_TREE)
+    return NULL_TREE;
+
+  return tree_cons (NULL_TREE, field, NULL_TREE);
+}
+
 } // namespace Backend
diff --git a/gcc/rust/rust-lang.cc b/gcc/rust/rust-lang.cc
index f3a155d..93ce041 100644
--- a/gcc/rust/rust-lang.cc
+++ b/gcc/rust/rust-lang.cc
@@ -51,10 +51,10 @@
 // FIXME: test saving intellisense
 #include "options.h"
 
-// version check to stop compiling if c++ isn't c++11 or higher
-#if __cplusplus < 201103
+// version check to stop compiling if c++ isn't c++14 or higher
+#if __cplusplus < 201402
 #error                                                                         \
-  "GCC Rust frontend requires C++11 or higher. You can compile the g++ frontend first and then compile the Rust frontend using that."
+  "GCC Rust frontend requires C++14 or higher. You can compile the g++ frontend first and then compile the Rust frontend using that."
 #endif
 // TODO: is this best way to do it? Is it allowed? (should be)
 
@@ -373,7 +373,13 @@ rust_localize_identifier (const char *ident)
   return identifier_to_locale (ident);
 }
 
-extern const attribute_spec grs_langhook_common_attribute_table[];
+extern const struct scoped_attribute_specs grs_langhook_gnu_attribute_table;
+extern const struct scoped_attribute_specs grs_langhook_common_attribute_table;
+
+const scoped_attribute_specs *const grs_langhook_attribute_table[] = {
+  &grs_langhook_gnu_attribute_table,
+  &grs_langhook_common_attribute_table,
+};
 
 /* The language hooks data structure. This is the main interface between the GCC
  * front-end and the GCC middle-end/back-end. A list of language hooks could be
@@ -394,8 +400,7 @@ extern const attribute_spec grs_langhook_common_attribute_table[];
 #undef LANG_HOOKS_WRITE_GLOBALS
 #undef LANG_HOOKS_GIMPLIFY_EXPR
 #undef LANG_HOOKS_EH_PERSONALITY
-
-#undef LANG_HOOKS_COMMON_ATTRIBUTE_TABLE
+#undef LANG_HOOKS_ATTRIBUTE_TABLE
 
 #define LANG_HOOKS_NAME "GNU Rust"
 #define LANG_HOOKS_INIT grs_langhook_init
@@ -417,7 +422,7 @@ extern const attribute_spec grs_langhook_common_attribute_table[];
 #define LANG_HOOKS_GIMPLIFY_EXPR grs_langhook_gimplify_expr
 #define LANG_HOOKS_EH_PERSONALITY grs_langhook_eh_personality
 
-#define LANG_HOOKS_COMMON_ATTRIBUTE_TABLE grs_langhook_common_attribute_table
+#define LANG_HOOKS_ATTRIBUTE_TABLE grs_langhook_attribute_table
 
 #if CHECKING_P
 
diff --git a/gcc/rust/rust-object-export.h b/gcc/rust/rust-object-export.h
index fe055c3..784fef3 100644
--- a/gcc/rust/rust-object-export.h
+++ b/gcc/rust/rust-object-export.h
@@ -21,13 +21,10 @@
 
 #include "rust-system.h"
 
-extern unsigned int
-rust_field_alignment (tree t);
-
-extern const char *
-rust_read_export_data (int fd, off_t offset, char **pbuf, size_t *plen,
-		       int *perr);
-extern void
-rust_write_export_data (const char *bytes, unsigned int size);
+extern unsigned int rust_field_alignment (tree t);
+
+extern const char *rust_read_export_data (int fd, off_t offset, char **pbuf,
+					  size_t *plen, int *perr);
+extern void rust_write_export_data (const char *bytes, unsigned int size);
 
 #endif // RUST_OBJECT_EXPORT_H
diff --git a/gcc/rust/rust-session-manager.cc b/gcc/rust/rust-session-manager.cc
index 48acbf34..17f9c06 100644
--- a/gcc/rust/rust-session-manager.cc
+++ b/gcc/rust/rust-session-manager.cc
@@ -20,7 +20,9 @@
 #include "rust-collect-lang-items.h"
 #include "rust-desugar-for-loops.h"
 #include "rust-desugar-question-mark.h"
+#include "rust-desugar-apit.h"
 #include "rust-diagnostics.h"
+#include "rust-expression-yeast.h"
 #include "rust-hir-pattern-analysis.h"
 #include "rust-immutable-name-resolution-context.h"
 #include "rust-unsafe-checker.h"
@@ -61,11 +63,9 @@
 #include "tm.h"
 #include "rust-target.h"
 
-extern bool
-saw_errors (void);
+extern bool saw_errors (void);
 
-extern Linemap *
-rust_get_linemap ();
+extern Linemap *rust_get_linemap ();
 
 namespace Rust {
 
@@ -150,9 +150,9 @@ validate_crate_name (const std::string &crate_name, Error &error)
     {
       if (!(is_alphabetic (c.value) || is_numeric (c.value) || c.value == '_'))
 	{
-	  error = Error (UNDEF_LOCATION,
-			 "invalid character %qs in crate name: %qs",
-			 c.as_string ().c_str (), crate_name.c_str ());
+	  error
+	    = Error (UNDEF_LOCATION, "invalid character %qs in crate name: %qs",
+		     c.as_string ().c_str (), crate_name.c_str ());
 	  return false;
 	}
     }
@@ -203,14 +203,16 @@ Session::handle_option (
   switch (code)
     {
     case OPT_I:
-      case OPT_L: {
+    case OPT_L:
+      {
 	// TODO: add search path
 	const std::string p = std::string (arg);
 	add_search_path (p);
       }
       break;
 
-      case OPT_frust_extern_: {
+    case OPT_frust_extern_:
+      {
 	std::string input (arg);
 	ret = handle_extern_option (input);
       }
@@ -251,7 +253,8 @@ Session::handle_option (
       Compile::Mangler::set_mangling (flag_rust_mangling);
       break;
 
-      case OPT_frust_cfg_: {
+    case OPT_frust_cfg_:
+      {
 	auto string_arg = std::string (arg);
 	ret = handle_cfg_option (string_arg);
 	break;
@@ -617,9 +620,6 @@ Session::compile_crate (const char *filename)
 
   expansion (parsed_crate, name_resolution_ctx);
 
-  AST::DesugarForLoops ().go (parsed_crate);
-  AST::DesugarQuestionMark ().go (parsed_crate);
-
   rust_debug ("\033[0;31mSUCCESSFULLY FINISHED EXPANSION \033[0m");
   if (options.dump_option_enabled (CompileOptions::EXPANSION_DUMP))
     {
@@ -682,6 +682,7 @@ Session::compile_crate (const char *filename)
   Resolver2_0::ImmutableNameResolutionContext::init (name_resolution_ctx);
 
   // type resolve
+  Compile::Context *ctx = Compile::Context::get ();
   Resolver::TypeResolution::Resolve (hir);
 
   Resolver::TypeCheckContext::get ()->get_variance_analysis_ctx ().solve ();
@@ -729,16 +730,15 @@ Session::compile_crate (const char *filename)
     return;
 
   // do compile to gcc generic
-  Compile::Context ctx;
-  Compile::CompileCrate::Compile (hir, &ctx);
+  Compile::CompileCrate::Compile (hir, ctx);
 
   // we can't do static analysis if there are errors to worry about
   if (!saw_errors ())
     {
       // lints
       Analysis::ScanDeadcode::Scan (hir);
-      Analysis::UnusedVariables::Lint (ctx);
-      Analysis::ReadonlyCheck::Lint (ctx);
+      Analysis::UnusedVariables::Lint (*ctx);
+      Analysis::ReadonlyCheck::Lint (*ctx);
 
       // metadata
       bool specified_emit_metadata
@@ -759,7 +759,7 @@ Session::compile_crate (const char *filename)
     }
 
   // pass to GCC middle-end
-  ctx.write_to_backend ();
+  ctx->write_to_backend ();
 }
 
 void
@@ -983,6 +983,20 @@ Session::expansion (AST::Crate &crate, Resolver2_0::NameResolutionContext &ctx)
       rust_error_at (range, "reached recursion limit");
     }
 
+  // handle AST desugaring
+  if (!saw_errors ())
+    {
+      AST::ExpressionYeast ().go (crate);
+
+      AST::DesugarApit ().go (crate);
+
+      // HACK: we may need a final TopLevel pass
+      // however, this should not count towards the recursion limit
+      // and we don't need a full Early pass
+      if (flag_name_resolution_2_0)
+	Resolver2_0::TopLevel (ctx).go (crate);
+    }
+
   // error reporting - check unused macros, get missing fragment specifiers
 
   // build test harness
@@ -1108,8 +1122,7 @@ Session::load_extern_crate (const std::string &crate_name, location_t locus)
   if (stream == NULL	       // No stream and
       && proc_macros.empty ()) // no proc macros
     {
-      rust_error_at (locus, "failed to locate crate %qs",
-		     import_name.c_str ());
+      rust_error_at (locus, "failed to locate crate %qs", import_name.c_str ());
       return UNKNOWN_NODEID;
     }
 
@@ -1177,16 +1190,13 @@ Session::load_extern_crate (const std::string &crate_name, location_t locus)
   mappings.insert_bang_proc_macros (crate_num, bang_macros);
   mappings.insert_derive_proc_macros (crate_num, derive_macros);
 
-  // name resolve it
-  Resolver::NameResolution::Resolve (parsed_crate);
-
-  // perform hir lowering
-  std::unique_ptr<HIR::Crate> lowered
-    = HIR::ASTLowering::Resolve (parsed_crate);
-  HIR::Crate &hir = mappings.insert_hir_crate (std::move (lowered));
-
-  // perform type resolution
-  Resolver::TypeResolution::Resolve (hir);
+  // if flag_name_resolution_2_0 is enabled
+  // then we perform resolution later
+  if (!flag_name_resolution_2_0)
+    {
+      // name resolve it
+      Resolver::NameResolution::Resolve (parsed_crate);
+    }
 
   // always restore the crate_num
   mappings.set_current_crate (saved_crate_num);
diff --git a/gcc/rust/rust-session-manager.h b/gcc/rust/rust-session-manager.h
index 83ba121..9af103c 100644
--- a/gcc/rust/rust-session-manager.h
+++ b/gcc/rust/rust-session-manager.h
@@ -441,8 +441,7 @@ private:
 
 #if CHECKING_P
 namespace selftest {
-extern void
-rust_crate_name_validation_test (void);
+extern void rust_crate_name_validation_test (void);
 }
 #endif // CHECKING_P
 
diff --git a/gcc/rust/rust-system.h b/gcc/rust/rust-system.h
index 986428b..a455123 100644
--- a/gcc/rust/rust-system.h
+++ b/gcc/rust/rust-system.h
@@ -88,10 +88,8 @@ constexpr static const char *file_separator = "/";
  */
 #define rust_unreachable() (fancy_abort (__FILE__, __LINE__, __FUNCTION__))
 
-extern void
-rust_preserve_from_gc (tree t);
+extern void rust_preserve_from_gc (tree t);
 
-extern const char *
-rust_localize_identifier (const char *ident);
+extern const char *rust_localize_identifier (const char *ident);
 
 #endif // !defined(RUST_SYSTEM_H)
diff --git a/gcc/rust/rust-target.h b/gcc/rust/rust-target.h
index dbc2baf..e61ea51 100644
--- a/gcc/rust/rust-target.h
+++ b/gcc/rust/rust-target.h
@@ -26,8 +26,7 @@
 #include "rust-target.def"
 
 /* Used by target to add target-related info.  */
-extern void
-rust_add_target_info (const char *, const char *);
+extern void rust_add_target_info (const char *, const char *);
 
 /* Each target can provide their own.  */
 extern struct gcc_targetrustm targetrustm;
diff --git a/gcc/rust/typecheck/rust-autoderef.cc b/gcc/rust/typecheck/rust-autoderef.cc
index 6aa20a8..10a59bd 100644
--- a/gcc/rust/typecheck/rust-autoderef.cc
+++ b/gcc/rust/typecheck/rust-autoderef.cc
@@ -26,8 +26,7 @@
 namespace Rust {
 namespace Resolver {
 
-static bool
-resolve_operator_overload_fn (
+static bool resolve_operator_overload_fn (
   LangItem::Kind lang_item_type, TyTy::BaseType *ty, TyTy::FnType **resolved_fn,
   Adjustment::AdjustmentType *requires_ref_adjustment);
 
@@ -248,7 +247,6 @@ resolve_operator_overload_fn (
 	  const TyTy::ADTType *adt = static_cast<const TyTy::ADTType *> (lhs);
 
 	  auto s = fn->get_self_type ()->get_root ();
-	  rust_assert (s->can_eq (adt, false));
 	  rust_assert (s->get_kind () == TyTy::TypeKind::ADT);
 	  const TyTy::ADTType *self_adt
 	    = static_cast<const TyTy::ADTType *> (s);
diff --git a/gcc/rust/typecheck/rust-casts.cc b/gcc/rust/typecheck/rust-casts.cc
index 90bdef1..f06d9ed 100644
--- a/gcc/rust/typecheck/rust-casts.cc
+++ b/gcc/rust/typecheck/rust-casts.cc
@@ -17,6 +17,7 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-casts.h"
+#include "rust-tyty-util.h"
 
 namespace Rust {
 namespace Resolver {
@@ -28,15 +29,20 @@ TypeCastRules::TypeCastRules (location_t locus, TyTy::TyWithLocation from,
 
 TypeCoercionRules::CoercionResult
 TypeCastRules::resolve (location_t locus, TyTy::TyWithLocation from,
-			TyTy::TyWithLocation to)
+			TyTy::TyWithLocation to, bool emit_error)
 {
   TypeCastRules cast_rules (locus, from, to);
-  return cast_rules.check ();
+  return cast_rules.check (emit_error);
 }
 
 TypeCoercionRules::CoercionResult
-TypeCastRules::check ()
+TypeCastRules::check (bool emit_error)
 {
+  // try the simple cast rules
+  auto simple_cast = cast_rules ();
+  if (!simple_cast.is_error ())
+    return simple_cast;
+
   // https://github.com/rust-lang/rust/blob/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/compiler/rustc_typeck/src/check/cast.rs#L565-L582
   auto possible_coercion
     = TypeCoercionRules::TryCoerce (from.get_ty (), to.get_ty (), locus,
@@ -51,13 +57,9 @@ TypeCastRules::check ()
 					true /*is_cast_site*/);
     }
 
-  // try the simple cast rules
-  auto simple_cast = cast_rules ();
-  if (!simple_cast.is_error ())
-    return simple_cast;
+  if (emit_error)
+    TypeCastRules::emit_cast_error (locus, from, to);
 
-  // failed to cast
-  emit_cast_error ();
   return TypeCoercionRules::CoercionResult::get_error ();
 }
 
@@ -73,7 +75,8 @@ TypeCastRules::cast_rules ()
 	      to.get_ty ()->debug_str ().c_str ());
   switch (from_type->get_kind ())
     {
-      case TyTy::TypeKind::INFER: {
+    case TyTy::TypeKind::INFER:
+      {
 	TyTy::InferType *from_infer
 	  = static_cast<TyTy::InferType *> (from_type);
 	switch (from_infer->get_infer_kind ())
@@ -85,7 +88,8 @@ TypeCastRules::cast_rules ()
 	  case TyTy::InferType::InferTypeKind::INTEGRAL:
 	    switch (to.get_ty ()->get_kind ())
 	      {
-		case TyTy::TypeKind::CHAR: {
+	      case TyTy::TypeKind::CHAR:
+		{
 		  // only u8 and char
 		  bool was_uint
 		    = from.get_ty ()->get_kind () == TyTy::TypeKind::UINT;
@@ -108,7 +112,8 @@ TypeCastRules::cast_rules ()
 		return TypeCoercionRules::CoercionResult{
 		  {}, to.get_ty ()->clone ()};
 
-		case TyTy::TypeKind::INFER: {
+	      case TyTy::TypeKind::INFER:
+		{
 		  TyTy::InferType *to_infer
 		    = static_cast<TyTy::InferType *> (to.get_ty ());
 
@@ -140,7 +145,8 @@ TypeCastRules::cast_rules ()
 		return TypeCoercionRules::CoercionResult{
 		  {}, to.get_ty ()->clone ()};
 
-		case TyTy::TypeKind::INFER: {
+	      case TyTy::TypeKind::INFER:
+		{
 		  TyTy::InferType *to_infer
 		    = static_cast<TyTy::InferType *> (to.get_ty ());
 
@@ -187,7 +193,8 @@ TypeCastRules::cast_rules ()
     case TyTy::TypeKind::INT:
       switch (to.get_ty ()->get_kind ())
 	{
-	  case TyTy::TypeKind::CHAR: {
+	case TyTy::TypeKind::CHAR:
+	  {
 	    // only u8 and char
 	    bool was_uint = from.get_ty ()->get_kind () == TyTy::TypeKind::UINT;
 	    bool was_u8 = was_uint
@@ -200,7 +207,8 @@ TypeCastRules::cast_rules ()
 	  }
 	  break;
 
-	  case TyTy::TypeKind::FLOAT: {
+	case TyTy::TypeKind::FLOAT:
+	  {
 	    // can only do this for number types not char
 	    bool from_char
 	      = from.get_ty ()->get_kind () == TyTy::TypeKind::CHAR;
@@ -210,7 +218,8 @@ TypeCastRules::cast_rules ()
 	  }
 	  break;
 
-	  case TyTy::TypeKind::POINTER: {
+	case TyTy::TypeKind::POINTER:
+	  {
 	    // char can't be casted as a ptr
 	    bool from_char
 	      = from.get_ty ()->get_kind () == TyTy::TypeKind::CHAR;
@@ -244,7 +253,8 @@ TypeCastRules::cast_rules ()
 	case TyTy::TypeKind::FLOAT:
 	  return TypeCoercionRules::CoercionResult{{}, to.get_ty ()->clone ()};
 
-	  case TyTy::TypeKind::INFER: {
+	case TyTy::TypeKind::INFER:
+	  {
 	    TyTy::InferType *to_infer
 	      = static_cast<TyTy::InferType *> (to.get_ty ());
 
@@ -273,7 +283,8 @@ TypeCastRules::cast_rules ()
 	case TyTy::TypeKind::USIZE:
 	case TyTy::TypeKind::ISIZE:
 	case TyTy::TypeKind::UINT:
-	  case TyTy::TypeKind::INT: {
+	case TyTy::TypeKind::INT:
+	  {
 	    // refs should not cast to numeric type
 	    bool from_ptr
 	      = from.get_ty ()->get_kind () == TyTy::TypeKind::POINTER;
@@ -320,7 +331,27 @@ TypeCastRules::check_ptr_ptr_cast ()
     }
   else if (from_is_ref && to_is_ref)
     {
-      // mutability must be coercedable
+      const auto &from_ref = *from.get_ty ()->as<TyTy::ReferenceType> ();
+      const auto &to_ref = *to.get_ty ()->as<TyTy::ReferenceType> ();
+
+      if (from_ref.is_dyn_object () != to_ref.is_dyn_object ())
+	{
+	  // this needs to be handled by coercion logic
+	  return TypeCoercionRules::CoercionResult::get_error ();
+	}
+
+      // are the underlying types safely simple castable?
+      const auto to_underly = to_ref.get_base ();
+      const auto from_underly = from_ref.get_base ();
+      auto res = resolve (locus, TyTy::TyWithLocation (from_underly),
+			  TyTy::TyWithLocation (to_underly), false);
+      if (res.is_error ())
+	{
+	  // this needs to be handled by coercion logic
+	  return TypeCoercionRules::CoercionResult::get_error ();
+	}
+
+      // mutability must be coerceable
       TyTy::ReferenceType &f
 	= static_cast<TyTy::ReferenceType &> (*from.get_ty ());
       TyTy::ReferenceType &t
@@ -337,7 +368,8 @@ TypeCastRules::check_ptr_ptr_cast ()
 }
 
 void
-TypeCastRules::emit_cast_error () const
+TypeCastRules::emit_cast_error (location_t locus, TyTy::TyWithLocation from,
+				TyTy::TyWithLocation to)
 {
   rich_location r (line_table, locus);
   r.add_range (from.get_locus ());
diff --git a/gcc/rust/typecheck/rust-casts.h b/gcc/rust/typecheck/rust-casts.h
index 0d6ed68..10bb006 100644
--- a/gcc/rust/typecheck/rust-casts.h
+++ b/gcc/rust/typecheck/rust-casts.h
@@ -30,15 +30,17 @@ class TypeCastRules
 public:
   static TypeCoercionRules::CoercionResult resolve (location_t locus,
 						    TyTy::TyWithLocation from,
-						    TyTy::TyWithLocation to);
+						    TyTy::TyWithLocation to,
+						    bool emit_error = true);
+
+  static void emit_cast_error (location_t locus, TyTy::TyWithLocation from,
+			       TyTy::TyWithLocation to);
 
 protected:
-  TypeCoercionRules::CoercionResult check ();
+  TypeCoercionRules::CoercionResult check (bool emit_error);
   TypeCoercionRules::CoercionResult cast_rules ();
   TypeCoercionRules::CoercionResult check_ptr_ptr_cast ();
 
-  void emit_cast_error () const;
-
 protected:
   TypeCastRules (location_t locus, TyTy::TyWithLocation from,
 		 TyTy::TyWithLocation to);
diff --git a/gcc/rust/typecheck/rust-coercion.cc b/gcc/rust/typecheck/rust-coercion.cc
index 5905992..fd12839 100644
--- a/gcc/rust/typecheck/rust-coercion.cc
+++ b/gcc/rust/typecheck/rust-coercion.cc
@@ -125,13 +125,15 @@ TypeCoercionRules::do_coercion (TyTy::BaseType *receiver)
   // pointers
   switch (expected->get_kind ())
     {
-      case TyTy::TypeKind::POINTER: {
+    case TyTy::TypeKind::POINTER:
+      {
 	TyTy::PointerType *ptr = static_cast<TyTy::PointerType *> (expected);
 	try_result = coerce_unsafe_ptr (receiver, ptr, ptr->mutability ());
 	return !try_result.is_error ();
       }
 
-      case TyTy::TypeKind::REF: {
+    case TyTy::TypeKind::REF:
+      {
 	TyTy::ReferenceType *ptr
 	  = static_cast<TyTy::ReferenceType *> (expected);
 	try_result
@@ -147,7 +149,8 @@ TypeCoercionRules::do_coercion (TyTy::BaseType *receiver)
   // https://github.com/rust-lang/rust/blob/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/compiler/rustc_typeck/src/check/coercion.rs#L210
   switch (receiver->get_kind ())
     {
-      default: {
+    default:
+      {
 	rust_debug (
 	  "do_coercion default unify and infer expected: %s receiver %s",
 	  receiver->debug_str ().c_str (), expected->debug_str ().c_str ());
@@ -182,7 +185,8 @@ TypeCoercionRules::coerce_unsafe_ptr (TyTy::BaseType *receiver,
   TyTy::BaseType *element = nullptr;
   switch (receiver->get_kind ())
     {
-      case TyTy::TypeKind::REF: {
+    case TyTy::TypeKind::REF:
+      {
 	TyTy::ReferenceType *ref
 	  = static_cast<TyTy::ReferenceType *> (receiver);
 	from_mutbl = ref->mutability ();
@@ -190,16 +194,19 @@ TypeCoercionRules::coerce_unsafe_ptr (TyTy::BaseType *receiver,
       }
       break;
 
-      case TyTy::TypeKind::POINTER: {
+    case TyTy::TypeKind::POINTER:
+      {
 	TyTy::PointerType *ref = static_cast<TyTy::PointerType *> (receiver);
 	from_mutbl = ref->mutability ();
 	element = ref->get_base ();
       }
       break;
 
-      default: {
-	// FIXME this can probably turn into a unify_and
-	if (receiver->can_eq (expected, false))
+    default:
+      {
+	if (types_compatable (TyTy::TyWithLocation (receiver),
+			      TyTy::TyWithLocation (expected), UNKNOWN_LOCATION,
+			      false))
 	  return CoercionResult{{}, expected->clone ()};
 
 	return CoercionResult::get_error ();
@@ -264,17 +271,16 @@ TypeCoercionRules::coerce_borrowed_pointer (TyTy::BaseType *receiver,
   Mutability from_mutbl = Mutability::Imm;
   switch (receiver->get_kind ())
     {
-      case TyTy::TypeKind::REF: {
+    case TyTy::TypeKind::REF:
+      {
 	TyTy::ReferenceType *from
 	  = static_cast<TyTy::ReferenceType *> (receiver);
 	from_mutbl = from->mutability ();
       }
       break;
 
-      default: {
-	// FIXME
-	// we might be able to replace this with a can_eq because we default
-	// back to a final unity anyway
+    default:
+      {
 	rust_debug ("coerce_borrowed_pointer -- unify");
 	TyTy::BaseType *result
 	  = unify_site_and (receiver->get_ref (),
@@ -393,7 +399,7 @@ TypeCoercionRules::coerce_unsized (TyTy::BaseType *source,
 
   if (expect_dyn && need_unsize)
     {
-      bool bounds_compatible = b->bounds_compatible (*a, locus, true);
+      bool bounds_compatible = b->bounds_compatible (*a, locus, false);
       if (!bounds_compatible)
 	{
 	  unsafe_error = true;
diff --git a/gcc/rust/typecheck/rust-hir-dot-operator.cc b/gcc/rust/typecheck/rust-hir-dot-operator.cc
index c1165e9..7b7944c 100644
--- a/gcc/rust/typecheck/rust-hir-dot-operator.cc
+++ b/gcc/rust/typecheck/rust-hir-dot-operator.cc
@@ -102,26 +102,16 @@ MethodResolver::try_hook (const TyTy::BaseType &r)
     }
 }
 
-bool
-MethodResolver::select (TyTy::BaseType &receiver)
+std::vector<MethodResolver::impl_item_candidate>
+MethodResolver::assemble_inherent_impl_candidates (
+  const TyTy::BaseType &receiver)
 {
-  rust_debug ("MethodResolver::select reciever=[%s] path=[%s]",
-	      receiver.debug_str ().c_str (),
-	      segment_name.as_string ().c_str ());
-
-  struct impl_item_candidate
-  {
-    HIR::Function *item;
-    HIR::ImplBlock *impl_block;
-    TyTy::FnType *ty;
-  };
-
+  std::vector<impl_item_candidate> inherent_impl_fns;
   const TyTy::BaseType *raw = receiver.destructure ();
   bool receiver_is_raw_ptr = raw->get_kind () == TyTy::TypeKind::POINTER;
   bool receiver_is_ref = raw->get_kind () == TyTy::TypeKind::REF;
 
-  // assemble inherent impl items
-  std::vector<impl_item_candidate> inherent_impl_fns;
+  // Assemble inherent impl items (non-trait impl blocks)
   mappings.iterate_impl_items (
     [&] (HirId id, HIR::ImplItem *item, HIR::ImplBlock *impl) mutable -> bool {
       bool is_trait_impl = impl->has_trait_ref ();
@@ -190,16 +180,19 @@ MethodResolver::select (TyTy::BaseType &receiver)
       return true;
     });
 
-  struct trait_item_candidate
-  {
-    const HIR::TraitItemFunc *item;
-    const HIR::Trait *trait;
-    TyTy::FnType *ty;
-    const TraitReference *reference;
-    const TraitItemReference *item_ref;
-  };
+  return inherent_impl_fns;
+}
+
+void
+MethodResolver::assemble_trait_impl_candidates (
+  const TyTy::BaseType &receiver,
+  std::vector<impl_item_candidate> &impl_candidates,
+  std::vector<trait_item_candidate> &trait_candidates)
+{
+  const TyTy::BaseType *raw = receiver.destructure ();
+  bool receiver_is_raw_ptr = raw->get_kind () == TyTy::TypeKind::POINTER;
+  bool receiver_is_ref = raw->get_kind () == TyTy::TypeKind::REF;
 
-  std::vector<trait_item_candidate> trait_fns;
   mappings.iterate_impl_blocks ([&] (HirId id,
 				     HIR::ImplBlock *impl) mutable -> bool {
     bool is_trait_impl = impl->has_trait_ref ();
@@ -266,7 +259,7 @@ MethodResolver::select (TyTy::BaseType &receiver)
 	      continue;
 	  }
 
-	inherent_impl_fns.push_back ({func, impl, fnty});
+	impl_candidates.push_back ({func, impl, fnty});
 	return true;
       }
 
@@ -293,26 +286,15 @@ MethodResolver::select (TyTy::BaseType &receiver)
     TyTy::FnType *fnty = static_cast<TyTy::FnType *> (ty);
 
     trait_item_candidate candidate{func, trait, fnty, trait_ref, item_ref};
-    trait_fns.push_back (candidate);
+    trait_candidates.push_back (candidate);
 
     return true;
   });
+}
 
-  // lookup specified bounds for an associated item
-  struct precdicate_candidate
-  {
-    TyTy::TypeBoundPredicateItem lookup;
-    TyTy::FnType *fntype;
-  };
-
-  // https://github.com/rust-lang/rust/blob/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/compiler/rustc_typeck/src/check/method/probe.rs#L580-L694
-
-  rust_debug ("inherent_impl_fns found {%lu}, trait_fns found {%lu}, "
-	      "predicate_items found {%lu}",
-	      (unsigned long) inherent_impl_fns.size (),
-	      (unsigned long) trait_fns.size (),
-	      (unsigned long) predicate_items.size ());
-
+bool
+MethodResolver::try_select_predicate_candidates (TyTy::BaseType &receiver)
+{
   bool found_possible_candidate = false;
   for (const auto &predicate : predicate_items)
     {
@@ -346,60 +328,33 @@ MethodResolver::select (TyTy::BaseType &receiver)
 	  found_possible_candidate = true;
 	}
     }
-  if (found_possible_candidate)
-    {
-      return true;
-    }
+  return found_possible_candidate;
+}
 
-  for (auto &impl_item : inherent_impl_fns)
+bool
+MethodResolver::try_select_inherent_impl_candidates (
+  TyTy::BaseType &receiver, const std::vector<impl_item_candidate> &candidates,
+  bool trait_impl_blocks_only)
+{
+  bool found_possible_candidate = false;
+  for (auto &impl_item : candidates)
     {
       bool is_trait_impl_block = impl_item.impl_block->has_trait_ref ();
-      if (is_trait_impl_block)
+      if (trait_impl_blocks_only && !is_trait_impl_block)
 	continue;
-
-      TyTy::FnType *fn = impl_item.ty;
-      rust_assert (fn->is_method ());
-
-      TyTy::BaseType *fn_self = fn->get_self_type ();
-      rust_debug ("dot-operator impl_item fn_self={%s} can_eq receiver={%s}",
-		  fn_self->debug_str ().c_str (),
-		  receiver.debug_str ().c_str ());
-
-      auto res
-	= TypeCoercionRules::TryCoerce (&receiver, fn_self, UNDEF_LOCATION,
-					false /*allow-autoderef*/);
-      bool ok = !res.is_error ();
-      if (ok)
-	{
-	  std::vector<Adjustment> adjs = append_adjustments (res.adjustments);
-	  PathProbeCandidate::ImplItemCandidate c{impl_item.item,
-						  impl_item.impl_block};
-	  auto try_result = MethodCandidate{
-	    PathProbeCandidate (PathProbeCandidate::CandidateType::IMPL_FUNC,
-				fn, impl_item.item->get_locus (), c),
-	    adjs};
-	  result.insert (std::move (try_result));
-	  found_possible_candidate = true;
-	}
-    }
-  if (found_possible_candidate)
-    {
-      return true;
-    }
-
-  for (auto &impl_item : inherent_impl_fns)
-    {
-      bool is_trait_impl_block = impl_item.impl_block->has_trait_ref ();
-      if (!is_trait_impl_block)
+      if (!trait_impl_blocks_only && is_trait_impl_block)
 	continue;
 
       TyTy::FnType *fn = impl_item.ty;
       rust_assert (fn->is_method ());
 
       TyTy::BaseType *fn_self = fn->get_self_type ();
-      rust_debug (
-	"dot-operator trait_impl_item fn_self={%s} can_eq receiver={%s}",
-	fn_self->debug_str ().c_str (), receiver.debug_str ().c_str ());
+
+      const char *debug_prefix
+	= trait_impl_blocks_only ? "trait_impl_item" : "impl_item";
+      rust_debug ("dot-operator %s fn_self={%s} can_eq receiver={%s}",
+		  debug_prefix, fn_self->debug_str ().c_str (),
+		  receiver.debug_str ().c_str ());
 
       auto res
 	= TypeCoercionRules::TryCoerce (&receiver, fn_self, UNDEF_LOCATION,
@@ -418,12 +373,15 @@ MethodResolver::select (TyTy::BaseType &receiver)
 	  found_possible_candidate = true;
 	}
     }
-  if (found_possible_candidate)
-    {
-      return true;
-    }
+  return found_possible_candidate;
+}
 
-  for (auto trait_item : trait_fns)
+bool
+MethodResolver::try_select_trait_impl_candidates (
+  TyTy::BaseType &receiver, const std::vector<trait_item_candidate> &candidates)
+{
+  bool found_possible_candidate = false;
+  for (auto trait_item : candidates)
     {
       TyTy::FnType *fn = trait_item.ty;
       rust_assert (fn->is_method ());
@@ -451,10 +409,53 @@ MethodResolver::select (TyTy::BaseType &receiver)
 	  found_possible_candidate = true;
 	}
     }
-
   return found_possible_candidate;
 }
 
+bool
+MethodResolver::select (TyTy::BaseType &receiver)
+{
+  rust_debug ("MethodResolver::select reciever=[%s] path=[%s]",
+	      receiver.debug_str ().c_str (),
+	      segment_name.as_string ().c_str ());
+
+  // Assemble candidates
+  std::vector<impl_item_candidate> inherent_impl_fns
+    = assemble_inherent_impl_candidates (receiver);
+  std::vector<impl_item_candidate> trait_impl_fns;
+  std::vector<trait_item_candidate> trait_fns;
+  assemble_trait_impl_candidates (receiver, trait_impl_fns, trait_fns);
+
+  // Combine inherent and trait impl functions
+  inherent_impl_fns.insert (inherent_impl_fns.end (), trait_impl_fns.begin (),
+			    trait_impl_fns.end ());
+
+  // https://github.com/rust-lang/rust/blob/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/compiler/rustc_typeck/src/check/method/probe.rs#L580-L694
+
+  rust_debug ("inherent_impl_fns found {%lu}, trait_fns found {%lu}, "
+	      "predicate_items found {%lu}",
+	      (unsigned long) inherent_impl_fns.size (),
+	      (unsigned long) trait_fns.size (),
+	      (unsigned long) predicate_items.size ());
+
+  // Try selection in the priority order defined by Rust's method resolution:
+
+  // 1. Try predicate candidates first (highest priority)
+  if (try_select_predicate_candidates (receiver))
+    return true;
+
+  // 2. Try inherent impl functions (non-trait impl blocks)
+  if (try_select_inherent_impl_candidates (receiver, inherent_impl_fns, false))
+    return true;
+
+  // 3. Try inherent impl functions from trait impl blocks
+  if (try_select_inherent_impl_candidates (receiver, inherent_impl_fns, true))
+    return true;
+
+  // 4. Try trait functions (lowest priority)
+  return try_select_trait_impl_candidates (receiver, trait_fns);
+}
+
 std::vector<MethodResolver::predicate_candidate>
 MethodResolver::get_predicate_items (
   const HIR::PathIdentSegment &segment_name, const TyTy::BaseType &receiver,
diff --git a/gcc/rust/typecheck/rust-hir-dot-operator.h b/gcc/rust/typecheck/rust-hir-dot-operator.h
index ab95a5a..cc40472 100644
--- a/gcc/rust/typecheck/rust-hir-dot-operator.h
+++ b/gcc/rust/typecheck/rust-hir-dot-operator.h
@@ -65,6 +65,22 @@ public:
     const HIR::PathIdentSegment &segment_name, const TyTy::BaseType &receiver,
     const std::vector<TyTy::TypeBoundPredicate> &specified_bounds);
 
+  struct impl_item_candidate
+  {
+    HIR::Function *item;
+    HIR::ImplBlock *impl_block;
+    TyTy::FnType *ty;
+  };
+
+  struct trait_item_candidate
+  {
+    const HIR::TraitItemFunc *item;
+    const HIR::Trait *trait;
+    TyTy::FnType *ty;
+    const TraitReference *reference;
+    const TraitItemReference *item_ref;
+  };
+
 protected:
   MethodResolver (bool autoderef_flag,
 		  const HIR::PathIdentSegment &segment_name);
@@ -77,6 +93,25 @@ private:
   std::vector<Adjustment>
   append_adjustments (const std::vector<Adjustment> &adjustments) const;
 
+  std::vector<impl_item_candidate>
+  assemble_inherent_impl_candidates (const TyTy::BaseType &receiver);
+
+  void assemble_trait_impl_candidates (
+    const TyTy::BaseType &receiver,
+    std::vector<impl_item_candidate> &impl_candidates,
+    std::vector<trait_item_candidate> &trait_candidates);
+
+  bool try_select_predicate_candidates (TyTy::BaseType &receiver);
+
+  bool try_select_inherent_impl_candidates (
+    TyTy::BaseType &receiver,
+    const std::vector<impl_item_candidate> &candidates,
+    bool trait_impl_blocks_only);
+
+  bool try_select_trait_impl_candidates (
+    TyTy::BaseType &receiver,
+    const std::vector<trait_item_candidate> &candidates);
+
 private:
   // search
   const HIR::PathIdentSegment &segment_name;
diff --git a/gcc/rust/typecheck/rust-hir-inherent-impl-overlap.h b/gcc/rust/typecheck/rust-hir-inherent-impl-overlap.h
index 5537b14..a66396f 100644
--- a/gcc/rust/typecheck/rust-hir-inherent-impl-overlap.h
+++ b/gcc/rust/typecheck/rust-hir-inherent-impl-overlap.h
@@ -79,27 +79,29 @@ public:
 	    if (query == candidate)
 	      continue;
 
-	    if (query->can_eq (candidate, false))
+	    if (!types_compatable (TyTy::TyWithLocation (query),
+				   TyTy::TyWithLocation (candidate),
+				   UNKNOWN_LOCATION, false))
+	      continue;
+
+	    // we might be in the case that we have:
+	    //
+	    // *const T vs *const [T]
+	    //
+	    // so lets use an equality check when the
+	    // candidates are both generic to be sure we dont emit a false
+	    // positive
+
+	    bool a = query->is_concrete ();
+	    bool b = candidate->is_concrete ();
+	    bool both_generic = !a && !b;
+	    if (both_generic)
 	      {
-		// we might be in the case that we have:
-		//
-		// *const T vs *const [T]
-		//
-		// so lets use an equality check when the
-		// candidates are both generic to be sure we dont emit a false
-		// positive
-
-		bool a = query->is_concrete ();
-		bool b = candidate->is_concrete ();
-		bool both_generic = !a && !b;
-		if (both_generic)
-		  {
-		    if (!query->is_equal (*candidate))
-		      continue;
-		  }
-
-		possible_collision (it->second, iy->second);
+		if (!query->is_equal (*candidate))
+		  continue;
 	      }
+
+	    possible_collision (it->second, iy->second);
 	  }
       }
   }
diff --git a/gcc/rust/typecheck/rust-hir-path-probe.cc b/gcc/rust/typecheck/rust-hir-path-probe.cc
index 32e2399..c02702f 100644
--- a/gcc/rust/typecheck/rust-hir-path-probe.cc
+++ b/gcc/rust/typecheck/rust-hir-path-probe.cc
@@ -137,7 +137,7 @@ PathProbeCandidate::operator< (const PathProbeCandidate &c) const
 
 // PathProbeType
 
-PathProbeType::PathProbeType (const TyTy::BaseType *receiver,
+PathProbeType::PathProbeType (TyTy::BaseType *receiver,
 			      const HIR::PathIdentSegment &query,
 			      DefId specific_trait_id)
   : TypeCheckBase (), receiver (receiver), search (query),
@@ -145,7 +145,7 @@ PathProbeType::PathProbeType (const TyTy::BaseType *receiver,
 {}
 
 std::set<PathProbeCandidate>
-PathProbeType::Probe (const TyTy::BaseType *receiver,
+PathProbeType::Probe (TyTy::BaseType *receiver,
 		      const HIR::PathIdentSegment &segment_name,
 		      bool probe_impls, bool probe_bounds,
 		      bool ignore_mandatory_trait_items,
@@ -443,7 +443,7 @@ PathProbeType::is_receiver_generic () const
 
 // PathProbImplTrait
 
-PathProbeImplTrait::PathProbeImplTrait (const TyTy::BaseType *receiver,
+PathProbeImplTrait::PathProbeImplTrait (TyTy::BaseType *receiver,
 					const HIR::PathIdentSegment &query,
 					const TraitReference *trait_reference)
   : PathProbeType (receiver, query, UNKNOWN_DEFID),
@@ -451,7 +451,7 @@ PathProbeImplTrait::PathProbeImplTrait (const TyTy::BaseType *receiver,
 {}
 
 std::set<PathProbeCandidate>
-PathProbeImplTrait::Probe (const TyTy::BaseType *receiver,
+PathProbeImplTrait::Probe (TyTy::BaseType *receiver,
 			   const HIR::PathIdentSegment &segment_name,
 			   const TraitReference *trait_reference)
 {
diff --git a/gcc/rust/typecheck/rust-hir-path-probe.h b/gcc/rust/typecheck/rust-hir-path-probe.h
index 59ffeb1..936bcb9 100644
--- a/gcc/rust/typecheck/rust-hir-path-probe.h
+++ b/gcc/rust/typecheck/rust-hir-path-probe.h
@@ -108,9 +108,8 @@ class PathProbeType : public TypeCheckBase, public HIR::HIRImplVisitor
 {
 public:
   static std::set<PathProbeCandidate>
-  Probe (const TyTy::BaseType *receiver,
-	 const HIR::PathIdentSegment &segment_name, bool probe_impls,
-	 bool probe_bounds, bool ignore_mandatory_trait_items,
+  Probe (TyTy::BaseType *receiver, const HIR::PathIdentSegment &segment_name,
+	 bool probe_impls, bool probe_bounds, bool ignore_mandatory_trait_items,
 	 DefId specific_trait_id = UNKNOWN_DEFID);
 
   void visit (HIR::TypeAlias &alias) override;
@@ -135,8 +134,8 @@ protected:
 				    bool ignore_mandatory_trait_items);
 
 protected:
-  PathProbeType (const TyTy::BaseType *receiver,
-		 const HIR::PathIdentSegment &query, DefId specific_trait_id);
+  PathProbeType (TyTy::BaseType *receiver, const HIR::PathIdentSegment &query,
+		 DefId specific_trait_id);
 
   std::vector<std::pair<const TraitReference *, HIR::ImplBlock *>>
   union_bounds (
@@ -147,7 +146,7 @@ protected:
 
   bool is_receiver_generic () const;
 
-  const TyTy::BaseType *receiver;
+  TyTy::BaseType *receiver;
   const HIR::PathIdentSegment &search;
   std::set<PathProbeCandidate> candidates;
   HIR::ImplBlock *current_impl;
@@ -178,12 +177,11 @@ class PathProbeImplTrait : public PathProbeType
 {
 public:
   static std::set<PathProbeCandidate>
-  Probe (const TyTy::BaseType *receiver,
-	 const HIR::PathIdentSegment &segment_name,
+  Probe (TyTy::BaseType *receiver, const HIR::PathIdentSegment &segment_name,
 	 const TraitReference *trait_reference);
 
 private:
-  PathProbeImplTrait (const TyTy::BaseType *receiver,
+  PathProbeImplTrait (TyTy::BaseType *receiver,
 		      const HIR::PathIdentSegment &query,
 		      const TraitReference *trait_reference);
 
diff --git a/gcc/rust/typecheck/rust-hir-trait-reference.cc b/gcc/rust/typecheck/rust-hir-trait-reference.cc
index 83985f0..74856f0 100644
--- a/gcc/rust/typecheck/rust-hir-trait-reference.cc
+++ b/gcc/rust/typecheck/rust-hir-trait-reference.cc
@@ -342,7 +342,15 @@ TraitReference::on_resolved ()
 {
   for (auto &item : item_refs)
     {
-      item.on_resolved ();
+      if (item.get_trait_item_type ()
+	  == TraitItemReference::TraitItemType::TYPE)
+	item.on_resolved ();
+    }
+  for (auto &item : item_refs)
+    {
+      if (item.get_trait_item_type ()
+	  != TraitItemReference::TraitItemType::TYPE)
+	item.on_resolved ();
     }
 }
 
@@ -424,7 +432,13 @@ TraitReference::trait_has_generics () const
   return !trait_substs.empty ();
 }
 
-std::vector<TyTy::SubstitutionParamMapping>
+std::vector<TyTy::SubstitutionParamMapping> &
+TraitReference::get_trait_substs ()
+{
+  return trait_substs;
+}
+
+const std::vector<TyTy::SubstitutionParamMapping> &
 TraitReference::get_trait_substs () const
 {
   return trait_substs;
diff --git a/gcc/rust/typecheck/rust-hir-trait-reference.h b/gcc/rust/typecheck/rust-hir-trait-reference.h
index 8b1ac7d..473513e 100644
--- a/gcc/rust/typecheck/rust-hir-trait-reference.h
+++ b/gcc/rust/typecheck/rust-hir-trait-reference.h
@@ -224,7 +224,9 @@ public:
 
   bool trait_has_generics () const;
 
-  std::vector<TyTy::SubstitutionParamMapping> get_trait_substs () const;
+  std::vector<TyTy::SubstitutionParamMapping> &get_trait_substs ();
+
+  const std::vector<TyTy::SubstitutionParamMapping> &get_trait_substs () const;
 
   bool satisfies_bound (const TraitReference &reference) const;
 
diff --git a/gcc/rust/typecheck/rust-hir-trait-resolve.cc b/gcc/rust/typecheck/rust-hir-trait-resolve.cc
index 032bb58..0fd0147 100644
--- a/gcc/rust/typecheck/rust-hir-trait-resolve.cc
+++ b/gcc/rust/typecheck/rust-hir-trait-resolve.cc
@@ -22,9 +22,6 @@
 #include "rust-type-util.h"
 #include "rust-immutable-name-resolution-context.h"
 
-// used for flag_name_resolution_2_0
-#include "options.h"
-
 namespace Rust {
 namespace Resolver {
 
@@ -123,28 +120,16 @@ bool
 TraitResolver::resolve_path_to_trait (const HIR::TypePath &path,
 				      HIR::Trait **resolved) const
 {
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+
   NodeId ref;
-  bool ok;
-  if (flag_name_resolution_2_0)
+  if (auto ref_opt = nr_ctx.lookup (path.get_mappings ().get_nodeid ()))
     {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      auto ref_opt = nr_ctx.lookup (path.get_mappings ().get_nodeid ());
-
-      if ((ok = ref_opt.has_value ()))
-	ref = *ref_opt;
+      ref = *ref_opt;
     }
   else
     {
-      auto path_nodeid = path.get_mappings ().get_nodeid ();
-      ok = resolver->lookup_resolved_type (path_nodeid, &ref)
-	   || resolver->lookup_resolved_name (path_nodeid, &ref)
-	   || resolver->lookup_resolved_macro (path_nodeid, &ref);
-    }
-
-  if (!ok)
-    {
       rust_error_at (path.get_locus (), "Failed to resolve path to node-id");
       return false;
     }
@@ -224,7 +209,8 @@ TraitResolver::resolve_trait (HIR::Trait *trait_reference)
 	  // handling.
 	  break;
 
-	  case HIR::GenericParam::GenericKind::TYPE: {
+	case HIR::GenericParam::GenericKind::TYPE:
+	  {
 	    auto &typaram = static_cast<HIR::TypeParam &> (*generic_param);
 	    bool is_self
 	      = typaram.get_type_representation ().as_string ().compare ("Self")
@@ -287,7 +273,8 @@ TraitResolver::resolve_trait (HIR::Trait *trait_reference)
 
 	      auto predicate = get_predicate_from_bound (
 		b->get_path (),
-		tl::nullopt /*this will setup a PLACEHOLDER for self*/);
+		tl::nullopt /*this will setup a PLACEHOLDER for self*/,
+		BoundPolarity::RegularBound, false, true);
 	      if (predicate.is_error ())
 		return &TraitReference::error_node ();
 
@@ -443,11 +430,27 @@ TraitItemReference::associated_type_set (TyTy::BaseType *ty) const
 {
   rust_assert (get_trait_item_type () == TraitItemType::TYPE);
 
+  // this isnt super safe there are cases like the FnTraits where the type is
+  // set to the impls placeholder associated type. For example
+  //
+  // type Output = F::Output; -- see the fn trait impls in libcore
+  //
+  // then this projection ends up resolving back to this placeholder so it just
+  // ends up being cyclical
+
   TyTy::BaseType *item_ty = get_tyty ();
   rust_assert (item_ty->get_kind () == TyTy::TypeKind::PLACEHOLDER);
   TyTy::PlaceholderType *placeholder
     = static_cast<TyTy::PlaceholderType *> (item_ty);
 
+  if (ty->is<TyTy::ProjectionType> ())
+    {
+      const auto &projection = *static_cast<const TyTy::ProjectionType *> (ty);
+      const auto resolved = projection.get ();
+      if (resolved == item_ty)
+	return;
+    }
+
   placeholder->set_associated_type (ty->get_ty_ref ());
 }
 
@@ -543,7 +546,8 @@ AssociatedImplTrait::setup_associated_types (
 	  // handling.
 	  break;
 
-	  case HIR::GenericParam::GenericKind::TYPE: {
+	case HIR::GenericParam::GenericKind::TYPE:
+	  {
 	    TyTy::BaseType *l = nullptr;
 	    bool ok = context->lookup_type (
 	      generic_param->get_mappings ().get_hirid (), &l);
@@ -580,8 +584,8 @@ AssociatedImplTrait::setup_associated_types (
 	}
       else
 	{
-	  TyTy::ParamType *param = p.get_param_ty ();
-	  TyTy::BaseType *resolved = param->destructure ();
+	  auto param = p.get_param_ty ();
+	  auto resolved = param->destructure ();
 	  subst_args.push_back (TyTy::SubstitutionArg (&p, resolved));
 	  param_mappings[param->get_symbol ()] = resolved->get_ref ();
 	}
@@ -609,8 +613,8 @@ AssociatedImplTrait::setup_associated_types (
       if (i == 0)
 	continue;
 
-      const TyTy::ParamType *p = arg.get_param_ty ();
-      TyTy::BaseType *r = p->resolve ();
+      const auto p = arg.get_param_ty ();
+      auto r = p->resolve ();
       if (!r->is_concrete ())
 	{
 	  r = SubstMapperInternal::Resolve (r, infer_arguments);
@@ -626,8 +630,8 @@ AssociatedImplTrait::setup_associated_types (
       if (i == 0)
 	continue;
 
-      const TyTy::ParamType *p = arg.get_param_ty ();
-      TyTy::BaseType *r = p->resolve ();
+      const auto p = arg.get_param_ty ();
+      auto r = p->resolve ();
       if (!r->is_concrete ())
 	{
 	  r = SubstMapperInternal::Resolve (r, infer_arguments);
@@ -753,7 +757,8 @@ TraitItemReference::is_object_safe () const
   // https://doc.rust-lang.org/reference/items/traits.html#object-safety
   switch (get_trait_item_type ())
     {
-      case TraitItemReference::TraitItemType::FN: {
+    case TraitItemReference::TraitItemType::FN:
+      {
 	// lets be boring and just check that this is indeed a method will do
 	// for now
 	const HIR::TraitItem *item = get_hir_trait_item ();
diff --git a/gcc/rust/typecheck/rust-hir-type-bounds.h b/gcc/rust/typecheck/rust-hir-type-bounds.h
index 82333f1..5384700 100644
--- a/gcc/rust/typecheck/rust-hir-type-bounds.h
+++ b/gcc/rust/typecheck/rust-hir-type-bounds.h
@@ -37,7 +37,11 @@ public:
 
 private:
   void scan ();
-  void assemble_sized_builtin ();
+  bool
+  process_impl_block (HirId id, HIR::ImplBlock *impl,
+		      std::vector<std::pair<HIR::TypePath *, HIR::ImplBlock *>>
+			&possible_trait_paths);
+  void assemble_marker_builtins ();
   void add_trait_bound (HIR::Trait *trait);
   void assemble_builtin_candidate (LangItem::Kind item);
 
diff --git a/gcc/rust/typecheck/rust-hir-type-check-base.cc b/gcc/rust/typecheck/rust-hir-type-check-base.cc
index 14b8ab8..68001bf 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-base.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-base.cc
@@ -17,28 +17,33 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-hir-type-check-base.h"
+#include "rust-compile-base.h"
+#include "rust-hir-item.h"
 #include "rust-hir-type-check-expr.h"
 #include "rust-hir-type-check-type.h"
 #include "rust-hir-trait-resolve.h"
 #include "rust-type-util.h"
 #include "rust-attribute-values.h"
+#include "rust-tyty.h"
+#include "tree.h"
 
 namespace Rust {
 namespace Resolver {
 
 TypeCheckBase::TypeCheckBase ()
-  : mappings (Analysis::Mappings::get ()), resolver (Resolver::get ()),
-    context (TypeCheckContext::get ())
+  : mappings (Analysis::Mappings::get ()), context (TypeCheckContext::get ())
 {}
 
 void
 TypeCheckBase::ResolveGenericParams (
+  const HIR::Item::ItemKind item_kind, location_t item_locus,
   const std::vector<std::unique_ptr<HIR::GenericParam>> &generic_params,
   std::vector<TyTy::SubstitutionParamMapping> &substitutions, bool is_foreign,
   ABI abi)
 {
   TypeCheckBase ctx;
-  ctx.resolve_generic_params (generic_params, substitutions, is_foreign, abi);
+  ctx.resolve_generic_params (item_kind, item_locus, generic_params,
+			      substitutions, is_foreign, abi);
 }
 
 static void
@@ -47,10 +52,10 @@ walk_types_to_constrain (std::set<HirId> &constrained_symbols,
 {
   for (const auto &c : constraints.get_mappings ())
     {
-      const TyTy::BaseType *arg = c.get_tyty ();
+      auto arg = c.get_tyty ();
       if (arg != nullptr)
 	{
-	  const TyTy::BaseType *p = arg->get_root ();
+	  const auto p = arg->get_root ();
 	  constrained_symbols.insert (p->get_ty_ref ());
 	  if (p->has_substitutions_defined ())
 	    {
@@ -66,7 +71,7 @@ TypeCheckBase::check_for_unconstrained (
   const std::vector<TyTy::SubstitutionParamMapping> &params_to_constrain,
   const TyTy::SubstitutionArgumentMappings &constraint_a,
   const TyTy::SubstitutionArgumentMappings &constraint_b,
-  const TyTy::BaseType *reference)
+  TyTy::BaseType *reference)
 {
   bool check_result = false;
   bool check_completed
@@ -124,7 +129,8 @@ TypeCheckBase::resolve_literal (const Analysis::NodeMapping &expr_mappings,
   TyTy::BaseType *infered = nullptr;
   switch (literal.get_lit_type ())
     {
-      case HIR::Literal::LitType::INT: {
+    case HIR::Literal::LitType::INT:
+      {
 	bool ok = false;
 
 	switch (literal.get_type_hint ())
@@ -191,7 +197,8 @@ TypeCheckBase::resolve_literal (const Analysis::NodeMapping &expr_mappings,
       }
       break;
 
-      case HIR::Literal::LitType::FLOAT: {
+    case HIR::Literal::LitType::FLOAT:
+      {
 	bool ok = false;
 
 	switch (literal.get_type_hint ())
@@ -216,25 +223,29 @@ TypeCheckBase::resolve_literal (const Analysis::NodeMapping &expr_mappings,
       }
       break;
 
-      case HIR::Literal::LitType::BOOL: {
+    case HIR::Literal::LitType::BOOL:
+      {
 	auto ok = context->lookup_builtin ("bool", &infered);
 	rust_assert (ok);
       }
       break;
 
-      case HIR::Literal::LitType::CHAR: {
+    case HIR::Literal::LitType::CHAR:
+      {
 	auto ok = context->lookup_builtin ("char", &infered);
 	rust_assert (ok);
       }
       break;
 
-      case HIR::Literal::LitType::BYTE: {
+    case HIR::Literal::LitType::BYTE:
+      {
 	auto ok = context->lookup_builtin ("u8", &infered);
 	rust_assert (ok);
       }
       break;
 
-      case HIR::Literal::LitType::STRING: {
+    case HIR::Literal::LitType::STRING:
+      {
 	TyTy::BaseType *base = nullptr;
 	auto ok = context->lookup_builtin ("str", &base);
 	rust_assert (ok);
@@ -246,7 +257,8 @@ TypeCheckBase::resolve_literal (const Analysis::NodeMapping &expr_mappings,
       }
       break;
 
-      case HIR::Literal::LitType::BYTE_STRING: {
+    case HIR::Literal::LitType::BYTE_STRING:
+      {
 	/* This is an arraytype of u8 reference (&[u8;size]). It isn't in
 	   UTF-8, but really just a byte array. Code to construct the array
 	   reference copied from ArrayElemsValues and ArrayType. */
@@ -280,10 +292,21 @@ TypeCheckBase::resolve_literal (const Analysis::NodeMapping &expr_mappings,
 					       crate_num),
 					     UNKNOWN_LOCAL_DEFID);
 
+	auto ctx = Compile::Context::get ();
+	tree capacity = Compile::HIRCompileBase::query_compile_const_expr (
+	  ctx, expected_ty, *literal_capacity);
+
+	TyTy::ConstType *capacity_expr
+	  = new TyTy::ConstType (TyTy::ConstType::ConstKind::Value, "",
+				 expected_ty, capacity, {},
+				 literal_capacity->get_locus (),
+				 literal_capacity->get_mappings ().get_hirid (),
+				 literal_capacity->get_mappings ().get_hirid (),
+				 {});
+
 	TyTy::ArrayType *array
 	  = new TyTy::ArrayType (array_mapping.get_hirid (), locus,
-				 *literal_capacity,
-				 TyTy::TyVar (u8->get_ref ()));
+				 capacity_expr, TyTy::TyVar (u8->get_ref ()));
 	context->insert_type (array_mapping, array);
 
 	infered = new TyTy::ReferenceType (expr_mappings.get_hirid (),
@@ -432,6 +455,7 @@ TypeCheckBase::parse_repr_options (const AST::AttrVec &attrs, location_t locus)
 
 void
 TypeCheckBase::resolve_generic_params (
+  const HIR::Item::ItemKind item_kind, location_t item_locus,
   const std::vector<std::unique_ptr<HIR::GenericParam>> &generic_params,
   std::vector<TyTy::SubstitutionParamMapping> &substitutions, bool is_foreign,
   ABI abi)
@@ -440,7 +464,8 @@ TypeCheckBase::resolve_generic_params (
     {
       switch (generic_param->get_kind ())
 	{
-	  case HIR::GenericParam::GenericKind::LIFETIME: {
+	case HIR::GenericParam::GenericKind::LIFETIME:
+	  {
 	    auto lifetime_param
 	      = static_cast<HIR::LifetimeParam &> (*generic_param);
 	    auto lifetime = lifetime_param.get_lifetime ();
@@ -449,7 +474,8 @@ TypeCheckBase::resolve_generic_params (
 	  }
 	  break;
 
-	  case HIR::GenericParam::GenericKind::CONST: {
+	case HIR::GenericParam::GenericKind::CONST:
+	  {
 	    if (is_foreign && abi != Rust::ABI::INTRINSIC)
 	      {
 		rust_error_at (generic_param->get_locus (), ErrorCode::E0044,
@@ -462,6 +488,27 @@ TypeCheckBase::resolve_generic_params (
 
 	    if (param.has_default_expression ())
 	      {
+		switch (item_kind)
+		  {
+		  case HIR::Item::ItemKind::Struct:
+		  case HIR::Item::ItemKind::Enum:
+		  case HIR::Item::ItemKind::TypeAlias:
+		  case HIR::Item::ItemKind::Trait:
+		  case HIR::Item::ItemKind::Union:
+		    break;
+
+		  default:
+		    {
+		      rich_location r (line_table, item_locus);
+		      r.add_fixit_remove (param.get_locus ());
+		      rust_error_at (
+			r,
+			"default values for const generic parameters are not "
+			"allowed here");
+		    }
+		    break;
+		  }
+
 		auto expr_type
 		  = TypeCheckExpr::Resolve (param.get_default_expression ());
 
@@ -471,14 +518,39 @@ TypeCheckBase::resolve_generic_params (
 				 expr_type,
 				 param.get_default_expression ().get_locus ()),
 			       param.get_locus ());
+
+		// fold the default value
+		auto ctx = Compile::Context::get ();
+		auto &expr = param.get_default_expression ();
+		tree default_value
+		  = Compile::HIRCompileBase::query_compile_const_expr (
+		    ctx, specified_type, expr);
+
+		TyTy::ConstType *default_const_decl
+		  = new TyTy::ConstType (TyTy::ConstType::ConstKind::Value,
+					 param.get_name (), specified_type,
+					 default_value, {}, param.get_locus (),
+					 expr.get_mappings ().get_hirid (),
+					 expr.get_mappings ().get_hirid (), {});
+
+		context->insert_type (expr.get_mappings (), default_const_decl);
 	      }
 
-	    context->insert_type (generic_param->get_mappings (),
-				  specified_type);
+	    TyTy::ConstType *const_decl
+	      = new TyTy::ConstType (TyTy::ConstType::ConstKind::Decl,
+				     param.get_name (), specified_type,
+				     error_mark_node, {}, param.get_locus (),
+				     param.get_mappings ().get_hirid (),
+				     param.get_mappings ().get_hirid (), {});
+
+	    context->insert_type (generic_param->get_mappings (), const_decl);
+	    TyTy::SubstitutionParamMapping p (*generic_param, const_decl);
+	    substitutions.push_back (p);
 	  }
 	  break;
 
-	  case HIR::GenericParam::GenericKind::TYPE: {
+	case HIR::GenericParam::GenericKind::TYPE:
+	  {
 	    if (is_foreign && abi != Rust::ABI::INTRINSIC)
 	      {
 		rust_error_at (generic_param->get_locus (), ErrorCode::E0044,
@@ -489,8 +561,7 @@ TypeCheckBase::resolve_generic_params (
 	      *generic_param, false /*resolve_trait_bounds*/);
 	    context->insert_type (generic_param->get_mappings (), param_type);
 
-	    auto &param = static_cast<HIR::TypeParam &> (*generic_param);
-	    TyTy::SubstitutionParamMapping p (param, param_type);
+	    TyTy::SubstitutionParamMapping p (*generic_param, param_type);
 	    substitutions.push_back (p);
 	  }
 	  break;
@@ -500,9 +571,16 @@ TypeCheckBase::resolve_generic_params (
   // now walk them to setup any specified type param bounds
   for (auto &subst : substitutions)
     {
-      auto pty = subst.get_param_ty ();
-      TypeResolveGenericParam::ApplyAnyTraitBounds (subst.get_generic_param (),
-						    pty);
+      auto &generic = subst.get_generic_param ();
+      if (generic.get_kind () != HIR::GenericParam::GenericKind::TYPE)
+	continue;
+
+      auto &type_param = static_cast<HIR::TypeParam &> (generic);
+      auto bpty = subst.get_param_ty ();
+      rust_assert (bpty->get_kind () == TyTy::TypeKind::PARAM);
+      auto pty = static_cast<TyTy::ParamType *> (bpty);
+
+      TypeResolveGenericParam::ApplyAnyTraitBounds (type_param, pty);
     }
 }
 
diff --git a/gcc/rust/typecheck/rust-hir-type-check-base.h b/gcc/rust/typecheck/rust-hir-type-check-base.h
index 580082a..6430089 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-base.h
+++ b/gcc/rust/typecheck/rust-hir-type-check-base.h
@@ -33,6 +33,7 @@ public:
   virtual ~TypeCheckBase () {}
 
   static void ResolveGenericParams (
+    const HIR::Item::ItemKind item_kind, location_t item_locus,
     const std::vector<std::unique_ptr<HIR::GenericParam>> &generic_params,
     std::vector<TyTy::SubstitutionParamMapping> &substitutions, bool is_foreign,
     ABI abi);
@@ -46,13 +47,13 @@ protected:
     HIR::TypePath &path,
     tl::optional<std::reference_wrapper<HIR::Type>> associated_self,
     BoundPolarity polarity = BoundPolarity::RegularBound,
-    bool is_qualified_type = false);
+    bool is_qualified_type = false, bool is_super_trait = false);
 
   bool check_for_unconstrained (
     const std::vector<TyTy::SubstitutionParamMapping> &params_to_constrain,
     const TyTy::SubstitutionArgumentMappings &constraint_a,
     const TyTy::SubstitutionArgumentMappings &constraint_b,
-    const TyTy::BaseType *reference);
+    TyTy::BaseType *reference);
 
   TyTy::BaseType *resolve_literal (const Analysis::NodeMapping &mappings,
 				   HIR::Literal &literal, location_t locus);
@@ -61,6 +62,7 @@ protected:
 						 location_t locus);
 
   void resolve_generic_params (
+    const HIR::Item::ItemKind item_kind, location_t item_locus,
     const std::vector<std::unique_ptr<HIR::GenericParam>> &generic_params,
     std::vector<TyTy::SubstitutionParamMapping> &substitutions,
     bool is_foreign = false, ABI abi = ABI::RUST);
@@ -69,7 +71,6 @@ protected:
 						 location_t locus);
 
   Analysis::Mappings &mappings;
-  Resolver *resolver;
   TypeCheckContext *context;
 };
 
diff --git a/gcc/rust/typecheck/rust-hir-type-check-enumitem.cc b/gcc/rust/typecheck/rust-hir-type-check-enumitem.cc
index c80a12f..23a8cca 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-enumitem.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-enumitem.cc
@@ -23,9 +23,6 @@
 #include "rust-type-util.h"
 #include "rust-immutable-name-resolution-context.h"
 
-// for flag_name_resolution_2_0
-#include "options.h"
-
 namespace Rust {
 namespace Resolver {
 
@@ -79,25 +76,13 @@ TypeCheckEnumItem::visit (HIR::EnumItem &item)
   rust_assert (ok);
   context->insert_type (mapping, isize);
 
-  tl::optional<CanonicalPath> canonical_path;
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.types.to_canonical_path (item.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path
-	= mappings.lookup_canonical_path (item.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path.has_value ());
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (item.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, item.get_locus ()};
+  RustIdent ident{canonical_path, item.get_locus ()};
   variant = new TyTy::VariantDef (item.get_mappings ().get_hirid (),
 				  item.get_mappings ().get_defid (),
 				  item.get_identifier ().as_string (), ident,
@@ -123,25 +108,13 @@ TypeCheckEnumItem::visit (HIR::EnumItemDiscriminant &item)
 	      TyTy::TyWithLocation (expected_ty),
 	      TyTy::TyWithLocation (capacity_type), item.get_locus ());
 
-  tl::optional<CanonicalPath> canonical_path;
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.types.to_canonical_path (item.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path
-	= mappings.lookup_canonical_path (item.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path.has_value ());
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (item.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, item.get_locus ()};
+  RustIdent ident{canonical_path, item.get_locus ()};
   variant
     = new TyTy::VariantDef (item.get_mappings ().get_hirid (),
 			    item.get_mappings ().get_defid (),
@@ -185,25 +158,13 @@ TypeCheckEnumItem::visit (HIR::EnumItemTuple &item)
   rust_assert (ok);
   context->insert_type (mapping, isize);
 
-  tl::optional<CanonicalPath> canonical_path;
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.types.to_canonical_path (item.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path
-	= mappings.lookup_canonical_path (item.get_mappings ().get_nodeid ());
-    }
-
-  rust_assert (canonical_path.has_value ());
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (item.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, item.get_locus ()};
+  RustIdent ident{canonical_path, item.get_locus ()};
   variant = new TyTy::VariantDef (item.get_mappings ().get_hirid (),
 				  item.get_mappings ().get_defid (),
 				  item.get_identifier ().as_string (), ident,
@@ -245,25 +206,13 @@ TypeCheckEnumItem::visit (HIR::EnumItemStruct &item)
   rust_assert (ok);
   context->insert_type (mapping, isize);
 
-  tl::optional<CanonicalPath> canonical_path;
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.types.to_canonical_path (item.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path
-	= mappings.lookup_canonical_path (item.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path.has_value ());
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (item.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, item.get_locus ()};
+  RustIdent ident{canonical_path, item.get_locus ()};
   variant = new TyTy::VariantDef (item.get_mappings ().get_hirid (),
 				  item.get_mappings ().get_defid (),
 				  item.get_identifier ().as_string (), ident,
diff --git a/gcc/rust/typecheck/rust-hir-type-check-expr.cc b/gcc/rust/typecheck/rust-hir-type-check-expr.cc
index cbf529a7..438200b 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-expr.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-expr.cc
@@ -17,6 +17,7 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "optional.h"
+#include "rust-common.h"
 #include "rust-hir-expr.h"
 #include "rust-system.h"
 #include "rust-tyty-call.h"
@@ -31,9 +32,9 @@
 #include "rust-hir-type-check-item.h"
 #include "rust-type-util.h"
 #include "rust-immutable-name-resolution-context.h"
-
-// for flag_name_resolution_2_0
-#include "options.h"
+#include "rust-compile-base.h"
+#include "rust-tyty-util.h"
+#include "tree.h"
 
 namespace Rust {
 namespace Resolver {
@@ -58,6 +59,19 @@ TypeCheckExpr::Resolve (HIR::Expr &expr)
   return resolver.infered;
 }
 
+TyTy::BaseType *
+TypeCheckExpr::ResolveOpOverload (LangItem::Kind lang_item_type,
+				  HIR::OperatorExprMeta expr,
+				  TyTy::BaseType *lhs, TyTy::BaseType *rhs,
+				  HIR::PathIdentSegment specified_segment)
+{
+  TypeCheckExpr resolver;
+
+  resolver.resolve_operator_overload (lang_item_type, expr, lhs, rhs,
+				      specified_segment);
+  return resolver.infered;
+}
+
 void
 TypeCheckExpr::visit (HIR::TupleIndexExpr &expr)
 {
@@ -373,7 +387,8 @@ TypeCheckExpr::visit (HIR::ArithmeticOrLogicalExpr &expr)
   switch (expr.get_expr_type ())
     {
     case ArithmeticOrLogicalOperator::LEFT_SHIFT:
-      case ArithmeticOrLogicalOperator::RIGHT_SHIFT: {
+    case ArithmeticOrLogicalOperator::RIGHT_SHIFT:
+      {
 	TyTy::TyWithLocation from (rhs, expr.get_rhs ().get_locus ());
 	TyTy::TyWithLocation to (lhs, expr.get_lhs ().get_locus ());
 	infered = cast_site (expr.get_mappings ().get_hirid (), from, to,
@@ -381,7 +396,8 @@ TypeCheckExpr::visit (HIR::ArithmeticOrLogicalExpr &expr)
       }
       break;
 
-      default: {
+    default:
+      {
 	infered = unify_site (
 	  expr.get_mappings ().get_hirid (),
 	  TyTy::TyWithLocation (lhs, expr.get_lhs ().get_locus ()),
@@ -470,7 +486,8 @@ TypeCheckExpr::visit (HIR::NegationExpr &expr)
   // https://doc.rust-lang.org/reference/expressions/operator-expr.html#negation-operators
   switch (expr.get_expr_type ())
     {
-      case NegationOperator::NEGATE: {
+    case NegationOperator::NEGATE:
+      {
 	bool valid
 	  = (negated_expr_ty->get_kind () == TyTy::TypeKind::INT)
 	    || (negated_expr_ty->get_kind () == TyTy::TypeKind::UINT)
@@ -492,7 +509,8 @@ TypeCheckExpr::visit (HIR::NegationExpr &expr)
       }
       break;
 
-      case NegationOperator::NOT: {
+    case NegationOperator::NOT:
+      {
 	bool valid
 	  = (negated_expr_ty->get_kind () == TyTy::TypeKind::BOOL)
 	    || (negated_expr_ty->get_kind () == TyTy::TypeKind::INT)
@@ -643,6 +661,33 @@ TypeCheckExpr::visit (HIR::BlockExpr &expr)
 }
 
 void
+TypeCheckExpr::visit (HIR::AnonConst &expr)
+{
+  if (!expr.is_deferred ())
+    {
+      infered = TypeCheckExpr::Resolve (expr.get_inner_expr ());
+      return;
+    }
+
+  auto locus = expr.get_locus ();
+  auto infer_ty_var = TyTy::TyVar::get_implicit_infer_var (locus);
+
+  HirId next = mappings.get_next_hir_id ();
+  infered = new TyTy::ConstType (TyTy::ConstType::ConstKind::Infer, "",
+				 infer_ty_var.get_tyty (), error_mark_node, {},
+				 locus, next, next, {});
+
+  context->insert_implicit_type (infered->get_ref (), infered);
+  mappings.insert_location (infered->get_ref (), locus);
+}
+
+void
+TypeCheckExpr::visit (HIR::ConstBlock &expr)
+{
+  infered = TypeCheckExpr::Resolve (expr.get_const_expr ());
+}
+
+void
 TypeCheckExpr::visit (HIR::RangeFromToExpr &expr)
 {
   auto lang_item_type = LangItem::Kind::RANGE;
@@ -790,38 +835,45 @@ typecheck_inline_asm_operand (HIR::InlineAsm &expr)
     {
       switch (operand.get_register_type ())
 	{
-	  case RegisterType::In: {
+	case RegisterType::In:
+	  {
 	    auto in = operand.get_in ();
 	    TypeCheckExpr::Resolve (*in.expr);
 	    break;
 	  }
-	  case RegisterType::Out: {
+	case RegisterType::Out:
+	  {
 	    auto out = operand.get_out ();
 	    TypeCheckExpr::Resolve (*out.expr);
 	    break;
 	  }
-	  case RegisterType::InOut: {
+	case RegisterType::InOut:
+	  {
 	    auto in_out = operand.get_in_out ();
 	    TypeCheckExpr::Resolve (*in_out.expr);
 	    break;
 	  }
-	  case RegisterType::SplitInOut: {
+	case RegisterType::SplitInOut:
+	  {
 	    auto split_in_out = operand.get_split_in_out ();
 	    TypeCheckExpr::Resolve (*split_in_out.in_expr);
 	    TypeCheckExpr::Resolve (*split_in_out.out_expr);
 	    break;
 	  }
-	  case RegisterType::Const: {
+	case RegisterType::Const:
+	  {
 	    auto anon_const = operand.get_const ().anon_const;
-	    TypeCheckExpr::Resolve (*anon_const.expr);
+	    TypeCheckExpr::Resolve (anon_const.get_inner_expr ());
 	    break;
 	  }
-	  case RegisterType::Sym: {
+	case RegisterType::Sym:
+	  {
 	    auto sym = operand.get_sym ();
 	    TypeCheckExpr::Resolve (*sym.expr);
 	    break;
 	  }
-	  case RegisterType::Label: {
+	case RegisterType::Label:
+	  {
 	    auto label = operand.get_label ();
 	    TypeCheckExpr::Resolve (*label.expr);
 	    break;
@@ -837,7 +889,7 @@ TypeCheckExpr::visit (HIR::InlineAsm &expr)
   // NOTE: Hoise out if we have noreturn as an option
   // to return a never type
   // TODO : new keyword for memory seems sooooo shaky
-  if (expr.options.count (AST::InlineAsmOption::NORETURN) == 1)
+  if (expr.options.count (AST::InlineAsm::Option::NORETURN) == 1)
     infered = new TyTy::NeverType (expr.get_mappings ().get_hirid ());
   else
     infered = TyTy::TupleType::get_unit_type ();
@@ -857,6 +909,19 @@ TypeCheckExpr::visit (HIR::LlvmInlineAsm &expr)
 }
 
 void
+TypeCheckExpr::visit (HIR::OffsetOf &expr)
+{
+  TypeCheckType::Resolve (expr.get_type ());
+
+  // FIXME: Does offset_of always return a usize?
+  TyTy::BaseType *size_ty;
+  bool ok = context->lookup_builtin ("usize", &size_ty);
+  rust_assert (ok);
+
+  infered = size_ty;
+}
+
+void
 TypeCheckExpr::visit (HIR::RangeFullExpr &expr)
 {
   auto lang_item_type = LangItem::Kind::RANGE_FULL;
@@ -960,7 +1025,10 @@ TypeCheckExpr::visit (HIR::ArrayIndexExpr &expr)
   bool ok = context->lookup_builtin ("usize", &size_ty);
   rust_assert (ok);
 
-  bool maybe_simple_array_access = index_expr_ty->can_eq (size_ty, false);
+  bool maybe_simple_array_access
+    = types_compatable (TyTy::TyWithLocation (index_expr_ty),
+			TyTy::TyWithLocation (size_ty), expr.get_locus (),
+			false);
   if (maybe_simple_array_access
       && direct_array_expr_ty->get_kind () == TyTy::TypeKind::ARRAY)
     {
@@ -996,8 +1064,7 @@ TypeCheckExpr::visit (HIR::ArrayIndexExpr &expr)
   rich_location r (line_table, expr.get_locus ());
   r.add_range (expr.get_array_expr ().get_locus ());
   r.add_range (expr.get_index_expr ().get_locus ());
-  rust_error_at (r, ErrorCode::E0277,
-		 "the type %qs cannot be indexed by %qs",
+  rust_error_at (r, ErrorCode::E0277, "the type %qs cannot be indexed by %qs",
 		 array_expr_ty->get_name ().c_str (),
 		 index_expr_ty->get_name ().c_str ());
 }
@@ -1007,35 +1074,40 @@ TypeCheckExpr::visit (HIR::ArrayExpr &expr)
 {
   auto &elements = expr.get_internal_elements ();
 
+  TyTy::BaseType *expected_ty = nullptr;
+  bool ok = context->lookup_builtin ("usize", &expected_ty);
+  rust_assert (ok);
+
   HIR::Expr *capacity_expr = nullptr;
   TyTy::BaseType *element_type = nullptr;
+  TyTy::BaseType *capacity_type = nullptr;
   switch (elements.get_array_expr_type ())
     {
-      case HIR::ArrayElems::ArrayExprType::COPIED: {
+    case HIR::ArrayElems::ArrayExprType::COPIED:
+      {
 	HIR::ArrayElemsCopied &elems
 	  = static_cast<HIR::ArrayElemsCopied &> (elements);
 	element_type = TypeCheckExpr::Resolve (elems.get_elem_to_copy ());
 
-	auto capacity_type
+	auto capacity_expr_ty
 	  = TypeCheckExpr::Resolve (elems.get_num_copies_expr ());
 
-	TyTy::BaseType *expected_ty = nullptr;
-	bool ok = context->lookup_builtin ("usize", &expected_ty);
-	rust_assert (ok);
 	context->insert_type (elems.get_num_copies_expr ().get_mappings (),
 			      expected_ty);
 
-	unify_site (expr.get_mappings ().get_hirid (),
-		    TyTy::TyWithLocation (expected_ty),
-		    TyTy::TyWithLocation (
-		      capacity_type, elems.get_num_copies_expr ().get_locus ()),
-		    expr.get_locus ());
+	unify_site (
+	  expr.get_mappings ().get_hirid (), TyTy::TyWithLocation (expected_ty),
+	  TyTy::TyWithLocation (capacity_expr_ty,
+				elems.get_num_copies_expr ().get_locus ()),
+	  expr.get_locus ());
 
 	capacity_expr = &elems.get_num_copies_expr ();
+	capacity_type = expected_ty;
       }
       break;
 
-      case HIR::ArrayElems::ArrayExprType::VALUES: {
+    case HIR::ArrayElems::ArrayExprType::VALUES:
+      {
 	HIR::ArrayElemsValues &elems
 	  = static_cast<HIR::ArrayElemsValues &> (elements);
 
@@ -1068,17 +1140,26 @@ TypeCheckExpr::visit (HIR::ArrayExpr &expr)
 					      UNDEF_LOCATION, {});
 
 	// mark the type for this implicit node
-	TyTy::BaseType *expected_ty = nullptr;
-	bool ok = context->lookup_builtin ("usize", &expected_ty);
-	rust_assert (ok);
 	context->insert_type (mapping, expected_ty);
+	capacity_type = expected_ty;
       }
       break;
     }
 
-  infered = new TyTy::ArrayType (expr.get_mappings ().get_hirid (),
-				 expr.get_locus (), *capacity_expr,
-				 TyTy::TyVar (element_type->get_ref ()));
+  rust_assert (capacity_expr);
+  rust_assert (capacity_type);
+  auto ctx = Compile::Context::get ();
+  tree capacity_value
+    = Compile::HIRCompileBase::query_compile_const_expr (ctx, capacity_type,
+							 *capacity_expr);
+  HirId size_id = capacity_expr->get_mappings ().get_hirid ();
+  TyTy::ConstType *const_type
+    = new TyTy::ConstType (TyTy::ConstType::ConstKind::Value, "", expected_ty,
+			   capacity_value, {}, capacity_expr->get_locus (),
+			   size_id, size_id);
+  infered
+    = new TyTy::ArrayType (expr.get_mappings ().get_hirid (), expr.get_locus (),
+			   const_type, TyTy::TyVar (element_type->get_ref ()));
 }
 
 // empty struct
@@ -1408,26 +1489,11 @@ TypeCheckExpr::visit (HIR::MethodCallExpr &expr)
   // store the expected fntype
   context->insert_type (expr.get_method_name ().get_mappings (), lookup);
 
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
-	Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
+  auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
+    Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
 
-      nr_ctx.map_usage (Resolver2_0::Usage (expr.get_mappings ().get_nodeid ()),
-			Resolver2_0::Definition (resolved_node_id));
-    }
-  // set up the resolved name on the path
-  else if (resolver->get_name_scope ().decl_was_declared_here (
-	     resolved_node_id))
-    {
-      resolver->insert_resolved_name (expr.get_mappings ().get_nodeid (),
-				      resolved_node_id);
-    }
-  else
-    {
-      resolver->insert_resolved_misc (expr.get_mappings ().get_nodeid (),
-				      resolved_node_id);
-    }
+  nr_ctx.map_usage (Resolver2_0::Usage (expr.get_mappings ().get_nodeid ()),
+		    Resolver2_0::Definition (resolved_node_id));
 
   // return the result of the function back
   infered = function_ret_tyty;
@@ -1702,16 +1768,22 @@ TypeCheckExpr::visit (HIR::MatchExpr &expr)
 void
 TypeCheckExpr::visit (HIR::ClosureExpr &expr)
 {
-  TypeCheckContextItem current_context = context->peek_context ();
-  TyTy::FnType *current_context_fndecl = current_context.get_context_type ();
-
+  std::vector<TyTy::SubstitutionParamMapping> subst_refs;
   HirId ref = expr.get_mappings ().get_hirid ();
   DefId id = expr.get_mappings ().get_defid ();
-  RustIdent ident{current_context_fndecl->get_ident ().path, expr.get_locus ()};
+  RustIdent ident{CanonicalPath::create_empty (), expr.get_locus ()};
 
-  // get from parent context
-  std::vector<TyTy::SubstitutionParamMapping> subst_refs
-    = current_context_fndecl->clone_substs ();
+  if (context->have_function_context ())
+    {
+      TypeCheckContextItem current_context = context->peek_context ();
+      TyTy::FnType *current_context_fndecl
+	= current_context.get_context_type ();
+
+      ident = RustIdent{current_context_fndecl->get_ident ().path,
+			expr.get_locus ()};
+
+      subst_refs = current_context_fndecl->clone_substs ();
+    }
 
   std::vector<TyTy::TyVar> parameter_types;
   for (auto &p : expr.get_params ())
@@ -1763,19 +1835,12 @@ TypeCheckExpr::visit (HIR::ClosureExpr &expr)
   // Resolve closure captures
 
   std::set<NodeId> captures;
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
-	Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
+  auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
+    Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
 
-      if (auto opt_cap = nr_ctx.mappings.lookup_captures (closure_node_id))
-	for (auto cap : opt_cap.value ())
-	  captures.insert (cap);
-    }
-  else
-    {
-      captures = resolver->get_captures (closure_node_id);
-    }
+  if (auto opt_cap = nr_ctx.mappings.lookup_captures (closure_node_id))
+    for (auto cap : opt_cap.value ())
+      captures.insert (cap);
 
   infered = new TyTy::ClosureType (ref, id, ident, closure_args, result_type,
 				   subst_refs, captures);
@@ -1826,7 +1891,7 @@ TypeCheckExpr::visit (HIR::ClosureExpr &expr)
   args.get_type_args ().push_back (std::unique_ptr<HIR::Type> (implicit_tuple));
 
   // apply the arguments
-  predicate.apply_generic_arguments (&args, false);
+  predicate.apply_generic_arguments (&args, false, false);
 
   // finally inherit the trait bound
   infered->inherit_bounds ({predicate});
@@ -1845,7 +1910,16 @@ TypeCheckExpr::resolve_operator_overload (
   // probe for the lang-item
   if (!lang_item_defined)
     return false;
+
   DefId &respective_lang_item_id = lang_item_defined.value ();
+  auto def_lookup = mappings.lookup_defid (respective_lang_item_id);
+  rust_assert (def_lookup.has_value ());
+
+  HIR::Item *def_item = def_lookup.value ();
+  rust_assert (def_item->get_item_kind () == HIR::Item::ItemKind::Trait);
+  HIR::Trait &trait = *static_cast<HIR::Trait *> (def_item);
+  TraitReference *defid_trait_reference = TraitResolver::Resolve (trait);
+  rust_assert (!defid_trait_reference->is_error ());
 
   // we might be in a static or const context and unknown is fine
   TypeCheckContextItem current_context = TypeCheckContextItem::get_error ();
@@ -1889,15 +1963,49 @@ TypeCheckExpr::resolve_operator_overload (
 
   if (selected_candidates.size () > 1)
     {
-      // mutliple candidates
-      rich_location r (line_table, expr.get_locus ());
-      for (auto &c : resolved_candidates)
-	r.add_range (c.candidate.locus);
+      auto infer
+	= TyTy::TyVar::get_implicit_infer_var (expr.get_locus ()).get_tyty ();
+      auto trait_subst = defid_trait_reference->get_trait_substs ();
+      rust_assert (trait_subst.size () > 0);
 
-      rust_error_at (
-	r, "multiple candidates found for possible operator overload");
+      TyTy::TypeBoundPredicate pred (respective_lang_item_id, trait_subst,
+				     BoundPolarity::RegularBound,
+				     expr.get_locus ());
 
-      return false;
+      std::vector<TyTy::SubstitutionArg> mappings;
+      auto &self_param_mapping = trait_subst[0];
+      mappings.push_back (TyTy::SubstitutionArg (&self_param_mapping, lhs));
+
+      if (rhs != nullptr)
+	{
+	  rust_assert (trait_subst.size () == 2);
+	  auto &rhs_param_mapping = trait_subst[1];
+	  mappings.push_back (TyTy::SubstitutionArg (&rhs_param_mapping, lhs));
+	}
+
+      std::map<std::string, TyTy::BaseType *> binding_args;
+      binding_args["Output"] = infer;
+
+      TyTy::SubstitutionArgumentMappings arg_mappings (mappings, binding_args,
+						       TyTy::RegionParamList (
+							 trait_subst.size ()),
+						       expr.get_locus ());
+      pred.apply_argument_mappings (arg_mappings, false);
+
+      infer->inherit_bounds ({pred});
+      DeferredOpOverload defer (expr.get_mappings ().get_hirid (),
+				lang_item_type, specified_segment, pred, expr);
+      context->insert_deferred_operator_overload (std::move (defer));
+
+      if (rhs != nullptr)
+	lhs = unify_site (expr.get_mappings ().get_hirid (),
+			  TyTy::TyWithLocation (lhs),
+			  TyTy::TyWithLocation (rhs), expr.get_locus ());
+
+      infered = unify_site (expr.get_mappings ().get_hirid (),
+			    TyTy::TyWithLocation (lhs),
+			    TyTy::TyWithLocation (infer), expr.get_locus ());
+      return true;
     }
 
   // Get the adjusted self
@@ -2040,19 +2148,11 @@ TypeCheckExpr::resolve_operator_overload (
   context->insert_operator_overload (expr.get_mappings ().get_hirid (), type);
 
   // set up the resolved name on the path
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
-	Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
+  auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
+    Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
 
-      nr_ctx.map_usage (Resolver2_0::Usage (expr.get_mappings ().get_nodeid ()),
-			Resolver2_0::Definition (resolved_node_id));
-    }
-  else
-    {
-      resolver->insert_resolved_name (expr.get_mappings ().get_nodeid (),
-				      resolved_node_id);
-    }
+  nr_ctx.map_usage (Resolver2_0::Usage (expr.get_mappings ().get_nodeid ()),
+		    Resolver2_0::Definition (resolved_node_id));
 
   // return the result of the function back
   infered = function_ret_tyty;
@@ -2062,16 +2162,13 @@ TypeCheckExpr::resolve_operator_overload (
 
 HIR::PathIdentSegment
 TypeCheckExpr::resolve_possible_fn_trait_call_method_name (
-  TyTy::BaseType &receiver, TyTy::TypeBoundPredicate *associated_predicate)
+  const TyTy::BaseType &receiver,
+  TyTy::TypeBoundPredicate *associated_predicate)
 {
-  // Question
-  // do we need to probe possible bounds here? I think not, i think when we
-  // support Fn traits they are explicitly specified
-
   // FIXME
   // the logic to map the FnTrait to their respective call trait-item is
   // duplicated over in the backend/rust-compile-expr.cc
-  for (auto &bound : receiver.get_specified_bounds ())
+  for (const auto &bound : receiver.get_specified_bounds ())
     {
       bool found_fn = bound.get_name ().compare ("Fn") == 0;
       bool found_fn_mut = bound.get_name ().compare ("FnMut") == 0;
@@ -2094,6 +2191,34 @@ TypeCheckExpr::resolve_possible_fn_trait_call_method_name (
 	}
     }
 
+  if (receiver.is<TyTy::ReferenceType> ())
+    {
+      const auto &ref = static_cast<const TyTy::ReferenceType &> (receiver);
+      const auto &underlying = *ref.get_base ();
+      for (const auto &bound : underlying.get_specified_bounds ())
+	{
+	  bool found_fn = bound.get_name ().compare ("Fn") == 0;
+	  bool found_fn_mut = bound.get_name ().compare ("FnMut") == 0;
+	  bool found_fn_once = bound.get_name ().compare ("FnOnce") == 0;
+
+	  if (found_fn)
+	    {
+	      *associated_predicate = bound;
+	      return HIR::PathIdentSegment ("call");
+	    }
+	  else if (found_fn_mut)
+	    {
+	      *associated_predicate = bound;
+	      return HIR::PathIdentSegment ("call_mut");
+	    }
+	  else if (found_fn_once)
+	    {
+	      *associated_predicate = bound;
+	      return HIR::PathIdentSegment ("call_once");
+	    }
+	}
+    }
+
   // nothing
   *associated_predicate = TyTy::TypeBoundPredicate::error ();
   return HIR::PathIdentSegment ("");
@@ -2220,32 +2345,15 @@ TypeCheckExpr::resolve_fn_trait_call (HIR::CallExpr &expr,
   context->insert_operator_overload (expr.get_mappings ().get_hirid (), fn);
 
   // set up the resolved name on the path
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
-	Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
+  auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
+    Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
 
-      auto existing = nr_ctx.lookup (expr.get_mappings ().get_nodeid ());
-      if (existing)
-	rust_assert (*existing == resolved_node_id);
-      else
-	nr_ctx.map_usage (Resolver2_0::Usage (
-			    expr.get_mappings ().get_nodeid ()),
-			  Resolver2_0::Definition (resolved_node_id));
-    }
+  auto existing = nr_ctx.lookup (expr.get_mappings ().get_nodeid ());
+  if (existing)
+    rust_assert (*existing == resolved_node_id);
   else
-    {
-      NodeId existing = UNKNOWN_NODEID;
-      bool ok
-	= resolver->lookup_resolved_name (expr.get_mappings ().get_nodeid (),
-					  &existing);
-
-      if (ok)
-	rust_assert (existing == resolved_node_id);
-      else
-	resolver->insert_resolved_name (expr.get_mappings ().get_nodeid (),
-					resolved_node_id);
-    }
+    nr_ctx.map_usage (Resolver2_0::Usage (expr.get_mappings ().get_nodeid ()),
+		      Resolver2_0::Definition (resolved_node_id));
 
   // return the result of the function back
   *result = function_ret_tyty;
diff --git a/gcc/rust/typecheck/rust-hir-type-check-expr.h b/gcc/rust/typecheck/rust-hir-type-check-expr.h
index 79121b3..0343922 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-expr.h
+++ b/gcc/rust/typecheck/rust-hir-type-check-expr.h
@@ -19,6 +19,7 @@
 #ifndef RUST_HIR_TYPE_CHECK_EXPR
 #define RUST_HIR_TYPE_CHECK_EXPR
 
+#include "rust-hir-expr.h"
 #include "rust-hir-type-check-base.h"
 #include "rust-hir-visitor.h"
 #include "rust-tyty.h"
@@ -31,6 +32,11 @@ class TypeCheckExpr : private TypeCheckBase, private HIR::HIRExpressionVisitor
 public:
   static TyTy::BaseType *Resolve (HIR::Expr &expr);
 
+  static TyTy::BaseType *
+  ResolveOpOverload (LangItem::Kind lang_item_type, HIR::OperatorExprMeta expr,
+		     TyTy::BaseType *lhs, TyTy::BaseType *rhs,
+		     HIR::PathIdentSegment specified_segment);
+
   void visit (HIR::TupleIndexExpr &expr) override;
   void visit (HIR::TupleExpr &expr) override;
   void visit (HIR::ReturnExpr &expr) override;
@@ -46,6 +52,8 @@ public:
   void visit (HIR::IfExpr &expr) override;
   void visit (HIR::IfExprConseqElse &expr) override;
   void visit (HIR::BlockExpr &expr) override;
+  void visit (HIR::AnonConst &expr) override;
+  void visit (HIR::ConstBlock &expr) override;
   void visit (HIR::UnsafeBlockExpr &expr) override;
   void visit (HIR::ArrayIndexExpr &expr) override;
   void visit (HIR::ArrayExpr &expr) override;
@@ -71,6 +79,7 @@ public:
   void visit (HIR::ClosureExpr &expr) override;
   void visit (HIR::InlineAsm &expr) override;
   void visit (HIR::LlvmInlineAsm &expr) override;
+  void visit (HIR::OffsetOf &expr) override;
 
   // TODO
   void visit (HIR::ErrorPropagationExpr &) override {}
@@ -107,7 +116,8 @@ protected:
 			      TyTy::BaseType **result);
 
   HIR::PathIdentSegment resolve_possible_fn_trait_call_method_name (
-    TyTy::BaseType &receiver, TyTy::TypeBoundPredicate *associated_predicate);
+    const TyTy::BaseType &receiver,
+    TyTy::TypeBoundPredicate *associated_predicate);
 
 private:
   TypeCheckExpr ();
diff --git a/gcc/rust/typecheck/rust-hir-type-check-implitem.cc b/gcc/rust/typecheck/rust-hir-type-check-implitem.cc
index bc7f6dc..c8544a1 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-implitem.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-implitem.cc
@@ -28,9 +28,6 @@
 #include "rust-tyty.h"
 #include "rust-immutable-name-resolution-context.h"
 
-// for flag_name_resolution_2_0
-#include "options.h"
-
 namespace Rust {
 namespace Resolver {
 
@@ -73,7 +70,9 @@ TypeCheckTopLevelExternItem::visit (HIR::ExternalFunctionItem &function)
   std::vector<TyTy::SubstitutionParamMapping> substitutions;
   if (function.has_generics ())
     {
-      resolve_generic_params (function.get_generic_params (), substitutions,
+      resolve_generic_params (HIR::Item::ItemKind::Function,
+			      function.get_locus (),
+			      function.get_generic_params (), substitutions,
 			      true /*is_foreign*/, parent.get_abi ());
     }
 
@@ -203,7 +202,9 @@ TypeCheckImplItem::visit (HIR::Function &function)
   auto binder_pin = context->push_lifetime_binder ();
 
   if (function.has_generics ())
-    resolve_generic_params (function.get_generic_params (), substitutions);
+    resolve_generic_params (HIR::Item::ItemKind::Function,
+			    function.get_locus (),
+			    function.get_generic_params (), substitutions);
 
   TyTy::RegionConstraints region_constraints;
   for (auto &where_clause_item : function.get_where_clause ().get_items ())
@@ -266,7 +267,8 @@ TypeCheckImplItem::visit (HIR::Function &function)
 	      self_type = self->clone ();
 	      break;
 
-	      case HIR::SelfParam::IMM_REF: {
+	    case HIR::SelfParam::IMM_REF:
+	      {
 		tl::optional<TyTy::Region> region;
 		if (self_param.has_lifetime ())
 		  {
@@ -290,7 +292,8 @@ TypeCheckImplItem::visit (HIR::Function &function)
 	      }
 	      break;
 
-	      case HIR::SelfParam::MUT_REF: {
+	    case HIR::SelfParam::MUT_REF:
+	      {
 		tl::optional<TyTy::Region> region;
 		if (self_param.has_lifetime ())
 		  {
@@ -336,25 +339,13 @@ TypeCheckImplItem::visit (HIR::Function &function)
 	TyTy::FnParam (param.get_param_name ().clone_pattern (), param_tyty));
     }
 
-  tl::optional<CanonicalPath> canonical_path;
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path = nr_ctx.values.to_canonical_path (
-	function.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path = mappings.lookup_canonical_path (
-	function.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path.has_value ());
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (function.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, function.get_locus ()};
+  RustIdent ident{canonical_path, function.get_locus ()};
   auto fnType = new TyTy::FnType (
     function.get_mappings ().get_hirid (),
     function.get_mappings ().get_defid (),
@@ -410,7 +401,8 @@ TypeCheckImplItem::visit (HIR::TypeAlias &alias)
   auto binder_pin = context->push_lifetime_binder ();
 
   if (alias.has_generics ())
-    resolve_generic_params (alias.get_generic_params (), substitutions);
+    resolve_generic_params (HIR::Item::ItemKind::TypeAlias, alias.get_locus (),
+			    alias.get_generic_params (), substitutions);
 
   TyTy::BaseType *actual_type
     = TypeCheckType::Resolve (alias.get_type_aliased ());
@@ -494,10 +486,9 @@ TypeCheckImplItemWithTrait::visit (HIR::ConstantItem &constant)
       rich_location r (line_table, constant.get_locus ());
       r.add_range (resolved_trait_item.get_locus ());
 
-      rust_error_at (
-	r, "constant %qs has an incompatible type for trait %qs",
-	constant.get_identifier ().as_string ().c_str (),
-	trait_reference.get_name ().c_str ());
+      rust_error_at (r, "constant %qs has an incompatible type for trait %qs",
+		     constant.get_identifier ().as_string ().c_str (),
+		     trait_reference.get_name ().c_str ());
     }
 }
 
@@ -545,10 +536,9 @@ TypeCheckImplItemWithTrait::visit (HIR::TypeAlias &type)
       rich_location r (line_table, type.get_locus ());
       r.add_range (resolved_trait_item.get_locus ());
 
-      rust_error_at (
-	r, "type alias %qs has an incompatible type for trait %qs",
-	type.get_new_type_name ().as_string ().c_str (),
-	trait_reference.get_name ().c_str ());
+      rust_error_at (r, "type alias %qs has an incompatible type for trait %qs",
+		     type.get_new_type_name ().as_string ().c_str (),
+		     trait_reference.get_name ().c_str ());
     }
 
   // its actually a projection, since we need a way to actually bind the
diff --git a/gcc/rust/typecheck/rust-hir-type-check-item.cc b/gcc/rust/typecheck/rust-hir-type-check-item.cc
index aaa04af..3ba607b 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-item.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-item.cc
@@ -20,6 +20,7 @@
 #include "optional.h"
 #include "rust-canonical-path.h"
 #include "rust-diagnostics.h"
+#include "rust-hir-item.h"
 #include "rust-hir-type-check-enumitem.h"
 #include "rust-hir-type-check-implitem.h"
 #include "rust-hir-type-check-type.h"
@@ -33,9 +34,6 @@
 #include "rust-type-util.h"
 #include "rust-tyty-variance-analysis.h"
 
-// for flag_name_resolution_2_0
-#include "options.h"
-
 namespace Rust {
 namespace Resolver {
 
@@ -121,8 +119,8 @@ TypeCheckItem::ResolveImplBlockSelfWithInference (
 	}
       else
 	{
-	  TyTy::ParamType *param = p.get_param_ty ();
-	  TyTy::BaseType *resolved = param->destructure ();
+	  auto param = p.get_param_ty ();
+	  auto resolved = param->destructure ();
 	  args.push_back (TyTy::SubstitutionArg (&p, resolved));
 	}
     }
@@ -170,7 +168,9 @@ TypeCheckItem::visit (HIR::TupleStruct &struct_decl)
 
   std::vector<TyTy::SubstitutionParamMapping> substitutions;
   if (struct_decl.has_generics ())
-    resolve_generic_params (struct_decl.get_generic_params (), substitutions);
+    resolve_generic_params (HIR::Item::ItemKind::Struct,
+			    struct_decl.get_locus (),
+			    struct_decl.get_generic_params (), substitutions);
 
   TyTy::RegionConstraints region_constraints;
   for (auto &where_clause_item : struct_decl.get_where_clause ().get_items ())
@@ -195,25 +195,11 @@ TypeCheckItem::visit (HIR::TupleStruct &struct_decl)
 
   // get the path
 
-  auto path = CanonicalPath::create_empty ();
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  // FIXME: HACK: ARTHUR: Disgusting
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      path = nr_ctx.values
-	       .to_canonical_path (struct_decl.get_mappings ().get_nodeid ())
-	       .value ();
-    }
-  else
-    {
-      path
-	= mappings
-	    .lookup_canonical_path (struct_decl.get_mappings ().get_nodeid ())
-	    .value ();
-    }
+  CanonicalPath path
+    = nr_ctx.to_canonical_path (struct_decl.get_mappings ().get_nodeid ());
 
   RustIdent ident{path, struct_decl.get_locus ()};
 
@@ -255,7 +241,9 @@ TypeCheckItem::visit (HIR::StructStruct &struct_decl)
 
   std::vector<TyTy::SubstitutionParamMapping> substitutions;
   if (struct_decl.has_generics ())
-    resolve_generic_params (struct_decl.get_generic_params (), substitutions);
+    resolve_generic_params (HIR::Item::ItemKind::Struct,
+			    struct_decl.get_locus (),
+			    struct_decl.get_generic_params (), substitutions);
 
   TyTy::RegionConstraints region_constraints;
   for (auto &where_clause_item : struct_decl.get_where_clause ().get_items ())
@@ -276,27 +264,11 @@ TypeCheckItem::visit (HIR::StructStruct &struct_decl)
       context->insert_type (field.get_mappings (), ty_field->get_field_type ());
     }
 
-  auto path = CanonicalPath::create_empty ();
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  // FIXME: HACK: ARTHUR: Disgusting
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-      auto canonical_path = nr_ctx.types.to_canonical_path (
-	struct_decl.get_mappings ().get_nodeid ());
-
-      if (!canonical_path.has_value ())
-	rust_unreachable ();
-      path = canonical_path.value ();
-    }
-  else
-    {
-      path
-	= mappings
-	    .lookup_canonical_path (struct_decl.get_mappings ().get_nodeid ())
-	    .value ();
-    }
+  CanonicalPath path
+    = nr_ctx.to_canonical_path (struct_decl.get_mappings ().get_nodeid ());
 
   RustIdent ident{path, struct_decl.get_locus ()};
 
@@ -337,7 +309,8 @@ TypeCheckItem::visit (HIR::Enum &enum_decl)
   auto lifetime_pin = context->push_clean_lifetime_resolver ();
   std::vector<TyTy::SubstitutionParamMapping> substitutions;
   if (enum_decl.has_generics ())
-    resolve_generic_params (enum_decl.get_generic_params (), substitutions);
+    resolve_generic_params (HIR::Item::ItemKind::Enum, enum_decl.get_locus (),
+			    enum_decl.get_generic_params (), substitutions);
 
   // Process #[repr(X)] attribute, if any
   const AST::AttrVec &attrs = enum_decl.get_outer_attrs ();
@@ -367,26 +340,14 @@ TypeCheckItem::visit (HIR::Enum &enum_decl)
 	}
     }
 
-  // get the path
-  tl::optional<CanonicalPath> canonical_path;
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path = nr_ctx.types.to_canonical_path (
-	enum_decl.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path = mappings.lookup_canonical_path (
-	enum_decl.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path.has_value ());
+  // get the path
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (enum_decl.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, enum_decl.get_locus ()};
+  RustIdent ident{canonical_path, enum_decl.get_locus ()};
 
   // multi variant ADT
   auto *type
@@ -409,7 +370,8 @@ TypeCheckItem::visit (HIR::Union &union_decl)
   auto lifetime_pin = context->push_clean_lifetime_resolver ();
   std::vector<TyTy::SubstitutionParamMapping> substitutions;
   if (union_decl.has_generics ())
-    resolve_generic_params (union_decl.get_generic_params (), substitutions);
+    resolve_generic_params (HIR::Item::ItemKind::Union, union_decl.get_locus (),
+			    union_decl.get_generic_params (), substitutions);
 
   TyTy::RegionConstraints region_constraints;
   for (auto &where_clause_item : union_decl.get_where_clause ().get_items ())
@@ -431,26 +393,14 @@ TypeCheckItem::visit (HIR::Union &union_decl)
 			    ty_variant->get_field_type ());
     }
 
-  // get the path
-  tl::optional<CanonicalPath> canonical_path;
-
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path = nr_ctx.types.to_canonical_path (
-	union_decl.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path = mappings.lookup_canonical_path (
-	union_decl.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path.has_value ());
+  // get the path
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (union_decl.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, union_decl.get_locus ()};
+  RustIdent ident{canonical_path, union_decl.get_locus ()};
 
   // there is only a single variant
   std::vector<TyTy::VariantDef *> variants;
@@ -569,8 +519,9 @@ TypeCheckItem::visit (HIR::Function &function)
   auto lifetime_pin = context->push_clean_lifetime_resolver ();
   std::vector<TyTy::SubstitutionParamMapping> substitutions;
   if (function.has_generics ())
-    resolve_generic_params (function.get_generic_params (),
-			    substitutions); // TODO resolve constraints
+    resolve_generic_params (HIR::Item::ItemKind::Function,
+			    function.get_locus (),
+			    function.get_generic_params (), substitutions);
 
   TyTy::RegionConstraints region_constraints;
   for (auto &where_clause_item : function.get_where_clause ().get_items ())
@@ -607,24 +558,11 @@ TypeCheckItem::visit (HIR::Function &function)
 	TyTy::FnParam (param.get_param_name ().clone_pattern (), param_tyty));
     }
 
-  auto path = CanonicalPath::create_empty ();
-
-  // FIXME: HACK: ARTHUR: Disgusting
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-      auto canonical_path = nr_ctx.values.to_canonical_path (
-	function.get_mappings ().get_nodeid ());
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-      path = canonical_path.value ();
-    }
-  else
-    {
-      path = mappings
-	       .lookup_canonical_path (function.get_mappings ().get_nodeid ())
-	       .value ();
-    }
+  CanonicalPath path
+    = nr_ctx.to_canonical_path (function.get_mappings ().get_nodeid ());
 
   RustIdent ident{path, function.get_locus ()};
 
@@ -745,13 +683,33 @@ TypeCheckItem::visit (HIR::ExternBlock &extern_block)
     }
 }
 
+void
+TypeCheckItem::visit (HIR::ExternCrate &extern_crate)
+{
+  if (extern_crate.references_self ())
+    return;
+
+  auto &mappings = Analysis::Mappings::get ();
+  CrateNum num
+    = mappings.lookup_crate_name (extern_crate.get_referenced_crate ())
+	.value ();
+  HIR::Crate &crate = mappings.get_hir_crate (num);
+
+  CrateNum saved_crate_num = mappings.get_current_crate ();
+  mappings.set_current_crate (num);
+  for (auto &item : crate.get_items ())
+    TypeCheckItem::Resolve (*item);
+  mappings.set_current_crate (saved_crate_num);
+}
+
 std::pair<std::vector<TyTy::SubstitutionParamMapping>, TyTy::RegionConstraints>
 TypeCheckItem::resolve_impl_block_substitutions (HIR::ImplBlock &impl_block,
 						 bool &failure_flag)
 {
   std::vector<TyTy::SubstitutionParamMapping> substitutions;
   if (impl_block.has_generics ())
-    resolve_generic_params (impl_block.get_generic_params (), substitutions);
+    resolve_generic_params (HIR::Item::ItemKind::Impl, impl_block.get_locus (),
+			    impl_block.get_generic_params (), substitutions);
 
   TyTy::RegionConstraints region_constraints;
   for (auto &where_clause_item : impl_block.get_where_clause ().get_items ())
diff --git a/gcc/rust/typecheck/rust-hir-type-check-item.h b/gcc/rust/typecheck/rust-hir-type-check-item.h
index 56832e7..414694b 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-item.h
+++ b/gcc/rust/typecheck/rust-hir-type-check-item.h
@@ -51,9 +51,9 @@ public:
   void visit (HIR::ImplBlock &impl_block) override;
   void visit (HIR::ExternBlock &extern_block) override;
   void visit (HIR::Trait &trait_block) override;
+  void visit (HIR::ExternCrate &extern_crate) override;
 
   // nothing to do
-  void visit (HIR::ExternCrate &) override {}
   void visit (HIR::UseDeclaration &) override {}
 
 protected:
diff --git a/gcc/rust/typecheck/rust-hir-type-check-path.cc b/gcc/rust/typecheck/rust-hir-type-check-path.cc
index 5662da5..cc5c412 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-path.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-path.cc
@@ -157,20 +157,11 @@ TypeCheckExpr::visit (HIR::QualifiedPathInExpression &expr)
   bool fully_resolved = expr.get_segments ().size () <= 1;
   if (fully_resolved)
     {
-      if (flag_name_resolution_2_0)
-	{
-	  auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
-	    Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
+      auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
+	Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
 
-	  nr_ctx.map_usage (Resolver2_0::Usage (
-			      expr.get_mappings ().get_nodeid ()),
-			    Resolver2_0::Definition (root_resolved_node_id));
-	}
-      else
-	{
-	  resolver->insert_resolved_name (expr.get_mappings ().get_nodeid (),
-					  root_resolved_node_id);
-	}
+      nr_ctx.map_usage (Resolver2_0::Usage (expr.get_mappings ().get_nodeid ()),
+			Resolver2_0::Definition (root_resolved_node_id));
       return;
     }
 
@@ -264,24 +255,16 @@ TypeCheckExpr::resolve_root_path (HIR::PathInExpression &expr, size_t *offset,
       bool is_root = *offset == 0;
       NodeId ast_node_id = seg.get_mappings ().get_nodeid ();
 
-      // then lookup the reference_node_id
-      NodeId ref_node_id = UNKNOWN_NODEID;
+      auto &nr_ctx
+	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-      if (flag_name_resolution_2_0)
+      // lookup the reference_node_id
+      NodeId ref_node_id;
+      if (auto res = nr_ctx.lookup (ast_node_id))
 	{
-	  auto &nr_ctx
-	    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-	  // assign the ref_node_id if we've found something
-	  nr_ctx.lookup (ast_node_id).map ([&ref_node_id] (NodeId resolved) {
-	    ref_node_id = resolved;
-	  });
+	  ref_node_id = *res;
 	}
-      else if (!resolver->lookup_resolved_name (ast_node_id, &ref_node_id))
-	resolver->lookup_resolved_type (ast_node_id, &ref_node_id);
-
-      // ref_node_id is the NodeId that the segments refers to.
-      if (ref_node_id == UNKNOWN_NODEID)
+      else
 	{
 	  if (root_tyty != nullptr && *offset > 0)
 	    {
@@ -561,33 +544,12 @@ TypeCheckExpr::resolve_segments (NodeId root_resolved_node_id,
     }
 
   rust_assert (resolved_node_id != UNKNOWN_NODEID);
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
-	Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
 
-      nr_ctx.map_usage (Resolver2_0::Usage (expr_mappings.get_nodeid ()),
-			Resolver2_0::Definition (resolved_node_id));
-    }
-  // name scope first
-  else if (resolver->get_name_scope ().decl_was_declared_here (
-	     resolved_node_id))
-    {
-      resolver->insert_resolved_name (expr_mappings.get_nodeid (),
-				      resolved_node_id);
-    }
-  // check the type scope
-  else if (resolver->get_type_scope ().decl_was_declared_here (
-	     resolved_node_id))
-    {
-      resolver->insert_resolved_type (expr_mappings.get_nodeid (),
-				      resolved_node_id);
-    }
-  else
-    {
-      resolver->insert_resolved_misc (expr_mappings.get_nodeid (),
-				      resolved_node_id);
-    }
+  auto &nr_ctx = const_cast<Resolver2_0::NameResolutionContext &> (
+    Resolver2_0::ImmutableNameResolutionContext::get ().resolver ());
+
+  nr_ctx.map_usage (Resolver2_0::Usage (expr_mappings.get_nodeid ()),
+		    Resolver2_0::Definition (resolved_node_id));
 
   infered = tyseg;
 }
diff --git a/gcc/rust/typecheck/rust-hir-type-check-pattern.cc b/gcc/rust/typecheck/rust-hir-type-check-pattern.cc
index bd13f7a..be926fc 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-pattern.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-pattern.cc
@@ -22,9 +22,6 @@
 #include "rust-type-util.h"
 #include "rust-immutable-name-resolution-context.h"
 
-// for flag_name_resolution_2_0
-#include "options.h"
-
 namespace Rust {
 namespace Resolver {
 
@@ -54,23 +51,13 @@ TypeCheckPattern::visit (HIR::PathInExpression &pattern)
   NodeId ref_node_id = UNKNOWN_NODEID;
   bool maybe_item = false;
 
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-      if (auto id = nr_ctx.lookup (pattern.get_mappings ().get_nodeid ()))
-	{
-	  ref_node_id = *id;
-	  maybe_item = true;
-	}
-    }
-  else
+  if (auto id = nr_ctx.lookup (pattern.get_mappings ().get_nodeid ()))
     {
-      maybe_item |= resolver->lookup_resolved_name (
-	pattern.get_mappings ().get_nodeid (), &ref_node_id);
-      maybe_item |= resolver->lookup_resolved_type (
-	pattern.get_mappings ().get_nodeid (), &ref_node_id);
+      ref_node_id = *id;
+      maybe_item = true;
     }
 
   bool path_is_const_item = false;
@@ -213,13 +200,15 @@ TypeCheckPattern::visit (HIR::TupleStructPattern &pattern)
   auto &items = pattern.get_items ();
   switch (items.get_item_type ())
     {
-      case HIR::TupleStructItems::RANGED: {
+    case HIR::TupleStructItems::RANGED:
+      {
 	// TODO
 	rust_unreachable ();
       }
       break;
 
-      case HIR::TupleStructItems::MULTIPLE: {
+    case HIR::TupleStructItems::MULTIPLE:
+      {
 	HIR::TupleStructItemsNoRange &items_no_range
 	  = static_cast<HIR::TupleStructItemsNoRange &> (items);
 
@@ -333,13 +322,15 @@ TypeCheckPattern::visit (HIR::StructPattern &pattern)
     {
       switch (field->get_item_type ())
 	{
-	  case HIR::StructPatternField::ItemType::TUPLE_PAT: {
+	case HIR::StructPatternField::ItemType::TUPLE_PAT:
+	  {
 	    // TODO
 	    rust_unreachable ();
 	  }
 	  break;
 
-	  case HIR::StructPatternField::ItemType::IDENT_PAT: {
+	case HIR::StructPatternField::ItemType::IDENT_PAT:
+	  {
 	    HIR::StructPatternFieldIdentPat &ident
 	      = static_cast<HIR::StructPatternFieldIdentPat &> (*field);
 
@@ -358,7 +349,8 @@ TypeCheckPattern::visit (HIR::StructPattern &pattern)
 	  }
 	  break;
 
-	  case HIR::StructPatternField::ItemType::IDENT: {
+	case HIR::StructPatternField::ItemType::IDENT:
+	  {
 	    HIR::StructPatternFieldIdent &ident
 	      = static_cast<HIR::StructPatternFieldIdent &> (*field);
 
@@ -397,7 +389,8 @@ TypeCheckPattern::visit (HIR::StructPattern &pattern)
 	    case HIR::StructPatternField::ItemType::IDENT:
 	    case HIR::StructPatternField::ItemType::IDENT_PAT:
 	      break;
-	      default: {
+	    default:
+	      {
 		auto first_elem
 		  = struct_pattern_elems.get_struct_pattern_fields ()
 		      .at (0)
@@ -457,25 +450,27 @@ void
 TypeCheckPattern::visit (HIR::TuplePattern &pattern)
 {
   std::unique_ptr<HIR::TuplePatternItems> items;
+
+  // Check whether parent is tuple
+  auto resolved_parent = parent->destructure ();
+  if (resolved_parent->get_kind () != TyTy::TUPLE)
+    {
+      rust_error_at (pattern.get_locus (), "expected %s, found tuple",
+		     parent->as_string ().c_str ());
+      return;
+    }
+  TyTy::TupleType &par = *static_cast<TyTy::TupleType *> (resolved_parent);
+
   switch (pattern.get_items ().get_item_type ())
     {
-      case HIR::TuplePatternItems::ItemType::MULTIPLE: {
+    case HIR::TuplePatternItems::ItemType::MULTIPLE:
+      {
 	auto &ref = static_cast<HIR::TuplePatternItemsMultiple &> (
 	  pattern.get_items ());
 
-	auto resolved_parent = parent->destructure ();
-	if (resolved_parent->get_kind () != TyTy::TUPLE)
-	  {
-	    rust_error_at (pattern.get_locus (), "expected %s, found tuple",
-			   parent->as_string ().c_str ());
-	    break;
-	  }
-
 	const auto &patterns = ref.get_patterns ();
 	size_t nitems_to_resolve = patterns.size ();
 
-	TyTy::TupleType &par
-	  = *static_cast<TyTy::TupleType *> (resolved_parent);
 	if (patterns.size () != par.get_fields ().size ())
 	  {
 	    emit_pattern_size_error (pattern, par.get_fields ().size (),
@@ -498,12 +493,55 @@ TypeCheckPattern::visit (HIR::TuplePattern &pattern)
       }
       break;
 
-      case HIR::TuplePatternItems::ItemType::RANGED: {
-	// HIR::TuplePatternItemsRanged &ref
-	//   = *static_cast<HIR::TuplePatternItemsRanged *> (
-	//     pattern.get_items ().get ());
-	// TODO
-	rust_unreachable ();
+    case HIR::TuplePatternItems::ItemType::RANGED:
+      {
+	HIR::TuplePatternItemsRanged &ref
+	  = static_cast<HIR::TuplePatternItemsRanged &> (pattern.get_items ());
+
+	const auto &lower = ref.get_lower_patterns ();
+	const auto &upper = ref.get_upper_patterns ();
+	size_t min_size_required = lower.size () + upper.size ();
+
+	// Ensure that size of lower and upper patterns <= parent size
+	if (min_size_required > par.get_fields ().size ())
+	  {
+	    emit_pattern_size_error (pattern, par.get_fields ().size (),
+				     min_size_required);
+	    // TODO attempt to continue to do typechecking even after wrong size
+	    break;
+	  }
+
+	// Resolve lower patterns
+	std::vector<TyTy::TyVar> pattern_elems;
+	for (size_t i = 0; i < lower.size (); i++)
+	  {
+	    auto &p = lower[i];
+	    TyTy::BaseType *par_type = par.get_field (i);
+
+	    TyTy::BaseType *elem = TypeCheckPattern::Resolve (*p, par_type);
+	    pattern_elems.push_back (TyTy::TyVar (elem->get_ref ()));
+	  }
+
+	// Pad pattern_elems until needing to resolve upper patterns
+	size_t rest_end = par.get_fields ().size () - upper.size ();
+	for (size_t i = lower.size (); i < rest_end; i++)
+	  {
+	    TyTy::BaseType *par_type = par.get_field (i);
+	    pattern_elems.push_back (TyTy::TyVar (par_type->get_ref ()));
+	  }
+
+	// Resolve upper patterns
+	for (size_t i = 0; i < upper.size (); i++)
+	  {
+	    auto &p = upper[i];
+	    TyTy::BaseType *par_type = par.get_field (rest_end + i);
+
+	    TyTy::BaseType *elem = TypeCheckPattern::Resolve (*p, par_type);
+	    pattern_elems.push_back (TyTy::TyVar (elem->get_ref ()));
+	  }
+
+	infered = new TyTy::TupleType (pattern.get_mappings ().get_hirid (),
+				       pattern.get_locus (), pattern_elems);
       }
       break;
     }
@@ -512,8 +550,18 @@ TypeCheckPattern::visit (HIR::TuplePattern &pattern)
 void
 TypeCheckPattern::visit (HIR::LiteralPattern &pattern)
 {
-  infered = resolve_literal (pattern.get_mappings (), pattern.get_literal (),
-			     pattern.get_locus ());
+  TyTy::BaseType *resolved
+    = resolve_literal (pattern.get_mappings (), pattern.get_literal (),
+		       pattern.get_locus ());
+  if (resolved->get_kind () == TyTy::TypeKind::ERROR)
+    {
+      infered = resolved;
+      return;
+    }
+
+  infered = unify_site (pattern.get_mappings ().get_hirid (),
+			TyTy::TyWithLocation (parent),
+			TyTy::TyWithLocation (resolved), pattern.get_locus ());
 }
 
 void
@@ -538,6 +586,11 @@ TypeCheckPattern::visit (HIR::RangePattern &pattern)
 void
 TypeCheckPattern::visit (HIR::IdentifierPattern &pattern)
 {
+  if (pattern.has_subpattern ())
+    {
+      TypeCheckPattern::Resolve (pattern.get_subpattern (), parent);
+    }
+
   if (!pattern.get_is_ref ())
     {
       infered = parent;
@@ -580,8 +633,72 @@ TypeCheckPattern::visit (HIR::ReferencePattern &pattern)
 void
 TypeCheckPattern::visit (HIR::SlicePattern &pattern)
 {
-  rust_sorry_at (pattern.get_locus (),
-		 "type checking qualified path patterns not supported");
+  auto resolved_parent = parent->destructure ();
+  TyTy::BaseType *parent_element_ty = nullptr;
+  switch (resolved_parent->get_kind ())
+    {
+    case TyTy::ARRAY:
+      {
+	auto &array_ty_ty = static_cast<TyTy::ArrayType &> (*parent);
+	parent_element_ty = array_ty_ty.get_element_type ();
+	auto capacity = array_ty_ty.get_capacity ();
+	tree cap = capacity->get_value ();
+	if (error_operand_p (cap))
+	  {
+	    rust_error_at (parent->get_locus (),
+			   "capacity of array %qs is not known at compile time",
+			   array_ty_ty.get_name ().c_str ());
+	    break;
+	  }
+	auto cap_wi = wi::to_wide (cap).to_uhwi ();
+	if (cap_wi != pattern.get_items ().size ())
+	  {
+	    rust_error_at (pattern.get_locus (), ErrorCode::E0527,
+			   "pattern requires %lu elements but array has %lu",
+			   (unsigned long) pattern.get_items ().size (),
+			   (unsigned long) cap_wi);
+	    break;
+	  }
+	break;
+      }
+    case TyTy::SLICE:
+      {
+	auto &slice_ty_ty = static_cast<TyTy::SliceType &> (*parent);
+	parent_element_ty = slice_ty_ty.get_element_type ();
+	break;
+      }
+    case TyTy::REF:
+      {
+	auto &ref_ty_ty = static_cast<TyTy::ReferenceType &> (*parent);
+	const TyTy::SliceType *slice = nullptr;
+	if (!ref_ty_ty.is_dyn_slice_type (&slice))
+	  {
+	    rust_error_at (pattern.get_locus (), "expected %s, found slice",
+			   parent->as_string ().c_str ());
+	    return;
+	  }
+	parent_element_ty = slice->get_element_type ();
+	break;
+      }
+    default:
+      {
+	rust_error_at (pattern.get_locus (), "expected %s, found slice",
+		       parent->as_string ().c_str ());
+	return;
+      }
+    }
+
+  rust_assert (parent_element_ty != nullptr);
+  // infered inherits array/slice typing from parent
+  infered = parent->clone ();
+  infered->set_ref (pattern.get_mappings ().get_hirid ());
+
+  // Type check every item in the SlicePattern against parent's element ty
+  // TODO update this after adding support for RestPattern in SlicePattern
+  for (const auto &item : pattern.get_items ())
+    {
+      TypeCheckPattern::Resolve (*item, parent_element_ty);
+    }
 }
 
 void
@@ -608,7 +725,8 @@ TypeCheckPattern::typecheck_range_pattern_bound (
   TyTy::BaseType *resolved_bound = nullptr;
   switch (bound.get_bound_type ())
     {
-      case HIR::RangePatternBound::RangePatternBoundType::LITERAL: {
+    case HIR::RangePatternBound::RangePatternBoundType::LITERAL:
+      {
 	auto &ref = static_cast<HIR::RangePatternBoundLiteral &> (bound);
 
 	HIR::Literal lit = ref.get_literal ();
@@ -617,14 +735,16 @@ TypeCheckPattern::typecheck_range_pattern_bound (
       }
       break;
 
-      case HIR::RangePatternBound::RangePatternBoundType::PATH: {
+    case HIR::RangePatternBound::RangePatternBoundType::PATH:
+      {
 	auto &ref = static_cast<HIR::RangePatternBoundPath &> (bound);
 
 	resolved_bound = TypeCheckExpr::Resolve (ref.get_path ());
       }
       break;
 
-      case HIR::RangePatternBound::RangePatternBoundType::QUALPATH: {
+    case HIR::RangePatternBound::RangePatternBoundType::QUALPATH:
+      {
 	auto &ref = static_cast<HIR::RangePatternBoundQualPath &> (bound);
 
 	resolved_bound = TypeCheckExpr::Resolve (ref.get_qualified_path ());
diff --git a/gcc/rust/typecheck/rust-hir-type-check-stmt.cc b/gcc/rust/typecheck/rust-hir-type-check-stmt.cc
index 4e53856..87141af 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-stmt.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-stmt.cc
@@ -60,6 +60,12 @@ void
 TypeCheckStmt::visit (HIR::ConstantItem &constant)
 {
   TyTy::BaseType *type = TypeCheckType::Resolve (constant.get_type ());
+  if (!constant.has_expr ())
+    {
+      infered = type;
+      return;
+    }
+
   TyTy::BaseType *expr_type = TypeCheckExpr::Resolve (constant.get_expr ());
 
   infered = coercion_site (
diff --git a/gcc/rust/typecheck/rust-hir-type-check-struct.cc b/gcc/rust/typecheck/rust-hir-type-check-struct.cc
index df1636a..4ef8348 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-struct.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-struct.cc
@@ -329,8 +329,7 @@ TypeCheckStructExpr::visit (HIR::StructExprFieldIndexValue &field)
       repeat_location.add_range (prev_field_locus);
 
       rust_error_at (repeat_location, ErrorCode::E0062,
-		     "field %qs specified more than once",
-		     field_name.c_str ());
+		     "field %qs specified more than once", field_name.c_str ());
       return false;
     }
 
@@ -363,7 +362,7 @@ TypeCheckStructExpr::visit (HIR::StructExprFieldIdentifier &field)
   if (!ok)
     {
       rust_error_at (field.get_locus (), "unknown field");
-      return true;
+      return false;
     }
 
   auto it = adtFieldIndexToField.find (field_index);
diff --git a/gcc/rust/typecheck/rust-hir-type-check-type.cc b/gcc/rust/typecheck/rust-hir-type-check-type.cc
index 6919093..78037bd 100644
--- a/gcc/rust/typecheck/rust-hir-type-check-type.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check-type.cc
@@ -29,6 +29,7 @@
 #include "rust-substitution-mapper.h"
 #include "rust-type-util.h"
 #include "rust-system.h"
+#include "rust-compile-base.h"
 
 namespace Rust {
 namespace Resolver {
@@ -335,19 +336,13 @@ TypeCheckType::resolve_root_path (HIR::TypePath &path, size_t *offset,
 	  seg->get_lang_item ());
       else
 	{
-	  // FIXME: HACK: ARTHUR: Remove this
-	  if (flag_name_resolution_2_0)
-	    {
-	      auto &nr_ctx = Resolver2_0::ImmutableNameResolutionContext::get ()
-			       .resolver ();
+	  auto &nr_ctx
+	    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-	      // assign the ref_node_id if we've found something
-	      nr_ctx.lookup (ast_node_id)
-		.map (
-		  [&ref_node_id] (NodeId resolved) { ref_node_id = resolved; });
-	    }
-	  else if (!resolver->lookup_resolved_name (ast_node_id, &ref_node_id))
-	    resolver->lookup_resolved_type (ast_node_id, &ref_node_id);
+	  // assign the ref_node_id if we've found something
+	  nr_ctx.lookup (ast_node_id).map ([&ref_node_id] (NodeId resolved) {
+	    ref_node_id = resolved;
+	  });
 	}
 
       // ref_node_id is the NodeId that the segments refers to.
@@ -549,8 +544,7 @@ TypeCheckType::resolve_segments (
       bool selfResolveOk = false;
 
       if (first_segment && tySegIsBigSelf
-	  && context->block_context ().is_in_context ()
-	  && context->block_context ().peek ().is_impl_block ())
+	  && context->block_context ().is_in_context ())
 	{
 	  TypeCheckBlockContextItem ctx = context->block_context ().peek ();
 	  TyTy::BaseType *lookup = nullptr;
@@ -695,6 +689,7 @@ TypeCheckType::visit (HIR::ParenthesisedType &type)
 void
 TypeCheckType::visit (HIR::ArrayType &type)
 {
+  auto element_type = TypeCheckType::Resolve (type.get_element_type ());
   auto capacity_type = TypeCheckExpr::Resolve (type.get_size_expr ());
   if (capacity_type->get_kind () == TyTy::TypeKind::ERROR)
     return;
@@ -704,16 +699,38 @@ TypeCheckType::visit (HIR::ArrayType &type)
   rust_assert (ok);
   context->insert_type (type.get_size_expr ().get_mappings (), expected_ty);
 
-  unify_site (type.get_size_expr ().get_mappings ().get_hirid (),
-	      TyTy::TyWithLocation (expected_ty),
-	      TyTy::TyWithLocation (capacity_type,
-				    type.get_size_expr ().get_locus ()),
-	      type.get_size_expr ().get_locus ());
+  TyTy::ConstType *const_type = nullptr;
+  if (capacity_type->get_kind () == TyTy::TypeKind::CONST)
+    {
+      const_type = static_cast<TyTy::ConstType *> (capacity_type);
 
-  TyTy::BaseType *base = TypeCheckType::Resolve (type.get_element_type ());
-  translated = new TyTy::ArrayType (type.get_mappings ().get_hirid (),
-				    type.get_locus (), type.get_size_expr (),
-				    TyTy::TyVar (base->get_ref ()));
+      unify_site (type.get_size_expr ().get_mappings ().get_hirid (),
+		  TyTy::TyWithLocation (expected_ty),
+		  TyTy::TyWithLocation (const_type->get_ty (),
+					type.get_size_expr ().get_locus ()),
+		  type.get_size_expr ().get_locus ());
+    }
+  else
+    {
+      HirId size_id = type.get_size_expr ().get_mappings ().get_hirid ();
+      unify_site (size_id, TyTy::TyWithLocation (expected_ty),
+		  TyTy::TyWithLocation (capacity_type,
+					type.get_size_expr ().get_locus ()),
+		  type.get_size_expr ().get_locus ());
+
+      auto ctx = Compile::Context::get ();
+      tree capacity_expr = Compile::HIRCompileBase::query_compile_const_expr (
+	ctx, capacity_type, type.get_size_expr ());
+
+      const_type = new TyTy::ConstType (TyTy::ConstType::ConstKind::Value, "",
+					expected_ty, capacity_expr, {},
+					type.get_size_expr ().get_locus (),
+					size_id, size_id);
+    }
+
+  translated
+    = new TyTy::ArrayType (type.get_mappings ().get_hirid (), type.get_locus (),
+			   const_type, TyTy::TyVar (element_type->get_ref ()));
 }
 
 void
@@ -850,10 +867,9 @@ TypeResolveGenericParam::visit (HIR::TypeParam &param)
   if (param.has_type ())
     TypeCheckType::Resolve (param.get_type ());
 
-  resolved
-    = new TyTy::ParamType (param.get_type_representation ().as_string (),
-			   param.get_locus (),
-			   param.get_mappings ().get_hirid (), param, {});
+  resolved = new TyTy::ParamType (param.get_type_representation ().as_string (),
+				  param.get_locus (),
+				  param.get_mappings ().get_hirid (), {});
 
   if (resolve_trait_bounds)
     apply_trait_bounds (param, resolved);
@@ -872,7 +888,7 @@ TypeResolveGenericParam::apply_trait_bounds (HIR::TypeParam &param,
       HirId implicit_id = mappings.get_next_hir_id ();
       TyTy::ParamType *p
 	= new TyTy::ParamType (param.get_type_representation ().as_string (),
-			       param.get_locus (), implicit_id, param,
+			       param.get_locus (), implicit_id,
 			       {} /*empty specified bounds*/);
       context->insert_implicit_type (implicit_id, p);
 
@@ -908,7 +924,8 @@ TypeResolveGenericParam::apply_trait_bounds (HIR::TypeParam &param,
 	{
 	  switch (bound->get_bound_type ())
 	    {
-	      case HIR::TypeParamBound::BoundType::TRAITBOUND: {
+	    case HIR::TypeParamBound::BoundType::TRAITBOUND:
+	      {
 		HIR::TraitBound &b = static_cast<HIR::TraitBound &> (*bound);
 
 		TyTy::TypeBoundPredicate predicate = get_predicate_from_bound (
@@ -920,7 +937,8 @@ TypeResolveGenericParam::apply_trait_bounds (HIR::TypeParam &param,
 		  {
 		    switch (predicate.get_polarity ())
 		      {
-			case BoundPolarity::AntiBound: {
+		      case BoundPolarity::AntiBound:
+			{
 			  bool found = predicates.find (predicate.get_id ())
 				       != predicates.end ();
 			  if (found)
@@ -937,7 +955,8 @@ TypeResolveGenericParam::apply_trait_bounds (HIR::TypeParam &param,
 			}
 			break;
 
-			default: {
+		      default:
+			{
 			  if (predicates.find (predicate.get_id ())
 			      == predicates.end ())
 			    {
@@ -1033,7 +1052,8 @@ ResolveWhereClauseItem::visit (HIR::TypeBoundWhereClauseItem &item)
     {
       switch (bound->get_bound_type ())
 	{
-	  case HIR::TypeParamBound::BoundType::TRAITBOUND: {
+	case HIR::TypeParamBound::BoundType::TRAITBOUND:
+	  {
 	    auto *b = static_cast<HIR::TraitBound *> (bound.get ());
 
 	    TyTy::TypeBoundPredicate predicate
@@ -1042,7 +1062,8 @@ ResolveWhereClauseItem::visit (HIR::TypeBoundWhereClauseItem &item)
 	      specified_bounds.push_back (std::move (predicate));
 	  }
 	  break;
-	  case HIR::TypeParamBound::BoundType::LIFETIME: {
+	case HIR::TypeParamBound::BoundType::LIFETIME:
+	  {
 	    if (auto param = binding->try_as<TyTy::ParamType> ())
 	      {
 		auto *b = static_cast<HIR::Lifetime *> (bound.get ());
@@ -1071,23 +1092,15 @@ ResolveWhereClauseItem::visit (HIR::TypeBoundWhereClauseItem &item)
 
   // then lookup the reference_node_id
   NodeId ref_node_id = UNKNOWN_NODEID;
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-      if (auto id = nr_ctx.lookup (ast_node_id))
-	ref_node_id = *id;
-    }
-  else
-    {
-      NodeId id = UNKNOWN_NODEID;
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-      if (resolver->lookup_resolved_type (ast_node_id, &id))
-	ref_node_id = id;
+  if (auto id = nr_ctx.lookup (ast_node_id))
+    {
+      ref_node_id = *id;
     }
-
-  if (ref_node_id == UNKNOWN_NODEID)
+  else
     {
       // FIXME
       rust_error_at (UNDEF_LOCATION,
diff --git a/gcc/rust/typecheck/rust-hir-type-check.cc b/gcc/rust/typecheck/rust-hir-type-check.cc
index fbaf323..aba4ab5 100644
--- a/gcc/rust/typecheck/rust-hir-type-check.cc
+++ b/gcc/rust/typecheck/rust-hir-type-check.cc
@@ -26,11 +26,7 @@
 #include "rust-hir-type-check-struct-field.h"
 #include "rust-immutable-name-resolution-context.h"
 
-// for flag_name_resolution_2_0
-#include "options.h"
-
-extern bool
-saw_errors (void);
+extern bool saw_errors (void);
 
 namespace Rust {
 namespace Resolver {
@@ -165,7 +161,9 @@ TraitItemReference::get_type_from_fn (/*const*/ HIR::TraitItemFunc &fn) const
   HIR::TraitFunctionDecl &function = fn.get_decl ();
   if (function.has_generics ())
     {
-      TypeCheckBase::ResolveGenericParams (function.get_generic_params (),
+      TypeCheckBase::ResolveGenericParams (HIR::Item::ItemKind::Function,
+					   fn.get_locus (),
+					   function.get_generic_params (),
 					   substitutions, false /*is_foreign*/,
 					   ABI::RUST);
     }
@@ -231,7 +229,8 @@ TraitItemReference::get_type_from_fn (/*const*/ HIR::TraitItemFunc &fn) const
 	      break;
 
 	    case HIR::SelfParam::IMM_REF:
-	      case HIR::SelfParam::MUT_REF: {
+	    case HIR::SelfParam::MUT_REF:
+	      {
 		auto mutability
 		  = self_param.get_self_kind () == HIR::SelfParam::IMM_REF
 		      ? Mutability::Imm
@@ -275,26 +274,13 @@ TraitItemReference::get_type_from_fn (/*const*/ HIR::TraitItemFunc &fn) const
 	TyTy::FnParam (param.get_param_name ().clone_pattern (), param_tyty));
     }
 
-  auto &mappings = Analysis::Mappings::get ();
-
-  tl::optional<CanonicalPath> canonical_path;
-  if (flag_name_resolution_2_0)
-    {
-      auto &nr_ctx
-	= Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
-
-      canonical_path
-	= nr_ctx.values.to_canonical_path (fn.get_mappings ().get_nodeid ());
-    }
-  else
-    {
-      canonical_path
-	= mappings.lookup_canonical_path (fn.get_mappings ().get_nodeid ());
-    }
+  auto &nr_ctx
+    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-  rust_assert (canonical_path);
+  CanonicalPath canonical_path
+    = nr_ctx.to_canonical_path (fn.get_mappings ().get_nodeid ());
 
-  RustIdent ident{*canonical_path, fn.get_locus ()};
+  RustIdent ident{canonical_path, fn.get_locus ()};
   auto resolved = new TyTy::FnType (
     fn.get_mappings ().get_hirid (), fn.get_mappings ().get_defid (),
     function.get_function_name ().as_string (), ident,
diff --git a/gcc/rust/typecheck/rust-hir-type-check.h b/gcc/rust/typecheck/rust-hir-type-check.h
index 18a65fe..e5a6e9e 100644
--- a/gcc/rust/typecheck/rust-hir-type-check.h
+++ b/gcc/rust/typecheck/rust-hir-type-check.h
@@ -20,6 +20,7 @@
 #define RUST_HIR_TYPE_CHECK
 
 #include "rust-hir-map.h"
+#include "rust-mapping-common.h"
 #include "rust-tyty.h"
 #include "rust-hir-trait-reference.h"
 #include "rust-stacked-contexts.h"
@@ -157,6 +158,39 @@ public:
   WARN_UNUSED_RESULT Lifetime next () { return Lifetime (interner_index++); }
 };
 
+struct DeferredOpOverload
+{
+  HirId expr_id;
+  LangItem::Kind lang_item_type;
+  HIR::PathIdentSegment specified_segment;
+  TyTy::TypeBoundPredicate predicate;
+  HIR::OperatorExprMeta op;
+
+  DeferredOpOverload (HirId expr_id, LangItem::Kind lang_item_type,
+		      HIR::PathIdentSegment specified_segment,
+		      TyTy::TypeBoundPredicate &predicate,
+		      HIR::OperatorExprMeta op)
+    : expr_id (expr_id), lang_item_type (lang_item_type),
+      specified_segment (specified_segment), predicate (predicate), op (op)
+  {}
+
+  DeferredOpOverload (const struct DeferredOpOverload &other)
+    : expr_id (other.expr_id), lang_item_type (other.lang_item_type),
+      specified_segment (other.specified_segment), predicate (other.predicate),
+      op (other.op)
+  {}
+
+  DeferredOpOverload &operator= (struct DeferredOpOverload const &other)
+  {
+    expr_id = other.expr_id;
+    lang_item_type = other.lang_item_type;
+    specified_segment = other.specified_segment;
+    op = other.op;
+
+    return *this;
+  }
+};
+
 class TypeCheckContext
 {
 public:
@@ -215,10 +249,10 @@ public:
   bool lookup_associated_type_mapping (HirId id, HirId *mapping);
 
   void insert_associated_impl_mapping (HirId trait_id,
-				       const TyTy::BaseType *impl_type,
+				       TyTy::BaseType *impl_type,
 				       HirId impl_id);
   bool lookup_associated_impl_mapping_for_self (HirId trait_id,
-						const TyTy::BaseType *self,
+						TyTy::BaseType *self,
 						HirId *mapping);
 
   void insert_autoderef_mappings (HirId id,
@@ -237,6 +271,13 @@ public:
   void insert_operator_overload (HirId id, TyTy::FnType *call_site);
   bool lookup_operator_overload (HirId id, TyTy::FnType **call);
 
+  void insert_deferred_operator_overload (DeferredOpOverload deferred);
+  bool lookup_deferred_operator_overload (HirId id,
+					  DeferredOpOverload *deferred);
+
+  void iterate_deferred_operator_overloads (
+    std::function<bool (HirId, DeferredOpOverload &)> cb);
+
   void insert_unconstrained_check_marker (HirId id, bool status);
   bool have_checked_for_unconstrained (HirId id, bool *result);
 
@@ -263,13 +304,16 @@ public:
   WARN_UNUSED_RESULT std::vector<TyTy::Region>
   regions_from_generic_args (const HIR::GenericArgs &args) const;
 
-  void compute_inference_variables (bool error);
+  void compute_inference_variables (bool emit_error);
 
   TyTy::VarianceAnalysis::CrateCtx &get_variance_analysis_ctx ();
 
 private:
   TypeCheckContext ();
 
+  bool compute_infer_var (HirId id, TyTy::BaseType *ty, bool emit_error);
+  bool compute_ambigious_op_overload (HirId id, DeferredOpOverload &op);
+
   std::map<NodeId, HirId> node_id_refs;
   std::map<HirId, TyTy::BaseType *> resolved;
   std::vector<std::unique_ptr<TyTy::BaseType>> builtins;
@@ -281,7 +325,7 @@ private:
   std::map<HirId, AssociatedImplTrait> associated_impl_traits;
 
   // trait-id -> list of < self-tyty:impl-id>
-  std::map<HirId, std::vector<std::pair<const TyTy::BaseType *, HirId>>>
+  std::map<HirId, std::vector<std::pair<TyTy::BaseType *, HirId>>>
     associated_traits_to_impls;
 
   std::map<HirId, HirId> associated_type_mappings;
@@ -306,6 +350,9 @@ private:
   std::set<HirId> querys_in_progress;
   std::set<DefId> trait_queries_in_progress;
 
+  // deferred operator overload
+  std::map<HirId, DeferredOpOverload> deferred_operator_overloads;
+
   // variance analysis
   TyTy::VarianceAnalysis::CrateCtx variance_analysis_ctx;
 
diff --git a/gcc/rust/typecheck/rust-substitution-mapper.cc b/gcc/rust/typecheck/rust-substitution-mapper.cc
index f0bd1f8..c5b823e 100644
--- a/gcc/rust/typecheck/rust-substitution-mapper.cc
+++ b/gcc/rust/typecheck/rust-substitution-mapper.cc
@@ -268,6 +268,12 @@ SubstMapperInternal::visit (TyTy::ParamType &type)
 }
 
 void
+SubstMapperInternal::visit (TyTy::ConstType &type)
+{
+  resolved = type.handle_substitions (mappings);
+}
+
+void
 SubstMapperInternal::visit (TyTy::PlaceholderType &type)
 {
   rust_assert (type.can_resolve ());
@@ -374,7 +380,7 @@ SubstMapperInternal::visit (TyTy::DynamicObjectType &type)
 void
 SubstMapperInternal::visit (TyTy::OpaqueType &type)
 {
-  resolved = type.handle_substitions (mappings);
+  resolved = type.clone ();
 }
 
 // SubstMapperFromExisting
diff --git a/gcc/rust/typecheck/rust-substitution-mapper.h b/gcc/rust/typecheck/rust-substitution-mapper.h
index 32ab71c..2389d83 100644
--- a/gcc/rust/typecheck/rust-substitution-mapper.h
+++ b/gcc/rust/typecheck/rust-substitution-mapper.h
@@ -61,6 +61,7 @@ public:
   void visit (TyTy::ReferenceType &) override { rust_unreachable (); }
   void visit (TyTy::PointerType &) override { rust_unreachable (); }
   void visit (TyTy::ParamType &) override { rust_unreachable (); }
+  void visit (TyTy::ConstType &) override { rust_unreachable (); }
   void visit (TyTy::StrType &) override { rust_unreachable (); }
   void visit (TyTy::NeverType &) override { rust_unreachable (); }
   void visit (TyTy::DynamicObjectType &) override { rust_unreachable (); }
@@ -92,6 +93,7 @@ public:
   void visit (TyTy::ReferenceType &type) override;
   void visit (TyTy::PointerType &type) override;
   void visit (TyTy::ParamType &type) override;
+  void visit (TyTy::ConstType &type) override;
   void visit (TyTy::PlaceholderType &type) override;
   void visit (TyTy::ProjectionType &type) override;
   void visit (TyTy::ClosureType &type) override;
@@ -145,6 +147,7 @@ public:
   void visit (TyTy::ReferenceType &) override { rust_unreachable (); }
   void visit (TyTy::PointerType &) override { rust_unreachable (); }
   void visit (TyTy::ParamType &) override { rust_unreachable (); }
+  void visit (TyTy::ConstType &) override { rust_unreachable (); }
   void visit (TyTy::StrType &) override { rust_unreachable (); }
   void visit (TyTy::NeverType &) override { rust_unreachable (); }
   void visit (TyTy::PlaceholderType &) override { rust_unreachable (); }
@@ -185,12 +188,13 @@ public:
   void visit (const TyTy::ReferenceType &) override {}
   void visit (const TyTy::PointerType &) override {}
   void visit (const TyTy::ParamType &) override {}
+  void visit (const TyTy::ConstType &) override {}
   void visit (const TyTy::StrType &) override {}
   void visit (const TyTy::NeverType &) override {}
   void visit (const TyTy::PlaceholderType &) override {}
   void visit (const TyTy::ProjectionType &) override {}
   void visit (const TyTy::DynamicObjectType &) override {}
-  void visit (const TyTy::OpaqueType &type) override {}
+  void visit (const TyTy::OpaqueType &) override {}
 
 private:
   GetUsedSubstArgs ();
diff --git a/gcc/rust/typecheck/rust-type-util.cc b/gcc/rust/typecheck/rust-type-util.cc
index c6c5b4b..2d66166 100644
--- a/gcc/rust/typecheck/rust-type-util.cc
+++ b/gcc/rust/typecheck/rust-type-util.cc
@@ -24,6 +24,7 @@
 #include "rust-hir-type-check.h"
 #include "rust-hir-type-check-type.h"
 #include "rust-casts.h"
+#include "rust-mapping-common.h"
 #include "rust-unify.h"
 #include "rust-coercion.h"
 #include "rust-hir-type-bounds.h"
@@ -37,15 +38,14 @@ bool
 query_type (HirId reference, TyTy::BaseType **result)
 {
   auto &mappings = Analysis::Mappings::get ();
-  auto &resolver = *Resolver::get ();
   TypeCheckContext *context = TypeCheckContext::get ();
 
-  if (context->query_in_progress (reference))
-    return false;
-
   if (context->lookup_type (reference, result))
     return true;
 
+  if (context->query_in_progress (reference))
+    return false;
+
   context->insert_query (reference);
 
   std::pair<HIR::Enum *, HIR::EnumItem *> enum_candidiate
@@ -103,18 +103,13 @@ query_type (HirId reference, TyTy::BaseType **result)
 	  NodeId ref_node_id = UNKNOWN_NODEID;
 	  NodeId ast_node_id = ty.get_mappings ().get_nodeid ();
 
-	  if (flag_name_resolution_2_0)
-	    {
-	      auto &nr_ctx = Resolver2_0::ImmutableNameResolutionContext::get ()
-			       .resolver ();
+	  auto &nr_ctx
+	    = Resolver2_0::ImmutableNameResolutionContext::get ().resolver ();
 
-	      // assign the ref_node_id if we've found something
-	      nr_ctx.lookup (ast_node_id)
-		.map (
-		  [&ref_node_id] (NodeId resolved) { ref_node_id = resolved; });
-	    }
-	  else if (!resolver.lookup_resolved_name (ast_node_id, &ref_node_id))
-	    resolver.lookup_resolved_type (ast_node_id, &ref_node_id);
+	  // assign the ref_node_id if we've found something
+	  nr_ctx.lookup (ast_node_id).map ([&ref_node_id] (NodeId resolved) {
+	    ref_node_id = resolved;
+	  });
 
 	  if (ref_node_id != UNKNOWN_NODEID)
 	    {
@@ -192,10 +187,12 @@ unify_site_and (HirId id, TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
   TyTy::BaseType *expected = lhs.get_ty ();
   TyTy::BaseType *expr = rhs.get_ty ();
 
-  rust_debug (
-    "unify_site_and commit %s infer %s id={%u} expected={%s} expr={%s}",
-    commit_if_ok ? "true" : "false", implicit_infer_vars ? "true" : "false", id,
-    expected->debug_str ().c_str (), expr->debug_str ().c_str ());
+  rust_debug_loc (
+    unify_locus,
+    "begin unify_site_and commit %s infer %s id={%u} expected={%s} expr={%s}",
+    commit_if_ok ? "true" : "false", implicit_infer_vars ? "true" : "false",
+    id == UNKNOWN_HIRID ? 0 : id, expected->debug_str ().c_str (),
+    expr->debug_str ().c_str ());
 
   std::vector<UnifyRules::CommitSite> commits;
   std::vector<UnifyRules::InferenceSite> infers;
@@ -203,6 +200,15 @@ unify_site_and (HirId id, TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
     = UnifyRules::Resolve (lhs, rhs, unify_locus, false /*commit inline*/,
 			   emit_errors, implicit_infer_vars, commits, infers);
   bool ok = result->get_kind () != TyTy::TypeKind::ERROR;
+
+  rust_debug_loc (unify_locus,
+		  "unify_site_and done ok=%s commit %s infer %s id={%u} "
+		  "expected={%s} expr={%s}",
+		  ok ? "true" : "false", commit_if_ok ? "true" : "false",
+		  implicit_infer_vars ? "true" : "false",
+		  id == UNKNOWN_HIRID ? 0 : id, expected->debug_str ().c_str (),
+		  expr->debug_str ().c_str ());
+
   if (ok && commit_if_ok)
     {
       for (auto &c : commits)
diff --git a/gcc/rust/typecheck/rust-type-util.h b/gcc/rust/typecheck/rust-type-util.h
index 03874a4..dd97f1e 100644
--- a/gcc/rust/typecheck/rust-type-util.h
+++ b/gcc/rust/typecheck/rust-type-util.h
@@ -25,33 +25,30 @@
 namespace Rust {
 namespace Resolver {
 
-bool
-query_type (HirId reference, TyTy::BaseType **result);
+bool query_type (HirId reference, TyTy::BaseType **result);
 
-bool
-types_compatable (TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
-		  location_t unify_locus, bool emit_errors);
+bool types_compatable (TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
+		       location_t unify_locus, bool emit_errors);
 
-TyTy::BaseType *
-unify_site (HirId id, TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
-	    location_t unify_locus);
+TyTy::BaseType *unify_site (HirId id, TyTy::TyWithLocation lhs,
+			    TyTy::TyWithLocation rhs, location_t unify_locus);
 
-TyTy::BaseType *
-unify_site_and (HirId id, TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
-		location_t unify_locus, bool emit_errors, bool commit_if_ok,
-		bool implicit_infer_vars, bool cleanup);
+TyTy::BaseType *unify_site_and (HirId id, TyTy::TyWithLocation lhs,
+				TyTy::TyWithLocation rhs,
+				location_t unify_locus, bool emit_errors,
+				bool commit_if_ok, bool implicit_infer_vars,
+				bool cleanup);
 
-TyTy::BaseType *
-coercion_site (HirId id, TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
-	       location_t coercion_locus);
+TyTy::BaseType *coercion_site (HirId id, TyTy::TyWithLocation lhs,
+			       TyTy::TyWithLocation rhs,
+			       location_t coercion_locus);
 
-TyTy::BaseType *
-try_coercion (HirId id, TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
-	      location_t coercion_locus);
+TyTy::BaseType *try_coercion (HirId id, TyTy::TyWithLocation lhs,
+			      TyTy::TyWithLocation rhs,
+			      location_t coercion_locus);
 
-TyTy::BaseType *
-cast_site (HirId id, TyTy::TyWithLocation from, TyTy::TyWithLocation to,
-	   location_t cast_locus);
+TyTy::BaseType *cast_site (HirId id, TyTy::TyWithLocation from,
+			   TyTy::TyWithLocation to, location_t cast_locus);
 
 AssociatedImplTrait *
 lookup_associated_impl_block (const TyTy::TypeBoundPredicate &bound,
diff --git a/gcc/rust/typecheck/rust-typecheck-context.cc b/gcc/rust/typecheck/rust-typecheck-context.cc
index 9112b99..c74a920 100644
--- a/gcc/rust/typecheck/rust-typecheck-context.cc
+++ b/gcc/rust/typecheck/rust-typecheck-context.cc
@@ -18,6 +18,7 @@
 
 #include "rust-hir-type-check.h"
 #include "rust-type-util.h"
+#include "rust-hir-type-check-expr.h"
 
 namespace Rust {
 namespace Resolver {
@@ -299,8 +300,9 @@ TypeCheckContext::lookup_associated_type_mapping (HirId id, HirId *mapping)
 }
 
 void
-TypeCheckContext::insert_associated_impl_mapping (
-  HirId trait_id, const TyTy::BaseType *impl_type, HirId impl_id)
+TypeCheckContext::insert_associated_impl_mapping (HirId trait_id,
+						  TyTy::BaseType *impl_type,
+						  HirId impl_id)
 {
   auto it = associated_traits_to_impls.find (trait_id);
   if (it == associated_traits_to_impls.end ())
@@ -312,8 +314,9 @@ TypeCheckContext::insert_associated_impl_mapping (
 }
 
 bool
-TypeCheckContext::lookup_associated_impl_mapping_for_self (
-  HirId trait_id, const TyTy::BaseType *self, HirId *mapping)
+TypeCheckContext::lookup_associated_impl_mapping_for_self (HirId trait_id,
+							   TyTy::BaseType *self,
+							   HirId *mapping)
 {
   auto it = associated_traits_to_impls.find (trait_id);
   if (it == associated_traits_to_impls.end ())
@@ -321,7 +324,9 @@ TypeCheckContext::lookup_associated_impl_mapping_for_self (
 
   for (auto &item : it->second)
     {
-      if (item.first->can_eq (self, false))
+      if (types_compatable (TyTy::TyWithLocation (item.first),
+			    TyTy::TyWithLocation (self), UNKNOWN_LOCATION,
+			    false))
 	{
 	  *mapping = item.second;
 	  return true;
@@ -409,6 +414,38 @@ TypeCheckContext::lookup_operator_overload (HirId id, TyTy::FnType **call)
 }
 
 void
+TypeCheckContext::insert_deferred_operator_overload (
+  DeferredOpOverload deferred)
+{
+  HirId expr_id = deferred.expr_id;
+  deferred_operator_overloads.emplace (std::make_pair (expr_id, deferred));
+}
+
+bool
+TypeCheckContext::lookup_deferred_operator_overload (
+  HirId id, DeferredOpOverload *deferred)
+{
+  auto it = deferred_operator_overloads.find (id);
+  if (it == deferred_operator_overloads.end ())
+    return false;
+
+  *deferred = it->second;
+  return true;
+}
+
+void
+TypeCheckContext::iterate_deferred_operator_overloads (
+  std::function<bool (HirId, DeferredOpOverload &)> cb)
+{
+  for (auto it = deferred_operator_overloads.begin ();
+       it != deferred_operator_overloads.end (); it++)
+    {
+      if (!cb (it->first, it->second))
+	return;
+    }
+}
+
+void
 TypeCheckContext::insert_unconstrained_check_marker (HirId id, bool status)
 {
   unconstrained[id] = status;
@@ -574,44 +611,77 @@ TypeCheckContext::regions_from_generic_args (const HIR::GenericArgs &args) const
   return regions;
 }
 
+bool
+TypeCheckContext::compute_ambigious_op_overload (HirId id,
+						 DeferredOpOverload &op)
+{
+  rust_debug ("attempting resolution of op overload: %s",
+	      op.predicate.as_string ().c_str ());
+
+  TyTy::BaseType *lhs = nullptr;
+  bool ok = lookup_type (op.op.get_lvalue_mappings ().get_hirid (), &lhs);
+  rust_assert (ok);
+
+  TyTy::BaseType *rhs = nullptr;
+  if (op.op.has_rvalue_mappings ())
+    {
+      bool ok = lookup_type (op.op.get_rvalue_mappings ().get_hirid (), &rhs);
+      rust_assert (ok);
+    }
+
+  TypeCheckExpr::ResolveOpOverload (op.lang_item_type, op.op, lhs, rhs,
+				    op.specified_segment);
+
+  return true;
+}
+
 void
-TypeCheckContext::compute_inference_variables (bool error)
+TypeCheckContext::compute_inference_variables (bool emit_error)
 {
-  auto &mappings = Analysis::Mappings::get ();
+  iterate_deferred_operator_overloads (
+    [&] (HirId id, DeferredOpOverload &op) mutable -> bool {
+      return compute_ambigious_op_overload (id, op);
+    });
 
-  // default inference variables if possible
   iterate ([&] (HirId id, TyTy::BaseType *ty) mutable -> bool {
-    // nothing to do
-    if (ty->get_kind () != TyTy::TypeKind::INFER)
-      return true;
+    return compute_infer_var (id, ty, emit_error);
+  });
+}
 
-    TyTy::InferType *infer_var = static_cast<TyTy::InferType *> (ty);
-    TyTy::BaseType *default_type;
-
-    rust_debug_loc (mappings.lookup_location (id),
-		    "trying to default infer-var: %s",
-		    infer_var->as_string ().c_str ());
-    bool ok = infer_var->default_type (&default_type);
-    if (!ok)
-      {
-	if (error)
-	  rust_error_at (mappings.lookup_location (id), ErrorCode::E0282,
-			 "type annotations needed");
-	return true;
-      }
-
-    auto result
-      = unify_site (id, TyTy::TyWithLocation (ty),
-		    TyTy::TyWithLocation (default_type), UNDEF_LOCATION);
-    rust_assert (result);
-    rust_assert (result->get_kind () != TyTy::TypeKind::ERROR);
-    result->set_ref (id);
-    insert_type (Analysis::NodeMapping (mappings.get_current_crate (), 0, id,
-					UNKNOWN_LOCAL_DEFID),
-		 result);
+bool
+TypeCheckContext::compute_infer_var (HirId id, TyTy::BaseType *ty,
+				     bool emit_error)
+{
+  auto &mappings = Analysis::Mappings::get ();
 
+  // nothing to do
+  if (ty->get_kind () != TyTy::TypeKind::INFER)
     return true;
-  });
+
+  TyTy::InferType *infer_var = static_cast<TyTy::InferType *> (ty);
+  TyTy::BaseType *default_type;
+
+  rust_debug_loc (mappings.lookup_location (id),
+		  "trying to default infer-var: %s",
+		  infer_var->as_string ().c_str ());
+  bool ok = infer_var->default_type (&default_type);
+  if (!ok)
+    {
+      if (emit_error)
+	rust_error_at (mappings.lookup_location (id), ErrorCode::E0282,
+		       "type annotations needed");
+      return true;
+    }
+
+  auto result
+    = unify_site (id, TyTy::TyWithLocation (ty),
+		  TyTy::TyWithLocation (default_type), UNDEF_LOCATION);
+  rust_assert (result);
+  rust_assert (result->get_kind () != TyTy::TypeKind::ERROR);
+  result->set_ref (id);
+  insert_implicit_type (id, result);
+
+  return true;
 }
 
 TyTy::VarianceAnalysis::CrateCtx &
diff --git a/gcc/rust/typecheck/rust-tyty-bounds.cc b/gcc/rust/typecheck/rust-tyty-bounds.cc
index e028a0a..6cf9b04 100644
--- a/gcc/rust/typecheck/rust-tyty-bounds.cc
+++ b/gcc/rust/typecheck/rust-tyty-bounds.cc
@@ -61,6 +61,39 @@ TypeBoundsProbe::is_bound_satisfied_for_type (TyTy::BaseType *receiver,
   return false;
 }
 
+bool
+TypeBoundsProbe::process_impl_block (
+  HirId id, HIR::ImplBlock *impl,
+  std::vector<std::pair<HIR::TypePath *, HIR::ImplBlock *>>
+    &possible_trait_paths)
+{
+  // we are filtering for trait-impl-blocks
+  if (!impl->has_trait_ref ())
+    return true;
+
+  // can be recursive trait resolution
+  HIR::Trait *t = TraitResolver::ResolveHirItem (impl->get_trait_ref ());
+  if (t == nullptr)
+    return true;
+  // DefId trait_id = t->get_mappings ().get_defid ();
+  // if (context->trait_query_in_progress (trait_id))
+  //   return true;
+
+  HirId impl_ty_id = impl->get_type ().get_mappings ().get_hirid ();
+  TyTy::BaseType *impl_type = nullptr;
+  if (!query_type (impl_ty_id, &impl_type))
+    return true;
+
+  if (!receiver->can_eq (impl_type, false))
+    {
+      if (!impl_type->can_eq (receiver, false))
+	return true;
+    }
+
+  possible_trait_paths.push_back ({&impl->get_trait_ref (), impl});
+  return true;
+}
+
 void
 TypeBoundsProbe::scan ()
 {
@@ -68,31 +101,7 @@ TypeBoundsProbe::scan ()
     possible_trait_paths;
   mappings.iterate_impl_blocks (
     [&] (HirId id, HIR::ImplBlock *impl) mutable -> bool {
-      // we are filtering for trait-impl-blocks
-      if (!impl->has_trait_ref ())
-	return true;
-
-      // can be recursive trait resolution
-      HIR::Trait *t = TraitResolver::ResolveHirItem (impl->get_trait_ref ());
-      if (t == nullptr)
-	return true;
-      DefId trait_id = t->get_mappings ().get_defid ();
-      if (context->trait_query_in_progress (trait_id))
-	return true;
-
-      HirId impl_ty_id = impl->get_type ().get_mappings ().get_hirid ();
-      TyTy::BaseType *impl_type = nullptr;
-      if (!query_type (impl_ty_id, &impl_type))
-	return true;
-
-      if (!receiver->can_eq (impl_type, false))
-	{
-	  if (!impl_type->can_eq (receiver, false))
-	    return true;
-	}
-
-      possible_trait_paths.push_back ({&impl->get_trait_ref (), impl});
-      return true;
+      return process_impl_block (id, impl, possible_trait_paths);
     });
 
   for (auto &path : possible_trait_paths)
@@ -105,7 +114,7 @@ TypeBoundsProbe::scan ()
     }
 
   // marker traits...
-  assemble_sized_builtin ();
+  assemble_marker_builtins ();
 
   // add auto trait bounds
   for (auto *auto_trait : mappings.get_auto_traits ())
@@ -113,7 +122,7 @@ TypeBoundsProbe::scan ()
 }
 
 void
-TypeBoundsProbe::assemble_sized_builtin ()
+TypeBoundsProbe::assemble_marker_builtins ()
 {
   const TyTy::BaseType *raw = receiver->destructure ();
 
@@ -132,7 +141,6 @@ TypeBoundsProbe::assemble_sized_builtin ()
     case TyTy::POINTER:
     case TyTy::PARAM:
     case TyTy::FNDEF:
-    case TyTy::FNPTR:
     case TyTy::BOOL:
     case TyTy::CHAR:
     case TyTy::INT:
@@ -140,7 +148,6 @@ TypeBoundsProbe::assemble_sized_builtin ()
     case TyTy::FLOAT:
     case TyTy::USIZE:
     case TyTy::ISIZE:
-    case TyTy::CLOSURE:
     case TyTy::INFER:
     case TyTy::NEVER:
     case TyTy::PLACEHOLDER:
@@ -149,6 +156,14 @@ TypeBoundsProbe::assemble_sized_builtin ()
       assemble_builtin_candidate (LangItem::Kind::SIZED);
       break;
 
+    case TyTy::FNPTR:
+    case TyTy::CLOSURE:
+      assemble_builtin_candidate (LangItem::Kind::SIZED);
+      assemble_builtin_candidate (LangItem::Kind::FN_ONCE);
+      assemble_builtin_candidate (LangItem::Kind::FN);
+      assemble_builtin_candidate (LangItem::Kind::FN_MUT);
+      break;
+
       // FIXME str and slice need to be moved and test cases updated
     case TyTy::SLICE:
     case TyTy::STR:
@@ -158,6 +173,7 @@ TypeBoundsProbe::assemble_sized_builtin ()
       assemble_builtin_candidate (LangItem::Kind::SIZED);
       break;
 
+    case TyTy::CONST:
     case TyTy::DYNAMIC:
     case TyTy::ERROR:
       break;
@@ -206,7 +222,7 @@ TyTy::TypeBoundPredicate
 TypeCheckBase::get_predicate_from_bound (
   HIR::TypePath &type_path,
   tl::optional<std::reference_wrapper<HIR::Type>> associated_self,
-  BoundPolarity polarity, bool is_qualified_type_path)
+  BoundPolarity polarity, bool is_qualified_type_path, bool is_super_trait)
 {
   TyTy::TypeBoundPredicate lookup = TyTy::TypeBoundPredicate::error ();
   bool already_resolved
@@ -226,7 +242,8 @@ TypeCheckBase::get_predicate_from_bound (
   auto &final_seg = type_path.get_final_segment ();
   switch (final_seg.get_type ())
     {
-      case HIR::TypePathSegment::SegmentType::GENERIC: {
+    case HIR::TypePathSegment::SegmentType::GENERIC:
+      {
 	auto &final_generic_seg
 	  = static_cast<HIR::TypePathSegmentGeneric &> (final_seg);
 	if (final_generic_seg.has_generic_args ())
@@ -251,7 +268,8 @@ TypeCheckBase::get_predicate_from_bound (
       }
       break;
 
-      case HIR::TypePathSegment::SegmentType::FUNCTION: {
+    case HIR::TypePathSegment::SegmentType::FUNCTION:
+      {
 	auto &final_function_seg
 	  = static_cast<HIR::TypePathSegmentFunction &> (final_seg);
 	auto &fn = final_function_seg.get_function_path ();
@@ -327,7 +345,8 @@ TypeCheckBase::get_predicate_from_bound (
   if (!args.is_empty () || predicate.requires_generic_args ())
     {
       // this is applying generic arguments to a trait reference
-      predicate.apply_generic_arguments (&args, associated_self.has_value ());
+      predicate.apply_generic_arguments (&args, associated_self.has_value (),
+					 is_super_trait);
     }
 
   context->insert_resolved_predicate (type_path.get_mappings ().get_hirid (),
@@ -508,7 +527,8 @@ TypeBoundPredicate::is_object_safe (bool emit_error, location_t locus) const
 
 void
 TypeBoundPredicate::apply_generic_arguments (HIR::GenericArgs *generic_args,
-					     bool has_associated_self)
+					     bool has_associated_self,
+					     bool is_super_trait)
 {
   rust_assert (!substitutions.empty ());
   if (has_associated_self)
@@ -529,23 +549,26 @@ TypeBoundPredicate::apply_generic_arguments (HIR::GenericArgs *generic_args,
     Resolver::TypeCheckContext::get ()->regions_from_generic_args (
       *generic_args));
 
-  apply_argument_mappings (args);
+  apply_argument_mappings (args, is_super_trait);
 }
 
 void
 TypeBoundPredicate::apply_argument_mappings (
-  SubstitutionArgumentMappings &arguments)
+  SubstitutionArgumentMappings &arguments, bool is_super_trait)
 {
   used_arguments = arguments;
   error_flag |= used_arguments.is_error ();
   auto &subst_mappings = used_arguments;
+
+  bool substs_need_bounds_check = !is_super_trait;
   for (auto &sub : get_substs ())
     {
       SubstitutionArg arg = SubstitutionArg::error ();
       bool ok
 	= subst_mappings.get_argument_for_symbol (sub.get_param_ty (), &arg);
       if (ok && arg.get_tyty () != nullptr)
-	sub.fill_param_ty (subst_mappings, subst_mappings.get_locus ());
+	sub.fill_param_ty (subst_mappings, subst_mappings.get_locus (),
+			   substs_need_bounds_check);
     }
 
   // associated argument mappings
@@ -566,7 +589,7 @@ TypeBoundPredicate::apply_argument_mappings (
       auto adjusted
 	= super_trait.adjust_mappings_for_this (used_arguments,
 						true /*trait mode*/);
-      super_trait.apply_argument_mappings (adjusted);
+      super_trait.apply_argument_mappings (adjusted, is_super_trait);
     }
 }
 
@@ -699,7 +722,7 @@ TypeBoundPredicate::handle_substitions (
       if (sub.get_param_ty () == nullptr)
 	continue;
 
-      ParamType *p = sub.get_param_ty ();
+      auto p = sub.get_param_ty ();
       BaseType *r = p->resolve ();
       BaseType *s = Resolver::SubstMapperInternal::Resolve (r, subst_mappings);
 
@@ -746,16 +769,34 @@ size_t
 TypeBoundPredicate::get_num_associated_bindings () const
 {
   size_t count = 0;
+
+  get_trait_hierachy ([&count] (const Resolver::TraitReference &ref) {
+    for (const auto &trait_item : ref.get_trait_items ())
+      {
+	bool is_associated_type
+	  = trait_item.get_trait_item_type ()
+	    == Resolver::TraitItemReference::TraitItemType::TYPE;
+	if (is_associated_type)
+	  count++;
+      }
+  });
+
+  return count;
+}
+
+void
+TypeBoundPredicate::get_trait_hierachy (
+  std::function<void (const Resolver::TraitReference &)> callback) const
+{
   auto trait_ref = get ();
-  for (const auto &trait_item : trait_ref->get_trait_items ())
+  callback (*trait_ref);
+
+  for (auto &super : super_traits)
     {
-      bool is_associated_type
-	= trait_item.get_trait_item_type ()
-	  == Resolver::TraitItemReference::TraitItemType::TYPE;
-      if (is_associated_type)
-	count++;
+      const auto &super_trait_ref = *super.get ();
+      callback (super_trait_ref);
+      super.get_trait_hierachy (callback);
     }
-  return count;
 }
 
 TypeBoundPredicateItem
@@ -808,21 +849,19 @@ TypeBoundPredicate::is_equal (const TypeBoundPredicate &other) const
   // then match the generics applied
   for (size_t i = 0; i < get_num_substitutions (); i++)
     {
-      const SubstitutionParamMapping &a = substitutions.at (i);
-      const SubstitutionParamMapping &b = other.substitutions.at (i);
+      SubstitutionParamMapping a = substitutions.at (i);
+      SubstitutionParamMapping b = other.substitutions.at (i);
 
-      const ParamType *ap = a.get_param_ty ();
-      const ParamType *bp = b.get_param_ty ();
+      auto ap = a.get_param_ty ();
+      auto bp = b.get_param_ty ();
 
-      const BaseType *apd = ap->destructure ();
-      const BaseType *bpd = bp->destructure ();
+      BaseType *apd = ap->destructure ();
+      BaseType *bpd = bp->destructure ();
 
-      // FIXME use the unify_and infer inteface or try coerce
-      if (!apd->can_eq (bpd, false /*emit_errors*/))
-	{
-	  if (!bpd->can_eq (apd, false /*emit_errors*/))
-	    return false;
-	}
+      if (!Resolver::types_compatable (TyTy::TyWithLocation (apd),
+				       TyTy::TyWithLocation (bpd),
+				       UNKNOWN_LOCATION, false))
+	return false;
     }
 
   return true;
diff --git a/gcc/rust/typecheck/rust-tyty-call.cc b/gcc/rust/typecheck/rust-tyty-call.cc
index 2e0830e..63bb1ff 100644
--- a/gcc/rust/typecheck/rust-tyty-call.cc
+++ b/gcc/rust/typecheck/rust-tyty-call.cc
@@ -171,7 +171,8 @@ TypeCheckCallExpr::visit (FnType &type)
 	    {
 	    case TyTy::TypeKind::ERROR:
 	      return;
-	      case TyTy::TypeKind::INT: {
+	    case TyTy::TypeKind::INT:
+	      {
 		auto &int_ty
 		  = static_cast<TyTy::IntType &> (*argument_expr_tyty);
 		if ((int_ty.get_int_kind () == TyTy::IntType::IntKind::I8)
@@ -186,7 +187,8 @@ TypeCheckCallExpr::visit (FnType &type)
 		  }
 		break;
 	      }
-	      case TyTy::TypeKind::UINT: {
+	    case TyTy::TypeKind::UINT:
+	      {
 		auto &uint_ty
 		  = static_cast<TyTy::UintType &> (*argument_expr_tyty);
 		if ((uint_ty.get_uint_kind () == TyTy::UintType::UintKind::U8)
@@ -202,7 +204,8 @@ TypeCheckCallExpr::visit (FnType &type)
 		  }
 		break;
 	      }
-	      case TyTy::TypeKind::FLOAT: {
+	    case TyTy::TypeKind::FLOAT:
+	      {
 		if (static_cast<TyTy::FloatType &> (*argument_expr_tyty)
 		      .get_float_kind ()
 		    == TyTy::FloatType::FloatKind::F32)
@@ -216,14 +219,16 @@ TypeCheckCallExpr::visit (FnType &type)
 		  }
 		break;
 	      }
-	      case TyTy::TypeKind::BOOL: {
+	    case TyTy::TypeKind::BOOL:
+	      {
 		rich_location richloc (line_table, arg_locus);
 		richloc.add_fixit_replace ("cast the value to c_int: as c_int");
 		rust_error_at (arg_locus, ErrorCode::E0617,
 			       "expected %<c_int%> variadic argument");
 		return;
 	      }
-	      case TyTy::TypeKind::FNDEF: {
+	    case TyTy::TypeKind::FNDEF:
+	      {
 		rust_error_at (
 		  arg_locus, ErrorCode::E0617,
 		  "unexpected function definition type as variadic "
@@ -246,7 +251,7 @@ TypeCheckCallExpr::visit (FnType &type)
     }
 
   type.monomorphize ();
-  resolved = type.get_return_type ()->clone ();
+  resolved = type.get_return_type ()->monomorphized_clone ();
 }
 
 void
diff --git a/gcc/rust/typecheck/rust-tyty-call.h b/gcc/rust/typecheck/rust-tyty-call.h
index c42fdcd..9e4aab5 100644
--- a/gcc/rust/typecheck/rust-tyty-call.h
+++ b/gcc/rust/typecheck/rust-tyty-call.h
@@ -62,6 +62,7 @@ public:
   void visit (DynamicObjectType &) override { rust_unreachable (); }
   void visit (ClosureType &type) override { rust_unreachable (); }
   void visit (OpaqueType &type) override { rust_unreachable (); }
+  void visit (ConstType &type) override { rust_unreachable (); }
 
   // tuple-structs
   void visit (ADTType &type) override;
diff --git a/gcc/rust/typecheck/rust-tyty-cmp.h b/gcc/rust/typecheck/rust-tyty-cmp.h
index c897c13..c22dfdd 100644
--- a/gcc/rust/typecheck/rust-tyty-cmp.h
+++ b/gcc/rust/typecheck/rust-tyty-cmp.h
@@ -447,6 +447,22 @@ public:
       }
   }
 
+  virtual void visit (const ConstType &type) override
+  {
+    ok = false;
+    if (emit_error_flag)
+      {
+	location_t ref_locus = mappings.lookup_location (type.get_ref ());
+	location_t base_locus
+	  = mappings.lookup_location (get_base ()->get_ref ());
+	rich_location r (line_table, ref_locus);
+	r.add_range (base_locus);
+	rust_error_at (r, "expected [%s] got [%s]",
+		       get_base ()->as_string ().c_str (),
+		       type.as_string ().c_str ());
+      }
+  }
+
 protected:
   BaseCmp (const BaseType *base, bool emit_errors)
     : mappings (Analysis::Mappings::get ()),
@@ -621,7 +637,8 @@ public:
 	ok = true;
 	return;
 
-	case InferType::InferTypeKind::INTEGRAL: {
+      case InferType::InferTypeKind::INTEGRAL:
+	{
 	  if (type.get_infer_kind () == InferType::InferTypeKind::INTEGRAL)
 	    {
 	      ok = true;
@@ -635,7 +652,8 @@ public:
 	}
 	break;
 
-	case InferType::InferTypeKind::FLOAT: {
+      case InferType::InferTypeKind::FLOAT:
+	{
 	  if (type.get_infer_kind () == InferType::InferTypeKind::FLOAT)
 	    {
 	      ok = true;
@@ -1604,6 +1622,23 @@ private:
   const OpaqueType *base;
 };
 
+class ConstCmp : public BaseCmp
+{
+  using Rust::TyTy::BaseCmp::visit;
+
+public:
+  ConstCmp (const ConstType *base, bool emit_errors)
+    : BaseCmp (base, emit_errors), base (base)
+  {}
+
+  // TODO
+
+private:
+  const BaseType *get_base () const override { return base; }
+
+  const ConstType *base;
+};
+
 } // namespace TyTy
 } // namespace Rust
 
diff --git a/gcc/rust/typecheck/rust-tyty-subst.cc b/gcc/rust/typecheck/rust-tyty-subst.cc
index bdb6474..817910b 100644
--- a/gcc/rust/typecheck/rust-tyty-subst.cc
+++ b/gcc/rust/typecheck/rust-tyty-subst.cc
@@ -18,18 +18,22 @@
 
 #include "rust-tyty-subst.h"
 
+#include "rust-hir-generic-param.h"
 #include "rust-system.h"
 #include "rust-tyty.h"
 #include "rust-hir-type-check.h"
 #include "rust-substitution-mapper.h"
 #include "rust-hir-type-check-type.h"
+#include "rust-hir-type-check-expr.h"
+#include "rust-compile-base.h"
 #include "rust-type-util.h"
+#include "tree.h"
 
 namespace Rust {
 namespace TyTy {
 
-SubstitutionParamMapping::SubstitutionParamMapping (HIR::TypeParam &generic,
-						    ParamType *param)
+SubstitutionParamMapping::SubstitutionParamMapping (HIR::GenericParam &generic,
+						    BaseGeneric *param)
   : generic (generic), param (param)
 {}
 
@@ -54,30 +58,42 @@ SubstitutionParamMapping::clone () const
 				   static_cast<ParamType *> (param->clone ()));
 }
 
-ParamType *
+BaseGeneric *
 SubstitutionParamMapping::get_param_ty ()
 {
   return param;
 }
 
-const ParamType *
+const BaseGeneric *
 SubstitutionParamMapping::get_param_ty () const
 {
   return param;
 }
 
-HIR::TypeParam &
+HIR::GenericParam &
 SubstitutionParamMapping::get_generic_param ()
 {
   return generic;
 }
 
+const HIR::GenericParam &
+SubstitutionParamMapping::get_generic_param () const
+{
+  return generic;
+}
+
 bool
 SubstitutionParamMapping::needs_substitution () const
 {
   return !(get_param_ty ()->is_concrete ());
 }
 
+Identifier
+SubstitutionParamMapping::get_type_representation () const
+{
+  return param->get_symbol ();
+}
+
 location_t
 SubstitutionParamMapping::get_param_locus () const
 {
@@ -87,13 +103,35 @@ SubstitutionParamMapping::get_param_locus () const
 bool
 SubstitutionParamMapping::param_has_default_ty () const
 {
-  return generic.has_type ();
+  if (generic.get_kind () == HIR::GenericParam::GenericKind::TYPE)
+    {
+      const auto &type_param = static_cast<const HIR::TypeParam &> (generic);
+      return type_param.has_type ();
+    }
+
+  rust_assert (generic.get_kind () == HIR::GenericParam::GenericKind::CONST);
+  const auto &const_param
+    = static_cast<const HIR::ConstGenericParam &> (generic);
+  return const_param.has_default_expression ();
 }
 
 BaseType *
 SubstitutionParamMapping::get_default_ty () const
 {
-  TyVar var (generic.get_type_mappings ().get_hirid ());
+  if (generic.get_kind () == HIR::GenericParam::GenericKind::TYPE)
+    {
+      const auto &type_param = static_cast<const HIR::TypeParam &> (generic);
+      TyVar var (type_param.get_type_mappings ().get_hirid ());
+      return var.get_tyty ();
+    }
+
+  rust_assert (generic.get_kind () == HIR::GenericParam::GenericKind::CONST);
+  const auto &const_param
+    = static_cast<const HIR::ConstGenericParam &> (generic);
+  rust_assert (const_param.has_default_expression ());
+
+  const auto &expr = const_param.get_default_expression ();
+  TyVar var (expr.get_mappings ().get_hirid ());
   return var.get_tyty ();
 }
 
@@ -109,7 +147,8 @@ SubstitutionParamMapping::need_substitution () const
 
 bool
 SubstitutionParamMapping::fill_param_ty (
-  SubstitutionArgumentMappings &subst_mappings, location_t locus)
+  SubstitutionArgumentMappings &subst_mappings, location_t locus,
+  bool needs_bounds_check)
 {
   SubstitutionArg arg = SubstitutionArg::error ();
   bool ok = subst_mappings.get_argument_for_symbol (get_param_ty (), &arg);
@@ -124,17 +163,21 @@ SubstitutionParamMapping::fill_param_ty (
 
   if (type.get_kind () == TypeKind::PARAM)
     {
-      // delete param;
-      param = static_cast<ParamType *> (type.clone ());
+      param = static_cast<BaseGeneric *> (type.clone ());
     }
-  else
+  else if (type.get_kind () == TyTy::TypeKind::CONST)
     {
+      param = static_cast<BaseGeneric *> (type.clone ());
+    }
+  else if (param->get_kind () == TypeKind::PARAM)
+    {
+      auto &p = *static_cast<TyTy::ParamType *> (param);
+
       // check the substitution is compatible with bounds
       rust_debug_loc (locus,
 		      "fill_param_ty bounds_compatible: param %s type %s",
 		      param->get_name ().c_str (), type.get_name ().c_str ());
-
-      if (!param->is_implicit_self_trait ())
+      if (needs_bounds_check && !p.is_implicit_self_trait ())
 	{
 	  if (!param->bounds_compatible (type, locus, true))
 	    return false;
@@ -145,7 +188,7 @@ SubstitutionParamMapping::fill_param_ty (
 	bound.handle_substitions (subst_mappings);
 
       param->set_ty_ref (type.get_ref ());
-      subst_mappings.on_param_subst (*param, arg);
+      subst_mappings.on_param_subst (p, arg);
     }
 
   return true;
@@ -191,12 +234,6 @@ SubstitutionArg::operator= (const SubstitutionArg &other)
 }
 
 BaseType *
-SubstitutionArg::get_tyty ()
-{
-  return argument;
-}
-
-const BaseType *
 SubstitutionArg::get_tyty () const
 {
   return argument;
@@ -208,7 +245,7 @@ SubstitutionArg::get_param_mapping () const
   return param;
 }
 
-const ParamType *
+const BaseGeneric *
 SubstitutionArg::get_param_ty () const
 {
   return original_param;
@@ -313,11 +350,11 @@ SubstitutionArgumentMappings::is_error () const
 
 bool
 SubstitutionArgumentMappings::get_argument_for_symbol (
-  const ParamType *param_to_find, SubstitutionArg *argument) const
+  const BaseGeneric *param_to_find, SubstitutionArg *argument) const
 {
   for (const auto &mapping : mappings)
     {
-      const ParamType *p = mapping.get_param_ty ();
+      const auto *p = mapping.get_param_ty ();
       if (p->get_symbol () == param_to_find->get_symbol ())
 	{
 	  *argument = mapping;
@@ -618,7 +655,6 @@ SubstitutionRef::get_mappings_from_generic_args (
 	  if (args.get_binding_args ().size () > get_num_associated_bindings ())
 	    {
 	      rich_location r (line_table, args.get_locus ());
-
 	      rust_error_at (r,
 			     "generic item takes at most %lu type binding "
 			     "arguments but %lu were supplied",
@@ -666,11 +702,17 @@ SubstitutionRef::get_mappings_from_generic_args (
 
   // for inherited arguments
   size_t offs = used_arguments.size ();
-  if (args.get_type_args ().size () + offs > substitutions.size ())
+  size_t total_arguments
+    = args.get_type_args ().size () + args.get_const_args ().size () + offs;
+  if (total_arguments > substitutions.size ())
     {
       rich_location r (line_table, args.get_locus ());
       if (!substitutions.empty ())
-	r.add_range (substitutions.front ().get_param_locus ());
+	{
+	  const auto &subst = substitutions.front ();
+	  const auto &generic = subst.get_generic_param ();
+	  r.add_range (generic.get_locus ());
+	}
 
       rust_error_at (
 	r,
@@ -680,10 +722,15 @@ SubstitutionRef::get_mappings_from_generic_args (
       return SubstitutionArgumentMappings::error ();
     }
 
-  if (args.get_type_args ().size () + offs < min_required_substitutions ())
+  if (total_arguments < min_required_substitutions ())
     {
       rich_location r (line_table, args.get_locus ());
-      r.add_range (substitutions.front ().get_param_locus ());
+      if (!substitutions.empty ())
+	{
+	  const auto &subst = substitutions.front ();
+	  const auto &generic = subst.get_generic_param ();
+	  r.add_range (generic.get_locus ());
+	}
 
       rust_error_at (
 	r, ErrorCode::E0107,
@@ -702,7 +749,91 @@ SubstitutionRef::get_mappings_from_generic_args (
 	  return SubstitutionArgumentMappings::error ();
 	}
 
-      SubstitutionArg subst_arg (&substitutions.at (offs), resolved);
+      const auto &param_mapping = substitutions.at (offs);
+      const auto &generic = param_mapping.get_generic_param ();
+      if (generic.get_kind () == HIR::GenericParam::GenericKind::TYPE)
+	{
+	  const auto &type_param
+	    = static_cast<const HIR::TypeParam &> (generic);
+	  if (type_param.from_impl_trait ())
+	    {
+	      rich_location r (line_table, arg->get_locus ());
+	      r.add_fixit_remove (arg->get_locus ());
+	      rust_error_at (r, ErrorCode::E0632,
+			     "cannot provide explicit generic arguments when "
+			     "%<impl Trait%> is used in argument position");
+	      return SubstitutionArgumentMappings::error ();
+	    }
+	}
+      else if (generic.get_kind () == HIR::GenericParam::GenericKind::CONST)
+	{
+	  if (!resolved->is<ConstType> ())
+	    {
+	      rich_location r (line_table, arg->get_locus ());
+	      r.add_fixit_remove (arg->get_locus ());
+	      rust_error_at (r, ErrorCode::E0747,
+			     "type provided when a constant was expected");
+	      return SubstitutionArgumentMappings::error ();
+	    }
+	}
+
+      SubstitutionArg subst_arg (&param_mapping, resolved);
+      offs++;
+      mappings.push_back (std::move (subst_arg));
+    }
+
+  for (auto &arg : args.get_const_args ())
+    {
+      auto &expr = *arg.get_expression ().get ();
+      BaseType *expr_type = Resolver::TypeCheckExpr::Resolve (expr);
+      if (expr_type == nullptr || expr_type->is<ErrorType> ())
+	return SubstitutionArgumentMappings::error ();
+
+      // validate this param is really a const generic
+      const auto &param_mapping = substitutions.at (offs);
+      const auto &generic = param_mapping.get_generic_param ();
+      if (generic.get_kind () != HIR::GenericParam::GenericKind::CONST)
+	{
+	  rich_location r (line_table, arg.get_locus ());
+	  r.add_fixit_remove (expr.get_locus ());
+	  rust_error_at (r, "invalid position for a const generic argument");
+	  return SubstitutionArgumentMappings::error ();
+	}
+
+      // get the const generic specified type
+      const auto base_generic = param_mapping.get_param_ty ();
+      rust_assert (base_generic->is<ConstType> ());
+      const auto const_param
+	= static_cast<const TyTy::ConstType *> (base_generic);
+      auto specified_type = const_param->get_ty ();
+
+      // validate this const generic is of the correct type
+      auto coereced_type
+	= Resolver::coercion_site (expr.get_mappings ().get_hirid (),
+				   TyTy::TyWithLocation (specified_type),
+				   TyTy::TyWithLocation (expr_type,
+							 expr.get_locus ()),
+				   arg.get_locus ());
+      if (coereced_type->is<ErrorType> ())
+	return SubstitutionArgumentMappings::error ();
+
+      // const fold it
+      auto ctx = Compile::Context::get ();
+      tree folded
+	= Compile::HIRCompileBase::query_compile_const_expr (ctx, coereced_type,
+							     expr);
+
+      if (folded == error_mark_node)
+	return SubstitutionArgumentMappings::error ();
+
+      // create const type
+      auto const_value
+	= new TyTy::ConstType (TyTy::ConstType::ConstKind::Value, "",
+			       coereced_type, folded, {}, expr.get_locus (),
+			       expr.get_mappings ().get_hirid (),
+			       expr.get_mappings ().get_hirid (), {});
+
+      SubstitutionArg subst_arg (&param_mapping, const_value);
       offs++;
       mappings.push_back (std::move (subst_arg));
     }
@@ -754,6 +885,7 @@ SubstitutionRef::infer_substitions (location_t locus)
     {
       if (p.needs_substitution ())
 	{
+	  const HIR::GenericParam &generic = p.get_generic_param ();
 	  const std::string &symbol = p.get_param_ty ()->get_symbol ();
 	  auto it = argument_mappings.find (symbol);
 	  bool have_mapping = it != argument_mappings.end ();
@@ -762,12 +894,24 @@ SubstitutionRef::infer_substitions (location_t locus)
 	    {
 	      args.push_back (SubstitutionArg (&p, it->second));
 	    }
-	  else
+	  else if (generic.get_kind () == HIR::GenericParam::GenericKind::TYPE)
 	    {
 	      TyVar infer_var = TyVar::get_implicit_infer_var (locus);
 	      args.push_back (SubstitutionArg (&p, infer_var.get_tyty ()));
 	      argument_mappings[symbol] = infer_var.get_tyty ();
 	    }
+	  else if (generic.get_kind () == HIR::GenericParam::GenericKind::CONST)
+	    {
+	      const auto const_param = p.get_param_ty ();
+	      rust_assert (const_param->is<TyTy::ConstType> ());
+	      const auto &const_type
+		= *static_cast<const TyTy::ConstType *> (const_param);
+
+	      TyVar infer_var
+		= TyVar::get_implicit_const_infer_var (const_type, locus);
+	      args.push_back (SubstitutionArg (&p, infer_var.get_tyty ()));
+	      argument_mappings[symbol] = infer_var.get_tyty ();
+	    }
 	}
       else
 	{
@@ -905,7 +1049,7 @@ SubstitutionRef::prepare_higher_ranked_bounds ()
 {
   for (const auto &subst : get_substs ())
     {
-      const TyTy::ParamType *pty = subst.get_param_ty ();
+      const auto pty = subst.get_param_ty ();
       for (const auto &bound : pty->get_specified_bounds ())
 	{
 	  const auto ref = bound.get ();
@@ -919,8 +1063,7 @@ SubstitutionRef::monomorphize ()
 {
   for (const auto &subst : get_substs ())
     {
-      const TyTy::ParamType *pty = subst.get_param_ty ();
-
+      const auto pty = subst.get_param_ty ();
       if (!pty->can_resolve ())
 	continue;
 
diff --git a/gcc/rust/typecheck/rust-tyty-subst.h b/gcc/rust/typecheck/rust-tyty-subst.h
index e6ed1fc..c1bc96a 100644
--- a/gcc/rust/typecheck/rust-tyty-subst.h
+++ b/gcc/rust/typecheck/rust-tyty-subst.h
@@ -24,12 +24,14 @@
 #include "rust-hir-full-decls.h"
 #include "rust-tyty-bounds.h"
 #include "rust-tyty-region.h"
+#include "rust-ast.h"
 #include "optional.h"
 
 namespace Rust {
 namespace TyTy {
 
 class ParamType;
+class BaseGeneric;
 
 struct RegionConstraints
 {
@@ -44,22 +46,24 @@ class SubstitutionArgumentMappings;
 class SubstitutionParamMapping
 {
 public:
-  SubstitutionParamMapping (HIR::TypeParam &generic, ParamType *param);
+  SubstitutionParamMapping (HIR::GenericParam &generic, BaseGeneric *param);
 
   SubstitutionParamMapping (const SubstitutionParamMapping &other);
 
   std::string as_string () const;
 
   bool fill_param_ty (SubstitutionArgumentMappings &subst_mappings,
-		      location_t locus);
+		      location_t locus, bool needs_bounds_check = true);
 
   SubstitutionParamMapping clone () const;
 
-  ParamType *get_param_ty ();
+  BaseGeneric *get_param_ty ();
+  const BaseGeneric *get_param_ty () const;
 
-  const ParamType *get_param_ty () const;
+  HIR::GenericParam &get_generic_param ();
+  const HIR::GenericParam &get_generic_param () const;
 
-  HIR::TypeParam &get_generic_param ();
+  Identifier get_type_representation () const;
 
   // this is used for the backend to override the HirId ref of the param to
   // what the concrete type is for the rest of the context
@@ -76,8 +80,8 @@ public:
   bool need_substitution () const;
 
 private:
-  HIR::TypeParam &generic;
-  ParamType *param;
+  HIR::GenericParam &generic;
+  BaseGeneric *param;
 };
 
 /**
@@ -147,13 +151,11 @@ public:
 
   SubstitutionArg &operator= (const SubstitutionArg &other);
 
-  BaseType *get_tyty ();
-
-  const BaseType *get_tyty () const;
+  BaseType *get_tyty () const;
 
   const SubstitutionParamMapping *get_param_mapping () const;
 
-  const ParamType *get_param_ty () const;
+  const BaseGeneric *get_param_ty () const;
 
   static SubstitutionArg error ();
 
@@ -165,7 +167,7 @@ public:
 
 private:
   const SubstitutionParamMapping *param;
-  const ParamType *original_param;
+  const BaseGeneric *original_param;
   BaseType *argument;
 };
 
@@ -205,7 +207,7 @@ public:
 
   bool is_error () const;
 
-  bool get_argument_for_symbol (const ParamType *param_to_find,
+  bool get_argument_for_symbol (const BaseGeneric *param_to_find,
 				SubstitutionArg *argument) const;
 
   /** Return type parameter index for symbol */
diff --git a/gcc/rust/typecheck/rust-tyty-util.cc b/gcc/rust/typecheck/rust-tyty-util.cc
index ff210ce..4bc1723 100644
--- a/gcc/rust/typecheck/rust-tyty-util.cc
+++ b/gcc/rust/typecheck/rust-tyty-util.cc
@@ -17,6 +17,8 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-hir-type-check.h"
+#include "rust-mapping-common.h"
+#include "rust-system.h"
 #include "rust-tyty.h"
 
 namespace Rust {
@@ -47,14 +49,30 @@ TyVar::get_implicit_infer_var (location_t locus)
   auto &mappings = Analysis::Mappings::get ();
   auto context = Resolver::TypeCheckContext::get ();
 
-  InferType *infer = new InferType (mappings.get_next_hir_id (),
-				    InferType::InferTypeKind::GENERAL,
-				    InferType::TypeHint::Default (), locus);
-  context->insert_type (Analysis::NodeMapping (mappings.get_current_crate (),
-					       UNKNOWN_NODEID,
-					       infer->get_ref (),
-					       UNKNOWN_LOCAL_DEFID),
-			infer);
+  HirId next = mappings.get_next_hir_id ();
+  auto infer = new InferType (next, InferType::InferTypeKind::GENERAL,
+			      InferType::TypeHint::Default (), locus);
+
+  context->insert_implicit_type (infer->get_ref (), infer);
+  mappings.insert_location (infer->get_ref (), locus);
+
+  return TyVar (infer->get_ref ());
+}
+
+TyVar
+TyVar::get_implicit_const_infer_var (const ConstType &const_type,
+				     location_t locus)
+{
+  auto &mappings = Analysis::Mappings::get ();
+  auto context = Resolver::TypeCheckContext::get ();
+
+  HirId next = mappings.get_next_hir_id ();
+  auto infer
+    = new ConstType (ConstType::ConstKind::Infer, const_type.get_symbol (),
+		     const_type.get_ty (), error_mark_node,
+		     const_type.get_specified_bounds (), locus, next, next, {});
+
+  context->insert_implicit_type (infer->get_ref (), infer);
   mappings.insert_location (infer->get_ref (), locus);
 
   return TyVar (infer->get_ref ());
diff --git a/gcc/rust/typecheck/rust-tyty-util.h b/gcc/rust/typecheck/rust-tyty-util.h
index cbb3e8e..1c8fd72 100644
--- a/gcc/rust/typecheck/rust-tyty-util.h
+++ b/gcc/rust/typecheck/rust-tyty-util.h
@@ -25,6 +25,7 @@ namespace Rust {
 namespace TyTy {
 
 class BaseType;
+class ConstType;
 
 // this is a placeholder for types that can change like inference variables
 class TyVar
@@ -42,6 +43,9 @@ public:
 
   static TyVar get_implicit_infer_var (location_t locus);
 
+  static TyVar get_implicit_const_infer_var (const TyTy::ConstType &const_type,
+					     location_t locus);
+
   static TyVar subst_covariant_var (TyTy::BaseType *orig,
 				    TyTy::BaseType *subst);
 
diff --git a/gcc/rust/typecheck/rust-tyty-variance-analysis-private.h b/gcc/rust/typecheck/rust-tyty-variance-analysis-private.h
index d36afc8..deb76a7 100644
--- a/gcc/rust/typecheck/rust-tyty-variance-analysis-private.h
+++ b/gcc/rust/typecheck/rust-tyty-variance-analysis-private.h
@@ -170,6 +170,8 @@ public:
   }
 
   void visit (OpaqueType &type) override {}
+
+  void visit (ConstType &type) override {}
 };
 
 /** Per crate context for generic type variance analysis. */
diff --git a/gcc/rust/typecheck/rust-tyty-variance-analysis.cc b/gcc/rust/typecheck/rust-tyty-variance-analysis.cc
index 38f9d52..7971ccf 100644
--- a/gcc/rust/typecheck/rust-tyty-variance-analysis.cc
+++ b/gcc/rust/typecheck/rust-tyty-variance-analysis.cc
@@ -199,9 +199,7 @@ GenericTyPerCrateCtx::debug_print_solutions ()
 	    {
 	      if (i > solution_index)
 		result += ", ";
-	      result += param.get_generic_param ()
-			  .get_type_representation ()
-			  .as_string ();
+	      result += param.get_type_representation ().as_string ();
 	      result += "=";
 	      result += solutions[i].as_string ();
 	      i++;
@@ -239,8 +237,7 @@ GenericTyVisitorCtx::process_type (ADTType &ty)
   first_type = first_lifetime + ty.get_used_arguments ().get_regions ().size ();
 
   for (auto &param : ty.get_substs ())
-    param_names.push_back (
-      param.get_generic_param ().get_type_representation ().as_string ());
+    param_names.push_back (param.get_type_representation ().as_string ());
 
   for (const auto &variant : ty.get_variants ())
     {
diff --git a/gcc/rust/typecheck/rust-tyty-variance-analysis.h b/gcc/rust/typecheck/rust-tyty-variance-analysis.h
index 9059a2f..282c6f3 100644
--- a/gcc/rust/typecheck/rust-tyty-variance-analysis.h
+++ b/gcc/rust/typecheck/rust-tyty-variance-analysis.h
@@ -41,9 +41,10 @@ private:
   std::unique_ptr<GenericTyPerCrateCtx> private_ctx;
 };
 
-std::vector<size_t>
-query_field_regions (const ADTType *parent, size_t variant_index,
-		     size_t field_index, const FreeRegions &parent_regions);
+std::vector<size_t> query_field_regions (const ADTType *parent,
+					 size_t variant_index,
+					 size_t field_index,
+					 const FreeRegions &parent_regions);
 
 /** Variance semilattice */
 class Variance
diff --git a/gcc/rust/typecheck/rust-tyty-visitor.h b/gcc/rust/typecheck/rust-tyty-visitor.h
index 4f8e7856..7783075 100644
--- a/gcc/rust/typecheck/rust-tyty-visitor.h
+++ b/gcc/rust/typecheck/rust-tyty-visitor.h
@@ -45,6 +45,7 @@ public:
   virtual void visit (ReferenceType &type) = 0;
   virtual void visit (PointerType &type) = 0;
   virtual void visit (ParamType &type) = 0;
+  virtual void visit (ConstType &type) = 0;
   virtual void visit (StrType &type) = 0;
   virtual void visit (NeverType &type) = 0;
   virtual void visit (PlaceholderType &type) = 0;
@@ -75,6 +76,7 @@ public:
   virtual void visit (const ReferenceType &type) = 0;
   virtual void visit (const PointerType &type) = 0;
   virtual void visit (const ParamType &type) = 0;
+  virtual void visit (const ConstType &type) = 0;
   virtual void visit (const StrType &type) = 0;
   virtual void visit (const NeverType &type) = 0;
   virtual void visit (const PlaceholderType &type) = 0;
diff --git a/gcc/rust/typecheck/rust-tyty.cc b/gcc/rust/typecheck/rust-tyty.cc
index f0f4a07..db96773 100644
--- a/gcc/rust/typecheck/rust-tyty.cc
+++ b/gcc/rust/typecheck/rust-tyty.cc
@@ -19,6 +19,7 @@
 #include "rust-tyty.h"
 
 #include "optional.h"
+#include "rust-tyty-subst.h"
 #include "rust-tyty-visitor.h"
 #include "rust-hir-map.h"
 #include "rust-location.h"
@@ -30,9 +31,14 @@
 #include "rust-tyty-cmp.h"
 #include "rust-type-util.h"
 #include "rust-hir-type-bounds.h"
+#include "print-tree.h"
+#include "tree-pretty-print.h"
 
 #include "options.h"
 #include "rust-system.h"
+#include "tree.h"
+#include "fold-const.h"
+#include <string>
 
 namespace Rust {
 namespace TyTy {
@@ -114,6 +120,9 @@ TypeKindFormat::to_string (TypeKind kind)
     case TypeKind::OPAQUE:
       return "Opaque";
 
+    case TypeKind::CONST:
+      return "Const";
+
     case TypeKind::ERROR:
       return "ERROR";
     }
@@ -223,6 +232,7 @@ BaseType::is_unit () const
     case OPAQUE:
     case STR:
     case DYNAMIC:
+    case CONST:
     case ERROR:
       return false;
 
@@ -230,11 +240,13 @@ BaseType::is_unit () const
     case NEVER:
       return true;
 
-      case TUPLE: {
+    case TUPLE:
+      {
 	return x->as<const TupleType> ()->num_fields () == 0;
       }
 
-      case ADT: {
+    case ADT:
+      {
 	auto adt = x->as<const ADTType> ();
 	if (adt->is_enum ())
 	  return false;
@@ -438,11 +450,10 @@ BaseType::inherit_bounds (
     }
 }
 
-const BaseType *
-BaseType::get_root () const
+BaseType *
+BaseType::get_root ()
 {
-  // FIXME this needs to be it its own visitor class with a vector adjustments
-  const TyTy::BaseType *root = this;
+  TyTy::BaseType *root = this;
 
   if (const auto r = root->try_as<const ReferenceType> ())
     {
@@ -546,17 +557,14 @@ BaseType::destructure () const
 	{
 	  x = p->get ();
 	}
-      // else if (auto p = x->try_as<const OpaqueType> ())
-      //   {
-      //     auto pr = p->resolve ();
-
-      //     rust_debug ("XXXXXX")
-
-      //     if (pr == x)
-      //       return pr;
+      else if (auto p = x->try_as<const OpaqueType> ())
+	{
+	  auto pr = p->resolve ();
+	  if (pr == x)
+	    return pr;
 
-      //     x = pr;
-      //   }
+	  x = pr;
+	}
       else
 	{
 	  return x;
@@ -575,7 +583,7 @@ BaseType::monomorphized_clone () const
     {
       TyVar elm = arr->get_var_element_type ().monomorphized_clone ();
       return new ArrayType (arr->get_ref (), arr->get_ty_ref (), ident.locus,
-			    arr->get_capacity_expr (), elm,
+			    arr->get_capacity (), elm,
 			    arr->get_combined_refs ());
     }
   else if (auto slice = x->try_as<const SliceType> ())
@@ -824,7 +832,8 @@ BaseType::is_concrete () const
     }
   else if (auto arr = x->try_as<const ArrayType> ())
     {
-      return arr->get_element_type ()->is_concrete ();
+      return arr->get_element_type ()->is_concrete ()
+	     && arr->get_capacity ()->is_concrete ();
     }
   else if (auto slice = x->try_as<const SliceType> ())
     {
@@ -853,6 +862,10 @@ BaseType::is_concrete () const
 	return false;
       return closure->get_result_type ().is_concrete ();
     }
+  else if (auto const_type = x->try_as<const ConstType> ())
+    {
+      return const_type->get_value () != error_mark_node;
+    }
   else if (x->is<InferType> () || x->is<BoolType> () || x->is<CharType> ()
 	   || x->is<IntType> () || x->is<UintType> () || x->is<FloatType> ()
 	   || x->is<USizeType> () || x->is<ISizeType> () || x->is<NeverType> ()
@@ -891,31 +904,36 @@ BaseType::has_substitutions_defined () const
     case TUPLE:
     case PARAM:
     case PLACEHOLDER:
+    case CONST:
     case OPAQUE:
       return false;
 
-      case PROJECTION: {
+    case PROJECTION:
+      {
 	const ProjectionType &p = *static_cast<const ProjectionType *> (x);
 	const SubstitutionRef &ref = static_cast<const SubstitutionRef &> (p);
 	return ref.has_substitutions ();
       }
       break;
 
-      case FNDEF: {
+    case FNDEF:
+      {
 	const FnType &fn = *static_cast<const FnType *> (x);
 	const SubstitutionRef &ref = static_cast<const SubstitutionRef &> (fn);
 	return ref.has_substitutions ();
       }
       break;
 
-      case ADT: {
+    case ADT:
+      {
 	const ADTType &adt = *static_cast<const ADTType *> (x);
 	const SubstitutionRef &ref = static_cast<const SubstitutionRef &> (adt);
 	return ref.has_substitutions ();
       }
       break;
 
-      case CLOSURE: {
+    case CLOSURE:
+      {
 	const ClosureType &closure = *static_cast<const ClosureType *> (x);
 	const SubstitutionRef &ref
 	  = static_cast<const SubstitutionRef &> (closure);
@@ -953,31 +971,36 @@ BaseType::needs_generic_substitutions () const
     case TUPLE:
     case PARAM:
     case PLACEHOLDER:
+    case CONST:
     case OPAQUE:
       return false;
 
-      case PROJECTION: {
+    case PROJECTION:
+      {
 	const ProjectionType &p = *static_cast<const ProjectionType *> (x);
 	const SubstitutionRef &ref = static_cast<const SubstitutionRef &> (p);
 	return ref.needs_substitution ();
       }
       break;
 
-      case FNDEF: {
+    case FNDEF:
+      {
 	const FnType &fn = *static_cast<const FnType *> (x);
 	const SubstitutionRef &ref = static_cast<const SubstitutionRef &> (fn);
 	return ref.needs_substitution ();
       }
       break;
 
-      case ADT: {
+    case ADT:
+      {
 	const ADTType &adt = *static_cast<const ADTType *> (x);
 	const SubstitutionRef &ref = static_cast<const SubstitutionRef &> (adt);
 	return ref.needs_substitution ();
       }
       break;
 
-      case CLOSURE: {
+    case CLOSURE:
+      {
 	const ClosureType &closure = *static_cast<const ClosureType *> (x);
 	const SubstitutionRef &ref
 	  = static_cast<const SubstitutionRef &> (closure);
@@ -996,28 +1019,32 @@ BaseType::get_subst_argument_mappings () const
   const TyTy::BaseType *x = destructure ();
   switch (x->get_kind ())
     {
-      case PROJECTION: {
+    case PROJECTION:
+      {
 	const auto &p = *static_cast<const ProjectionType *> (x);
 	const auto &ref = static_cast<const SubstitutionRef &> (p);
 	return ref.get_substitution_arguments ();
       }
       break;
 
-      case FNDEF: {
+    case FNDEF:
+      {
 	const auto &fn = *static_cast<const FnType *> (x);
 	const auto &ref = static_cast<const SubstitutionRef &> (fn);
 	return ref.get_substitution_arguments ();
       }
       break;
 
-      case ADT: {
+    case ADT:
+      {
 	const auto &adt = *static_cast<const ADTType *> (x);
 	const auto &ref = static_cast<const SubstitutionRef &> (adt);
 	return ref.get_substitution_arguments ();
       }
       break;
 
-      case CLOSURE: {
+    case CLOSURE:
+      {
 	const auto &closure = *static_cast<const ClosureType *> (x);
 	const auto &ref = static_cast<const SubstitutionRef &> (closure);
 	return ref.get_substitution_arguments ();
@@ -1140,13 +1167,15 @@ InferType::default_type (BaseType **type) const
 	case GENERAL:
 	  return false;
 
-	  case INTEGRAL: {
+	case INTEGRAL:
+	  {
 	    ok = context->lookup_builtin ("i32", type);
 	    rust_assert (ok);
 	    return ok;
 	  }
 
-	  case FLOAT: {
+	case FLOAT:
+	  {
 	    ok = context->lookup_builtin ("f64", type);
 	    rust_assert (ok);
 	    return ok;
@@ -1269,7 +1298,8 @@ InferType::apply_primitive_type_hint (const BaseType &hint)
       default_hint.kind = hint.get_kind ();
       break;
 
-      case INT: {
+    case INT:
+      {
 	infer_kind = INTEGRAL;
 	default_hint.kind = hint.get_kind ();
 	default_hint.shint = TypeHint::SignedHint::SIGNED;
@@ -1294,7 +1324,8 @@ InferType::apply_primitive_type_hint (const BaseType &hint)
       }
       break;
 
-      case UINT: {
+    case UINT:
+      {
 	infer_kind = INTEGRAL;
 	default_hint.kind = hint.get_kind ();
 	default_hint.shint = TypeHint::SignedHint::UNSIGNED;
@@ -1319,7 +1350,8 @@ InferType::apply_primitive_type_hint (const BaseType &hint)
       }
       break;
 
-      case TypeKind::FLOAT: {
+    case TypeKind::FLOAT:
+      {
 	infer_kind = FLOAT;
 	default_hint.shint = TypeHint::SignedHint::SIGNED;
 	default_hint.kind = hint.get_kind ();
@@ -1800,11 +1832,9 @@ ADTType::is_equal (const BaseType &other) const
 	  const SubstitutionParamMapping &a = substitutions.at (i);
 	  const SubstitutionParamMapping &b = other2->substitutions.at (i);
 
-	  const ParamType *aa = a.get_param_ty ();
-	  const ParamType *bb = b.get_param_ty ();
-	  BaseType *aaa = aa->resolve ();
-	  BaseType *bbb = bb->resolve ();
-	  if (!aaa->is_equal (*bbb))
+	  const auto &aa = a.get_param_ty ();
+	  const auto &bb = b.get_param_ty ();
+	  if (!aa->is_equal (*bb))
 	    return false;
 	}
     }
@@ -1992,7 +2022,7 @@ TupleType::get_name () const
   std::string fields_buffer;
   for (const TyVar &field : get_fields ())
     {
-      fields_buffer += field.get_tyty ()->as_string ();
+      fields_buffer += field.get_tyty ()->get_name ();
       bool has_next = (i + 1) < get_fields ().size ();
       fields_buffer += has_next ? ", " : "";
       i++;
@@ -2129,9 +2159,8 @@ FnType::is_equal (const BaseType &other) const
 	  const SubstitutionParamMapping &a = get_substs ().at (i);
 	  const SubstitutionParamMapping &b = ofn.get_substs ().at (i);
 
-	  const ParamType *pa = a.get_param_ty ();
-	  const ParamType *pb = b.get_param_ty ();
-
+	  const auto *pa = a.get_param_ty ();
+	  const auto *pb = b.get_param_ty ();
 	  if (!pa->is_equal (*pb))
 	    return false;
 	}
@@ -2470,7 +2499,8 @@ ArrayType::accept_vis (TyConstVisitor &vis) const
 std::string
 ArrayType::as_string () const
 {
-  return "[" + get_element_type ()->as_string () + ":" + "CAPACITY" + "]";
+  return "[" + get_element_type ()->as_string () + "; " + capacity->as_string ()
+	 + "]";
 }
 
 bool
@@ -2509,7 +2539,7 @@ ArrayType::get_var_element_type () const
 BaseType *
 ArrayType::clone () const
 {
-  return new ArrayType (get_ref (), get_ty_ref (), ident.locus, capacity_expr,
+  return new ArrayType (get_ref (), get_ty_ref (), ident.locus, capacity,
 			element_type, get_combined_refs ());
 }
 
@@ -2526,6 +2556,13 @@ ArrayType::handle_substitions (SubstitutionArgumentMappings &mappings)
   BaseType *concrete = Resolver::SubstMapperInternal::Resolve (base, mappings);
   ref->element_type = TyVar::subst_covariant_var (base, concrete);
 
+  // handle capacity type
+  auto cap = ref->get_capacity ();
+  BaseType *concrete_cap
+    = Resolver::SubstMapperInternal::Resolve (cap, mappings);
+  rust_assert (concrete_cap->get_kind () == TyTy::TypeKind::CONST);
+  ref->capacity = static_cast<TyTy::ConstType *> (concrete_cap);
+
   return ref;
 }
 
@@ -3382,33 +3419,26 @@ PointerType::handle_substitions (SubstitutionArgumentMappings &mappings)
 // PARAM Type
 
 ParamType::ParamType (std::string symbol, location_t locus, HirId ref,
-		      HIR::GenericParam &param,
 		      std::vector<TypeBoundPredicate> specified_bounds,
 		      std::set<HirId> refs)
-  : BaseType (ref, ref, KIND,
-	      {Resolver::CanonicalPath::new_seg (UNKNOWN_NODEID, symbol),
-	       locus},
-	      specified_bounds, refs),
-    is_trait_self (false), symbol (symbol), param (param)
+  : BaseGeneric (ref, ref, KIND,
+		 {Resolver::CanonicalPath::new_seg (UNKNOWN_NODEID, symbol),
+		  locus},
+		 specified_bounds, refs),
+    is_trait_self (false), symbol (symbol)
 {}
 
 ParamType::ParamType (bool is_trait_self, std::string symbol, location_t locus,
-		      HirId ref, HirId ty_ref, HIR::GenericParam &param,
+		      HirId ref, HirId ty_ref,
 		      std::vector<TypeBoundPredicate> specified_bounds,
 		      std::set<HirId> refs)
-  : BaseType (ref, ty_ref, KIND,
-	      {Resolver::CanonicalPath::new_seg (UNKNOWN_NODEID, symbol),
-	       locus},
-	      specified_bounds, refs),
-    is_trait_self (is_trait_self), symbol (symbol), param (param)
+  : BaseGeneric (ref, ty_ref, KIND,
+		 {Resolver::CanonicalPath::new_seg (UNKNOWN_NODEID, symbol),
+		  locus},
+		 specified_bounds, refs),
+    is_trait_self (is_trait_self), symbol (symbol)
 {}
 
-HIR::GenericParam &
-ParamType::get_generic_param ()
-{
-  return param;
-}
-
 bool
 ParamType::can_resolve () const
 {
@@ -3459,7 +3489,7 @@ BaseType *
 ParamType::clone () const
 {
   return new ParamType (is_trait_self, get_symbol (), ident.locus, get_ref (),
-			get_ty_ref (), param, get_specified_bounds (),
+			get_ty_ref (), get_specified_bounds (),
 			get_combined_refs ());
 }
 
@@ -3529,12 +3559,12 @@ ParamType::handle_substitions (SubstitutionArgumentMappings &subst_mappings)
   ParamType *p = static_cast<ParamType *> (clone ());
   subst_mappings.on_param_subst (*p, arg);
 
-  // there are two cases one where we substitute directly to a new PARAM and
-  // otherwise
-  if (arg.get_tyty ()->get_kind () == TyTy::TypeKind::PARAM)
+  const BaseType *resolved = arg.get_tyty ();
+  if (resolved->get_kind () == TyTy::TypeKind::PARAM)
     {
-      p->set_ty_ref (arg.get_tyty ()->get_ref ());
-      return p;
+      const ParamType &pp = *static_cast<const ParamType *> (resolved);
+      if (pp.can_resolve ())
+	resolved = pp.resolve ();
     }
 
   // this is the new subst that this needs to pass
@@ -3556,6 +3586,157 @@ ParamType::is_implicit_self_trait () const
   return is_trait_self;
 }
 
+// ConstType
+
+ConstType::ConstType (ConstKind kind, std::string symbol, TyTy::BaseType *ty,
+		      tree value,
+		      std::vector<TypeBoundPredicate> specified_bounds,
+		      location_t locus, HirId ref, HirId ty_ref,
+		      std::set<HirId> refs)
+  : BaseGeneric (ref, ty_ref, KIND,
+		 {Resolver::CanonicalPath::new_seg (UNKNOWN_NODEID,
+						    symbol.empty () ? "<n/a>"
+								    : symbol),
+		  locus},
+		 specified_bounds, refs),
+    const_kind (kind), ty (ty), value (value), symbol (symbol)
+{}
+
+void
+ConstType::accept_vis (TyVisitor &vis)
+{
+  vis.visit (*this);
+}
+
+void
+ConstType::accept_vis (TyConstVisitor &vis) const
+{
+  vis.visit (*this);
+}
+
+void
+ConstType::set_value (tree v)
+{
+  value = v;
+  const_kind = ConstType::ConstKind::Value;
+}
+
+std::string
+ConstType::as_string () const
+{
+  return get_name ();
+}
+
+bool
+ConstType::can_eq (const BaseType *other, bool emit_errors) const
+{
+  ConstCmp r (this, emit_errors);
+  return r.can_eq (other);
+}
+
+BaseType *
+ConstType::clone () const
+{
+  return new ConstType (const_kind, symbol, ty, value, get_specified_bounds (),
+			ident.locus, ref, ty_ref, get_combined_refs ());
+}
+
+std::string
+ConstType::get_symbol () const
+{
+  return symbol;
+}
+
+bool
+ConstType::can_resolve () const
+{
+  return false;
+}
+
+BaseType *
+ConstType::resolve () const
+{
+  rust_unreachable ();
+  return nullptr;
+}
+
+static std::string
+generate_tree_str (tree value)
+{
+  char *buf = nullptr;
+  size_t size = 0;
+
+  FILE *stream = open_memstream (&buf, &size);
+  if (!stream)
+    return "<error>";
+
+  print_generic_stmt (stream, value, TDF_NONE);
+  fclose (stream);
+
+  std::string result = (buf ? std::string (buf, size) : "<error>");
+  free (buf);
+
+  if (!result.empty () && result.back () == '\n')
+    result.pop_back ();
+
+  return result;
+}
+
+std::string
+ConstType::get_name () const
+{
+  if (value == error_mark_node)
+    {
+      switch (get_const_kind ())
+	{
+	case Rust::TyTy::ConstType::Decl:
+	  return "ConstType:<" + get_ty ()->get_name () + " " + get_symbol ()
+		 + ">";
+
+	case Rust::TyTy::ConstType::Infer:
+	  return "ConstType:<" + get_ty ()->get_name () + " ?" + ">";
+
+	default:
+	  return "ConstType:<" + get_ty ()->get_name () + " - <error>" + ">";
+	}
+    }
+
+  return generate_tree_str (value);
+}
+
+bool
+ConstType::is_equal (const BaseType &other) const
+{
+  if (get_kind () != other.get_kind ())
+    {
+      return false;
+    }
+
+  const ConstType &rhs = static_cast<const ConstType &> (other);
+  if (!get_ty ()->is_equal (*rhs.get_ty ()))
+    return false;
+
+  tree lv = get_value ();
+  tree rv = rhs.get_value ();
+
+  return operand_equal_p (lv, rv, 0);
+}
+
+ConstType *
+ConstType::handle_substitions (SubstitutionArgumentMappings &mappings)
+{
+  SubstitutionArg arg = SubstitutionArg::error ();
+  bool found = mappings.get_argument_for_symbol (this, &arg);
+  if (found && !arg.is_error ())
+    {
+      TyTy::BaseType *subst = arg.get_tyty ();
+      rust_assert (subst->is<TyTy::ConstType> ());
+      return static_cast<TyTy::ConstType *> (subst);
+    }
+
+  return this;
+}
+
 // OpaqueType
 
 OpaqueType::OpaqueType (location_t locus, HirId ref,
@@ -3624,28 +3805,7 @@ BaseType *
 OpaqueType::resolve () const
 {
   TyVar var (get_ty_ref ());
-  BaseType *r = var.get_tyty ();
-
-  while (r->get_kind () == TypeKind::OPAQUE)
-    {
-      OpaqueType *rr = static_cast<OpaqueType *> (r);
-      if (!rr->can_resolve ())
-	break;
-
-      TyVar v (rr->get_ty_ref ());
-      BaseType *n = v.get_tyty ();
-
-      // fix infinite loop
-      if (r == n)
-	break;
-
-      r = n;
-    }
-
-  if (r->get_kind () == TypeKind::OPAQUE && (r->get_ref () == r->get_ty_ref ()))
-    return TyVar (r->get_ty_ref ()).get_tyty ();
-
-  return r;
+  return var.get_tyty ();
 }
 
 bool
@@ -3655,41 +3815,9 @@ OpaqueType::is_equal (const BaseType &other) const
   if (can_resolve () != other2.can_resolve ())
     return false;
 
-  if (can_resolve ())
-    return resolve ()->can_eq (other2.resolve (), false);
-
   return bounds_compatible (other, UNDEF_LOCATION, false);
 }
 
-OpaqueType *
-OpaqueType::handle_substitions (SubstitutionArgumentMappings &subst_mappings)
-{
-  // SubstitutionArg arg = SubstitutionArg::error ();
-  // bool ok = subst_mappings.get_argument_for_symbol (this, &arg);
-  // if (!ok || arg.is_error ())
-  //   return this;
-
-  // OpaqueType *p = static_cast<OpaqueType *> (clone ());
-  // subst_mappings.on_param_subst (*p, arg);
-
-  // // there are two cases one where we substitute directly to a new PARAM and
-  // // otherwise
-  // if (arg.get_tyty ()->get_kind () == TyTy::TypeKind::PARAM)
-  //   {
-  //     p->set_ty_ref (arg.get_tyty ()->get_ref ());
-  //     return p;
-  //   }
-
-  // // this is the new subst that this needs to pass
-  // p->set_ref (mappings.get_next_hir_id ());
-  // p->set_ty_ref (arg.get_tyty ()->get_ref ());
-
-  // return p;
-
-  rust_unreachable ();
-  return nullptr;
-}
-
 // StrType
 
 StrType::StrType (HirId ref, std::set<HirId> refs)
diff --git a/gcc/rust/typecheck/rust-tyty.h b/gcc/rust/typecheck/rust-tyty.h
index 1cada9a..49415ea 100644
--- a/gcc/rust/typecheck/rust-tyty.h
+++ b/gcc/rust/typecheck/rust-tyty.h
@@ -19,6 +19,7 @@
 #ifndef RUST_TYTY
 #define RUST_TYTY
 
+#include "optional.h"
 #include "rust-hir-map.h"
 #include "rust-common.h"
 #include "rust-identifier.h"
@@ -29,6 +30,7 @@
 #include "rust-tyty-region.h"
 #include "rust-system.h"
 #include "rust-hir.h"
+#include "tree.h"
 
 namespace Rust {
 
@@ -56,6 +58,7 @@ enum TypeKind
   REF,
   POINTER,
   PARAM,
+  CONST,
   ARRAY,
   SLICE,
   FNDEF,
@@ -78,8 +81,7 @@ enum TypeKind
   ERROR
 };
 
-extern bool
-is_primitive_type_kind (TypeKind kind);
+extern bool is_primitive_type_kind (TypeKind kind);
 
 class TypeKindFormat
 {
@@ -165,8 +167,7 @@ public:
 
   void debug () const;
 
-  // FIXME this will eventually go away
-  const BaseType *get_root () const;
+  BaseType *get_root ();
 
   // This will get the monomorphized type from Params, Placeholders or
   // Projections if available or error
@@ -268,8 +269,8 @@ public:
   {}
 
   WARN_UNUSED_RESULT virtual size_t get_num_params () const = 0;
-  WARN_UNUSED_RESULT virtual BaseType *
-  get_param_type_at (size_t index) const = 0;
+  WARN_UNUSED_RESULT virtual BaseType *get_param_type_at (size_t index) const
+    = 0;
   WARN_UNUSED_RESULT virtual BaseType *get_return_type () const = 0;
 };
 
@@ -365,18 +366,34 @@ public:
   std::string get_name () const override final;
 };
 
-class ParamType : public BaseType
+class BaseGeneric : public BaseType
+{
+public:
+  virtual std::string get_symbol () const = 0;
+
+  virtual bool can_resolve () const = 0;
+
+  virtual BaseType *resolve () const = 0;
+
+protected:
+  BaseGeneric (HirId ref, HirId ty_ref, TypeKind kind, RustIdent ident,
+	       std::vector<TypeBoundPredicate> specified_bounds,
+	       std::set<HirId> refs = std::set<HirId> ())
+    : BaseType (ref, ty_ref, kind, ident, specified_bounds, refs)
+  {}
+};
+
+class ParamType : public BaseGeneric
 {
 public:
   static constexpr auto KIND = TypeKind::PARAM;
 
   ParamType (std::string symbol, location_t locus, HirId ref,
-	     HIR::GenericParam &param,
 	     std::vector<TypeBoundPredicate> specified_bounds,
 	     std::set<HirId> refs = std::set<HirId> ());
 
   ParamType (bool is_trait_self, std::string symbol, location_t locus,
-	     HirId ref, HirId ty_ref, HIR::GenericParam &param,
+	     HirId ref, HirId ty_ref,
 	     std::vector<TypeBoundPredicate> specified_bounds,
 	     std::set<HirId> refs = std::set<HirId> ());
 
@@ -389,13 +406,11 @@ public:
 
   BaseType *clone () const final override;
 
-  std::string get_symbol () const;
-
-  HIR::GenericParam &get_generic_param ();
+  std::string get_symbol () const override final;
 
-  bool can_resolve () const;
+  bool can_resolve () const override final;
 
-  BaseType *resolve () const;
+  BaseType *resolve () const override final;
 
   std::string get_name () const override final;
 
@@ -409,7 +424,58 @@ public:
 private:
   bool is_trait_self;
   std::string symbol;
-  HIR::GenericParam &param;
+};
+
+class ConstType : public BaseGeneric
+{
+public:
+  static constexpr auto KIND = TypeKind::CONST;
+
+  enum ConstKind
+  {
+    Decl,
+    Value,
+    Infer,
+    Error
+  };
+
+  ConstType (ConstKind kind, std::string symbol, TyTy::BaseType *ty, tree value,
+	     std::vector<TypeBoundPredicate> specified_bounds, location_t locus,
+	     HirId ref, HirId ty_ref,
+	     std::set<HirId> refs = std::set<HirId> ());
+
+  void accept_vis (TyVisitor &vis) override;
+  void accept_vis (TyConstVisitor &vis) const override;
+
+  ConstKind get_const_kind () const { return const_kind; }
+  TyTy::BaseType *get_ty () const { return ty; }
+  tree get_value () const { return value; }
+
+  void set_value (tree value);
+
+  std::string as_string () const override;
+
+  bool can_eq (const BaseType *other, bool emit_errors) const override final;
+
+  BaseType *clone () const final override;
+
+  std::string get_symbol () const override final;
+
+  bool can_resolve () const override final;
+
+  BaseType *resolve () const override final;
+
+  std::string get_name () const override final;
+
+  bool is_equal (const BaseType &other) const override;
+
+  ConstType *handle_substitions (SubstitutionArgumentMappings &mappings);
+
+private:
+  ConstKind const_kind;
+  TyTy::BaseType *ty;
+  tree value;
+  std::string symbol;
 };
 
 class OpaqueType : public BaseType
@@ -441,8 +507,6 @@ public:
   std::string get_name () const override final;
 
   bool is_equal (const BaseType &other) const override;
-
-  OpaqueType *handle_substitions (SubstitutionArgumentMappings &mappings);
 };
 
 class StructFieldType
@@ -541,14 +605,15 @@ public:
 
   std::string get_name () const;
 
-  // check that this predicate is object-safe see:
+  // check that this  is object-safe see:
   // https://doc.rust-lang.org/reference/items/traits.html#object-safety
   bool is_object_safe (bool emit_error, location_t locus) const;
 
   void apply_generic_arguments (HIR::GenericArgs *generic_args,
-				bool has_associated_self);
+				bool has_associated_self, bool is_super_trait);
 
-  void apply_argument_mappings (SubstitutionArgumentMappings &arguments);
+  void apply_argument_mappings (SubstitutionArgumentMappings &arguments,
+				bool is_super_trait);
 
   bool contains_item (const std::string &search) const;
 
@@ -596,6 +661,9 @@ private:
 
   TypeBoundPredicate (mark_is_error);
 
+  void get_trait_hierachy (
+    std::function<void (const Resolver::TraitReference &)> callback) const;
+
   DefId reference;
   location_t locus;
   bool error_flag;
@@ -1156,19 +1224,18 @@ class ArrayType : public BaseType
 public:
   static constexpr auto KIND = TypeKind::ARRAY;
 
-  ArrayType (HirId ref, location_t locus, HIR::Expr &capacity_expr, TyVar base,
+  ArrayType (HirId ref, location_t locus, ConstType *capacity, TyVar base,
 	     std::set<HirId> refs = std::set<HirId> ())
     : BaseType (ref, ref, TypeKind::ARRAY,
 		{Resolver::CanonicalPath::create_empty (), locus}, refs),
-      element_type (base), capacity_expr (capacity_expr)
+      element_type (base), capacity (capacity)
   {}
 
-  ArrayType (HirId ref, HirId ty_ref, location_t locus,
-	     HIR::Expr &capacity_expr, TyVar base,
-	     std::set<HirId> refs = std::set<HirId> ())
+  ArrayType (HirId ref, HirId ty_ref, location_t locus, ConstType *capacity,
+	     TyVar base, std::set<HirId> refs = std::set<HirId> ())
     : BaseType (ref, ty_ref, TypeKind::ARRAY,
 		{Resolver::CanonicalPath::create_empty (), locus}, refs),
-      element_type (base), capacity_expr (capacity_expr)
+      element_type (base), capacity (capacity)
   {}
 
   void accept_vis (TyVisitor &vis) override;
@@ -1187,15 +1254,13 @@ public:
 
   BaseType *clone () const final override;
 
-  HIR::Expr &get_capacity_expr () const { return capacity_expr; }
+  ConstType *get_capacity () const { return capacity; }
 
   ArrayType *handle_substitions (SubstitutionArgumentMappings &mappings);
 
 private:
   TyVar element_type;
-  // FIXME: I dont think this should be in tyty - tyty should already be const
-  // evaluated
-  HIR::Expr &capacity_expr;
+  ConstType *capacity;
 };
 
 class SliceType : public BaseType
diff --git a/gcc/rust/typecheck/rust-unify.cc b/gcc/rust/typecheck/rust-unify.cc
index 294b677..30ead5b 100644
--- a/gcc/rust/typecheck/rust-unify.cc
+++ b/gcc/rust/typecheck/rust-unify.cc
@@ -17,6 +17,9 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-unify.h"
+#include "fold-const.h"
+#include "rust-tyty.h"
+#include "tree.h"
 
 namespace Rust {
 namespace Resolver {
@@ -53,6 +56,22 @@ UnifyRules::Resolve (TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
 }
 
 TyTy::BaseType *
+UnifyRules::resolve_subtype (TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs)
+{
+  TyTy::BaseType *result
+    = UnifyRules::Resolve (lhs, rhs, locus, commit_flag, emit_error, infer_flag,
+			   commits, infers);
+
+  // If the recursive call resulted in an error and would have emitted an error
+  // message, disable error emission for the current level to avoid duplicate
+  // errors
+  if (result->get_kind () == TyTy::TypeKind::ERROR && emit_error)
+    emit_error = false;
+
+  return result;
+}
+
+TyTy::BaseType *
 UnifyRules::get_base ()
 {
   return lhs.get_ty ()->destructure ();
@@ -69,7 +88,6 @@ UnifyRules::commit (TyTy::BaseType *base, TyTy::BaseType *other,
 		    TyTy::BaseType *resolved)
 {
   TypeCheckContext &context = *TypeCheckContext::get ();
-  Analysis::Mappings &mappings = Analysis::Mappings::get ();
 
   TyTy::BaseType *b = base->destructure ();
   TyTy::BaseType *o = other->destructure ();
@@ -102,13 +120,8 @@ UnifyRules::commit (TyTy::BaseType *base, TyTy::BaseType *other,
 	    continue;
 
 	  // if any of the types are inference variables lets fix them
-	  if (ref_tyty->get_kind () == TyTy::TypeKind::INFER)
-	    {
-	      auto node = Analysis::NodeMapping (mappings.get_current_crate (),
-						 UNKNOWN_NODEID, ref,
-						 UNKNOWN_LOCAL_DEFID);
-	      context.insert_type (node, resolved->clone ());
-	    }
+	  if (ref_tyty->is<TyTy::InferType> ())
+	    context.insert_implicit_type (ref, resolved);
 	}
     }
 }
@@ -315,6 +328,9 @@ UnifyRules::go ()
     case TyTy::OPAQUE:
       return expect_opaque (static_cast<TyTy::OpaqueType *> (ltype), rtype);
 
+    case TyTy::CONST:
+      return expect_const (static_cast<TyTy::ConstType *> (ltype), rtype);
+
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -328,30 +344,33 @@ UnifyRules::expect_inference_variable (TyTy::InferType *ltype,
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	switch (ltype->get_infer_kind ())
 	  {
 	  case TyTy::InferType::InferTypeKind::GENERAL:
 	    return rtype->clone ();
 
-	    case TyTy::InferType::InferTypeKind::INTEGRAL: {
+	  case TyTy::InferType::InferTypeKind::INTEGRAL:
+	    {
 	      bool is_valid = r->get_infer_kind ()
 				== TyTy::InferType::InferTypeKind::INTEGRAL
 			      || r->get_infer_kind ()
 				   == TyTy::InferType::InferTypeKind::GENERAL;
 	      if (is_valid)
-		return rtype->clone ();
+		return rtype;
 	    }
 	    break;
 
-	    case TyTy::InferType::InferTypeKind::FLOAT: {
+	  case TyTy::InferType::InferTypeKind::FLOAT:
+	    {
 	      bool is_valid
 		= r->get_infer_kind () == TyTy::InferType::InferTypeKind::FLOAT
 		  || r->get_infer_kind ()
 		       == TyTy::InferType::InferTypeKind::GENERAL;
 	      if (is_valid)
-		return rtype->clone ();
+		return rtype;
 	    }
 	    break;
 	  }
@@ -361,7 +380,8 @@ UnifyRules::expect_inference_variable (TyTy::InferType *ltype,
     case TyTy::INT:
     case TyTy::UINT:
     case TyTy::USIZE:
-      case TyTy::ISIZE: {
+    case TyTy::ISIZE:
+      {
 	bool is_valid = (ltype->get_infer_kind ()
 			 == TyTy::InferType::InferTypeKind::GENERAL)
 			|| (ltype->get_infer_kind ()
@@ -369,12 +389,13 @@ UnifyRules::expect_inference_variable (TyTy::InferType *ltype,
 	if (is_valid)
 	  {
 	    ltype->apply_primitive_type_hint (*rtype);
-	    return rtype->clone ();
+	    return rtype;
 	  }
       }
       break;
 
-      case TyTy::FLOAT: {
+    case TyTy::FLOAT:
+      {
 	bool is_valid = (ltype->get_infer_kind ()
 			 == TyTy::InferType::InferTypeKind::GENERAL)
 			|| (ltype->get_infer_kind ()
@@ -382,7 +403,7 @@ UnifyRules::expect_inference_variable (TyTy::InferType *ltype,
 	if (is_valid)
 	  {
 	    ltype->apply_primitive_type_hint (*rtype);
-	    return rtype->clone ();
+	    return rtype;
 	  }
       }
       break;
@@ -404,7 +425,9 @@ UnifyRules::expect_inference_variable (TyTy::InferType *ltype,
     case TyTy::PROJECTION:
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
-      case TyTy::OPAQUE: {
+    case TyTy::CONST:
+    case TyTy::OPAQUE:
+      {
 	bool is_valid = (ltype->get_infer_kind ()
 			 == TyTy::InferType::InferTypeKind::GENERAL);
 	if (is_valid)
@@ -424,7 +447,8 @@ UnifyRules::expect_adt (TyTy::ADTType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -433,7 +457,8 @@ UnifyRules::expect_adt (TyTy::ADTType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::ADT: {
+    case TyTy::ADT:
+      {
 	TyTy::ADTType &type = *static_cast<TyTy::ADTType *> (rtype);
 	if (ltype->get_adt_kind () != type.get_adt_kind ())
 	  {
@@ -469,11 +494,8 @@ UnifyRules::expect_adt (TyTy::ADTType *ltype, TyTy::BaseType *rtype)
 		TyTy::BaseType *other_field_ty = other_field->get_field_type ();
 
 		TyTy::BaseType *unified_ty
-		  = UnifyRules::Resolve (TyTy::TyWithLocation (this_field_ty),
-					 TyTy::TyWithLocation (other_field_ty),
-					 locus, commit_flag,
-					 false /* emit_error */, infer_flag,
-					 commits, infers);
+		  = resolve_subtype (TyTy::TyWithLocation (this_field_ty),
+				     TyTy::TyWithLocation (other_field_ty));
 		if (unified_ty->get_kind () == TyTy::TypeKind::ERROR)
 		  {
 		    return new TyTy::ErrorType (0);
@@ -495,11 +517,8 @@ UnifyRules::expect_adt (TyTy::ADTType *ltype, TyTy::BaseType *rtype)
 		auto pa = a.get_param_ty ();
 		auto pb = b.get_param_ty ();
 
-		auto res
-		  = UnifyRules::Resolve (TyTy::TyWithLocation (pa),
-					 TyTy::TyWithLocation (pb), locus,
-					 commit_flag, false /* emit_error */,
-					 infer_flag, commits, infers);
+		auto res = resolve_subtype (TyTy::TyWithLocation (pa),
+					    TyTy::TyWithLocation (pb));
 		if (res->get_kind () == TyTy::TypeKind::ERROR)
 		  {
 		    return new TyTy::ErrorType (0);
@@ -533,6 +552,7 @@ UnifyRules::expect_adt (TyTy::ADTType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -544,7 +564,8 @@ UnifyRules::expect_str (TyTy::StrType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -578,6 +599,7 @@ UnifyRules::expect_str (TyTy::StrType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -589,7 +611,8 @@ UnifyRules::expect_reference (TyTy::ReferenceType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -598,16 +621,15 @@ UnifyRules::expect_reference (TyTy::ReferenceType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::REF: {
+    case TyTy::REF:
+      {
 	TyTy::ReferenceType &type = *static_cast<TyTy::ReferenceType *> (rtype);
 	auto base_type = ltype->get_base ();
 	auto other_base_type = type.get_base ();
 
 	TyTy::BaseType *base_resolved
-	  = UnifyRules::Resolve (TyTy::TyWithLocation (base_type),
-				 TyTy::TyWithLocation (other_base_type), locus,
-				 commit_flag, false /* emit_error */,
-				 infer_flag, commits, infers);
+	  = resolve_subtype (TyTy::TyWithLocation (base_type),
+			     TyTy::TyWithLocation (other_base_type));
 	if (base_resolved->get_kind () == TyTy::TypeKind::ERROR)
 	  {
 	    return new TyTy::ErrorType (0);
@@ -649,6 +671,7 @@ UnifyRules::expect_reference (TyTy::ReferenceType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -660,7 +683,8 @@ UnifyRules::expect_pointer (TyTy::PointerType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -669,16 +693,15 @@ UnifyRules::expect_pointer (TyTy::PointerType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::POINTER: {
+    case TyTy::POINTER:
+      {
 	TyTy::PointerType &type = *static_cast<TyTy::PointerType *> (rtype);
 	auto base_type = ltype->get_base ();
 	auto other_base_type = type.get_base ();
 
 	TyTy::BaseType *base_resolved
-	  = UnifyRules::Resolve (TyTy::TyWithLocation (base_type),
-				 TyTy::TyWithLocation (other_base_type), locus,
-				 commit_flag, false /* emit_error */,
-				 infer_flag, commits, infers);
+	  = resolve_subtype (TyTy::TyWithLocation (base_type),
+			     TyTy::TyWithLocation (other_base_type));
 	if (base_resolved->get_kind () == TyTy::TypeKind::ERROR)
 	  {
 	    return new TyTy::ErrorType (0);
@@ -720,6 +743,7 @@ UnifyRules::expect_pointer (TyTy::PointerType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -731,7 +755,8 @@ UnifyRules::expect_param (TyTy::ParamType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -740,7 +765,8 @@ UnifyRules::expect_param (TyTy::ParamType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::PARAM: {
+    case TyTy::PARAM:
+      {
 	TyTy::ParamType &type = *static_cast<TyTy::ParamType *> (rtype);
 	// bool symbol_matches
 	//   = ltype->get_symbol ().compare (type.get_symbol ()) == 0;
@@ -782,6 +808,7 @@ UnifyRules::expect_param (TyTy::ParamType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -793,7 +820,8 @@ UnifyRules::expect_array (TyTy::ArrayType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -802,21 +830,32 @@ UnifyRules::expect_array (TyTy::ArrayType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::ARRAY: {
+    case TyTy::ARRAY:
+      {
 	TyTy::ArrayType &type = *static_cast<TyTy::ArrayType *> (rtype);
-	TyTy::BaseType *element_unify = UnifyRules::Resolve (
-	  TyTy::TyWithLocation (ltype->get_element_type ()),
-	  TyTy::TyWithLocation (type.get_element_type ()), locus, commit_flag,
-	  false /* emit_error*/, infer_flag, commits, infers);
-
-	if (element_unify->get_kind () != TyTy::TypeKind::ERROR)
-	  {
-	    return new TyTy::ArrayType (type.get_ref (), type.get_ty_ref (),
-					type.get_ident ().locus,
-					type.get_capacity_expr (),
-					TyTy::TyVar (
-					  element_unify->get_ref ()));
-	  }
+	TyTy::BaseType *element_unify
+	  = resolve_subtype (TyTy::TyWithLocation (ltype->get_element_type ()),
+			     TyTy::TyWithLocation (type.get_element_type ()));
+
+	if (element_unify->get_kind () == TyTy::TypeKind::ERROR)
+	  return new TyTy::ErrorType (0);
+
+	bool save_emit_error = emit_error;
+	emit_error = false;
+	TyTy::BaseType *capacity_unify
+	  = resolve_subtype (TyTy::TyWithLocation (ltype->get_capacity ()),
+			     TyTy::TyWithLocation (type.get_capacity ()));
+	emit_error = save_emit_error;
+
+	if (capacity_unify->get_kind () != TyTy::TypeKind::CONST)
+	  return new TyTy::ErrorType (0);
+
+	TyTy::ConstType *capacity_type_unify
+	  = static_cast<TyTy::ConstType *> (capacity_unify);
+	return new TyTy::ArrayType (type.get_ref (), type.get_ty_ref (),
+				    type.get_ident ().locus,
+				    capacity_type_unify,
+				    TyTy::TyVar (element_unify->get_ref ()));
       }
       break;
 
@@ -842,6 +881,7 @@ UnifyRules::expect_array (TyTy::ArrayType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -853,7 +893,8 @@ UnifyRules::expect_slice (TyTy::SliceType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -862,12 +903,12 @@ UnifyRules::expect_slice (TyTy::SliceType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::SLICE: {
+    case TyTy::SLICE:
+      {
 	TyTy::SliceType &type = *static_cast<TyTy::SliceType *> (rtype);
-	TyTy::BaseType *element_unify = UnifyRules::Resolve (
-	  TyTy::TyWithLocation (ltype->get_element_type ()),
-	  TyTy::TyWithLocation (type.get_element_type ()), locus, commit_flag,
-	  false /* emit_error*/, infer_flag, commits, infers);
+	TyTy::BaseType *element_unify
+	  = resolve_subtype (TyTy::TyWithLocation (ltype->get_element_type ()),
+			     TyTy::TyWithLocation (type.get_element_type ()));
 
 	if (element_unify->get_kind () != TyTy::TypeKind::ERROR)
 	  {
@@ -901,6 +942,7 @@ UnifyRules::expect_slice (TyTy::SliceType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -912,7 +954,8 @@ UnifyRules::expect_fndef (TyTy::FnType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -921,7 +964,8 @@ UnifyRules::expect_fndef (TyTy::FnType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::FNDEF: {
+    case TyTy::FNDEF:
+      {
 	TyTy::FnType &type = *static_cast<TyTy::FnType *> (rtype);
 	if (ltype->num_params () != type.num_params ())
 	  {
@@ -933,21 +977,17 @@ UnifyRules::expect_fndef (TyTy::FnType *ltype, TyTy::BaseType *rtype)
 	    auto a = ltype->param_at (i).get_type ();
 	    auto b = type.param_at (i).get_type ();
 
-	    auto unified_param
-	      = UnifyRules::Resolve (TyTy::TyWithLocation (a),
-				     TyTy::TyWithLocation (b), locus,
-				     commit_flag, false /* emit_errors */,
-				     infer_flag, commits, infers);
+	    auto unified_param = resolve_subtype (TyTy::TyWithLocation (a),
+						  TyTy::TyWithLocation (b));
 	    if (unified_param->get_kind () == TyTy::TypeKind::ERROR)
 	      {
 		return new TyTy::ErrorType (0);
 	      }
 	  }
 
-	auto unified_return = UnifyRules::Resolve (
-	  TyTy::TyWithLocation (ltype->get_return_type ()),
-	  TyTy::TyWithLocation (type.get_return_type ()), locus, commit_flag,
-	  false /* emit_errors */, infer_flag, commits, infers);
+	auto unified_return
+	  = resolve_subtype (TyTy::TyWithLocation (ltype->get_return_type ()),
+			     TyTy::TyWithLocation (type.get_return_type ()));
 	if (unified_return->get_kind () == TyTy::TypeKind::ERROR)
 	  {
 	    return new TyTy::ErrorType (0);
@@ -992,6 +1032,7 @@ UnifyRules::expect_fndef (TyTy::FnType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1003,7 +1044,8 @@ UnifyRules::expect_fnptr (TyTy::FnPtr *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1012,7 +1054,8 @@ UnifyRules::expect_fnptr (TyTy::FnPtr *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::FNPTR: {
+    case TyTy::FNPTR:
+      {
 	TyTy::FnPtr &type = *static_cast<TyTy::FnPtr *> (rtype);
 	if (ltype->num_params () != type.num_params ())
 	  {
@@ -1024,21 +1067,17 @@ UnifyRules::expect_fnptr (TyTy::FnPtr *ltype, TyTy::BaseType *rtype)
 	    auto a = ltype->get_param_type_at (i);
 	    auto b = type.get_param_type_at (i);
 
-	    auto unified_param
-	      = UnifyRules::Resolve (TyTy::TyWithLocation (a),
-				     TyTy::TyWithLocation (b), locus,
-				     commit_flag, false /* emit_errors */,
-				     infer_flag, commits, infers);
+	    auto unified_param = resolve_subtype (TyTy::TyWithLocation (a),
+						  TyTy::TyWithLocation (b));
 	    if (unified_param->get_kind () == TyTy::TypeKind::ERROR)
 	      {
 		return new TyTy::ErrorType (0);
 	      }
 	  }
 
-	auto unified_return = UnifyRules::Resolve (
-	  TyTy::TyWithLocation (ltype->get_return_type ()),
-	  TyTy::TyWithLocation (type.get_return_type ()), locus, commit_flag,
-	  false /* emit_errors */, infer_flag, commits, infers);
+	auto unified_return
+	  = resolve_subtype (TyTy::TyWithLocation (ltype->get_return_type ()),
+			     TyTy::TyWithLocation (type.get_return_type ()));
 	if (unified_return->get_kind () == TyTy::TypeKind::ERROR)
 	  {
 	    return new TyTy::ErrorType (0);
@@ -1048,16 +1087,15 @@ UnifyRules::expect_fnptr (TyTy::FnPtr *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::FNDEF: {
+    case TyTy::FNDEF:
+      {
 	TyTy::FnType &type = *static_cast<TyTy::FnType *> (rtype);
 	auto this_ret_type = ltype->get_return_type ();
 	auto other_ret_type = type.get_return_type ();
 
 	auto unified_result
-	  = UnifyRules::Resolve (TyTy::TyWithLocation (this_ret_type),
-				 TyTy::TyWithLocation (other_ret_type), locus,
-				 commit_flag, false /*emit_errors*/, infer_flag,
-				 commits, infers);
+	  = resolve_subtype (TyTy::TyWithLocation (this_ret_type),
+			     TyTy::TyWithLocation (other_ret_type));
 	if (unified_result->get_kind () == TyTy::TypeKind::ERROR)
 	  {
 	    return new TyTy::ErrorType (0);
@@ -1074,10 +1112,45 @@ UnifyRules::expect_fnptr (TyTy::FnPtr *ltype, TyTy::BaseType *rtype)
 	    auto other_param = type.param_at (i).get_type ();
 
 	    auto unified_param
-	      = UnifyRules::Resolve (TyTy::TyWithLocation (this_param),
-				     TyTy::TyWithLocation (other_param), locus,
-				     commit_flag, false /* emit_errors */,
-				     infer_flag, commits, infers);
+	      = resolve_subtype (TyTy::TyWithLocation (this_param),
+				 TyTy::TyWithLocation (other_param));
+	    if (unified_param->get_kind () == TyTy::TypeKind::ERROR)
+	      {
+		return new TyTy::ErrorType (0);
+	      }
+	  }
+
+	return ltype->clone ();
+      }
+      break;
+
+    case TyTy::CLOSURE:
+      {
+	TyTy::ClosureType &type = *static_cast<TyTy::ClosureType *> (rtype);
+	auto this_ret_type = ltype->get_return_type ();
+	auto other_ret_type = type.get_return_type ();
+
+	auto unified_result
+	  = resolve_subtype (TyTy::TyWithLocation (this_ret_type),
+			     TyTy::TyWithLocation (other_ret_type));
+	if (unified_result->get_kind () == TyTy::TypeKind::ERROR)
+	  {
+	    return new TyTy::ErrorType (0);
+	  }
+
+	if (ltype->num_params () != type.get_num_params ())
+	  {
+	    return new TyTy::ErrorType (0);
+	  }
+
+	for (size_t i = 0; i < ltype->num_params (); i++)
+	  {
+	    auto this_param = ltype->get_param_type_at (i);
+	    auto other_param = type.get_param_type_at (i);
+
+	    auto unified_param
+	      = resolve_subtype (TyTy::TyWithLocation (this_param),
+				 TyTy::TyWithLocation (other_param));
 	    if (unified_param->get_kind () == TyTy::TypeKind::ERROR)
 	      {
 		return new TyTy::ErrorType (0);
@@ -1107,8 +1180,8 @@ UnifyRules::expect_fnptr (TyTy::FnPtr *ltype, TyTy::BaseType *rtype)
     case TyTy::PLACEHOLDER:
     case TyTy::PROJECTION:
     case TyTy::DYNAMIC:
-    case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1120,7 +1193,8 @@ UnifyRules::expect_tuple (TyTy::TupleType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1129,7 +1203,8 @@ UnifyRules::expect_tuple (TyTy::TupleType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::TUPLE: {
+    case TyTy::TUPLE:
+      {
 	TyTy::TupleType &type = *static_cast<TyTy::TupleType *> (rtype);
 	if (ltype->num_fields () != type.num_fields ())
 	  {
@@ -1143,10 +1218,8 @@ UnifyRules::expect_tuple (TyTy::TupleType *ltype, TyTy::BaseType *rtype)
 	    TyTy::BaseType *fo = type.get_field (i);
 
 	    TyTy::BaseType *unified_ty
-	      = UnifyRules::Resolve (TyTy::TyWithLocation (bo),
-				     TyTy::TyWithLocation (fo), locus,
-				     commit_flag, false /* emit_errors */,
-				     infer_flag, commits, infers);
+	      = resolve_subtype (TyTy::TyWithLocation (bo),
+				 TyTy::TyWithLocation (fo));
 	    if (unified_ty->get_kind () == TyTy::TypeKind::ERROR)
 	      return new TyTy::ErrorType (0);
 
@@ -1180,6 +1253,7 @@ UnifyRules::expect_tuple (TyTy::TupleType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1191,7 +1265,8 @@ UnifyRules::expect_bool (TyTy::BoolType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1228,6 +1303,7 @@ UnifyRules::expect_bool (TyTy::BoolType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1239,7 +1315,8 @@ UnifyRules::expect_char (TyTy::CharType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1276,6 +1353,7 @@ UnifyRules::expect_char (TyTy::CharType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1287,7 +1365,8 @@ UnifyRules::expect_int (TyTy::IntType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL
@@ -1300,7 +1379,8 @@ UnifyRules::expect_int (TyTy::IntType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::INT: {
+    case TyTy::INT:
+      {
 	TyTy::IntType &type = *static_cast<TyTy::IntType *> (rtype);
 	bool is_valid = ltype->get_int_kind () == type.get_int_kind ();
 	if (is_valid)
@@ -1331,6 +1411,7 @@ UnifyRules::expect_int (TyTy::IntType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1342,7 +1423,8 @@ UnifyRules::expect_uint (TyTy::UintType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL
@@ -1355,7 +1437,8 @@ UnifyRules::expect_uint (TyTy::UintType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::UINT: {
+    case TyTy::UINT:
+      {
 	TyTy::UintType &type = *static_cast<TyTy::UintType *> (rtype);
 	bool is_valid = ltype->get_uint_kind () == type.get_uint_kind ();
 	if (is_valid)
@@ -1386,6 +1469,7 @@ UnifyRules::expect_uint (TyTy::UintType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1397,7 +1481,8 @@ UnifyRules::expect_float (TyTy::FloatType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL
@@ -1410,7 +1495,8 @@ UnifyRules::expect_float (TyTy::FloatType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::FLOAT: {
+    case TyTy::FLOAT:
+      {
 	TyTy::FloatType &type = *static_cast<TyTy::FloatType *> (rtype);
 	bool is_valid = ltype->get_float_kind () == type.get_float_kind ();
 	if (is_valid)
@@ -1441,6 +1527,7 @@ UnifyRules::expect_float (TyTy::FloatType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1452,7 +1539,8 @@ UnifyRules::expect_isize (TyTy::ISizeType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () != TyTy::InferType::InferTypeKind::FLOAT;
@@ -1489,6 +1577,7 @@ UnifyRules::expect_isize (TyTy::ISizeType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1500,7 +1589,8 @@ UnifyRules::expect_usize (TyTy::USizeType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () != TyTy::InferType::InferTypeKind::FLOAT;
@@ -1537,6 +1627,7 @@ UnifyRules::expect_usize (TyTy::USizeType *ltype, TyTy::BaseType *rtype)
     case TyTy::DYNAMIC:
     case TyTy::CLOSURE:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1548,7 +1639,8 @@ UnifyRules::expect_never (TyTy::NeverType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1569,7 +1661,8 @@ UnifyRules::expect_placeholder (TyTy::PlaceholderType *ltype,
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1607,6 +1700,7 @@ UnifyRules::expect_placeholder (TyTy::PlaceholderType *ltype,
 	return rtype->clone ();
       gcc_fallthrough ();
 
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1619,7 +1713,8 @@ UnifyRules::expect_projection (TyTy::ProjectionType *ltype,
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1655,6 +1750,7 @@ UnifyRules::expect_projection (TyTy::ProjectionType *ltype,
     case TyTy::NEVER:
     case TyTy::PLACEHOLDER:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1666,7 +1762,8 @@ UnifyRules::expect_dyn (TyTy::DynamicObjectType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1675,7 +1772,8 @@ UnifyRules::expect_dyn (TyTy::DynamicObjectType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::DYNAMIC: {
+    case TyTy::DYNAMIC:
+      {
 	TyTy::DynamicObjectType &type
 	  = *static_cast<TyTy::DynamicObjectType *> (rtype);
 	if (ltype->num_specified_bounds () != type.num_specified_bounds ())
@@ -1714,6 +1812,7 @@ UnifyRules::expect_dyn (TyTy::DynamicObjectType *ltype, TyTy::BaseType *rtype)
     case TyTy::PLACEHOLDER:
     case TyTy::PROJECTION:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1725,7 +1824,8 @@ UnifyRules::expect_closure (TyTy::ClosureType *ltype, TyTy::BaseType *rtype)
 {
   switch (rtype->get_kind ())
     {
-      case TyTy::INFER: {
+    case TyTy::INFER:
+      {
 	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
 	bool is_valid
 	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
@@ -1734,26 +1834,25 @@ UnifyRules::expect_closure (TyTy::ClosureType *ltype, TyTy::BaseType *rtype)
       }
       break;
 
-      case TyTy::CLOSURE: {
+    case TyTy::CLOSURE:
+      {
 	TyTy::ClosureType &type = *static_cast<TyTy::ClosureType *> (rtype);
 	if (ltype->get_def_id () != type.get_def_id ())
 	  {
 	    return new TyTy::ErrorType (0);
 	  }
 
-	TyTy::BaseType *args_res = UnifyRules::Resolve (
-	  TyTy::TyWithLocation (&ltype->get_parameters ()),
-	  TyTy::TyWithLocation (&type.get_parameters ()), locus, commit_flag,
-	  false /* emit_error */, infer_flag, commits, infers);
+	TyTy::BaseType *args_res
+	  = resolve_subtype (TyTy::TyWithLocation (&ltype->get_parameters ()),
+			     TyTy::TyWithLocation (&type.get_parameters ()));
 	if (args_res->get_kind () == TyTy::TypeKind::ERROR)
 	  {
 	    return new TyTy::ErrorType (0);
 	  }
 
-	TyTy::BaseType *res = UnifyRules::Resolve (
-	  TyTy::TyWithLocation (&ltype->get_result_type ()),
-	  TyTy::TyWithLocation (&type.get_result_type ()), locus, commit_flag,
-	  false /* emit_error */, infer_flag, commits, infers);
+	TyTy::BaseType *res
+	  = resolve_subtype (TyTy::TyWithLocation (&ltype->get_result_type ()),
+			     TyTy::TyWithLocation (&type.get_result_type ()));
 	if (res == nullptr || res->get_kind () == TyTy::TypeKind::ERROR)
 	  {
 	    return new TyTy::ErrorType (0);
@@ -1785,6 +1884,7 @@ UnifyRules::expect_closure (TyTy::ClosureType *ltype, TyTy::BaseType *rtype)
     case TyTy::PROJECTION:
     case TyTy::DYNAMIC:
     case TyTy::OPAQUE:
+    case TyTy::CONST:
     case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
@@ -1794,58 +1894,113 @@ UnifyRules::expect_closure (TyTy::ClosureType *ltype, TyTy::BaseType *rtype)
 TyTy::BaseType *
 UnifyRules::expect_opaque (TyTy::OpaqueType *ltype, TyTy::BaseType *rtype)
 {
-  switch (rtype->get_kind ())
+  if (rtype->is<TyTy::OpaqueType> ())
     {
-      case TyTy::INFER: {
-	TyTy::InferType *r = static_cast<TyTy::InferType *> (rtype);
-	bool is_valid
-	  = r->get_infer_kind () == TyTy::InferType::InferTypeKind::GENERAL;
-	if (is_valid)
-	  return ltype->clone ();
-      }
-      break;
+      TyTy::OpaqueType *ro = rtype->as<TyTy::OpaqueType> ();
+      if (!ltype->is_equal (*ro))
+	return new TyTy::ErrorType (0);
 
-      case TyTy::OPAQUE: {
-	auto &type = *static_cast<TyTy::OpaqueType *> (rtype);
-	if (ltype->num_specified_bounds () != type.num_specified_bounds ())
-	  {
-	    return new TyTy::ErrorType (0);
-	  }
+      if (ltype->can_resolve () && ro->can_resolve ())
+	{
+	  auto lr = ltype->resolve ();
+	  auto rr = ro->resolve ();
 
-	if (!ltype->bounds_compatible (type, locus, true))
-	  {
+	  auto res = resolve_subtype (TyTy::TyWithLocation (lr),
+				      TyTy::TyWithLocation (rr));
+	  if (res->get_kind () == TyTy::TypeKind::ERROR)
 	    return new TyTy::ErrorType (0);
-	  }
+	}
+      else if (ltype->can_resolve ())
+	{
+	  auto lr = ltype->resolve ();
+	  ro->set_ty_ref (lr->get_ref ());
+	}
+      else if (ro->can_resolve ())
+	{
+	  auto rr = ro->resolve ();
+	  ltype->set_ty_ref (rr->get_ref ());
+	}
+    }
+  else if (ltype->can_resolve ())
+    {
+      auto underly = ltype->resolve ();
+      auto res = resolve_subtype (TyTy::TyWithLocation (underly),
+				  TyTy::TyWithLocation (rtype));
+      if (res->get_kind () == TyTy::TypeKind::ERROR)
+	return new TyTy::ErrorType (0);
+    }
+  else
+    {
+      ltype->set_ty_ref (rtype->get_ref ());
+    }
 
-	return ltype->clone ();
-      }
-      break;
+  return ltype;
+}
+
+TyTy::BaseType *
+UnifyRules::expect_const (TyTy::ConstType *ltype, TyTy::BaseType *rtype)
+{
+  if (rtype->get_kind () != TyTy::TypeKind::CONST)
+    return new TyTy::ErrorType (0);
+
+  TyTy::ConstType &lhs = *ltype;
+  TyTy::ConstType &rhs = *static_cast<TyTy::ConstType *> (rtype);
+
+  auto res = resolve_subtype (TyTy::TyWithLocation (lhs.get_ty ()),
+			      TyTy::TyWithLocation (rhs.get_ty ()));
+  if (res->get_kind () == TyTy::TypeKind::ERROR)
+    return new TyTy::ErrorType (0);
+
+  tree lv = lhs.get_value ();
+  tree rv = rhs.get_value ();
+
+  if (error_operand_p (lv) && error_operand_p (rv))
+    {
+      // this is only allowed for some silly senarios like:
+      // gcc/testsuite/rust/compile/issue-const_generics_5.rs
+      if (lhs.get_const_kind () == rhs.get_const_kind ())
+	{
+	  return new TyTy::ConstType (lhs.get_const_kind (), lhs.get_symbol (),
+				      res, error_mark_node,
+				      lhs.get_specified_bounds (),
+				      lhs.get_locus (), lhs.get_ref (),
+				      lhs.get_ty_ref (),
+				      lhs.get_combined_refs ());
+	}
 
-    case TyTy::CLOSURE:
-    case TyTy::SLICE:
-    case TyTy::PARAM:
-    case TyTy::POINTER:
-    case TyTy::STR:
-    case TyTy::ADT:
-    case TyTy::REF:
-    case TyTy::ARRAY:
-    case TyTy::FNDEF:
-    case TyTy::FNPTR:
-    case TyTy::TUPLE:
-    case TyTy::BOOL:
-    case TyTy::CHAR:
-    case TyTy::INT:
-    case TyTy::UINT:
-    case TyTy::FLOAT:
-    case TyTy::USIZE:
-    case TyTy::ISIZE:
-    case TyTy::NEVER:
-    case TyTy::PLACEHOLDER:
-    case TyTy::PROJECTION:
-    case TyTy::DYNAMIC:
-    case TyTy::ERROR:
       return new TyTy::ErrorType (0);
     }
+
+  bool equal = operand_equal_p (lv, rv, 0);
+  if (equal)
+    {
+      return new TyTy::ConstType (TyTy::ConstType::ConstKind::Value,
+				  lhs.get_symbol (), res, lv,
+				  lhs.get_specified_bounds (), lhs.get_locus (),
+				  lhs.get_ref (), lhs.get_ty_ref (),
+				  lhs.get_combined_refs ());
+    }
+
+  if (lhs.get_const_kind () == TyTy::ConstType::Infer && !error_operand_p (rv))
+    {
+      lhs.set_value (rv);
+      return new TyTy::ConstType (TyTy::ConstType::ConstKind::Value,
+				  lhs.get_symbol (), res, rv,
+				  lhs.get_specified_bounds (), lhs.get_locus (),
+				  lhs.get_ref (), lhs.get_ty_ref (),
+				  lhs.get_combined_refs ());
+    }
+  else if (rhs.get_const_kind () == TyTy::ConstType::Infer
+	   && !error_operand_p (lv))
+    {
+      rhs.set_value (lv);
+      return new TyTy::ConstType (TyTy::ConstType::ConstKind::Value,
+				  rhs.get_symbol (), res, lv,
+				  rhs.get_specified_bounds (), rhs.get_locus (),
+				  rhs.get_ref (), rhs.get_ty_ref (),
+				  rhs.get_combined_refs ());
+    }
+
   return new TyTy::ErrorType (0);
 }
 
diff --git a/gcc/rust/typecheck/rust-unify.h b/gcc/rust/typecheck/rust-unify.h
index 5ff3b7c..b8c9cbc 100644
--- a/gcc/rust/typecheck/rust-unify.h
+++ b/gcc/rust/typecheck/rust-unify.h
@@ -84,6 +84,7 @@ protected:
 				  TyTy::BaseType *rtype);
   TyTy::BaseType *expect_opaque (TyTy::OpaqueType *ltype,
 				 TyTy::BaseType *rtype);
+  TyTy::BaseType *expect_const (TyTy::ConstType *ltype, TyTy::BaseType *rtype);
 
 private:
   UnifyRules (TyTy::TyWithLocation lhs, TyTy::TyWithLocation rhs,
@@ -91,6 +92,9 @@ private:
 	      std::vector<CommitSite> &commits,
 	      std::vector<InferenceSite> &infers);
 
+  TyTy::BaseType *resolve_subtype (TyTy::TyWithLocation lhs,
+				   TyTy::TyWithLocation rhs);
+
   void emit_type_mismatch () const;
   void emit_abi_mismatch (const TyTy::FnType &expected,
 			  const TyTy::FnType &got) const;
diff --git a/gcc/rust/util/optional.h b/gcc/rust/util/optional.h
index 2c59459..9d2cd97 100644
--- a/gcc/rust/util/optional.h
+++ b/gcc/rust/util/optional.h
@@ -1364,7 +1364,7 @@ public:
       this->m_has_value = false;
     }
   }
-}; // namespace tl
+};
 
 /// Compares two optional objects
 template <class T, class U>
@@ -2110,7 +2110,7 @@ public:
 
 private:
   T *m_value;
-}; // namespace tl
+};
 
 
 
@@ -2128,4 +2128,4 @@ template <class T> struct hash<tl::optional<T>> {
 };
 } // namespace std
 
-#endif
\ No newline at end of file
+#endif
diff --git a/gcc/rust/util/rust-abi.h b/gcc/rust/util/rust-abi.h
index a0180ed..357a5db 100644
--- a/gcc/rust/util/rust-abi.h
+++ b/gcc/rust/util/rust-abi.h
@@ -34,11 +34,9 @@ enum ABI
   SYSV64
 };
 
-extern Rust::ABI
-get_abi_from_string (const std::string &abi);
+extern Rust::ABI get_abi_from_string (const std::string &abi);
 
-extern std::string
-get_string_from_abi (Rust::ABI abi);
+extern std::string get_string_from_abi (Rust::ABI abi);
 
 } // namespace Rust
 
diff --git a/gcc/rust/util/rust-attribute-values.h b/gcc/rust/util/rust-attribute-values.h
index 47e6a17..367044a 100644
--- a/gcc/rust/util/rust-attribute-values.h
+++ b/gcc/rust/util/rust-attribute-values.h
@@ -85,6 +85,13 @@ public:
   static constexpr auto &NON_EXHAUSTIVE = "non_exhaustive";
 
   static constexpr auto &RUSTFMT = "rustfmt";
+
+  static constexpr auto &TEST = "test";
+
+  static constexpr auto &SIMD_TEST = "simd_test";
+
+  static constexpr auto &RUSTC_ARGS_REQUIRED_CONST
+    = "rustc_args_required_const";
 };
 } // namespace Values
 } // namespace Rust
diff --git a/gcc/rust/util/rust-attributes.cc b/gcc/rust/util/rust-attributes.cc
index c77e99c..c846c2d 100644
--- a/gcc/rust/util/rust-attributes.cc
+++ b/gcc/rust/util/rust-attributes.cc
@@ -88,6 +88,9 @@ static const BuiltinAttrDefinition __definitions[]
 
      {Attrs::RUSTC_LAYOUT_SCALAR_VALID_RANGE_START, CODE_GENERATION},
 
+     // TODO: be careful about calling functions marked with this?
+     {Attrs::RUSTC_ARGS_REQUIRED_CONST, CODE_GENERATION},
+
      {Attrs::PRELUDE_IMPORT, NAME_RESOLUTION},
 
      {Attrs::RUSTC_DIAGNOSTIC_ITEM, STATIC_ANALYSIS},
@@ -95,7 +98,10 @@ static const BuiltinAttrDefinition __definitions[]
 
      {Attrs::FUNDAMENTAL, TYPE_CHECK},
      {Attrs::NON_EXHAUSTIVE, TYPE_CHECK},
-     {Attrs::RUSTFMT, EXTERNAL}};
+     {Attrs::RUSTFMT, EXTERNAL},
+
+     {Attrs::TEST, CODE_GENERATION},
+     {Attrs::SIMD_TEST, CODE_GENERATION}};
 
 BuiltinAttributeMappings *
 BuiltinAttributeMappings::get ()
@@ -221,7 +227,8 @@ check_doc_attribute (const AST::Attribute &attribute)
       break;
       // FIXME: Handle them as well
 
-      case AST::AttrInput::TOKEN_TREE: {
+    case AST::AttrInput::TOKEN_TREE:
+      {
 	// FIXME: This doesn't check for #[doc(alias(...))]
 	const auto &option = static_cast<const AST::DelimTokenTree &> (
 	  attribute.get_attr_input ());
@@ -388,7 +395,7 @@ AttributeChecker::visit (AST::MetaItemLitExpr &)
 {}
 
 void
-AttributeChecker::visit (AST::MetaItemPathLit &)
+AttributeChecker::visit (AST::MetaItemPathExpr &)
 {}
 
 void
diff --git a/gcc/rust/util/rust-attributes.h b/gcc/rust/util/rust-attributes.h
index 7c7a1fc..db8fe23 100644
--- a/gcc/rust/util/rust-attributes.h
+++ b/gcc/rust/util/rust-attributes.h
@@ -130,7 +130,7 @@ private:
   void visit (AST::AttrInputLiteral &attr_input) override;
   void visit (AST::AttrInputMacro &attr_input) override;
   void visit (AST::MetaItemLitExpr &meta_item) override;
-  void visit (AST::MetaItemPathLit &meta_item) override;
+  void visit (AST::MetaItemPathExpr &meta_item) override;
   void visit (AST::BorrowExpr &expr) override;
   void visit (AST::DereferenceExpr &expr) override;
   void visit (AST::ErrorPropagationExpr &expr) override;
diff --git a/gcc/rust/util/rust-base62.h b/gcc/rust/util/rust-base62.h
index e751c95..5fd7a37 100644
--- a/gcc/rust/util/rust-base62.h
+++ b/gcc/rust/util/rust-base62.h
@@ -26,8 +26,7 @@ namespace Rust {
 /**
  * Get the Base62 representation of an integer
  */
-std::string
-base62_integer (uint64_t value);
+std::string base62_integer (uint64_t value);
 
 } // namespace Rust
 
diff --git a/gcc/rust/util/rust-canonical-path.h b/gcc/rust/util/rust-canonical-path.h
index 4d8f954..079ae76 100644
--- a/gcc/rust/util/rust-canonical-path.h
+++ b/gcc/rust/util/rust-canonical-path.h
@@ -57,10 +57,11 @@ public:
     return *this;
   }
 
-  static CanonicalPath new_seg (NodeId id, const std::string &path)
+  static CanonicalPath new_seg (NodeId id, std::string path)
   {
     rust_assert (!path.empty ());
-    return CanonicalPath ({std::pair<NodeId, std::string> (id, path)},
+    return CanonicalPath ({std::pair<NodeId, std::string> (id,
+							   std::move (path))},
 			  UNKNOWN_CRATENUM);
   }
 
@@ -68,14 +69,18 @@ public:
   trait_impl_projection_seg (NodeId id, const CanonicalPath &trait_seg,
 			     const CanonicalPath &impl_type_seg)
   {
-    return CanonicalPath::new_seg (id, "<" + impl_type_seg.get () + " as "
+    // https://doc.rust-lang.org/reference/paths.html#canonical-paths
+    // should be "<X>"?
+    return CanonicalPath::new_seg (id, "<impl " + impl_type_seg.get () + " as "
 					 + trait_seg.get () + ">");
   }
 
   static CanonicalPath inherent_impl_seg (NodeId id,
 					  const CanonicalPath &impl_type_seg)
   {
-    return CanonicalPath::new_seg (id, "<" + impl_type_seg.get () + ">");
+    // https://doc.rust-lang.org/reference/paths.html#canonical-paths
+    // should be "<X as Y>"?
+    return CanonicalPath::new_seg (id, "<impl " + impl_type_seg.get () + ">");
   }
 
   std::string get () const
diff --git a/gcc/rust/util/rust-dir-owner.h b/gcc/rust/util/rust-dir-owner.h
index dcb45d8..fb72bb1 100644
--- a/gcc/rust/util/rust-dir-owner.h
+++ b/gcc/rust/util/rust-dir-owner.h
@@ -26,8 +26,7 @@
 namespace Rust {
 
 // extracts the owned subdirectory name from a file name
-bool
-get_file_subdir (const std::string &filename, std::string &subdir);
+bool get_file_subdir (const std::string &filename, std::string &subdir);
 
 } // namespace Rust
 
diff --git a/gcc/rust/util/rust-edition.h b/gcc/rust/util/rust-edition.h
index d034ea0..532fedb 100644
--- a/gcc/rust/util/rust-edition.h
+++ b/gcc/rust/util/rust-edition.h
@@ -33,8 +33,7 @@ enum class Edition
   E2021
 };
 
-Edition
-get_rust_edition ();
+Edition get_rust_edition ();
 
 } // namespace Rust
 
diff --git a/gcc/rust/util/rust-ggc.cc b/gcc/rust/util/rust-ggc.cc
new file mode 100644
index 0000000..0722af2
--- /dev/null
+++ b/gcc/rust/util/rust-ggc.cc
@@ -0,0 +1,41 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "rust-ggc.h"
+#include "stringpool.h"
+
+namespace Rust {
+
+namespace GGC {
+
+Ident::Ident (const char *str) : inner (get_identifier (str)) {}
+
+Ident::Ident (const std::string &str)
+  : inner (get_identifier_with_length (str.c_str (), str.length ()))
+{}
+
+bool
+Ident::operator== (const std::string &other) const
+{
+  // maybe_get_identifier_with_length doesn't seem to exist
+  return maybe_get_identifier (other.c_str ()) == inner;
+}
+
+} // namespace GGC
+
+} // namespace Rust
diff --git a/gcc/rust/util/rust-ggc.h b/gcc/rust/util/rust-ggc.h
new file mode 100644
index 0000000..da28ede
--- /dev/null
+++ b/gcc/rust/util/rust-ggc.h
@@ -0,0 +1,63 @@
+// Copyright (C) 2025 Free Software Foundation, Inc.
+
+// This file is part of GCC.
+
+// GCC is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3, or (at your option) any later
+// version.
+
+// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with GCC; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef RUST_GGC_H
+#define RUST_GGC_H
+
+#include "rust-system.h"
+#include "tree.h"
+
+namespace Rust {
+
+namespace GGC {
+
+class Ident
+{
+  tree inner;
+
+public:
+  Ident (const char *str);
+  Ident (const std::string &str);
+
+  bool operator== (const Ident &other) const { return inner == other.inner; }
+  bool operator== (const std::string &other) const;
+
+  const char *c_str () const { return IDENTIFIER_POINTER (inner); }
+  size_t size () const { return IDENTIFIER_LENGTH (inner); }
+
+  bool empty () const { return !size (); }
+
+  std::string as_string () const
+  {
+    return std::string (c_str (), c_str () + size ());
+  }
+
+  tree as_tree () const { return inner; }
+};
+
+static inline bool
+operator== (const std::string &a, const Ident &b)
+{
+  return b == a;
+}
+
+} // namespace GGC
+
+} // namespace Rust
+
+#endif // RUST_GGC_H
diff --git a/gcc/rust/util/rust-hir-map.cc b/gcc/rust/util/rust-hir-map.cc
index eaa640c..4629e6a 100644
--- a/gcc/rust/util/rust-hir-map.cc
+++ b/gcc/rust/util/rust-hir-map.cc
@@ -1148,17 +1148,19 @@ Mappings::lookup_module_children (NodeId module)
 }
 
 void
-Mappings::insert_ast_module (AST::Module *module)
+Mappings::insert_glob_container (AST::Item *container)
 {
-  rust_assert (modules.find (module->get_node_id ()) == modules.end ());
-  modules[module->get_node_id ()] = module;
+  rust_assert (glob_containers.find (container->get_node_id ())
+	       == glob_containers.end ());
+
+  glob_containers[container->get_node_id ()] = container;
 }
 
-tl::optional<AST::Module *>
-Mappings::lookup_ast_module (NodeId id)
+tl::optional<AST::Item *>
+Mappings::lookup_glob_container (NodeId id)
 {
-  auto it = modules.find (id);
-  if (it == modules.end ())
+  auto it = glob_containers.find (id);
+  if (it == glob_containers.end ())
     return tl::nullopt;
 
   return {it->second};
diff --git a/gcc/rust/util/rust-hir-map.h b/gcc/rust/util/rust-hir-map.h
index b523a36..c8fafa4 100644
--- a/gcc/rust/util/rust-hir-map.h
+++ b/gcc/rust/util/rust-hir-map.h
@@ -321,8 +321,8 @@ public:
   void insert_visibility (NodeId id, Privacy::ModuleVisibility visibility);
   tl::optional<Privacy::ModuleVisibility &> lookup_visibility (NodeId id);
 
-  void insert_ast_module (AST::Module *);
-  tl::optional<AST::Module *> lookup_ast_module (NodeId id);
+  void insert_glob_container (AST::Item *);
+  tl::optional<AST::Item *> lookup_glob_container (NodeId id);
   void insert_module_child (NodeId module, NodeId child);
   tl::optional<std::vector<NodeId> &> lookup_module_children (NodeId module);
 
@@ -436,7 +436,7 @@ private:
   std::map<NodeId, std::vector<NodeId>> module_child_map;
   std::map<NodeId, std::vector<Resolver::CanonicalPath>> module_child_items;
   std::map<NodeId, NodeId> child_to_parent_module_map;
-  std::map<NodeId, AST::Module *> modules;
+  std::map<NodeId, AST::Item *> glob_containers;
 
   // AST mappings
   std::map<NodeId, AST::Item *> ast_item_mappings;
diff --git a/gcc/rust/util/rust-punycode.h b/gcc/rust/util/rust-punycode.h
index a939f05..75260ce 100644
--- a/gcc/rust/util/rust-punycode.h
+++ b/gcc/rust/util/rust-punycode.h
@@ -27,8 +27,7 @@ namespace Rust {
 /* Encode a string as punycode. Returns a string if encoding is successful.
  * Returns nullopt otherwise. Note that a returned string contains only ASCII
  * characters and does not start with `xn--`. */
-tl::optional<std::string>
-encode_punycode (const Utf8String &src);
+tl::optional<std::string> encode_punycode (const Utf8String &src);
 
 } // namespace Rust
 
@@ -36,8 +35,7 @@ encode_punycode (const Utf8String &src);
 
 namespace selftest {
 
-void
-rust_punycode_encode_test ();
+void rust_punycode_encode_test ();
 
 } // namespace selftest
 
diff --git a/gcc/rust/util/rust-token-converter.cc b/gcc/rust/util/rust-token-converter.cc
index fc34adb..52172f4 100644
--- a/gcc/rust/util/rust-token-converter.cc
+++ b/gcc/rust/util/rust-token-converter.cc
@@ -202,7 +202,8 @@ convert (const std::vector<const_TokenPtr> &tokens)
 	case PERCENT_EQ:
 	case SCOPE_RESOLUTION:
 	case NOT_EQUAL:
-	  case EQUAL_EQUAL: {
+	case EQUAL_EQUAL:
+	  {
 	    auto str = token->as_string ();
 	    auto it = str.cbegin ();
 	    for (; it != str.cend () - 1; it++)
@@ -260,9 +261,8 @@ convert (const std::vector<const_TokenPtr> &tokens)
   return trees.back ();
 }
 
-static void
-from_tokenstream (const ProcMacro::TokenStream &ts,
-		  std::vector<const_TokenPtr> &result);
+static void from_tokenstream (const ProcMacro::TokenStream &ts,
+			      std::vector<const_TokenPtr> &result);
 
 /**
  * Append the token corresponding to a given Ident to a vector.
diff --git a/gcc/rust/util/rust-token-converter.h b/gcc/rust/util/rust-token-converter.h
index 5405d6e..6e4af50 100644
--- a/gcc/rust/util/rust-token-converter.h
+++ b/gcc/rust/util/rust-token-converter.h
@@ -23,14 +23,11 @@
 
 namespace Rust {
 
-ProcMacro::TokenStream
-convert (const std::vector<const_TokenPtr> &tokens);
+ProcMacro::TokenStream convert (const std::vector<const_TokenPtr> &tokens);
 
-std::vector<const_TokenPtr>
-convert (const ProcMacro::TokenStream &ts);
+std::vector<const_TokenPtr> convert (const ProcMacro::TokenStream &ts);
 
-ProcMacro::Literal
-convert_literal (const_TokenPtr lit);
+ProcMacro::Literal convert_literal (const_TokenPtr lit);
 
 } // namespace Rust
 
diff --git a/gcc/rust/util/rust-unicode.h b/gcc/rust/util/rust-unicode.h
index 6a6526d..6579b80 100644
--- a/gcc/rust/util/rust-unicode.h
+++ b/gcc/rust/util/rust-unicode.h
@@ -59,20 +59,15 @@ public:
   Utf8String nfc_normalize () const;
 };
 
-bool
-is_alphabetic (uint32_t codepoint);
+bool is_alphabetic (uint32_t codepoint);
 
-bool
-is_ascii_only (const std::string &str);
+bool is_ascii_only (const std::string &str);
 
-bool
-is_numeric (uint32_t codepoint);
+bool is_numeric (uint32_t codepoint);
 
-bool
-is_nfc_qc_no (uint32_t codepoint);
+bool is_nfc_qc_no (uint32_t codepoint);
 
-bool
-is_nfc_qc_maybe (uint32_t codepoint);
+bool is_nfc_qc_maybe (uint32_t codepoint);
 
 enum class QuickCheckResult
 {
@@ -81,8 +76,7 @@ enum class QuickCheckResult
   MAYBE
 };
 
-QuickCheckResult
-nfc_quick_check (const std::vector<Codepoint> &s);
+QuickCheckResult nfc_quick_check (const std::vector<Codepoint> &s);
 
 } // namespace Rust
 
@@ -90,14 +84,11 @@ nfc_quick_check (const std::vector<Codepoint> &s);
 
 namespace selftest {
 
-void
-rust_nfc_qc_test ();
+void rust_nfc_qc_test ();
 
-void
-rust_utf8_normalize_test ();
+void rust_utf8_normalize_test ();
 
-void
-rust_utf8_property_test ();
+void rust_utf8_property_test ();
 
 } // namespace selftest
 
diff --git a/gcc/rust/util/rust-unwrap-segment.h b/gcc/rust/util/rust-unwrap-segment.h
index bebdc3a..af3a237 100644
--- a/gcc/rust/util/rust-unwrap-segment.h
+++ b/gcc/rust/util/rust-unwrap-segment.h
@@ -83,14 +83,11 @@ public:
 /*
  * Used to get the node id of a path segment object
  */
-NodeId
-unwrap_segment_node_id (const AST::TypePathSegment &seg);
+NodeId unwrap_segment_node_id (const AST::TypePathSegment &seg);
 
-NodeId
-unwrap_segment_node_id (const AST::SimplePathSegment &seg);
+NodeId unwrap_segment_node_id (const AST::SimplePathSegment &seg);
 
-NodeId
-unwrap_segment_node_id (const AST::PathExprSegment &seg);
+NodeId unwrap_segment_node_id (const AST::PathExprSegment &seg);
 
 template <class T>
 NodeId
diff --git a/gcc/sarif-replay.cc b/gcc/sarif-replay.cc
index a96c97b..c740c29 100644
--- a/gcc/sarif-replay.cc
+++ b/gcc/sarif-replay.cc
@@ -93,6 +93,11 @@ static const char *const usage_msg = (
 "\n"
 "  --usage\n"
 "     Print this message and exit.\n"
+"\n"
+"Options for maintainers:\n"
+"\n"
+"  -fdebug-physical-locations\n"
+"    Dump debugging information about physical locations to stderr.\n"
 "\n");
 
 static void
@@ -182,7 +187,11 @@ parse_options (int argc, char **argv,
 	  print_version ();
 	  exit (0);
 	}
-
+      else if (strcmp (option, "-fdebug-physical-locations") == 0)
+	{
+	  opts.m_replay_opts.m_debug_physical_locations = true;
+	  handled = true;
+	}
       if (!handled)
 	{
 	  if (option[0] == '-')
@@ -245,6 +254,8 @@ main (int argc, char **argv)
 	  note.finish ("about to replay %qs...", filename);
 	}
       libgdiagnostics::manager playback_mgr;
+      playback_mgr.set_debug_physical_locations
+	(opts.m_replay_opts.m_debug_physical_locations);
       playback_mgr.add_text_sink (stderr,
 				  opts.m_replay_opts.m_diagnostics_colorize);
       for (auto spec : opts.m_extra_output_specs)
diff --git a/gcc/selftest-run-tests.cc b/gcc/selftest-run-tests.cc
index 3245453..c68d3e7 100644
--- a/gcc/selftest-run-tests.cc
+++ b/gcc/selftest-run-tests.cc
@@ -28,6 +28,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "stringpool.h"
 #include "attribs.h"
 #include "analyzer/analyzer-selftests.h"
+#include "diagnostics/diagnostics-selftests.h"
 #include "text-art/selftests.h"
 
 /* This function needed to be split out from selftest.cc as it references
@@ -94,23 +95,17 @@ selftest::run_tests ()
   digraph_cc_tests ();
   tristate_cc_tests ();
   ipa_modref_tree_cc_tests ();
-  selftest_logical_location_cc_tests ();
+
+  /* Run the diagnostics selftests.  */
+  diagnostics::selftest::run_diagnostics_selftests ();
 
   /* Higher-level tests, or for components that other selftests don't
      rely on.  */
-  diagnostic_color_cc_tests ();
-  diagnostic_show_locus_cc_tests ();
-  diagnostic_format_html_cc_tests ();
-  diagnostic_format_sarif_cc_tests ();
-  diagnostic_output_spec_cc_tests ();
-  edit_context_cc_tests ();
   fold_const_cc_tests ();
   spellcheck_cc_tests ();
   spellcheck_tree_cc_tests ();
   tree_cfg_cc_tests ();
-  diagnostic_path_output_cc_tests ();
   simple_diagnostic_path_cc_tests ();
-  lazy_diagnostic_path_cc_tests ();
   attribs_cc_tests ();
   path_coverage_cc_tests ();
 
diff --git a/gcc/selftest.cc b/gcc/selftest.cc
index 4fdce07..dc90340 100644
--- a/gcc/selftest.cc
+++ b/gcc/selftest.cc
@@ -21,6 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "selftest.h"
+#include "diagnostics/file-cache.h"
 #include "intl.h"
 
 #if CHECKING_P
@@ -186,7 +187,7 @@ assert_str_startswith (const location &loc,
 /* Constructor.  Generate a name for the file.  */
 
 named_temp_file::named_temp_file (const char *suffix,
-				  file_cache *fc)
+				  diagnostics::file_cache *fc)
 {
   m_filename = make_temp_file (suffix);
   ASSERT_NE (m_filename, NULL);
@@ -210,7 +211,7 @@ named_temp_file::~named_temp_file ()
 temp_source_file::temp_source_file (const location &loc,
 				    const char *suffix,
 				    const char *content,
-				    file_cache *fc)
+				    diagnostics::file_cache *fc)
 : named_temp_file (suffix, fc)
 {
   FILE *out = fopen (get_filename (), "w");
diff --git a/gcc/selftest.h b/gcc/selftest.h
index eb9e604..4501d34 100644
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -25,7 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #if CHECKING_P
 
-class file_cache;
+namespace diagnostics { class file_cache; }
 
 namespace selftest {
 
@@ -100,13 +100,13 @@ extern void assert_str_startswith (const location &loc,
 class named_temp_file
 {
  public:
-  named_temp_file (const char *suffix, file_cache *fc = nullptr);
+  named_temp_file (const char *suffix, diagnostics::file_cache *fc = nullptr);
   ~named_temp_file ();
   const char *get_filename () const { return m_filename; }
 
  private:
   char *m_filename;
-  file_cache *m_file_cache;
+  diagnostics::file_cache *m_file_cache;
 };
 
 /* A class for writing out a temporary sourcefile for use in selftests
@@ -116,7 +116,7 @@ class temp_source_file : public named_temp_file
 {
  public:
   temp_source_file (const location &loc, const char *suffix,
-		    const char *content, file_cache *fc = nullptr);
+		    const char *content, diagnostics::file_cache *fc = nullptr);
   temp_source_file (const location &loc, const char *suffix,
 		    const char *content, size_t sz);
 };
@@ -221,15 +221,8 @@ extern void bitmap_cc_tests ();
 extern void cgraph_cc_tests ();
 extern void convert_cc_tests ();
 extern void dbgcnt_cc_tests ();
-extern void diagnostic_color_cc_tests ();
-extern void diagnostic_format_html_cc_tests ();
-extern void diagnostic_format_sarif_cc_tests ();
-extern void diagnostic_output_spec_cc_tests ();
-extern void diagnostic_path_output_cc_tests ();
-extern void diagnostic_show_locus_cc_tests ();
 extern void digraph_cc_tests ();
 extern void dumpfile_cc_tests ();
-extern void edit_context_cc_tests ();
 extern void et_forest_cc_tests ();
 extern void fibonacci_heap_cc_tests ();
 extern void fold_const_cc_tests ();
@@ -246,7 +239,6 @@ extern void input_cc_tests ();
 extern void ipa_modref_tree_cc_tests ();
 extern void json_cc_tests ();
 extern void json_parser_cc_tests ();
-extern void lazy_diagnostic_path_cc_tests ();
 extern void opt_suggestions_cc_tests ();
 extern void optinfo_emit_json_cc_tests ();
 extern void opts_cc_tests ();
@@ -261,7 +253,6 @@ extern void relation_tests ();
 extern void rtl_tests_cc_tests ();
 extern void sbitmap_cc_tests ();
 extern void selftest_cc_tests ();
-extern void selftest_logical_location_cc_tests ();
 extern void simple_diagnostic_path_cc_tests ();
 extern void simplify_rtx_cc_tests ();
 extern void spellcheck_cc_tests ();
diff --git a/gcc/simple-diagnostic-path.cc b/gcc/simple-diagnostic-path.cc
index d64f7b6..2e112b5 100644
--- a/gcc/simple-diagnostic-path.cc
+++ b/gcc/simple-diagnostic-path.cc
@@ -1,4 +1,4 @@
-/* Concrete classes for implementing diagnostic paths.
+/* Concrete classes for implementing diagnostic paths, using tree.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>
 
@@ -32,29 +32,31 @@ along with GCC; see the file COPYING3.  If not see
 #include "simple-diagnostic-path.h"
 #include "selftest.h"
 
-/* class simple_diagnostic_path : public diagnostic_path.  */
+using namespace diagnostics::paths;
+
+/* class simple_diagnostic_path : public diagnostics::paths::path.  */
 
 simple_diagnostic_path::
 simple_diagnostic_path (const tree_logical_location_manager &logical_loc_mgr,
 			pretty_printer *event_pp)
-: diagnostic_path (logical_loc_mgr),
+: path (logical_loc_mgr),
   m_event_pp (event_pp),
   m_localize_events (true)
 {
   add_thread ("main");
 }
 
-/* Implementation of diagnostic_path::get_event vfunc for
+/* Implementation of path::get_event vfunc for
    simple_diagnostic_path: simply return the event in the vec.  */
 
-const diagnostic_event &
+const event &
 simple_diagnostic_path::get_event (int idx) const
 {
   return *m_events[idx];
 }
 
-const diagnostic_thread &
-simple_diagnostic_path::get_thread (diagnostic_thread_id_t idx) const
+const thread &
+simple_diagnostic_path::get_thread (thread_id_t idx) const
 {
   return *m_threads[idx];
 }
@@ -67,7 +69,7 @@ simple_diagnostic_path::same_function_p (int event_idx_a,
 	  == m_events[event_idx_b]->get_fndecl ());
 }
 
-diagnostic_thread_id_t
+thread_id_t
 simple_diagnostic_path::add_thread (const char *name)
 {
   m_threads.safe_push (new simple_diagnostic_thread (name));
@@ -82,7 +84,7 @@ simple_diagnostic_path::add_thread (const char *name)
 
    Return the id of the new event.  */
 
-diagnostic_event_id_t
+event_id_t
 simple_diagnostic_path::add_event (location_t loc, tree fndecl, int depth,
 				   const char *fmt, ...)
 {
@@ -108,11 +110,11 @@ simple_diagnostic_path::add_event (location_t loc, tree fndecl, int depth,
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
-diagnostic_event_id_t
-simple_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
+event_id_t
+simple_diagnostic_path::add_thread_event (thread_id_t thread_id,
 					  location_t loc,
 					  tree fndecl,
 					  int depth,
@@ -141,7 +143,7 @@ simple_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
 /* Mark the most recent event on this path (which must exist) as being
@@ -163,7 +165,7 @@ simple_diagnostic_event (location_t loc,
 			 tree fndecl,
 			 int depth,
 			 const char *desc,
-			 diagnostic_thread_id_t thread_id)
+			 thread_id_t thread_id)
 : m_loc (loc), m_fndecl (fndecl),
   m_logical_loc (tree_logical_location_manager::key_from_tree (fndecl)),
   m_depth (depth), m_desc (xstrdup (desc)),
diff --git a/gcc/simple-diagnostic-path.h b/gcc/simple-diagnostic-path.h
index d2b366e..d549c24 100644
--- a/gcc/simple-diagnostic-path.h
+++ b/gcc/simple-diagnostic-path.h
@@ -1,4 +1,4 @@
-/* Concrete classes for implementing diagnostic paths.
+/* Concrete classes for implementing diagnostic paths, using tree.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>
 
@@ -21,26 +21,30 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_SIMPLE_DIAGNOSTIC_PATH_H
 #define GCC_SIMPLE_DIAGNOSTIC_PATH_H
 
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 #include "tree-logical-location.h"
 
 /* Concrete subclasses of the abstract base classes
    declared in diagnostic-path.h.  */
 
-/* A simple implementation of diagnostic_event.  */
+/* A simple implementation of diagnostic event.
+   This uses "tree" and so is not in "namespace diagnostics".  */
 
-class simple_diagnostic_event : public diagnostic_event
+class simple_diagnostic_event : public diagnostics::paths::event
 {
  public:
+  using thread_id_t = diagnostics::paths::thread_id_t;
+
   simple_diagnostic_event (location_t loc, tree fndecl, int depth,
 			   const char *desc,
-			   diagnostic_thread_id_t thread_id = 0);
+			   thread_id_t thread_id = 0);
   ~simple_diagnostic_event ();
 
   location_t get_location () const final override { return m_loc; }
   int get_stack_depth () const final override { return m_depth; }
   void print_desc (pretty_printer &pp) const final override;
-  logical_location get_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_logical_location () const final override
   {
     return tree_logical_location_manager::key_from_tree (m_fndecl);
   }
@@ -52,7 +56,7 @@ class simple_diagnostic_event : public diagnostic_event
   {
     return m_connected_to_next_event;
   }
-  diagnostic_thread_id_t get_thread_id () const final override
+  thread_id_t get_thread_id () const final override
   {
     return m_thread_id;
   }
@@ -67,16 +71,16 @@ class simple_diagnostic_event : public diagnostic_event
  private:
   location_t m_loc;
   tree m_fndecl;
-  logical_location m_logical_loc;
+  diagnostics::logical_locations::key m_logical_loc;
   int m_depth;
   char *m_desc; // has been i18n-ed and formatted
   bool m_connected_to_next_event;
-  diagnostic_thread_id_t m_thread_id;
+  thread_id_t m_thread_id;
 };
 
-/* A simple implementation of diagnostic_thread.  */
+/* A simple implementation of diagnostics::paths::thread.  */
 
-class simple_diagnostic_thread : public diagnostic_thread
+class simple_diagnostic_thread : public diagnostics::paths::thread
 {
 public:
   simple_diagnostic_thread (const char *name) : m_name (name) {}
@@ -92,28 +96,33 @@ private:
 /* A simple implementation of diagnostic_path, as a vector of
    simple_diagnostic_event instances.  */
 
-class simple_diagnostic_path : public diagnostic_path
+class simple_diagnostic_path : public diagnostics::paths::path
 {
  public:
+  using thread = diagnostics::paths::thread;
+  using thread_id_t = diagnostics::paths::thread_id_t;
+  using event = diagnostics::paths::event;
+  using event_id_t = diagnostics::paths::event_id_t;
+
   simple_diagnostic_path (const tree_logical_location_manager &logical_loc_mgr,
 			  pretty_printer *event_pp);
 
   unsigned num_events () const final override { return m_events.length (); }
-  const diagnostic_event & get_event (int idx) const final override;
+  const event & get_event (int idx) const final override;
   unsigned num_threads () const final override { return m_threads.length (); }
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override;
+  const thread &
+  get_thread (thread_id_t) const final override;
   bool
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override;
 
-  diagnostic_thread_id_t add_thread (const char *name);
+  thread_id_t add_thread (const char *name);
 
-  diagnostic_event_id_t add_event (location_t loc, tree fndecl, int depth,
-				   const char *fmt, ...)
+  event_id_t add_event (location_t loc, tree fndecl, int depth,
+			const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(5,6);
-  diagnostic_event_id_t
-  add_thread_event (diagnostic_thread_id_t thread_id,
+  event_id_t
+  add_thread_event (thread_id_t thread_id,
 		    location_t loc, tree fndecl, int depth,
 		    const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(6,7);
diff --git a/gcc/simplify-rtx.cc b/gcc/simplify-rtx.cc
index cbe61b4..c723a07 100644
--- a/gcc/simplify-rtx.cc
+++ b/gcc/simplify-rtx.cc
@@ -8344,6 +8344,15 @@ simplify_context::simplify_subreg (machine_mode outermode, rtx op,
 	return simplify_gen_binary (GET_CODE (op), outermode, op0, op1);
     }
 
+  /* Attempt to simplify WORD_MODE SUBREGs of unary bitwise expression.  */
+  if (outermode == word_mode && GET_CODE (op) == NOT
+      && SCALAR_INT_MODE_P (innermode))
+    {
+      rtx op0 = simplify_subreg (outermode, XEXP (op, 0), innermode, byte);
+      if (op0)
+	return simplify_gen_unary (GET_CODE (op), outermode, op0, outermode);
+    }
+
   scalar_int_mode int_outermode, int_innermode;
   if (is_a <scalar_int_mode> (outermode, &int_outermode)
       && is_a <scalar_int_mode> (innermode, &int_innermode)
@@ -8394,9 +8403,45 @@ simplify_context::simplify_subreg (machine_mode outermode, rtx op,
       && VECTOR_MODE_P (innermode)
       && known_eq (GET_MODE_NUNITS (outermode), GET_MODE_NUNITS (innermode))
       && known_eq (GET_MODE_UNIT_SIZE (outermode),
-		    GET_MODE_UNIT_SIZE (innermode)))
+		   GET_MODE_UNIT_SIZE (innermode)))
     return simplify_gen_relational (GET_CODE (op), outermode, innermode,
 				    XEXP (op, 0), XEXP (op, 1));
+
+  /* Distribute non-paradoxical subregs through logic ops in cases where one term
+     disappears.
+
+     (subreg:M1 (and:M2 X C1)) -> (subreg:M1 X)
+     (subreg:M1 (ior:M2 X C1)) -> (subreg:M1 C1)
+     (subreg:M1 (xor:M2 X C1)) -> (subreg:M1 (not:M2 X))
+
+     if M2 is no smaller than M1 and (subreg:M1 C1) is all-ones.
+
+     (subreg:M1 (and:M2 X C2)) -> (subreg:M1 C2)
+     (subreg:M1 (ior/xor:M2 X C2)) -> (subreg:M1 X)
+
+     if M2 is no smaller than M1 and (subreg:M1 C2) is zero.  */
+  if (known_ge (innersize, outersize)
+      && GET_MODE_CLASS (outermode) == GET_MODE_CLASS (innermode)
+      && (GET_CODE (op) == AND || GET_CODE (op) == IOR || GET_CODE (op) == XOR)
+      && CONSTANT_P (XEXP (op, 1)))
+    {
+      rtx op1_subreg = simplify_subreg (outermode, XEXP (op, 1), innermode, byte);
+      if (op1_subreg == CONSTM1_RTX (outermode))
+	{
+	  if (GET_CODE (op) == IOR)
+	    return op1_subreg;
+	  rtx op0 = XEXP (op, 0);
+	  if (GET_CODE (op) == XOR)
+	    op0 = simplify_gen_unary (NOT, innermode, op0, innermode);
+	  return simplify_gen_subreg (outermode, op0, innermode, byte);
+	}
+
+      if (op1_subreg == CONST0_RTX (outermode))
+	return (GET_CODE (op) == AND
+		? op1_subreg
+		: simplify_gen_subreg (outermode, XEXP (op, 0), innermode, byte));
+    }
+
   return NULL_RTX;
 }
 
@@ -8668,6 +8713,43 @@ test_scalar_int_ext_ops (machine_mode bmode, machine_mode smode)
 				     lowpart_subreg (bmode, sreg, smode),
 				     bmode),
 		 sreg);
+
+  /* Test extensions, followed by logic ops, followed by truncations.  */
+  rtx bsubreg = lowpart_subreg (bmode, sreg, smode);
+  rtx smask = gen_int_mode (GET_MODE_MASK (smode), bmode);
+  rtx inv_smask = gen_int_mode (~GET_MODE_MASK (smode), bmode);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (AND, bmode,
+						      bsubreg, smask),
+				 bmode),
+		 sreg);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (AND, bmode,
+						      bsubreg, inv_smask),
+				 bmode),
+		 const0_rtx);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (IOR, bmode,
+						      bsubreg, smask),
+				 bmode),
+		 constm1_rtx);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (IOR, bmode,
+						      bsubreg, inv_smask),
+				 bmode),
+		 sreg);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (XOR, bmode,
+						      bsubreg, smask),
+				 bmode),
+		 lowpart_subreg (smode,
+				 gen_rtx_NOT (bmode, bsubreg),
+				 bmode));
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (XOR, bmode,
+						      bsubreg, inv_smask),
+				 bmode),
+		 sreg);
 }
 
 /* Verify more simplifications of integer extension/truncation.
diff --git a/gcc/spellcheck.cc b/gcc/spellcheck.cc
index 9a0a920..3fb0f50 100644
--- a/gcc/spellcheck.cc
+++ b/gcc/spellcheck.cc
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
+#define INCLUDE_ALGORITHM
 #include "config.h"
 #include "system.h"
 #include "coretypes.h"
@@ -25,11 +26,15 @@ along with GCC; see the file COPYING3.  If not see
 #include "spellcheck.h"
 #include "selftest.h"
 
+namespace {
+
 /* Cost of a case transformation.  */
-#define CASE_COST 1
+const edit_distance_t case_cost = 1;
 
 /* Cost of another kind of edit.  */
-#define BASE_COST 2
+const edit_distance_t base_cost = 2;
+
+} // anonymous namespace
 
 /* Get the edit distance between the two strings: the minimal
    number of edits that are needed to change one string into another,
@@ -55,9 +60,9 @@ get_edit_distance (const char *s, int len_s,
     }
 
   if (len_s == 0)
-    return BASE_COST * len_t;
+    return base_cost * len_t;
   if (len_t == 0)
-    return BASE_COST * len_s;
+    return base_cost * len_s;
 
   /* We effectively build a matrix where each (i, j) contains the
      distance between the prefix strings s[0:j] and t[0:i].
@@ -75,7 +80,7 @@ get_edit_distance (const char *s, int len_s,
   /* The first row is for the case of an empty target string, which
      we can reach by deleting every character in the source string.  */
   for (int i = 0; i < len_s + 1; i++)
-    v_one_ago[i] = i * BASE_COST;
+    v_one_ago[i] = i * base_cost;
 
   /* Build successive rows.  */
   for (int i = 0; i < len_t; i++)
@@ -91,7 +96,7 @@ get_edit_distance (const char *s, int len_s,
       /* The initial column is for the case of an empty source string; we
 	 can reach prefixes of the target string of length i
 	 by inserting i characters.  */
-      v_next[0] = (i + 1) * BASE_COST;
+      v_next[0] = (i + 1) * base_cost;
 
       /* Build the rest of the row by considering neighbors to
 	 the north, west and northwest.  */
@@ -102,18 +107,18 @@ get_edit_distance (const char *s, int len_s,
 	  if (s[j] == t[i])
 	    cost = 0;
 	  else if (TOLOWER (s[j]) == TOLOWER (t[i]))
-	    cost = CASE_COST;
+	    cost = case_cost;
 	  else
-	    cost = BASE_COST;
-	  edit_distance_t deletion     = v_next[j] + BASE_COST;
-	  edit_distance_t insertion    = v_one_ago[j + 1] + BASE_COST;
+	    cost = base_cost;
+	  edit_distance_t deletion     = v_next[j] + base_cost;
+	  edit_distance_t insertion    = v_one_ago[j + 1] + base_cost;
 	  edit_distance_t substitution = v_one_ago[j] + cost;
-	  edit_distance_t cheapest = MIN (deletion, insertion);
-	  cheapest = MIN (cheapest, substitution);
+	  edit_distance_t cheapest = std::min (deletion, insertion);
+	  cheapest = std::min (cheapest, substitution);
 	  if (i > 0 && j > 0 && s[j] == t[i - 1] && s[j - 1] == t[i])
 	    {
-	      edit_distance_t transposition = v_two_ago[j - 1] + BASE_COST;
-	      cheapest = MIN (cheapest, transposition);
+	      edit_distance_t transposition = v_two_ago[j - 1] + base_cost;
+	      cheapest = std::min (cheapest, transposition);
 	    }
 	  v_next[j + 1] = cheapest;
 	}
@@ -187,8 +192,8 @@ find_closest_string (const char *target,
 edit_distance_t
 get_edit_distance_cutoff (size_t goal_len, size_t candidate_len)
 {
-  size_t max_length = MAX (goal_len, candidate_len);
-  size_t min_length = MIN (goal_len, candidate_len);
+  size_t max_length = std::max (goal_len, candidate_len);
+  size_t min_length = std::min (goal_len, candidate_len);
 
   gcc_assert (max_length >= min_length);
 
@@ -200,11 +205,11 @@ get_edit_distance_cutoff (size_t goal_len, size_t candidate_len)
   /* If the lengths are close, then round down.  */
   if (max_length - min_length <= 1)
     /* ...but allow an edit distance of at least 1.  */
-    return BASE_COST * MAX (max_length / 3, 1);
+    return base_cost * std::max (max_length / 3, size_t {1});
 
   /* Otherwise, round up (thus giving a little extra leeway to some cases
      involving insertions/deletions).  */
-  return BASE_COST * (max_length + 2) / 3;
+  return base_cost * (max_length + 2) / 3;
 }
 
 #if CHECKING_P
@@ -244,49 +249,49 @@ static void
 test_edit_distances ()
 {
   test_get_edit_distance_both_ways ("", "nonempty",
-				    BASE_COST * strlen ("nonempty"));
+				    base_cost * strlen ("nonempty"));
   test_get_edit_distance_both_ways ("saturday", "sunday",
-				    BASE_COST * 3);
-  test_get_edit_distance_both_ways ("foo", "m_foo", BASE_COST * 2);
+				    base_cost * 3);
+  test_get_edit_distance_both_ways ("foo", "m_foo", base_cost * 2);
   test_get_edit_distance_both_ways ("hello_world", "HelloWorld", 4);
   test_get_edit_distance_both_ways
-    ("the quick brown fox jumps over the lazy dog", "dog", BASE_COST * 40);
+    ("the quick brown fox jumps over the lazy dog", "dog", base_cost * 40);
   test_get_edit_distance_both_ways
     ("the quick brown fox jumps over the lazy dog",
      "the quick brown dog jumps over the lazy fox",
-     BASE_COST * 4);
+     base_cost * 4);
   test_get_edit_distance_both_ways
     ("Lorem ipsum dolor sit amet, consectetur adipiscing elit,",
      "All your base are belong to us",
-     BASE_COST * 44);
+     base_cost * 44);
   test_get_edit_distance_both_ways ("foo", "FOO", 3);
-  test_get_edit_distance_both_ways ("fee", "deed", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("coorzd1", "coordx1", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("assert", "sqrt", BASE_COST * 3);
-  test_get_edit_distance_both_ways ("PATH_MAX", "INT8_MAX", BASE_COST * 3);
-  test_get_edit_distance_both_ways ("time", "nice", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("bar", "carg", BASE_COST * 2);
+  test_get_edit_distance_both_ways ("fee", "deed", base_cost * 2);
+  test_get_edit_distance_both_ways ("coorzd1", "coordx1", base_cost * 2);
+  test_get_edit_distance_both_ways ("assert", "sqrt", base_cost * 3);
+  test_get_edit_distance_both_ways ("PATH_MAX", "INT8_MAX", base_cost * 3);
+  test_get_edit_distance_both_ways ("time", "nice", base_cost * 2);
+  test_get_edit_distance_both_ways ("bar", "carg", base_cost * 2);
   test_get_edit_distance_both_ways ("gtk_widget_show_all",
 				    "GtkWidgetShowAll", 10);
-  test_get_edit_distance_both_ways ("m_bar", "bar", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("MACRO", "MACRAME", BASE_COST * 3);
-  test_get_edit_distance_both_ways ("ab", "ac", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("ab", "a", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("a", "b", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("nanl", "name", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("char", "bar", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("-optimize", "fsanitize", BASE_COST * 5);
-  test_get_edit_distance_both_ways ("__DATE__", "__i386__", BASE_COST * 4);
+  test_get_edit_distance_both_ways ("m_bar", "bar", base_cost * 2);
+  test_get_edit_distance_both_ways ("MACRO", "MACRAME", base_cost * 3);
+  test_get_edit_distance_both_ways ("ab", "ac", base_cost * 1);
+  test_get_edit_distance_both_ways ("ab", "a", base_cost * 1);
+  test_get_edit_distance_both_ways ("a", "b", base_cost * 1);
+  test_get_edit_distance_both_ways ("nanl", "name", base_cost * 2);
+  test_get_edit_distance_both_ways ("char", "bar", base_cost * 2);
+  test_get_edit_distance_both_ways ("-optimize", "fsanitize", base_cost * 5);
+  test_get_edit_distance_both_ways ("__DATE__", "__i386__", base_cost * 4);
 
   /* Examples where transposition helps.  */
-  test_get_edit_distance_both_ways ("ab", "ba", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("ba", "abc", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("coorzd1", "coordz1", BASE_COST * 1);
+  test_get_edit_distance_both_ways ("ab", "ba", base_cost * 1);
+  test_get_edit_distance_both_ways ("ba", "abc", base_cost * 2);
+  test_get_edit_distance_both_ways ("coorzd1", "coordz1", base_cost * 1);
   test_get_edit_distance_both_ways ("abcdefghijklmnopqrstuvwxyz",
 				    "bacdefghijklmnopqrstuvwxzy",
-				    BASE_COST * 2);
-  test_get_edit_distance_both_ways ("saturday", "sundya", BASE_COST * 4);
-  test_get_edit_distance_both_ways ("signed", "singed", BASE_COST * 1);
+				    base_cost * 2);
+  test_get_edit_distance_both_ways ("saturday", "sundya", base_cost * 4);
+  test_get_edit_distance_both_ways ("signed", "singed", base_cost * 1);
 }
 
 /* Subroutine of test_get_edit_distance_cutoff, for emulating the
@@ -295,7 +300,7 @@ test_edit_distances ()
 static edit_distance_t
 get_old_cutoff (size_t goal_len, size_t candidate_len)
 {
-  return BASE_COST * MAX (goal_len, candidate_len) / 2;
+  return base_cost * std::max (goal_len, candidate_len) / 2;
 }
 
 /* Verify that the cutoff for "meaningfulness" of suggestions is at least as
@@ -339,7 +344,7 @@ assert_not_suggested_for (const location &loc, const char *candidate,
 {
   auto_vec<const char *> candidates;
   candidates.safe_push (candidate);
-  ASSERT_EQ_AT (loc, NULL, find_closest_string (target, &candidates));
+  ASSERT_EQ_AT (loc, nullptr, find_closest_string (target, &candidates));
 }
 
 /* Assert that CANDIDATE is not offered as a suggestion for TARGET.  */
@@ -409,7 +414,7 @@ test_find_closest_string ()
   auto_vec<const char *> candidates;
 
   /* Verify that it can handle an empty vec.  */
-  ASSERT_EQ (NULL, find_closest_string ("", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("", &candidates));
 
   /* Verify that it works sanely for non-empty vecs.  */
   candidates.safe_push ("apple");
@@ -419,7 +424,7 @@ test_find_closest_string ()
   ASSERT_STREQ ("apple", find_closest_string ("app", &candidates));
   ASSERT_STREQ ("banana", find_closest_string ("banyan", &candidates));
   ASSERT_STREQ ("cherry", find_closest_string ("berry", &candidates));
-  ASSERT_EQ (NULL, find_closest_string ("not like the others", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("not like the others", &candidates));
 
   /* The order of the vec can matter, but it should not matter for these
      inputs.  */
@@ -430,12 +435,12 @@ test_find_closest_string ()
   ASSERT_STREQ ("apple", find_closest_string ("app", &candidates));
   ASSERT_STREQ ("banana", find_closest_string ("banyan", &candidates));
   ASSERT_STREQ ("cherry", find_closest_string ("berry", &candidates));
-  ASSERT_EQ (NULL, find_closest_string ("not like the others", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("not like the others", &candidates));
 
   /* If the goal string somehow makes it into the candidate list, offering
      it as a suggestion will be nonsensical.  Verify that we don't offer such
      suggestions.  */
-  ASSERT_EQ (NULL, find_closest_string ("banana", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("banana", &candidates));
 
   /* Example from PR 69968 where transposition helps.  */
   candidates.truncate (0);
diff --git a/gcc/spellcheck.h b/gcc/spellcheck.h
index b0c2c89..c5006ee 100644
--- a/gcc/spellcheck.h
+++ b/gcc/spellcheck.h
@@ -39,13 +39,13 @@ find_closest_string (const char *target,
    class best_match.
    Specializations should provide the implementations of the following:
 
-     static size_t get_length (TYPE);
-     static const char *get_string (TYPE);
+     static size_t get_length (StringlikeType);
+     static const char *get_string (StringlikeType);
 
    get_string should return a non-NULL ptr, which does not need to be
    0-terminated.  */
 
-template <typename TYPE>
+template <typename StringlikeType>
 struct edit_distance_traits {};
 
 /* Specialization of edit_distance_traits for C-style strings.  */
@@ -74,23 +74,23 @@ extern edit_distance_t get_edit_distance_cutoff (size_t goal_len,
    string-like types (const char *, frontend identifiers, and preprocessor
    macros).
 
-   This type accumulates the best possible match against GOAL_TYPE for
-   a sequence of elements of CANDIDATE_TYPE, whilst minimizing the
+   This type accumulates the best possible match against GoalType for
+   a sequence of elements of CandidateType, whilst minimizing the
    number of calls to get_edit_distance and to
    edit_distance_traits<T>::get_length.  */
 
-template <typename GOAL_TYPE, typename CANDIDATE_TYPE>
+template <typename GoalType, typename CandidateType>
 class best_match
 {
  public:
-  typedef GOAL_TYPE goal_t;
-  typedef CANDIDATE_TYPE candidate_t;
+  typedef GoalType goal_t;
+  typedef CandidateType candidate_t;
   typedef edit_distance_traits<goal_t> goal_traits;
   typedef edit_distance_traits<candidate_t> candidate_traits;
 
   /* Constructor.  */
 
-  best_match (GOAL_TYPE goal,
+  best_match (goal_t goal,
 	      edit_distance_t best_distance_so_far = MAX_EDIT_DISTANCE)
   : m_goal (goal_traits::get_string (goal)),
     m_goal_len (goal_traits::get_length (goal)),
@@ -163,7 +163,7 @@ class best_match
      m_best_candidate, update (without recomputing the edit distance to
      the goal).  */
 
-  void set_best_so_far (CANDIDATE_TYPE best_candidate,
+  void set_best_so_far (candidate_t best_candidate,
 			edit_distance_t best_distance,
 			size_t best_candidate_len)
   {
diff --git a/gcc/stmt.cc b/gcc/stmt.cc
index a510f8f..7942aa3 100644
--- a/gcc/stmt.cc
+++ b/gcc/stmt.cc
@@ -39,6 +39,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "emit-rtl.h"
 #include "pretty-print.h"
 #include "diagnostic-core.h"
+#include "output.h"
+#include "gimplify_reg_info.h"
 
 #include "fold-const.h"
 #include "varasm.h"
@@ -53,6 +55,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "dumpfile.h"
 #include "builtins.h"
 #include "cfgexpand.h"
+#include "output.h"
 
 
 /* Functions and data structures for expanding case statements.  */
@@ -175,6 +178,77 @@ expand_label (tree label)
     maybe_set_first_label_num (label_r);
 }
 
+/* Parse a hard register constraint and return its number or -1 in case of an
+   error.  BEGIN should point to a string of the form `{regname}`.  For the
+   sake of simplicity assume that a register name is not longer than 31
+   characters, if not error out.  */
+
+int
+decode_hard_reg_constraint (const char *begin)
+{
+  if (*begin != '{')
+    return -1;
+  ++begin;
+  const char *end = begin;
+  while (*end != '}' && *end != '\0')
+    ++end;
+  if (*end != '}' || end == begin)
+    return -1;
+  ptrdiff_t len = end - begin;
+  if (len >= 31)
+    return -1;
+  char regname[32];
+  memcpy (regname, begin, len);
+  regname[len] = '\0';
+  int regno = decode_reg_name (regname);
+  return regno;
+}
+
+static bool
+eliminable_regno_p (int regnum)
+{
+  static const struct
+  {
+    const int from;
+    const int to;
+  } eliminables[] = ELIMINABLE_REGS;
+  for (size_t i = 0; i < ARRAY_SIZE (eliminables); i++)
+    if (regnum == eliminables[i].from)
+      return true;
+  return false;
+}
+
+/* Perform a similar check as done in make_decl_rtl().  */
+
+static bool
+hardreg_ok_p (int reg_number, machine_mode mode, int operand_num)
+{
+  if (mode == BLKmode)
+    error ("data type isn%'t suitable for register %s of operand %i",
+	   reg_names[reg_number], operand_num);
+  else if (!in_hard_reg_set_p (accessible_reg_set, mode, reg_number))
+    error ("register %s for operand %i cannot be accessed"
+	   " by the current target", reg_names[reg_number], operand_num);
+  else if (!in_hard_reg_set_p (operand_reg_set, mode, reg_number))
+    error ("register %s for operand %i is not general enough"
+	   " to be used as a register variable", reg_names[reg_number], operand_num);
+  else if (!targetm.hard_regno_mode_ok (reg_number, mode))
+    error ("register %s for operand %i isn%'t suitable for data type",
+	   reg_names[reg_number], operand_num);
+  else if (reg_number != HARD_FRAME_POINTER_REGNUM
+	   && (reg_number == FRAME_POINTER_REGNUM
+#ifdef RETURN_ADDRESS_POINTER_REGNUM
+	       || reg_number == RETURN_ADDRESS_POINTER_REGNUM
+#endif
+	       || reg_number == ARG_POINTER_REGNUM)
+	   && eliminable_regno_p (reg_number))
+    error ("register for operand %i is an internal GCC "
+	   "implementation detail", operand_num);
+  else
+    return true;
+  return false;
+}
+
 /* Parse the output constraint pointed to by *CONSTRAINT_P.  It is the
    OPERAND_NUMth output operand, indexed from zero.  There are NINPUTS
    inputs and NOUTPUTS outputs to this extended-asm.  Upon return,
@@ -191,7 +265,8 @@ expand_label (tree label)
 bool
 parse_output_constraint (const char **constraint_p, int operand_num,
 			 int ninputs, int noutputs, bool *allows_mem,
-			 bool *allows_reg, bool *is_inout)
+			 bool *allows_reg, bool *is_inout,
+			 gimplify_reg_info *reg_info)
 {
   const char *constraint = *constraint_p;
   const char *p;
@@ -245,6 +320,9 @@ parse_output_constraint (const char **constraint_p, int operand_num,
       constraint = *constraint_p;
     }
 
+  unsigned int alt = 0;
+  bool early_clobbered = false;
+
   /* Loop through the constraint string.  */
   for (p = constraint + 1; *p; )
     {
@@ -264,12 +342,21 @@ parse_output_constraint (const char **constraint_p, int operand_num,
 	    }
 	  break;
 
-	case '?':  case '!':  case '*':  case '&':  case '#':
+	case '&':
+	  early_clobbered = true;
+	  break;
+
+	case '?':  case '!':  case '*':  case '#':
 	case '$':  case '^':
 	case 'E':  case 'F':  case 'G':  case 'H':
 	case 's':  case 'i':  case 'n':
 	case 'I':  case 'J':  case 'K':  case 'L':  case 'M':
-	case 'N':  case 'O':  case 'P':  case ',':  case '-':
+	case 'N':  case 'O':  case 'P':  case '-':
+	  break;
+
+	case ',':
+	  ++alt;
+	  early_clobbered = false;
 	  break;
 
 	case '0':  case '1':  case '2':  case '3':  case '4':
@@ -294,6 +381,60 @@ parse_output_constraint (const char **constraint_p, int operand_num,
 	  *allows_mem = true;
 	  break;
 
+	case '{':
+	  {
+	    if (!targetm.lra_p ())
+	      {
+		error ("hard register constraints are only supported while using LRA");
+		return false;
+	      }
+	    if (reg_info)
+	      {
+		int regno = decode_hard_reg_constraint (p);
+		if (regno < 0)
+		  {
+		    error ("invalid output constraint: %s", p);
+		    return false;
+		  }
+		int overlap_regno = reg_info->test_alt_output (alt, regno);
+		if (overlap_regno < 0)
+		  overlap_regno = reg_info->test_reg_asm_output (regno);
+		if (overlap_regno >= 0)
+		  {
+		    error ("multiple outputs to hard register: %s",
+			   reg_names[overlap_regno]);
+		    return false;
+		  }
+		reg_info->set_output (alt, regno);
+		if (early_clobbered)
+		  reg_info->set_early_clobbered (alt, operand_num, regno);
+		if (reg_info->is_clobbered (regno))
+		  {
+		    error ("hard register constraint for output %i conflicts "
+			   "with %<asm%> clobber list", operand_num);
+		    return false;
+		  }
+		if (VAR_P (reg_info->operand)
+		    && DECL_HARD_REGISTER (reg_info->operand))
+		  {
+		      tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+		      const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+		      int regno_op = decode_reg_name (asmspec);
+		      if (regno != regno_op)
+			{
+			  error ("constraint and register %<asm%> for output "
+				 "operand %i are unsatisfiable", operand_num);
+			  return false;
+			}
+		  }
+		machine_mode mode = TYPE_MODE (TREE_TYPE (reg_info->operand));
+		if (!hardreg_ok_p (regno, mode, operand_num))
+		  return false;
+	      }
+	    *allows_reg = true;
+	    break;
+	  }
+
 	default:
 	  if (!ISALPHA (*p))
 	    break;
@@ -305,6 +446,31 @@ parse_output_constraint (const char **constraint_p, int operand_num,
 	    *allows_mem = true;
 	  else
 	    insn_extra_constraint_allows_reg_mem (cn, allows_reg, allows_mem);
+	  if (reg_info && *allows_reg
+	      && VAR_P (reg_info->operand)
+	      && DECL_HARD_REGISTER (reg_info->operand))
+	    {
+		tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+		const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+		int regno = decode_reg_name (asmspec);
+		if (regno < 0)
+		  {
+		    location_t loc = DECL_SOURCE_LOCATION (reg_info->operand);
+		    error_at (loc, "invalid register name for %q+D",
+			      reg_info->operand);
+		    return false;
+		  }
+		int overlap_regno = reg_info->test_alt_output (alt, regno);
+		if (overlap_regno >= 0)
+		  {
+		    error ("multiple outputs to hard register: %s",
+			   reg_names[overlap_regno]);
+		    return false;
+		  }
+		reg_info->set_reg_asm_output (regno);
+		if (early_clobbered)
+		  reg_info->set_early_clobbered (alt, operand_num, regno);
+	    }
 	  break;
 	}
 
@@ -322,7 +488,8 @@ bool
 parse_input_constraint (const char **constraint_p, int input_num,
 			int ninputs, int noutputs, int ninout,
 			const char * const * constraints,
-			bool *allows_mem, bool *allows_reg)
+			bool *allows_mem, bool *allows_reg,
+			gimplify_reg_info *reg_info)
 {
   const char *constraint = *constraint_p;
   const char *orig_constraint = constraint;
@@ -338,6 +505,9 @@ parse_input_constraint (const char **constraint_p, int input_num,
 
   /* Make sure constraint has neither `=', `+', nor '&'.  */
 
+  unsigned int alt = 0;
+  unsigned long match = 0;
+
   for (j = 0; j < c_len; j += CONSTRAINT_LEN (constraint[j], constraint+j))
     switch (constraint[j])
       {
@@ -364,7 +534,7 @@ parse_input_constraint (const char **constraint_p, int input_num,
       case 'E':  case 'F':  case 'G':  case 'H':
       case 's':  case 'i':  case 'n':
       case 'I':  case 'J':  case 'K':  case 'L':  case 'M':
-      case 'N':  case 'O':  case 'P':  case ',':  case '-':
+      case 'N':  case 'O':  case 'P':  case '-':
 	break;
 
       case ':':
@@ -382,6 +552,10 @@ parse_input_constraint (const char **constraint_p, int input_num,
 	  }
 	break;
 
+      case ',':
+	++alt;
+	break;
+
 	/* Whether or not a numeric constraint allows a register is
 	   decided by the matching constraint, and so there is no need
 	   to do anything special with them.  We must handle them in
@@ -391,7 +565,6 @@ parse_input_constraint (const char **constraint_p, int input_num,
       case '5':  case '6':  case '7':  case '8':  case '9':
 	{
 	  char *end;
-	  unsigned long match;
 
 	  saw_match = true;
 
@@ -429,6 +602,65 @@ parse_input_constraint (const char **constraint_p, int input_num,
 	*allows_mem = true;
 	break;
 
+      case '{':
+	{
+	  if (!targetm.lra_p ())
+	    {
+	      error ("hard register constraints are only supported while using LRA");
+	      return false;
+	    }
+	  if (reg_info)
+	    {
+	      int regno = decode_hard_reg_constraint (constraint + j);
+	      if (regno < 0)
+		{
+		  error ("invalid input constraint: %s", constraint + j);
+		  return false;
+		}
+	      int overlap_regno = reg_info->test_alt_input (alt, regno);
+	      if (overlap_regno < 0)
+		overlap_regno = reg_info->test_reg_asm_input (regno);
+	      if (overlap_regno >= 0)
+		{
+		  error ("multiple inputs to hard register: %s",
+			    reg_names[overlap_regno]);
+		  return false;
+		}
+	      reg_info->set_input (alt, regno);
+	      if (reg_info->is_clobbered (regno))
+		{
+		  error ("hard register constraint for input %i conflicts "
+			 "with %<asm%> clobber list", input_num);
+		  return false;
+		}
+	      if (constraint == orig_constraint
+		  && reg_info->test_early_clobbered_alt (alt, regno))
+		{
+		  error ("invalid hard register usage between earlyclobber "
+			 "operand and input operand");
+		  return false;
+		}
+	      if (VAR_P (reg_info->operand)
+		  && DECL_HARD_REGISTER (reg_info->operand))
+		{
+		    tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+		    const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+		    int regno_op = decode_reg_name (asmspec);
+		    if (regno != regno_op)
+		      {
+			error ("constraint and register %<asm%> for input "
+			       "operand %i are unsatisfiable", input_num);
+			return false;
+		      }
+		}
+	      machine_mode mode = TYPE_MODE (TREE_TYPE (reg_info->operand));
+	      if (!hardreg_ok_p (regno, mode, input_num))
+		return false;
+	    }
+	  *allows_reg = true;
+	  break;
+	}
+
       default:
 	if (! ISALPHA (constraint[j]))
 	  {
@@ -445,6 +677,39 @@ parse_input_constraint (const char **constraint_p, int input_num,
 	  *allows_mem = true;
 	else
 	  insn_extra_constraint_allows_reg_mem (cn, allows_reg, allows_mem);
+	if (reg_info && *allows_reg
+	    && VAR_P (reg_info->operand)
+	    && DECL_HARD_REGISTER (reg_info->operand))
+	  {
+	      tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+	      const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+	      int regno = decode_reg_name (asmspec);
+	      if (regno < 0)
+		{
+		  location_t loc = DECL_SOURCE_LOCATION (reg_info->operand);
+		  error_at (loc, "invalid register name for %q+D",
+			    reg_info->operand);
+		  return false;
+		}
+	      int overlap_regno = reg_info->test_alt_input (alt, regno);
+	      if (overlap_regno >= 0)
+		{
+		  error ("multiple inputs to hard register: %s",
+			 reg_names[overlap_regno]);
+		  return false;
+		}
+	      reg_info->set_reg_asm_input (regno);
+	      if ((constraint == orig_constraint
+		   && reg_info->test_early_clobbered_alt (alt, regno))
+		  || (constraint != orig_constraint
+		      && reg_info->is_early_clobbered_in_any_output_unequal
+			  (match, regno)))
+		{
+		  error ("invalid hard register usage between earlyclobber "
+			 "operand and input operand");
+		  return false;
+		}
+	  }
 	break;
       }
 
diff --git a/gcc/stmt.h b/gcc/stmt.h
index 2beb4d5..c178036 100644
--- a/gcc/stmt.h
+++ b/gcc/stmt.h
@@ -20,11 +20,16 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_STMT_H
 #define GCC_STMT_H
 
+class gimplify_reg_info;
+
 extern void expand_label (tree);
 extern bool parse_output_constraint (const char **, int, int, int,
-				     bool *, bool *, bool *);
+				     bool *, bool *, bool *,
+				     gimplify_reg_info *);
 extern bool parse_input_constraint (const char **, int, int, int, int,
-				    const char * const *, bool *, bool *);
+				    const char * const *, bool *, bool *,
+				    gimplify_reg_info *);
+extern int decode_hard_reg_constraint (const char *);
 extern tree resolve_asm_operand_names (tree, tree, tree, tree);
 #ifdef HARD_CONST
 /* Silly ifdef to avoid having all includers depend on hard-reg-set.h.  */
diff --git a/gcc/stor-layout.cc b/gcc/stor-layout.cc
index 12071c9..63e830a 100644
--- a/gcc/stor-layout.cc
+++ b/gcc/stor-layout.cc
@@ -3167,7 +3167,7 @@ bit_field_mode_iterator::prefer_smaller_modes ()
    decide which of the above modes should be used.  */
 
 bool
-get_best_mode (int bitsize, int bitpos,
+get_best_mode (HOST_WIDE_INT bitsize, HOST_WIDE_INT bitpos,
 	       poly_uint64 bitregion_start, poly_uint64 bitregion_end,
 	       unsigned int align,
 	       unsigned HOST_WIDE_INT largest_mode_bitsize, bool volatilep,
diff --git a/gcc/substring-locations.cc b/gcc/substring-locations.cc
index 78ce722..2f920ca 100644
--- a/gcc/substring-locations.cc
+++ b/gcc/substring-locations.cc
@@ -151,7 +151,7 @@ format_string_diagnostic_t (const substring_loc &fmt_loc,
    Return true if a warning was emitted, false otherwise.  */
 
 bool
-format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning_n_va (diagnostics::option_id option_id,
 					       unsigned HOST_WIDE_INT n,
 					       const char *singular_gmsgid,
 					       const char *plural_gmsgid,
@@ -203,7 +203,7 @@ format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
   if (!err && m_corrected_substring && substring_within_range)
     richloc.add_fixit_replace (fmt_substring_range, m_corrected_substring);
 
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   if (singular_gmsgid != plural_gmsgid)
     {
       unsigned long gtn;
@@ -218,12 +218,12 @@ format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
 
       const char *text = ngettext (singular_gmsgid, plural_gmsgid, gtn);
       diagnostic_set_info_translated (&diagnostic, text, ap, &richloc,
-				      DK_WARNING);
+				      diagnostics::kind::warning);
     }
   else
     diagnostic_set_info (&diagnostic, singular_gmsgid, ap, &richloc,
-			 DK_WARNING);
-  diagnostic.option_id = option_id;
+			 diagnostics::kind::warning);
+  diagnostic.m_option_id = option_id;
   bool warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
 
   if (!err && fmt_substring_loc && !substring_within_range)
@@ -248,7 +248,7 @@ format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
 /* Singular-only version of the above.  */
 
 bool
-format_string_diagnostic_t::emit_warning_va (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning_va (diagnostics::option_id option_id,
 					     const char *gmsgid,
 					     va_list *ap) const
 {
@@ -258,7 +258,7 @@ format_string_diagnostic_t::emit_warning_va (diagnostic_option_id option_id,
 /* Variadic version of the above (singular only).  */
 
 bool
-format_string_diagnostic_t::emit_warning (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning (diagnostics::option_id option_id,
 					  const char *gmsgid,
 					  ...) const
 {
@@ -273,7 +273,7 @@ format_string_diagnostic_t::emit_warning (diagnostic_option_id option_id,
 /* Variadic version of the above (singular vs plural).  */
 
 bool
-format_string_diagnostic_t::emit_warning_n (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning_n (diagnostics::option_id option_id,
 					    unsigned HOST_WIDE_INT n,
 					    const char *singular_gmsgid,
 					    const char *plural_gmsgid,
diff --git a/gcc/substring-locations.h b/gcc/substring-locations.h
index fee7a89..94139e2 100644
--- a/gcc/substring-locations.h
+++ b/gcc/substring-locations.h
@@ -90,23 +90,23 @@ class format_string_diagnostic_t
 
   /* Functions for emitting a warning about a format string.  */
 
-  bool emit_warning_va (diagnostic_option_id option_id,
+  bool emit_warning_va (diagnostics::option_id option_id,
 			const char *gmsgid,
 			va_list *ap) const
     ATTRIBUTE_GCC_DIAG (3, 0);
 
-  bool emit_warning_n_va (diagnostic_option_id option_id,
+  bool emit_warning_n_va (diagnostics::option_id option_id,
 			  unsigned HOST_WIDE_INT n,
 			  const char *singular_gmsgid,
 			  const char *plural_gmsgid,
 			  va_list *ap) const
   ATTRIBUTE_GCC_DIAG (4, 0) ATTRIBUTE_GCC_DIAG (5, 0);
 
-  bool emit_warning (diagnostic_option_id option_id,
+  bool emit_warning (diagnostics::option_id option_id,
 		     const char *gmsgid, ...) const
     ATTRIBUTE_GCC_DIAG (3, 4);
 
-  bool emit_warning_n (diagnostic_option_id option_id,
+  bool emit_warning_n (diagnostics::option_id option_id,
 		       unsigned HOST_WIDE_INT n,
 		       const char *singular_gmsgid,
 		       const char *plural_gmsgid, ...) const
@@ -125,7 +125,7 @@ class format_string_diagnostic_t
    LANG_HOOKS_GET_SUBSTRING_LOCATION.  */
 
 extern const char *get_location_within_string (cpp_reader *pfile,
-					       file_cache &fc,
+					       diagnostics::file_cache &fc,
 					       string_concat_db *concats,
 					       location_t strloc,
 					       enum cpp_ttype type,
diff --git a/gcc/symtab.cc b/gcc/symtab.cc
index fc1155f..20dfe09 100644
--- a/gcc/symtab.cc
+++ b/gcc/symtab.cc
@@ -303,8 +303,14 @@ symbol_table::change_decl_assembler_name (tree decl, tree name)
 	warning (0, "%qD renamed after being referenced in assembly", decl);
 
       SET_DECL_ASSEMBLER_NAME (decl, name);
+      if (DECL_RTL_SET_P (decl))
+	{
+	  SET_DECL_RTL (decl, NULL);
+	  make_decl_rtl (decl);
+	}
       if (alias)
 	{
+	  gcc_assert (!IDENTIFIER_INTERNAL_P (name));
 	  IDENTIFIER_TRANSPARENT_ALIAS (name) = 1;
 	  TREE_CHAIN (name) = alias;
 	}
diff --git a/gcc/target.def b/gcc/target.def
index 38903eb..5dd8f25 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -1918,17 +1918,21 @@ implementation approaches itself.",
  NULL)
 
 /* Return true if the target supports misaligned store/load of a
-   specific factor denoted in the third parameter.  The last parameter
-   is true if the access is defined in a packed struct.  */
+   specific factor denoted in the third parameter.  The second to the last
+   parameter is true if the access is defined in a packed struct and
+   the last parameter is true if the access is a gather/scatter.  */
 DEFHOOK
 (support_vector_misalignment,
  "This hook should return true if the target supports misaligned vector\n\
 store/load of a specific factor denoted in the @var{misalignment}\n\
 parameter.  The vector store/load should be of machine mode @var{mode} and\n\
-the elements in the vectors should be of type @var{type}.  @var{is_packed}\n\
-parameter is true if the memory access is defined in a packed struct.",
+the elements in the vectors should be of type @var{type}.  The\n\
+@var{is_packed} parameter is true if the misalignment is unknown and the\n\
+memory access is defined in a packed struct.  @var{is_gather_scatter} is true\n\
+if the load/store is a gather or scatter.",
  bool,
- (machine_mode mode, const_tree type, int misalignment, bool is_packed),
+ (machine_mode mode, const_tree type, int misalignment, bool is_packed,
+  bool is_gather_scatter),
  default_builtin_support_vector_misalignment)
 
 /* Returns the preferred mode for SIMD operations for the specified
@@ -2056,6 +2060,20 @@ all zeros.  GCC can then try to branch around the instruction instead.",
  (unsigned ifn),
  default_empty_mask_is_expensive)
 
+/* Prefer gather/scatter loads/stores to e.g. elementwise accesses if\n\
+we cannot use a contiguous access.  */
+DEFHOOK
+(prefer_gather_scatter,
+ "This hook returns TRUE if gather loads or scatter stores are cheaper on\n\
+this target than a sequence of elementwise loads or stores.  The @var{mode}\n\
+and @var{scale} correspond to the @code{gather_load} and\n\
+@code{scatter_store} instruction patterns.  The @var{group_size} is the\n\
+number of scalar elements in each scalar loop iteration that are to be\n\
+combined into the vector.",
+ bool,
+ (machine_mode mode, int scale, unsigned int group_size),
+ hook_bool_mode_int_unsigned_false)
+
 /* Target builtin that implements vector gather operation.  */
 DEFHOOK
 (builtin_gather,
diff --git a/gcc/target.h b/gcc/target.h
index c9c7b52..47f1846 100644
--- a/gcc/target.h
+++ b/gcc/target.h
@@ -169,7 +169,6 @@ class predefined_function_abi;
 struct store_fwd_info;
 
 /* These are defined in tree-vect-stmts.cc.  */
-extern tree stmt_vectype (class _stmt_vec_info *);
 extern bool stmt_in_inner_loop_p (class vec_info *, class _stmt_vec_info *);
 
 /* Assembler instructions for creating various kinds of integer object.  */
diff --git a/gcc/targhooks.cc b/gcc/targhooks.cc
index c79458e..e723bbb 100644
--- a/gcc/targhooks.cc
+++ b/gcc/targhooks.cc
@@ -1556,6 +1556,8 @@ default_builtin_support_vector_misalignment (machine_mode mode,
 					     int misalignment
 					     ATTRIBUTE_UNUSED,
 					     bool is_packed
+					     ATTRIBUTE_UNUSED,
+					     bool is_gather_scatter
 					     ATTRIBUTE_UNUSED)
 {
   if (optab_handler (movmisalign_optab, mode) != CODE_FOR_nothing)
diff --git a/gcc/targhooks.h b/gcc/targhooks.h
index f16b587..3fa20af 100644
--- a/gcc/targhooks.h
+++ b/gcc/targhooks.h
@@ -114,7 +114,7 @@ extern bool default_builtin_vector_alignment_reachable (const_tree, bool);
 extern bool
 default_builtin_support_vector_misalignment (machine_mode mode,
 					     const_tree,
-					     int, bool);
+					     int, bool, bool);
 extern machine_mode default_preferred_simd_mode (scalar_mode mode);
 extern machine_mode default_split_reduction (machine_mode);
 extern unsigned int default_autovectorize_vector_modes (vector_modes *, bool);
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index ab15e88..f9f3de6 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,3216 @@
+2025-08-04  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121351
+	* g++.dg/cpp2a/concepts-using6.C: New test.
+
+2025-08-04  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121351
+	PR c++/119859
+	* g++.dg/cpp2a/concepts-using5.C: New test.
+	* g++.dg/cpp2a/concepts-using5a.C: New test.
+
+2025-08-04  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/120620
+	* g++.dg/cpp2a/constexpr-dynamic19.C: New test.
+	* g++.dg/cpp2a/constexpr-dynamic1a.C: New test.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	PR diagnostics/116253
+	* gcc.dg/plugin/diagnostic-test-nesting-html.c: New test.
+	* gcc.dg/plugin/diagnostic-test-nesting-html.py: New test script.
+	* gcc.dg/plugin/plugin.exp: Add it.
+	* libgdiagnostics.dg/test-multiple-lines.c: Update expected output
+	to show fix-it hint.
+	* sarif-replay.dg/2.1.0-valid/nested-diagnostics-1.sarif: New test.
+
+2025-08-04  David Malcolm  <dmalcolm@redhat.com>
+
+	PR diagnostics/116792
+	* gcc.dg/plugin/diagnostic-test-graphs-html.py: Remove trailing
+	space from expected text of message.
+	* sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py:
+	Likewise.
+	* sarif-replay.dg/2.1.0-valid/graphs-check-html.py: Likewise.
+
+2025-08-04  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/121303
+	* gcc.target/i386/pr121303.c: New test.
+
+2025-08-04  Jakub Jelinek  <jakub@redhat.com>
+
+	PR preprocessor/120778
+	* g++.dg/DRs/dr2580.C: New test.
+
+2025-08-04  Jakub Jelinek  <jakub@redhat.com>
+
+	PR preprocessor/120778
+	* g++.dg/DRs/dr2578.C: New test.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/dupq_13.c: New test.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/dup_1.c: New test.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/pnext_3.c: New test.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve2/acle/general/match_4.c: New test.
+	* gcc.target/aarch64/sve2/acle/general/nmatch_1.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/acge_1.c: New test.
+	* gcc.target/aarch64/sve/acle/general/acgt_1.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/acle_1.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/aclt_1.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/cmpeq_6.c: New test.
+	* gcc.target/aarch64/sve/acle/general/cmpge_9.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_9.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_9.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_9.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpne_5.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpuo_1.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/cmpeq_5.c: New test.
+	* gcc.target/aarch64/sve/acle/general/cmpge_7.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpge_8.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_7.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_8.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_7.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_8.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_7.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_8.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpne_4.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/cmpeq_1.c: Check the number
+	of PTRUEs.
+	* gcc.target/aarch64/sve/acle/general/cmpge_5.c: New test.
+	* gcc.target/aarch64/sve/acle/general/cmpge_6.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_5.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_6.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_5.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_6.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_5.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_6.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpne_3.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/cmpeq_1.c: Add more tests.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/pred_clobber_1.c: Disable combine.
+	* gcc.target/aarch64/sve/pred_clobber_2.c: Likewise.
+	* gcc.target/aarch64/sve/pred_clobber_3.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpeq_2.c: Add more cases.
+	* gcc.target/aarch64/sve/acle/general/cmpeq_4.c: New test.
+	* gcc.target/aarch64/sve/acle/general/cmpge_1.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpge_2.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpge_3.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpge_4.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_1.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_2.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_3.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpgt_4.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_1.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_2.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_3.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmple_4.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_1.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_2.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_3.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmplt_4.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpne_1.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/cmpne_2.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/acle/general/unpkhi_1.c: New test.
+	* gcc.target/aarch64/sve/acle/general/unpklo_1.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121294
+	* gcc.target/aarch64/sve/acle/general/rev_2.c: New test.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121294
+	* gcc.target/aarch64/sve/acle/general/perm_2.c: New test.
+	* gcc.target/aarch64/sve/acle/general/perm_3.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/perm_4.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/perm_5.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/perm_6.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/perm_7.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR testsuite/121118
+	* gcc.target/aarch64/sve/acle/general/pr121118_1.c: New test.
+	* gcc.target/aarch64/sve/acle/general/whilele_13.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/whilelt_6.c: Likewise.
+	* gcc.target/aarch64/sve2/acle/general/whilege_1.c: Likewise.
+	* gcc.target/aarch64/sve2/acle/general/whilegt_1.c: Likewise.
+	* gcc.target/aarch64/sve2/acle/general/whilerw_5.c: Likewise.
+	* gcc.target/aarch64/sve2/acle/general/whilewr_5.c: Likewise.
+
+2025-08-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121293
+	* gcc.target/aarch64/sve/acle/general/dupq_lane_9.c: New test.
+
+2025-08-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121362
+	* gcc.dg/tree-ssa/ssa-fre-105.c: New testcase.
+	* gcc.dg/tree-ssa/ssa-fre-106.c: Likewise.
+
+2025-08-04  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120941
+	* gcc.target/i386/pr120941-1.c: New test.
+
+2025-08-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR preprocessor/120778
+	* g++.dg/DRs/dr1709.C: New test.
+
+2025-08-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120845
+	* g++.dg/modules/cpp-21.C: New test.
+
+2025-08-02  Martin Uecker  <uecker@tugraz.at>
+
+	* gcc.dg/Warray-parameter-11.c: Change Warray-parameter to
+	-Wvla-parameter as these are VLAs.
+	* gcc.dg/Warray-parameter.c: Remove xfail.
+
+2025-08-01  Artemiy Granat  <a.granat@ispras.ru>
+
+	* gcc.target/i386/attributes-error.c: Change incorrect
+	sseregparm,fastcall combination to cdecl,fastcall.
+
+2025-08-01  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121322
+	* gcc.dg/pr121322.c: New test.
+
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/108080
+	* g++.dg/modules/pr108080.H: New test.
+
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121238
+	* g++.dg/modules/merge-19.h: New test.
+	* g++.dg/modules/merge-19_a.H: New test.
+	* g++.dg/modules/merge-19_b.C: New test.
+
+2025-07-31  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121314
+	* gm2/errors/fail/badindrtype.mod: New test.
+	* gm2/errors/fail/badindrtype2.mod: New test.
+
+2025-07-31  Mikael Morin  <morin-mikael@orange.fr>
+
+	PR fortran/121342
+	* gfortran.dg/class_elemental_1.f90: New test.
+
+2025-07-31  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120800
+	* g++.dg/cpp0x/constexpr-array30.C: New test.
+
+2025-07-31  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/120775
+	* g++.dg/cpp26/consteval-block1.C: New test.
+	* g++.dg/cpp26/consteval-block2.C: New test.
+	* g++.dg/cpp26/consteval-block3.C: New test.
+	* g++.dg/cpp26/consteval-block4.C: New test.
+	* g++.dg/cpp26/consteval-block5.C: New test.
+	* g++.dg/cpp26/consteval-block6.C: New test.
+	* g++.dg/cpp26/consteval-block7.C: New test.
+	* g++.dg/cpp26/consteval-block8.C: New test.
+
+2025-07-31  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check
+	for signed avg ceil.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add
+	test data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c: New test.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* gcc.target/i386/attributes-error.c: Add more attributes
+	combinations.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* g++.dg/abi/regparm1.C: Require ia32 target.
+	* gcc.target/i386/20020224-1.c: Likewise.
+	* gcc.target/i386/pr103785.c: Use regparm attribute only if
+	not in 64-bit mode.
+	* gcc.target/i386/pr36533.c: Likewise.
+	* gcc.target/i386/pr59099.c: Likewise.
+	* gcc.target/i386/sibcall-8.c: Likewise.
+	* gcc.target/i386/sw-1.c: Likewise.
+	* gcc.target/i386/pr15184-2.c: Fix invalid comment.
+	* gcc.target/i386/attributes-ignore.c: New test.
+
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* g++.target/aarch64/mv-cpu-features.C: new test.
+
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* gcc.target/aarch64/ifunc-resolver.in: add core test functions.
+	* gcc.target/aarch64/ifunc-resolver-0.c: new test.
+	* gcc.target/aarch64/ifunc-resolver-1.c: ditto.
+	* gcc.target/aarch64/ifunc-resolver-2.c: ditto.
+	* gcc.target/aarch64/ifunc-resolver-3.c: ditto.
+	* gcc.target/aarch64/ifunc-resolver-4.c: as above.
+
+2025-07-31  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/121028
+	* gcc.target/aarch64/sme/call_sm_switch_1.c: Tell check-function
+	-bodies not to ignore .inst directives, and replace the test for
+	"smstart sm" with one for it's encoding.
+	* gcc.target/aarch64/sme/call_sm_switch_11.c: Likewise.
+	* gcc.target/aarch64/sme/pr121028.c: New test.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121264
+	* gcc.dg/tree-ssa/pr121264.c: New test.
+
+2025-07-31  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c: Gate do-assemble on
+	assembler support for +faminmax and +sme2.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c: Likewise.
+	* lib/target-supports.exp: Split the extensions that require SME into
+	a separate set, and use armv9-a as their baseline.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	* gcc.target/i386/apx-1.c (apx_hanlder): Rename to ...
+	(apx_handler): ... this.
+	* gcc.target/i386/uintr-2.c (UINTR_hanlder): Rename to ...
+	(UINTR_handler): ... this.
+	* gcc.target/i386/uintr-5.c (UINTR_hanlder): Rename to ...
+	(UINTR_handler): ... this.
+
+2025-07-30  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121291
+	* g++.dg/ext/is_invocable7.C: New test.
+	* g++.dg/ext/is_nothrow_convertible5.C: New test.
+
+2025-07-30  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/tc1/dr49.C: Adjust diagnostic.
+	* g++.dg/template/func2.C: Likewise.
+	* g++.dg/cpp1z/nontype8.C: New test.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	PR tree-optimization/121295
+	* gcc.dg/torture/pr121236-1.c: New test.
+	* gcc.dg/torture/pr121295-1.c: New test.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	Revert:
+	2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* gcc.dg/torture/pr121236-1.c: New test.
+
+2025-07-30  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/spaceship-fp-1.c: New test.
+	* gcc.target/s390/spaceship-fp-2.c: New test.
+	* gcc.target/s390/spaceship-fp-3.c: New test.
+	* gcc.target/s390/spaceship-fp-4.c: New test.
+	* gcc.target/s390/spaceship-int-1.c: New test.
+	* gcc.target/s390/spaceship-int-2.c: New test.
+	* gcc.target/s390/spaceship-int-3.c: New test.
+
+2025-07-30  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120427
+	* gcc.target/i386/pr120427-5.c: New test.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* g++.dg/tree-prof/eh1.C: New test.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121130
+	* gcc.dg/vect/vect-simd-pr121130.c: New testcase.
+
+2025-07-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/121133
+	* g++.dg/warn/pr121133-1.C: New test.
+	* g++.dg/warn/pr121133-2.C: New test.
+	* g++.dg/warn/pr121133-3.C: New test.
+	* g++.dg/warn/pr121133-4.C: New test.
+
+2025-07-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120778
+	* g++.dg/cpp/if-comma-1.C: New test.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121020
+	* gcc.dg/vect/vect-early-break_138-pr121020.c: New test.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121190
+	* gcc.dg/vect/vect-early-break_52.c: Update an unsafe test.
+	* gcc.dg/vect/vect-early-break_137-pr121190.c: New test.
+
+2025-07-30  Alfie Richards  <alfie.richards@arm.com>
+
+	PR target/121300
+	* gcc.target/aarch64/pr121300.c: New test.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_fmla_1.c: Add test cases
+	for merging with multiplcand.
+	* gcc.target/aarch64/sve/unpacked_cond_fmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmla_2.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fmls_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c: Likewise..
+	* gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c: Likewise.
+	* g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C: Likewise.
+	* g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C: Likewise.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_fmla_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c: Likewise.
+
+2025-07-30  Yuao Ma  <c8ef@outlook.com>
+
+	* gfortran.dg/split_1.f90: New test.
+	* gfortran.dg/split_2.f90: New test.
+	* gfortran.dg/split_3.f90: New test.
+	* gfortran.dg/split_4.f90: New test.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_ternary_bf16_1.C: New test.
+	* g++.target/aarch64/sve/unpacked_ternary_bf16_2.C: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmla_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmla_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmls_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fnmla_1.c: Likeiwse.
+	* gcc.target/aarch64/sve/unpacked_fnmla_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fnmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fnmls_2.c: Likewise.
+
+2025-07-30  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr121274.c: New test.
+
+2025-07-30  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c: Add asm check
+	for unsigned avg ceil.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add
+	test data.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c: New test.
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/121215
+	* lib/profopt.exp (profopt-execute): Call cleanup-after-saved-dg-test
+	if returning early for the -fauto-profile case failing case.
+
+2025-07-29  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fadd_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmul_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c: Likewise.
+
+2025-07-29  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/121208
+	* gcc.target/i386/pr121208-1a.c (dg-options): Add -mno-80387.
+	* gcc.target/i386/pr121208-1b.c (dg-options): Likewise.
+
+2025-07-29  Juergen Christ  <jchrist@linux.ibm.com>
+
+	PR testsuite/121286
+	PR testsuite/121288
+	* gcc.dg/vect/pr112325.c: Adjust parameters for s390.
+	* gcc.dg/vect/pr117888-1.c: Ditto.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/saturating_arithmetic_1.c: Allow w0 and w1
+	to be duplicated in either order.
+	* gcc.target/aarch64/saturating_arithmetic_2.c: Likewise.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/cmpbr.c: Support both operand orders
+	for 8-bit and 16-bit comparisons.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/120660
+	* gcc.dg/pr120660.c: New test.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/119795
+	* gcc.target/i386/pr119795.c: New test.
+
+2025-07-29  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c: Add rv64
+	target for run.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c: New test.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120687
+	* gcc.dg/vect/pr120687-3.c: New testcase.
+
+2025-07-29  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR testsuite/121285
+	* g++.dg/modules/class-11_a.H: Make static_asserts valid for
+	C++14.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120687
+	* gcc.dg/vect/pr120687-1.c: New testcase.
+	* gcc.dg/vect/pr120687-2.c: Likewise.
+
+2025-07-29  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121289
+	* gm2/warnings/style/fail/badvarname.mod: New test.
+	* gm2/warnings/style/fail/warnings-style-fail.exp: New test.
+
+2025-07-29  Christophe Lyon  <christophe.lyon@linaro.org>
+
+	* gcc.dg/pr116906-1.c: Add 'dg-do run'.
+	* gcc.dg/pr116906-2.c: Likewise.
+	* gcc.dg/pr78185.c: Likewise.
+
+2025-07-29  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121159
+	* c-c++-common/pr121159.c: New test.
+	* gcc.dg/plugin/must-tail-call-2.c (test_5): Don't expect an error.
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120523
+	* gcc.dg/tree-ssa/cswtch-7.c: New test.
+
+2025-07-28  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* gcc.dg/torture/pr121236-1.c: New test.
+
+2025-07-28  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/121208
+	* gcc.target/i386/pr121208-1a.c: New test.
+	* gcc.target/i386/pr121208-1b.c: Likewise.
+	* gcc.target/i386/pr121208-2a.c: Likewise.
+	* gcc.target/i386/pr121208-2b.c: Likewise.
+	* gcc.target/i386/pr121208-3a.c: Likewise.
+	* gcc.target/i386/pr121208-3b.c: Likewise.
+
+2025-07-28  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* gcc.target/nvptx/march-map=sm_100.c: New.
+	* gcc.target/nvptx/march-map=sm_100a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_100f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_101.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_101a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_101f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_103.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_103a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_103f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_120.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_120a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_120f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_121.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_121a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_121f.c: Likewise.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121256
+	* gcc.dg/vect/vect-recurr-pr121256.c: New testcase.
+	* gcc.dg/vect/vect-recurr-pr121256-2.c: Likewise.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* gfortran.dg/assign_14.f90: New test.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* gfortran.dg/assign_13.f90: New test.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c: New test.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test helper
+	macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c: New test.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c:
+	Add zvfh requirements and options.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c:
+	Ditto.
+
+2025-07-27  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	* g++.dg/modules/class-11_a.H: New test.
+	* g++.dg/modules/class-11_b.C: New test.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: Eliminate
+	digraphs::lazy_digraph and digraphs::lazy_digraphs in favor of
+	lazily_created template.
+	* gcc.dg/plugin/diagnostic_plugin_test_metadata.cc: Define
+	INCLUDE_VECTOR since diagnostics/metadata.h now requires it.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_show_trees.cc: Make
+	diagnostics::context::m_source_printing private.
+	* gcc.dg/plugin/diagnostic_plugin_test_inlining.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc:
+	Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc:
+	Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/plugin/show_template_tree_color_plugin.cc: Update usage
+	of "diagnostic_info" to explicitly refer to
+	"diagnostics::diagnostic_info".
+	* gcc.dg/plugin/diagnostic_group_plugin.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Update for
+	file_cache and char_span moving from input.h to
+	diagnostics/file-cache.h and into the "diagnostics::" namespace.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Update for
+	diagnostic_t becoming enum class diagnostics::kind.
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+	* lib/gcc-dg.exp: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update for change
+	from edit-context.h to changes.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update for move of
+	selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/analyzer_cpython_plugin.cc: Update for move of
+	diagnostic-color.h to diagnostics/color.h.
+	* gcc.dg/plugin/analyzer_kernel_plugin.cc: Likewise.
+	* gcc.dg/plugin/analyzer_known_fns_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update for move of
+	selftest-diagnostic.h to diagnostics/selftest-context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/plugin/show_template_tree_color_plugin.cc: Update for
+	moves to namespace diagnostics.
+	* gcc.dg/plugin/diagnostic_group_plugin.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Update to
+	add "m_" prefix to fields of diagnostic_info throughout.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/plugin/show-template-tree-color-labels.C: Update for
+	moves to "source-printing".
+	* gcc.dg/plugin/diagnostic-test-show-locus.py: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_group_plugin.cc: Update for move of
+	diagnostics output formats into namespace "diagnostics" as
+	"sinks".
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/analyzer_gil_plugin.cc: Update #include for
+	"diagnostic-path.h" moving to "diagnostics/paths.h",
+	diagnostic_thread_id_t to diagnostics::paths::thread_id_t,
+	diagnostic_event_id_t to diagnostics::paths::event_id_t,
+	diagnostic_path to diagnostics::paths::path, and
+	diagnostic_thread to diagnostics::paths::thread, and
+	diagnostic_event to diagnostics::paths::event.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+	* lib/sarif.py (get_state_graph): Update property prefix for
+	threadFlowLocations from "gcc/diagnostic_event/" to
+	"gcc/diagnostics/paths/event/".
+	* gcc.dg/sarif-output/include-chain-2.h: Update comment.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update #include for
+	move of edit-context.h to diagnostics subdir.  Update
+	for move of edit_context to diagnostics::edit_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_text_art.cc: Update
+	#include for move of "diagnostic-diagram.h" to
+	"diagnostics/diagram.h". Update for move of diagnostic_diagram to
+	diagnostics::diagram.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/analyzer_cpython_plugin.cc: : Update #include for
+	move of "diagnostic-metadata.h" to "diagnostics/metadata.h"
+	* gcc.dg/plugin/analyzer_kernel_plugin.cc: Likewise.
+	* gcc.dg/plugin/analyzer_known_fns_plugin.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: Likewise.  Also
+	update for move of diagnostic_metadata to diagnostics::metadata.
+	* gcc.dg/plugin/diagnostic_plugin_test_metadata.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: Update #include
+	for move of "diagnostic-digraphs.h" to "diagnostics/digraphs.h".
+
+2025-07-25  Patrick Palka  <ppalka@redhat.com>
+
+	* g++.dg/lookup/operator-8.C: Remove XFAILs and properly
+	suppress all -Wunused-result warnings.
+
+2025-07-25  Iain Sandoe  <iain@sandoe.co.uk>
+
+	PR c++/121219
+	* g++.dg/coroutines/torture/pr121219.C: New test.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* gcc.target/riscv/interrupt-conflict-mode.c: Remove "user"
+	interrupts.
+	* gcc.target/riscv/xtheadint-push-pop.c: Likewise.
+	* gcc.target/riscv/interrupt-umode.c: Removed.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* gcc.target/riscv/interrupt-rnmi.c: New test.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/117294
+	PR c++/113854
+	* g++.dg/cpp2a/concepts-traits3.C: Adjust diagnostics.
+	* g++.dg/cpp2a/concepts-traits4.C: New test.
+	* g++.dg/diagnostic/static_assert5.C: New test.
+	* g++.dg/ext/has_virtual_destructor2.C: New test.
+	* g++.dg/ext/is_assignable2.C: New test.
+	* g++.dg/ext/is_constructible9.C: New test.
+	* g++.dg/ext/is_convertible7.C: New test.
+	* g++.dg/ext/is_destructible3.C: New test.
+	* g++.dg/ext/is_invocable6.C: New test.
+	* g++.dg/ext/is_virtual_base_of_diagnostic2.C: New test.
+
+2025-07-24  Jason Merrill  <jason@redhat.com>
+
+	PR c++/114632
+	PR c++/101233
+	* g++.dg/cpp23/explicit-obj-lambda18.C: New test.
+	* g++.dg/cpp23/static-operator-call7.C: New test.
+
+2025-07-24  Robert Dubner  <rdubner@symas.com>
+
+	* cobol.dg/group2/_-static__compilation.cob: Modify for -static warning.
+	* cobol.dg/group2/_-static__compilation.out: Removed.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* lib/target-supports.exp: Fix misalignment check.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fadd_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmul_1.c: Likewise..
+	* gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c: Likewise.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fdiv_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fdiv_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fdiv_3.c: Likewise.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_binary_bf16_1.C: New test.
+	* g++.target/aarch64/sve/unpacked_binary_bf16_2.C: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fadd_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fadd_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmaxnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmaxnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fminnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fminnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmul_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmul_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fsubr_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fsubr_2.c: Likewise.
+
+2025-07-24  Jeevitha Palanisamy  <jeevitha@linux.ibm.com>
+
+	PR testsuite/119382
+	* gcc.target/powerpc/vsx-builtin-7.c: Add '-fno-ipa-icf' to dg-options.
+
+2025-07-24  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h: New test.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/120412
+	* g++.dg/modules/internal-14_a.C: New test.
+	* g++.dg/modules/internal-14_b.C: New test.
+	* g++.dg/modules/internal-14_c.C: New test.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_fabs_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fneg_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frinta_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frinta_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frinti_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintp_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintx_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintz_1.c: Likewise.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fabs_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fneg_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinta_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinta_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinti_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinti_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintp_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintp_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintx_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintx_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintz_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintz_2.c: Likewise.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c: Likewise.
+
+2025-07-23  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121203
+	* gfortran.dg/function_charlen_4.f90: New test.
+
+2025-07-23  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/121073
+	* gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c: Adjust test
+	expectation.
+	* gcc.target/riscv/rvv/base/scalar_move-5.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-6.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-7.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-8.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-9.c: Ditto.
+	* gcc.target/riscv/rvv/pr121073.c: New test.
+
+2025-07-23  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c:
+	Add zvfh requirements and options.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c:
+	Ditto.
+	* lib/target-supports.exp: Add zvfh options.
+
+2025-07-23  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR target/120119
+	* g++.dg/torture/pr120119-1.C: New test.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121179
+	* g++.dg/lookup/operator-8.C: Strengthen test and remove one
+	XFAIL.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121055
+	* g++.dg/ext/is_invocable5.C: New test.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* lib/gcc-defs.exp (aarch64-arg-dg-options): Split add_tune into
+	add_tune and add_override, so that specifying -moverride does not
+	change the baseline tuning from the testuite's default (generic).
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121220
+	* gcc.dg/tree-ssa/ssa-sink-23.c: New testcase.
+
+2025-07-23  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/119085
+	* gcc.dg/tree-ssa/pr119085.c: New test.
+
+2025-07-23  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121164
+	* gm2/switches/pedantic-params/fail/arrayofchar.def: New test.
+	* gm2/switches/pedantic-params/fail/arrayofchar.mod: New test.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/121216
+	* gcc.dg/pr121216.c: New testcase.
+
+2025-07-23  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/120101
+	* gcc.dg/tree-ssa/pr81627.c (fn1): Mark as noinline.
+
+2025-07-23  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gcc.dg/vect/pr116125.c (mem_overlap): Expand A to 10 members.
+
+2025-07-22  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121068
+	* g++.dg/cpp2a/constexpr-union6.C: Expect x5 to work.
+	* g++.dg/cpp26/constexpr-new4.C: New test.
+
+2025-07-22  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/warn/Wmismatched-new-delete-5.C: Fix allocation.
+
+2025-07-22  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/109267
+	* lib/target-supports.exp (check_effective_target_trap): New proc.
+	* g++.dg/missing-return.C: Update testcase for the !trap case.
+	* gcc.dg/pr109267-1.c: New test.
+	* gcc.dg/pr109267-2.c: New test.
+
+2025-07-22  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/sve/mask_load_2.c: Update tests.
+
+2025-07-22  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/sve/mask_load_2.c: New test.
+
+2025-07-22  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121202
+	* gcc.dg/pr121202.c: New testcase.
+
+2025-07-22  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/vect/slp-28.c: Adjust.
+
+2025-07-21  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/110949
+	PR tree-optimization/95906
+	* gcc.dg/tree-ssa/cmp-2.c: New test.
+	* gcc.dg/tree-ssa/max-bitcmp-1.c: New test.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c: New test.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c: Ditto.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c: New test.
+
+2025-07-21  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/mf8_data_1.c (test_set_lane4,
+	test_setq_lane4): Relax allowed assembly.
+	* gcc.target/aarch64/vec-set-zero.c: Use -Os in flags.
+	* gcc.target/aarch64/inszero_split_1.c: New test.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121194
+	* gcc.dg/torture/pr121194.c: New testcase.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.dg/pr87600-2.c: Split test into two files since errors for
+	functions test{0,1} are thrown during expand, and for
+	test{2,3} during gimplification.
+	* lib/scanasm.exp: On s390, skip lines beginning with #.
+	* gcc.dg/asm-hard-reg-error-1.c: New test.
+	* gcc.dg/asm-hard-reg-error-2.c: New test.
+	* gcc.dg/asm-hard-reg-error-3.c: New test.
+	* gcc.dg/asm-hard-reg-error-4.c: New test.
+	* gcc.dg/asm-hard-reg-error-5.c: New test.
+	* gcc.dg/pr87600-3.c: New test.
+	* gcc.target/aarch64/asm-hard-reg-2.c: New test.
+	* gcc.target/s390/asm-hard-reg-7.c: New test.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.dg/asm-hard-reg-1.c: New test.
+	* gcc.dg/asm-hard-reg-2.c: New test.
+	* gcc.dg/asm-hard-reg-3.c: New test.
+	* gcc.dg/asm-hard-reg-4.c: New test.
+	* gcc.dg/asm-hard-reg-5.c: New test.
+	* gcc.dg/asm-hard-reg-6.c: New test.
+	* gcc.dg/asm-hard-reg-7.c: New test.
+	* gcc.dg/asm-hard-reg-8.c: New test.
+	* gcc.target/aarch64/asm-hard-reg-1.c: New test.
+	* gcc.target/i386/asm-hard-reg-1.c: New test.
+	* gcc.target/i386/asm-hard-reg-2.c: New test.
+	* gcc.target/s390/asm-hard-reg-1.c: New test.
+	* gcc.target/s390/asm-hard-reg-2.c: New test.
+	* gcc.target/s390/asm-hard-reg-3.c: New test.
+	* gcc.target/s390/asm-hard-reg-4.c: New test.
+	* gcc.target/s390/asm-hard-reg-5.c: New test.
+	* gcc.target/s390/asm-hard-reg-6.c: New test.
+	* gcc.target/s390/asm-hard-reg-longdouble.h: New test.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/vect/vect-127.c: New testcase.
+
+2025-07-21  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/119106
+	* gfortran.dg/array_constructor_58.f90: New test.
+
+2025-07-21  panciyan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Unsigned testcase form8 form9.
+	* gcc.target/riscv/sat/sat_u_add-8-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-8-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-8-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-8-u8.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u8.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u8.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u8.c: New test.
+
+2025-07-20  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/120859
+	* gcc.dg/tree-prof/afdo-crossmodule-1b.c: Add some dg-*
+	commands like what is in afdo-crossmodule-1.c
+
+2025-07-20  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c:
+	Leverage DEF_AVG_0_WRAP to generate the correct func name.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c: Ditto.
+
+2025-07-19  Dimitar Dimitrov  <dimitar@dinux.eu>
+
+	PR target/121124
+	* gcc.target/pru/pragma-ctable_entry-2.c: New test.
+
+2025-07-19  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c: Add vfwnmacc and
+	vfwnmsac.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c: New test.
+
+2025-07-18  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121145
+	* gfortran.dg/pointer_check_15.f90: New test.
+
+2025-07-18  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/121153
+	* gcc.dg/vect/vect-reduc-cond-1.c: Require vect_condition.
+	* gcc.dg/vect/vect-reduc-cond-2.c: Likewise.
+
+2025-07-18  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/avg_data.h: Adjust the test data.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c: New test.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c: New test.
+
+2025-07-18  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/117423
+	* gcc.dg/tree-ssa/pr117423.c: New test.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121126
+	* gcc.dg/vect/pr121126.c: New testcase.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120924
+	* gcc.dg/uninit-pr120924.c: New testcase.
+
+2025-07-18  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121131
+	* gcc.dg/bitint-124.c: New test.
+
+2025-07-17  Jason Merrill  <jason@redhat.com>
+
+	PR c++/87097
+	* g++.dg/cpp0x/constexpr-array29.C: New test.
+
+2025-07-17  Richard Sandiford  <richard.sandiford@arm.com>
+	    Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* lib/target-supports.exp (add_options_for_aarch64_sme)
+	(check_effective_target_aarch64_sme_hw): New procedures.
+	* g++.target/aarch64/sme/sme_throw_1.C: New test.
+	* g++.target/aarch64/sme/sme_throw_2.C: Likewise.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c: Adapt
+	scan assembler directives.
+	* gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c: Ditto.
+	* gcc.target/s390/signbit-1.c: New test.
+	* gcc.target/s390/signbit-2.c: New test.
+	* gcc.target/s390/signbit-3.c: New test.
+	* gcc.target/s390/signbit-4.c: New test.
+	* gcc.target/s390/signbit-5.c: New test.
+	* gcc.target/s390/signbit.h: New test.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/vector/vlgv-zero-extend-1.c: New test.
+
+2025-07-17  Xi Ruoyao  <xry111@xry111.site>
+
+	PR target/121064
+	* gcc.target/loongarch/pr121064.c: New test.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/vect/bb-slp-39.c: Adjust.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121035
+	* gcc.dg/pr121035.c: New testcase.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121062
+	* gcc.target/i386/pr121062-1.c: New test.
+	* gcc.target/i386/pr121062-2.c: Likewise.
+	* gcc.target/i386/pr121062-3a.c: Likewise.
+	* gcc.target/i386/pr121062-3b.c: Likewise.
+	* gcc.target/i386/pr121062-3c.c: Likewise.
+	* gcc.target/i386/pr121062-4.c: Likewise.
+	* gcc.target/i386/pr121062-5.c: Likewise.
+	* gcc.target/i386/pr121062-6.c: Likewise.
+	* gcc.target/i386/pr121062-7.c: Likewise.
+
+2025-07-16  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	PR testsuite/121078
+	* gcc.dg/20021014-1.c (dg-additional-options): Add -mfentry
+	-fno-pic only on gnu/x86 targets.
+	* gcc.dg/aru-2.c (dg-additional-options): Likewise.
+	* gcc.dg/nest.c (dg-additional-options): Likewise.
+	* gcc.dg/pr32450.c (dg-additional-options): Likewise.
+	* gcc.dg/pr43643.c (dg-additional-options): Likewise.
+	* gcc.target/i386/pr104447.c (dg-additional-options): Likewise.
+	* gcc.target/i386/pr113122-3.c(dg-additional-options): Likewise.
+	* gcc.target/i386/pr119386-1.c (dg-additional-options): Add
+	-mfentry only on gnu targets.
+	* gcc.target/i386/pr119386-2.c (dg-additional-options): Likewise.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121049
+	* gcc.dg/vect/pr121049.c: New testcase.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/119920
+	PR tree-optimization/112324
+	PR tree-optimization/110015
+	* gcc.dg/vect/vect-reduc-cond-1.c: New test.
+	* gcc.dg/vect/vect-reduc-cond-2.c: New test.
+	* gcc.dg/vect/vect-reduc-cond-3.c: New test.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121116
+	* gcc.dg/torture/pr121116.c: New testcase.
+
+2025-07-16  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/117850
+	* gcc.target/aarch64/simd/vabal_combine.c: Removed.  This is
+	covered by fold_to_highpart_1.c
+	* gcc.target/aarch64/simd/fold_to_highpart_1.c: New test.
+	* gcc.target/aarch64/simd/fold_to_highpart_2.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_3.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_4.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_5.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_6.c: Likewise.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* g++.dg/warn/Wformat-gcc_diag-1.C: Add string_slice "%B" format tests.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR middle-end/121065
+	* gcc.target/arm/pr121065.c: New test.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120297
+	* gcc.target/riscv/rvv/pr120297.c: New test.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/sve2/eon_bsl2n.c: New test.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c: New test.
+
+2025-07-16  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/121060
+	* gfortran.dg/associate_75.f90: New test.
+
+2025-07-16  Steve Kargl  <sgk@troutmask.apl.washington.edu>
+
+	* gfortran.dg/import13.f90: New test.
+
+2025-07-16  Jeremy Rifkin  <jeremy@rifkin.dev>
+
+	PR c/82134
+	* c-c++-common/attr-warn-unused-result-2.c: New test.
+
+2025-07-16  Haochen Jiang  <haochen.jiang@intel.com>
+
+	* gcc.target/i386/amxavx512-cvtrowd2ps-2.c: Add -mavx512fp16 to
+	use FP16 related intrins for convert.
+	* gcc.target/i386/amxavx512-cvtrowps2bf16-2.c: Ditto.
+	* gcc.target/i386/amxavx512-cvtrowps2ph-2.c: Ditto.
+	* gcc.target/i386/amxavx512-movrow-2.c: Ditto.
+
+2025-07-16  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_s_add-1-i16.c: Remove function-body
+	check and add no jmp label asm check.
+	* gcc.target/riscv/sat/sat_s_add-1-i32.c:
+	* gcc.target/riscv/sat/sat_s_add-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u8.c: Ditto.
+
+2025-07-16  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/avg.h: Add int128 type when
+	xlen == 64.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c:
+	Suppress __int128 warning for run test.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_data.h: Fix one incorrect
+	test data.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c: New test.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c: New test.
+
+2025-07-15  David Malcolm  <dmalcolm@redhat.com>
+
+	PR sarif-replay/120792
+	* libgdiagnostics.dg/sarif.py: Delete duplicate script.
+	* libgdiagnostics.dg/test-message-buffer-c.py: New test script.
+	* libgdiagnostics.dg/test-message-buffer.c: New test.
+	* libgdiagnostics.dg/test-warning-with-path-c.py: Update expected
+	output to reflect that SARIF for event messages now contains JSON
+	pointers when referring to other events by ID.
+	* sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif: Add
+	HTML and SARIF output, and call out to Python scripts to verify
+	the output.  Add example of a result with a link in its message.
+	* sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py: New
+	test script.
+	* sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py:
+	New test script.
+
+2025-07-15  Umesh Kalappa  <ukalappa.mips@gmail.com>
+
+	* gcc.target/riscv/mipscondmov.c: Test file for mips.ccmov insn.
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120577
+	* g++.dg/cpp2a/constexpr-union9.C: New test.
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/117784
+	* g++.dg/cpp26/decomp25.C: Add -fno-implicit-constexpr.
+
+2025-07-15  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	PR fortran/104428
+	* gfortran.dg/gomp/pr104428.f90: New.
+
+2025-07-15  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	* gcc.target/aarch64/avoid-store-forwarding-be.c: New test.
+
+2025-07-15  Soumya AR  <soumyaa@nvidia.com>
+
+	* gcc.target/aarch64/ldapr-sext.c: Update expected output to include
+	offsets.
+	* gcc.target/aarch64/ldapur.c: New test for LDAPUR.
+	* gcc.target/aarch64/ldapur_avoid.c: New test for AVOID_LDAPUR flag.
+
+2025-07-15  Richard Biener  <rguenther@suse.de>
+	    Richard Sandiford   <richard.sandiford@arm.com>
+
+	PR tree-optimization/121059
+	* gcc.dg/vect/pr121059.c: New testcase.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* c-c++-common/Wunused-parm-1.c: New test.
+	* c-c++-common/Wunused-parm-2.c: New test.
+	* c-c++-common/Wunused-parm-3.c: New test.
+	* c-c++-common/Wunused-parm-4.c: New test.
+	* c-c++-common/Wunused-parm-5.c: New test.
+	* c-c++-common/Wunused-parm-6.c: New test.
+	* c-c++-common/Wunused-var-7.c (bar, baz): Expect warning on a.
+	* c-c++-common/Wunused-var-19.c: New test.
+	* c-c++-common/Wunused-var-20.c: New test.
+	* c-c++-common/Wunused-var-21.c: New test.
+	* c-c++-common/Wunused-var-22.c: New test.
+	* c-c++-common/Wunused-var-23.c: New test.
+	* c-c++-common/Wunused-var-24.c: New test.
+	* g++.dg/cpp26/name-independent-decl1.C (foo): Expect one
+	set but not used warning.
+	* g++.dg/warn/Wunused-parm-12.C: New test.
+	* g++.dg/warn/Wunused-parm-13.C: New test.
+	* g++.dg/warn/Wunused-var-2.C (f2): Expect set but not used warning
+	on parameter x and variable a.
+	* g++.dg/warn/Wunused-var-40.C: New test.
+	* g++.dg/warn/Wunused-var-41.C: New test.
+	* gcc.dg/memchr-3.c (test_find): Change return type from void to int,
+	and add return n; statement.
+	* gcc.dg/unused-9.c (g): Move dg-bogus to the correct line and expect
+	a warning on i.
+
+2025-07-15  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	Revert:
+	2025-07-15  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/eor3_d.c: Add tests for DImode operands.
+
+2025-07-15  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fcm_combines_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fcm_combines_2.c: Likewise.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	Revert:
+	2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* gcc.dg/vect/pr121059.c: New testcase.
+
+2025-07-14  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* lib/target-supports.exp: Add s390 to vect_logical_reduc targets.
+	* gcc.target/s390/vector/reduc-binops-1.c: New test.
+	* gcc.target/s390/vector/reduc-minmax-1.c: New test.
+	* gcc.target/s390/vector/reduc-plus-1.c: New test.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* gcc.dg/vect/pr121059.c: New testcase.
+
+2025-07-14  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c: New test.
+
+2025-07-14  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121015
+	* gcc.target/i386/pr121015.c: New test.
+
+2025-07-14  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	* gcc.dg/20021014-1.c: Add additional -mfentry -fno-pic options
+	for x86.
+	* gcc.dg/aru-2.c: Likewise.
+	* gcc.dg/nest.c: Likewise.
+	* gcc.dg/pr32450.c: Likewise.
+	* gcc.dg/pr43643.c: Likewise.
+	* gcc.target/i386/pr104447.c: Likewise.
+	* gcc.target/i386/pr113122-3.c: Likewise.
+	* gcc.target/i386/pr119386-1.c: Add additional -mfentry if not
+	ia32.
+	* gcc.target/i386/pr119386-2.c: Likewise.
+	* gcc.target/i386/pr120881-1a.c: New test.
+	* gcc.target/i386/pr120881-1b.c: Likewise.
+	* gcc.target/i386/pr120881-1c.c: Likewise.
+	* gcc.target/i386/pr120881-1d.c: Likewise.
+	* gcc.target/i386/pr120881-2a.c: Likewise.
+	* gcc.target/i386/pr120881-2b.c: Likewise.
+	* gcc.target/i386/pr82699-1.c: Add additional -mfentry.
+	* lib/target-supports.exp (check_effective_target_fentry): New.
+
+2025-07-14  François-Xavier Coudert  <fxcoudert@gcc.gnu.org>
+
+	* gcc.dg/darwin-minversion-link.c: Account for macOS 26.
+
+2025-07-14  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c: Add vfwmacc and
+	vfwmsac.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c: Likewise. Also check
+	for fcvt and vfmv.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c: Add vfwmacc and
+	vfwmsac.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c: Likewise. Also check
+	for fcvt and vfmv.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h: Add support for
+	widening variants.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h: New test
+	helper.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c: New test.
+
+2025-07-14  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gnat.dg/deref4.adb: New test.
+	* gnat.dg/deref4_pkg.ads: New helper.
+
+2025-07-14  Alfie Richards  <alfie.richards@arm.com>
+
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c: New test.
+
+2025-07-14  panciyan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h: Unsigned vector SAT_SUB form11 form12.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c: New test.
+
+2025-07-12  Xi Ruoyao  <xry111@xry111.site>
+
+	PR rtl-optimization/87600
+	PR rtl-optimization/120983
+	* gcc.dg/pr87600.h [__loongarch__]: Define REG0 and REG1.
+	* gcc.dg/pr87600-1.c (dg-do): Add loongarch.
+	* gcc.dg/pr87600-2.c (dg-do): Likewise.
+
+2025-07-12  Tobias Burnus  <tburnus@baylibre.com>
+
+	* gfortran.dg/goacc/parameter.f95: Add -Wsurprising flag
+	and update expected diagnostic.
+	* gfortran.dg/goacc/parameter-3.f90: New test.
+	* gfortran.dg/goacc/parameter-4.f90: New test.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/analyzer/state-diagram-1-sarif.py (test_xml_state):
+	Rename to...
+	(test_state_graph): ...this.  Port from XML to SARIF graphs.
+	* gcc.dg/analyzer/state-diagram-1.c: Update sink option
+	from "sarif:xml-state=yes" to "sarif:state-graphs=yes".
+	* gcc.dg/analyzer/state-diagram-5-sarif.c: Likewise.
+	* gcc.dg/analyzer/state-diagram-5-sarif.py: Drop import of ET.
+	(test_nested_types_in_xml_state): Rename to...
+	(test_nested_types_in_state_graph): ...this.  Port from XML to
+	SARIF graphs.
+	* gcc.dg/plugin/diagnostic-test-graphs-html.c: New test.
+	* gcc.dg/plugin/diagnostic-test-graphs-html.py: New test script.
+	* gcc.dg/plugin/diagnostic-test-graphs-sarif.c: New test.
+	* gcc.dg/plugin/diagnostic-test-graphs-sarif.py: New test script.
+	* gcc.dg/plugin/diagnostic-test-graphs.c: New test.
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: New test plugin.
+	* gcc.dg/plugin/plugin.exp (plugin_test_list): Add the above.
+	* lib/sarif.py (get_xml_state): Delete.
+	(get_state_graph): New.
+	(def get_state_node_attr): New.
+	(get_state_node_kind): New.
+	(get_state_node_name): New.
+	(get_state_node_type): New.
+	(get_state_node_value): New.
+	* sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif:
+	New test.
+	* sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif:
+	New test.
+	* sarif-replay.dg/2.1.0-valid/graphs-check-html.py: New test
+	script.
+	* sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py: New
+	test script.
+	* sarif-replay.dg/2.1.0-valid/graphs.sarif: New test.
+
+2025-07-11  Vladimir N. Makarov  <vmakarov@redhat.com>
+
+	* gcc.target/powerpc/pr121007.c: New.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* g++.dg/cpp26/feat-cxx26.C: Add test for
+	__cpp_trivial_relocatability.
+	* g++.dg/cpp26/trivially-relocatable1.C: New test.
+	* g++.dg/cpp26/trivially-relocatable2.C: New test.
+	* g++.dg/cpp26/trivially-relocatable3.C: New test.
+	* g++.dg/cpp26/trivially-relocatable4.C: New test.
+	* g++.dg/cpp26/trivially-relocatable5.C: New test.
+	* g++.dg/cpp26/trivially-relocatable6.C: New test.
+	* g++.dg/cpp26/trivially-relocatable7.C: New test.
+	* g++.dg/cpp26/trivially-relocatable8.C: New test.
+	* g++.dg/cpp26/trivially-relocatable9.C: New test.
+	* g++.dg/cpp26/trivially-relocatable10.C: New test.
+	* g++.dg/cpp26/trivially-relocatable11.C: New test.
+
+2025-07-11  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121027
+	* gcc.target/aarch64/sve/acle/general/perm_1.c: New test.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/eor3_d.c: Add tests for DImode operands.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/bcax_d.c: Add tests for DImode arguments.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/eor3_d.c: New test.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/bcax_d.c: New test.
+
+2025-07-11  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121034
+	* gcc.dg/vect/pr121034.c: New testcase.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120954
+	* c-c++-common/Warray-bounds-11.c: New test.
+
+2025-07-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* g++.dg/lto/pr114790_0.C: New test.
+	* g++.dg/lto/pr114790_1.C: New test.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+	    Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/121023
+	* c-c++-common/musttail32.c: New test.
+
+2025-07-11  Hu, Lin1  <lin1.hu@intel.com>
+
+	PR target/91384
+	* gcc.target/i386/pr91384-1.c: New test.
+
+2025-07-11  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/106135
+	* gfortran.dg/import3.f90: Use -std=f2008 and comment on change
+	in error message texts with f2018.
+	* gfortran.dg/import12.f90: New test.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120628
+	* g++.dg/cpp0x/final1.C: New test.
+	* g++.dg/cpp0x/final2.C: New test.
+	* g++.dg/cpp0x/override6.C: New test.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120569
+	* g++.dg/cpp0x/override2.C: Expect different diagnostics with
+	override or duplicate final.
+	* g++.dg/cpp0x/override5.C: New test.
+	* g++.dg/cpp0x/duplicate1.C: Expect different diagnostics with
+	duplicate final.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117785
+	* g++.dg/cpp0x/constexpr-ellipsis2.C: Expect different diagnostics for
+	C++26.
+	* g++.dg/cpp0x/constexpr-throw.C: Likewise.
+	* g++.dg/cpp1y/constexpr-84192.C: Expect different diagnostics.
+	* g++.dg/cpp1y/constexpr-throw.C: Expect different diagnostics for
+	C++26.
+	* g++.dg/cpp1z/constexpr-asm-5.C: Likewise.
+	* g++.dg/cpp26/constexpr-eh1.C: New test.
+	* g++.dg/cpp26/constexpr-eh2.C: New test.
+	* g++.dg/cpp26/constexpr-eh3.C: New test.
+	* g++.dg/cpp26/constexpr-eh4.C: New test.
+	* g++.dg/cpp26/constexpr-eh5.C: New test.
+	* g++.dg/cpp26/constexpr-eh6.C: New test.
+	* g++.dg/cpp26/constexpr-eh7.C: New test.
+	* g++.dg/cpp26/constexpr-eh8.C: New test.
+	* g++.dg/cpp26/constexpr-eh9.C: New test.
+	* g++.dg/cpp26/constexpr-eh10.C: New test.
+	* g++.dg/cpp26/constexpr-eh11.C: New test.
+	* g++.dg/cpp26/constexpr-eh12.C: New test.
+	* g++.dg/cpp26/constexpr-eh13.C: New test.
+	* g++.dg/cpp26/constexpr-eh14.C: New test.
+	* g++.dg/cpp26/constexpr-eh15.C: New test.
+	* g++.dg/cpp26/feat-cxx26.C: Change formatting in __cpp_pack_indexing
+	and __cpp_pp_embed test.  Add __cpp_constexpr_exceptions test.
+	* g++.dg/cpp26/static_assert1.C: Expect different diagnostics for
+	C++26.
+	* g++.dg/cpp2a/consteval34.C: Likewise.
+	* g++.dg/cpp2a/consteval-memfn1.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic4.C: For C++26 add std::exception and
+	std::bad_cast definitions and expect different diagnostics.
+	* g++.dg/cpp2a/constexpr-dynamic6.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic7.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic8.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic9.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic11.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic14.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic18.C: Likewise.
+	* g++.dg/cpp2a/constexpr-new27.C: New test.
+	* g++.dg/cpp2a/constexpr-typeid5.C: New test.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	PR middle-end/121000
+	* gcc.dg/flex-array-counted-by-pr121000.c: New test.
+
+2025-07-10  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* gfortran.dg/asan/array_constructor_1.f90: Allocate array
+	before using it.
+
+2025-07-10  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/tree-prof/afdo-inline.c: Add user symbol names.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/pfalse-binary.c: Add -funwind-tables.
+	* gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_rotate.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binaryxn.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-clast.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-compare_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-count_pred.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-fold_left.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_ext.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_gather_sv.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_gather_vs.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_replicate.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-prefetch.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-ptest.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-rdffr.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-reduction.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-reduction_wide.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-shift_right_imm.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-store.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-store_scatter_index.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-store_scatter_offset.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-storexn.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-ternary_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-ternary_rotate.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_convertxn.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_pred.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_to_uint.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unaryxn.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_to_uint.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_wide.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-compare.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-load_gather_vs.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-shift_right_imm.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-unary.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-unary_convert.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-unary_to_int.c: Likewise.
+
+2025-07-10  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	PR target/120999
+	* gcc.target/aarch64/sve2/pr120999.c: New test.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* lib/target-supports.exp (check_effective_target_aarch64_sve2p1_hw):
+	New proc.
+	* gcc.target/aarch64/sve2/dupq_1.c: Extend to big-endian.  Add
+	noipa attributes.
+	* gcc.target/aarch64/sve2/extq_1.c: Likewise.
+	* gcc.target/aarch64/sve2/uzpq_1.c: Likewise.
+	* gcc.target/aarch64/sve2/zipq_1.c: Likewise.
+	* gcc.target/aarch64/sve2/dupq_1_run.c: New test.
+	* gcc.target/aarch64/sve2/extq_1_run.c: Likewise.
+	* gcc.target/aarch64/sve2/uzpq_1_run.c: Likewise.
+	* gcc.target/aarch64/sve2/zipq_1_run.c: Likewise.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* gfortran.dg/g77/980310-3.f: Comment spelling fix: bellow -> below.
+	* jit.dg/test-debuginfo.c: Likewise.
+
+2025-07-10  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_s_add-run-1-i16.c: Take rv32 || rv64
+	instead of riscv_v for scalar run test.
+	* gcc.target/riscv/sat/sat_s_add-run-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c: Ditto.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120243
+	* g++.dg/coroutines/torture/pr120243-unhandled-1.C: New test.
+	* g++.dg/coroutines/torture/pr120243-unhandled-2.C: New test.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121012
+	PR c++/120917
+	* g++.dg/cpp2a/lambda-targ17.C: New test.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121008
+	PR c++/113563
+	* g++.dg/cpp2a/lambda-uneval28.C: New test.
+
+2025-07-09  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/119838
+	* g++.dg/parse/template32.C: New test.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.dg/rtl/aarch64/vec-series-1.c: New test.
+	* gcc.dg/rtl/aarch64/vec-series-2.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/dupq_2.c: Fix expected
+	output for this big-endian test.
+	* gcc.target/aarch64/sve/acle/general/dupq_4.c: Likewise.
+	* gcc.target/aarch64/sve/vec_init_3.c: Restrict to little-endian
+	targets and add more tests.
+	* gcc.target/aarch64/sve/vec_init_4.c: New big-endian version
+	of vec_init_3.c.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c: New test.
+
+2025-07-09  Paul-Antoine Arras  <parras@baylibre.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h: Set
+	MAX_RELATIVE_DIFF depending on type.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c: Enable zvfh.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c: Likewise.
+
+2025-07-09  Ciyan Pan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h: Add vec_sat_u_sub_fmt wrap define.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h: Add vec_sat_u_sub test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c: Remove test data.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/pr118348_1.c: Require fstack_protector.
+	* gcc.target/aarch64/pr118348_2.c: Likewise.
+
+2025-07-09  Icen Zeyada  <Icen.Zeyada2@arm.com>
+
+	* gcc.target/aarch64/vector-compare-5.c: Add new test for vector compare simplification.
+
+2025-07-09  Jeff Law  <jlaw@ventanamicro.com>
+
+	PR target/120642
+	* gcc.target/riscv/rvv/xtheadvector/pr120642.c: New test.
+
+2025-07-09  Richard Biener  <rguenther@suse.de>
+
+	PR testsuite/120093
+	* gcc.dg/vect/pr101145.c: Use noipa instead of noinline
+	attribute.
+
+2025-07-09  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/pattern-avg-1.c: Fix on -m31.
+	* gcc.target/s390/vector/pattern-mulh-1.c: Fix on -m31.
+	* gcc.target/s390/vector/pattern-mulh-2.c: Fix on -m31.
+
+2025-07-09  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* gcc.dg/builtin-dynamic-object-size-pr120780.c: Fix 'main' function.
+
+2025-07-09  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120922
+	* gcc.dg/vect/pr120922.c: New test.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Add xlen check for
+	uint128_t.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c: Enable
+	run test for rv64 only.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c: Ditto.
+
+2025-07-08  Marek Polacek  <polacek@redhat.com>
+	    Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c++/83469
+	PR c++/93809
+	* g++.dg/template/error45.C: Adjust dg-error.
+	* g++.dg/warn/Wredundant-tags-3.C: Remove xfail.
+	* g++.dg/parse/union1.C: New test.
+	* g++.dg/parse/union2.C: New test.
+	* g++.dg/parse/union3.C: New test.
+	* g++.dg/parse/union4.C: New test.
+	* g++.dg/parse/union5.C: New test.
+	* g++.dg/parse/union6.C: New test.
+
+2025-07-08  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c: New.
+
+2025-07-08  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117784
+	* g++.dg/cpp1z/decomp3.C (test): For constexpr structured binding
+	initialize from constexpr var instead of non-constexpr and expect
+	just a pedwarn for C++23 and older instead of error always.
+	* g++.dg/cpp26/decomp9.C (foo): Likewise.
+	* g++.dg/cpp26/decomp22.C: New test.
+	* g++.dg/cpp26/decomp23.C: New test.
+	* g++.dg/cpp26/decomp24.C: New test.
+	* g++.dg/cpp26/decomp25.C: New test.
+
+2025-07-08  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/stack-protector-guard-tls-1.c: New test.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/guality/guality.h (guality_main): Declare noipa.
+	(guality_check): Likewise.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120461
+	* gcc.target/riscv/rvv/xtheadvector/pr120461.c: New test.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/113829
+	* gcc.target/riscv/rvv/base/pr113829.c: New test.
+
+2025-07-08  Andreas Schwab  <schwab@suse.de>
+
+	PR target/120995
+	* gcc.target/riscv/amo/zabha-zacas-atomic-cas.c: New test.
+
+2025-07-08  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/cpp1y/lambda-generic-variadic.C: Change to 'compile'.
+
+2025-07-08  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.target/i386/memcpy-pr120683-1.c (dg-options): Add
+	-fasynchronous-unwind-tables -fdwarf2-cfi-asm.
+	* gcc.target/i386/memcpy-pr120683-2.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-3.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-4.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-5.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-6.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-7.c: Likewise.
+	* gcc.target/i386/memcpy-strategy-12.c: Likewise.
+	* gcc.target/i386/memset-pr120683-1.c: Likewise.
+	* gcc.target/i386/memset-pr120683-10.c: Likewise.
+	* gcc.target/i386/memset-pr120683-11.c: Likewise.
+	* gcc.target/i386/memset-pr120683-12.c: Likewise.
+	* gcc.target/i386/memset-pr120683-13.c: Likewise.
+	* gcc.target/i386/memset-pr120683-14.c: Likewise.
+	* gcc.target/i386/memset-pr120683-15.c: Likewise.
+	* gcc.target/i386/memset-pr120683-16.c: Likewise.
+	* gcc.target/i386/memset-pr120683-17.c: Likewise.
+	* gcc.target/i386/memset-pr120683-18.c: Likewise.
+	* gcc.target/i386/memset-pr120683-19.c: Likewise.
+	* gcc.target/i386/memset-pr120683-2.c: Likewise.
+	* gcc.target/i386/memset-pr120683-20.c: Likewise.
+	* gcc.target/i386/memset-pr120683-21.c: Likewise.
+	* gcc.target/i386/memset-pr120683-22.c: Likewise.
+	* gcc.target/i386/memset-pr120683-23.c: Likewise.
+	* gcc.target/i386/memset-pr120683-3.c: Likewise.
+	* gcc.target/i386/memset-pr120683-4.c: Likewise.
+	* gcc.target/i386/memset-pr120683-5.c: Likewise.
+	* gcc.target/i386/memset-pr120683-6.c: Likewise.
+	* gcc.target/i386/memset-pr120683-7.c: Likewise.
+	* gcc.target/i386/memset-pr120683-8.c: Likewise.
+	* gcc.target/i386/memset-pr120683-9.c: Likewise.
+
+2025-07-08  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/pattern-avg-1.c: Split test.
+	* gcc.target/s390/vector/pattern-mulh-1.c: Split test.
+	* gcc.target/s390/vector/pattern-avg-2.c: New test.
+	* gcc.target/s390/vector/pattern-mulh-2.c: New test.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* gcc.target/i386/vect-mask-epilogue-1.c: New testcase.
+	* gcc.target/i386/vect-mask-epilogue-2.c: Likewise.
+	* gcc.target/i386/vect-epilogues-3.c: Adjust.
+
+2025-07-08  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120637
+	* gfortran.dg/asan/finalize_1.f90: New test.
+
+2025-07-08  Jeff Law  <jlaw@ventanamicro.com>
+
+	* gcc.dg/torture/pr120654.c: Use __builtin variants of malloc and free.
+
+2025-07-08  Jeff Law  <jlaw@ventanamicro.com>
+
+	* gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c: Adjust expected
+	output.
+	* gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c: Likewise.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* gcc.dg/flex-array-counted-by.c: Update test.
+	* gcc.dg/pointer-counted-by-1.c: New test.
+	* gcc.dg/pointer-counted-by-2.c: New test.
+	* gcc.dg/pointer-counted-by-3.c: New test.
+	* gcc.dg/pointer-counted-by.c: New test.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* gcc.dg/pointer-counted-by-4-char.c: New test.
+	* gcc.dg/pointer-counted-by-4-float.c: New test.
+	* gcc.dg/pointer-counted-by-4-struct.c: New test.
+	* gcc.dg/pointer-counted-by-4-union.c: New test.
+	* gcc.dg/pointer-counted-by-4.c: New test.
+	* gcc.dg/pointer-counted-by-5.c: New test.
+	* gcc.dg/pointer-counted-by-6.c: New test.
+	* gcc.dg/pointer-counted-by-7.c: New test.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* gcc.dg/ubsan/pointer-counted-by-bounds-2.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds-3.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds-4.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds-5.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds.c: New test.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR testsuite/120881
+	* lib/scanasm.exp (check-function-bodies): Allow "^[0-9]+:".
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120888
+	* gcc.target/xtensa/pr120888-1.c: New test.
+	* gcc.target/xtensa/pr120888-2.c: Likewise.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/fminmax-1.c: New test.
+	* gcc.target/s390/fminmax-2.c: New test.
+
+2025-07-07  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120817
+	* gcc.dg/vect/pr120817.c: Add SVE HW check.
+
+2025-07-07  Alfie Richards  <alfie.richards@arm.com>
+
+	PR c++/119498
+	* g++.target/aarch64/pr119498.C: New test.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* g++.dg/concepts/auto7a.C: Add diagnostic.
+	* g++.dg/concepts/auto7b.C: New test.
+	* g++.dg/concepts/auto7c.C: New test.
+	* g++.dg/cpp1y/pr85076.C: Expect 'auto' error.
+	* g++.dg/concepts/pr67249.C: Likewise.
+	* g++.dg/cpp1y/lambda-generic-variadic.C: Likewise.
+	* g++.dg/cpp2a/concepts-pr67210.C: Likewise.
+	* g++.dg/concepts/pr67249a.C: New test.
+	* g++.dg/cpp1y/lambda-generic-variadic-a.C: New test.
+	* g++.dg/cpp2a/concepts-pr67210a.C: New test.
+
+2025-07-07  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/popcnt9.c: Add +nosve to target pragma.
+	* gcc.target/aarch64/popcnt13.c: New test.
+
+2025-07-07  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120817
+	* gcc.dg/vect/pr120817.c: New testcase.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Add test helper macros.
+	* gcc.target/riscv/sat/sat_arith_data.h: Add test data for
+	run test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c: New test.
+
+2025-07-07  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* ada/acats-3/tests/c9/c94001c.ada: Tweak delay statements.
+	* ada/acats-4/tests/c9/c94001c.ada: Likewise.
+	* ada/acats-4/tests/c9/c94006a.ada: Likewise.
+	* ada/acats-4/tests/c9/c94008c.ada: Likewise.
+	* ada/acats-4/tests/c9/c951002.a: Likewise.
+	* ada/acats-4/tests/c9/c954a01.a: Likewise.
+	* ada/acats-4/tests/c9/c940005.a: Tweak duration constants.
+	* ada/acats-4/tests/c9/c940007.a: Likewise.
+	* ada/acats-4/tests/c9/c96001a.ada: Likewise.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/pattern-avg-1.c: New test.
+	* gcc.target/s390/vector/pattern-mulh-1.c: New test.
+
+2025-07-07  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fcm_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fcm_2.c: Likewise.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120670
+	PR target/120683
+	* gcc.target/i386/auto-init-padding-9.c: Updated.
+	* gcc.target/i386/memcpy-strategy-12.c: Likewise.
+	* gcc.target/i386/memset-strategy-25.c: Likewise.
+	* gcc.target/i386/memset-strategy-29.c: Likewise.
+	* gcc.target/i386/memset-strategy-30.c: Likewise.
+	* gcc.target/i386/memset-strategy-31.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-1.c: New test.
+	* gcc.target/i386/memcpy-pr120683-2.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-3.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-4.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-5.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-6.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-7.c: Likewise.
+	* gcc.target/i386/memset-pr120683-1.c: Likewise.
+	* gcc.target/i386/memset-pr120683-2.c: Likewise.
+	* gcc.target/i386/memset-pr120683-3.c: Likewise.
+	* gcc.target/i386/memset-pr120683-4.c: Likewise.
+	* gcc.target/i386/memset-pr120683-5.c: Likewise.
+	* gcc.target/i386/memset-pr120683-6.c: Likewise.
+	* gcc.target/i386/memset-pr120683-7.c: Likewise.
+	* gcc.target/i386/memset-pr120683-8.c: Likewise.
+	* gcc.target/i386/memset-pr120683-9.c: Likewise.
+	* gcc.target/i386/memset-pr120683-10.c: Likewise.
+	* gcc.target/i386/memset-pr120683-11.c: Likewise.
+	* gcc.target/i386/memset-pr120683-12.c: Likewise.
+	* gcc.target/i386/memset-pr120683-13.c: Likewise.
+	* gcc.target/i386/memset-pr120683-14.c: Likewise.
+	* gcc.target/i386/memset-pr120683-15.c: Likewise.
+	* gcc.target/i386/memset-pr120683-16.c: Likewise.
+	* gcc.target/i386/memset-pr120683-17.c: Likewise.
+	* gcc.target/i386/memset-pr120683-18.c: Likewise.
+	* gcc.target/i386/memset-pr120683-19.c: Likewise.
+	* gcc.target/i386/memset-pr120683-20.c: Likewise.
+	* gcc.target/i386/memset-pr120683-21.c: Likewise.
+	* gcc.target/i386/memset-pr120683-22.c: Likewise.
+	* gcc.target/i386/memset-pr120683-23.c: Likewise.
+
+2025-07-07  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/84009
+	* g++.dg/cpp0x/range-for40.C: New test.
+	* g++.dg/cpp0x/range-for41.C: New test.
+	* g++.dg/cpp0x/range-for42.C: New test.
+	* g++.dg/cpp0x/range-for43.C: New test.
+
+2025-07-07  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* gfortran.dg/move_alloc_20.f03: New test.
+
+2025-07-07  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120709
+	* gcc.dg/crc-non-cst-poly-1.c: New test.
+
+2025-07-06  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/120951
+	* gcc.dg/torture/pr120951-1.c: New test.
+
+2025-07-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/tree-prof/clone-merge-1.c:
+
+2025-07-04  Vineet Gupta  <vineetg@rivosinc.com>
+
+	PR target/118241
+	* gcc.target/riscv/pr118241-b.cc: New test.
+
+2025-07-04  Raphael Moreira Zinsly  <rzinsly@ventanamicro.com>
+
+	* gcc.target/sh/pr54236-2.c: Fix comments and expected output
+
+2025-07-04  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c/118948
+	* gcc.dg/pr118948-1.c: New test.
+
+2025-07-04  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120575
+	PR c++/116064
+	* g++.dg/template/permissive-error3.C: New test.
+
+2025-07-04  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+
+2025-07-04  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c: New test.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120944
+	* gcc.dg/torture/pr120944.c: New testcase.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120927
+	* gcc.dg/vect/vect-pr120927.c: New testcase.
+	* gcc.dg/vect/vect-pr120927-2.c: Likewise.
+
+2025-07-04  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c/120837
+	* gcc.dg/ubsan/pr120837.c: New test.
+
+2025-07-04  Xi Ruoyao  <xry111@xry111.site>
+
+	PR target/120807
+	* gcc.c-torture/compile/pr120708.c: Rename to ...
+	* gcc.c-torture/compile/pr120807.c: ... here.
+
+2025-07-04  Xi Ruoyao  <xry111@xry111.site>
+
+	* gcc.c-torture/compile/pr120708.c: New test.
+
+2025-07-04  panciyan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Add signed scalar SAT_ADD IMM form2.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i16.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i32.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i64.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i8.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c: New test.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120716
+	* g++.dg/cpp0x/lambda/lambda-constexpr3.C: New test.
+	* g++.dg/cpp0x/lambda/lambda-constexpr3a.C: New test.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120748
+	* g++.dg/cpp2a/lambda-targ16.C: New test.
+	* g++.dg/cpp0x/this1.C: Adjust diagnostics.
+
+2025-07-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120940
+	* g++.dg/parse/pr120940.C: New test.
+	* g++.dg/warn/Wduplicated-branches9.C: New test.
+
+2025-07-03  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/vec-perm-merge-1.c: New test.
+	* gcc.target/s390/vector/vec-perm-pack-1.c: New test.
+
+2025-07-03  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* c-c++-common/gomp/omp_get_num_devices_initial_device.c: Fix.
+	* gfortran.dg/gomp/omp_get_num_devices_initial_device.f90: Likewise.
+
+2025-07-03  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.dg/ipa/pr120295.c (glob): Rename to glob_.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/cmpbr.c: Update tests.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/cmpbr.c:
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* lib/target-supports.exp: Add `cmpbr` to the list of extensions.
+	* gcc.target/aarch64/cmpbr.c: New test.
+
+2025-07-03  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	PR tree-optimization/120780
+	* gcc.dg/builtin-dynamic-object-size-pr120780.c: New test case.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120936
+	* gcc.target/i386/pr120936-1.c: New test
+	* gcc.target/i386/pr120936-2.c: Likewise.
+	* gcc.target/i386/pr120936-3.c: Likewise.
+	* gcc.target/i386/pr120936-4.c: Likewise.
+	* gcc.target/i386/pr120936-5.c: Likewise.
+	* gcc.target/i386/pr120936-6.c: Likewise.
+	* gcc.target/i386/pr120936-7.c: Likewise.
+	* gcc.target/i386/pr120936-8.c: Likewise.
+	* gcc.target/i386/pr120936-9.c: Likewise.
+	* gcc.target/i386/pr120936-10.c: Likewise.
+	* gcc.target/i386/pr120936-11.c: Likewise.
+	* gcc.target/i386/pr120936-12.c: Likewise.
+	* gcc.target/i386/pr93492-3.c: Updated.
+	* gcc.target/i386/pr93492-5.c: Likewise.
+
+2025-07-03  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120843
+	* gfortran.dg/coarray/coindexed_6.f90: Enhance test to have
+	coarray components covered.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120908
+	* gcc.target/i386/pr120908.c: New test.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120684
+	PR c++/118856
+	* g++.dg/cpp23/range-for10.C: New test.
+
 2025-07-02  Dimitar Dimitrov  <dimitar@dinux.eu>
 
 	* gcc.target/riscv/mcpu-xt-c908.c: Disable for E ABI variants.
diff --git a/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada b/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada
index 0cc14f4..df38f99 100644
--- a/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada
+++ b/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada
@@ -211,7 +211,7 @@ BEGIN
 
      BEGIN -- (E)
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
@@ -254,7 +254,7 @@ BEGIN
 
      BEGIN
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c940005.a b/gcc/testsuite/ada/acats-4/tests/c9/c940005.a
index adb58b1..47a97bf 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c940005.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c940005.a
@@ -85,7 +85,7 @@ begin
       -- In reality one would expect a time of 5 to 10 seconds.  In
       -- the interests of speeding up the test suite a shorter time
       -- is used
-      Pulse_Time_Delta : constant duration := ImpDef.Switch_To_New_Task;
+      Pulse_Time_Delta : constant duration := ImpDef.Long_Switch_To_New_Task;
 
       -- control over stopping tasks
       protected Control is
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c940007.a b/gcc/testsuite/ada/acats-4/tests/c9/c940007.a
index c678463..41e80f4 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c940007.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c940007.a
@@ -90,7 +90,7 @@ begin
       -- In reality one would expect a time of 5 to 10 seconds.  In
       -- the interests of speeding up the test suite a shorter time
       -- is used
-      Pulse_Time_Delta : constant duration := ImpDef.Switch_To_New_Task;
+      Pulse_Time_Delta : constant duration := ImpDef.Long_Switch_To_New_Task;
 
 
       -- control over stopping tasks
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada b/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada
index 0cc14f4..df38f99 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada
@@ -211,7 +211,7 @@ BEGIN
 
      BEGIN -- (E)
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
@@ -254,7 +254,7 @@ BEGIN
 
      BEGIN
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada b/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada
index 6b9c85f..cac5fc6 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada
@@ -28,6 +28,7 @@
 -- TBN  9/17/86
 -- PWN 01/31/95  REMOVED PRAGMA PRIORITY FOR ADA 9X.
 
+with Impdef;
 WITH REPORT; USE REPORT;
 WITH SYSTEM; USE SYSTEM;
 PROCEDURE C94006A IS
@@ -41,7 +42,7 @@ PROCEDURE C94006A IS
           SELECT
                ACCEPT E;
           OR
-               DELAY 30.0;
+               DELAY 30.0 * Impdef.One_Long_Second;
           END SELECT;
      END TT;
 
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada b/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada
index 6d10e25..fb2eee9 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada
@@ -33,6 +33,7 @@
 -- JBG 8/29/86 ELIMINATED SHARED VARIABLES; ADDED GENERIC UNIT
 -- PWN 11/30/94 REMOVED PRAGMA PRIORITY INSTANCES FOR ADA 9X.
 
+with Impdef;
 WITH REPORT; USE REPORT;
 WITH SYSTEM; USE SYSTEM;
 PROCEDURE C94008C IS
@@ -198,10 +199,10 @@ BEGIN -- C94008C
                     OR WHEN ENTER_TERMINATE => TERMINATE;
                     END SELECT;
 
-                    DELAY 10.0;
+                    DELAY 10.0 * Impdef.One_Second;
 
                     IF TERMINATE_COUNT.GET /= 1 THEN
-                         DELAY 20.0;
+                         DELAY 20.0 * Impdef.One_Long_Second;
                     END IF;
 
                     IF TERMINATE_COUNT.GET /= 1 THEN
@@ -239,10 +240,10 @@ BEGIN -- C94008C
 
      BEGIN
 
-          DELAY 10.0; -- WAIT FOR T1, T2, AND T3 TO GET TO SELECT STMTS.
+          DELAY 10.0 * Impdef.One_Second; -- WAIT FOR T1, T2, AND T3 TO GET TO SELECT STMTS.
 
            IF TERMINATE_COUNT.GET /= 3 THEN
-                DELAY 20.0;
+                DELAY 20.0 * Impdef.One_Long_Second;
            END IF;
 
            IF TERMINATE_COUNT.GET /= 3 THEN
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c951002.a b/gcc/testsuite/ada/acats-4/tests/c9/c951002.a
index 8ccb2d0..65b696c 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c951002.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c951002.a
@@ -278,14 +278,14 @@ begin
       -- Wait until the message is queued on the entry before starting
       -- the Credit_Task
       while not Hold.TC_Message_is_Queued loop
-         delay ImpDef.Minimum_Task_Switch;   
+         delay ImpDef.Long_Minimum_Task_Switch;   
       end loop;
       --
       Credit_Task.TC_Start;
 
       -- Ensure the first part of the test is complete before continuing
       while not (Credit_Message'terminated and Credit_Task'terminated) loop
-         delay ImpDef.Minimum_Task_Switch;   
+         delay ImpDef.Long_Minimum_Task_Switch;   
       end loop;
 
       --======================================================
@@ -298,12 +298,12 @@ begin
       -- for it to reach the accept statement and call Hold.Set_DB_Overload
       -- before starting Debit_Message
       --
-      delay ImpDef.Switch_To_New_Task;
+      delay ImpDef.Long_Switch_To_New_Task;
 
       Debit_Message.TC_Start;
 
       while not Debit_Task'terminated loop
-         delay ImpDef.Minimum_Task_Switch;   
+         delay ImpDef.Long_Minimum_Task_Switch;   
       end loop;    
   
       Hold.Clear_DB_Overload;  -- Allow completion 
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a b/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a
index 34f48b2..3ea545a 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a
@@ -148,7 +148,7 @@ package body C954A01_0 is  -- Printer server abstraction.
          end select;
 
          -- Allow other tasks to get control
-         delay ImpDef.Minimum_Task_Switch;
+         delay ImpDef.Long_Minimum_Task_Switch;
 
       end loop;
 
@@ -175,7 +175,7 @@ use  F954A00;
 
 procedure C954A01 is
 
-   Long_Enough : constant Duration := ImpDef.Switch_To_New_Task;
+   Long_Enough : constant Duration := ImpDef.Long_Switch_To_New_Task;
 
                --==============================================--
 
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada b/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada
index 74374b9..f958ea1 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada
@@ -36,6 +36,7 @@
 --     RJW 11/13/87  ADDED CODE WHICH ALLOWS TEST TO REPORT "PASSED"
 --                   IF TICK > DURATION'SMALL.
 
+with Impdef;
 WITH CALENDAR;  USE CALENDAR;
 WITH SYSTEM;    USE SYSTEM;
 WITH REPORT;    USE REPORT;
@@ -50,7 +51,7 @@ BEGIN
      ---------------------------------------------
 
      DECLARE   -- (A)
-          X : DURATION := 5.0;
+          X : DURATION := 5.0 * Impdef.One_Second;
           OLD_TIME : TIME;
           LAPSE : DURATION;
      BEGIN     -- (A)
@@ -138,8 +139,8 @@ BEGIN
      ---------------------------------------------
 
      DECLARE   -- (E)
-          INC1 : DURATION := 2.0;
-          INC2 : DURATION := 3.0;
+          INC1 : DURATION := 2.0 * Impdef.One_Second;
+          INC2 : DURATION := 3.0 * Impdef.One_Second;
           OLD_TIME : TIME;
           LAPSE : DURATION;
      BEGIN     -- (E)
diff --git a/gcc/testsuite/c-c++-common/Warray-bounds-11.c b/gcc/testsuite/c-c++-common/Warray-bounds-11.c
new file mode 100644
index 0000000..686a94d
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Warray-bounds-11.c
@@ -0,0 +1,21 @@
+/* PR c++/120954 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Warray-bounds=2" } */
+
+static const int a[32] = { 11, 12, 13, 14, 15 };
+static const int b[32] = { 21, 22, 23, 24, 25 };
+static const int c[32] = { 31, 32, 33, 34, 35 };
+static const int d[32] = { 111, 112, 113, 114, 115 };
+static const int e[32] = { 121, 122, 123, 124, 125 };
+static const int f[32] = { 131, 132, 133, 134, 135 };
+
+int
+foo (int x, int y)
+{
+  int r = 0;
+  if (x >= 0 && x < 32)
+    r = (y >= 4 ? (y >= 0x65 ? a : b ) : c)[x];
+  else if (x >= 0x100 && x < 0x120)
+    r = (y >= 4 ? (y >= 0x65 ? d : e ) : f)[x - 0x100];
+  return r;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-1.c b/gcc/testsuite/c-c++-common/Wunused-parm-1.c
new file mode 100644
index 0000000..355fa4a
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-1.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,		/* { dg-warning "parameter 'f' set but not used" } */
+     int g,		/* { dg-warning "parameter 'g' set but not used" } */
+     int h,		/* { dg-warning "parameter 'h' set but not used" } */
+     int i,		/* { dg-warning "parameter 'i' set but not used" } */
+     int j,		/* { dg-warning "parameter 'j' set but not used" } */
+     int k,		/* { dg-warning "parameter 'k' set but not used" } */
+     int l,		/* { dg-warning "parameter 'l' set but not used" } */
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-2.c b/gcc/testsuite/c-c++-common/Wunused-parm-2.c
new file mode 100644
index 0000000..2caea94
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-2.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused -Wextra" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,		/* { dg-warning "parameter 'f' set but not used" } */
+     int g,		/* { dg-warning "parameter 'g' set but not used" } */
+     int h,		/* { dg-warning "parameter 'h' set but not used" } */
+     int i,		/* { dg-warning "parameter 'i' set but not used" } */
+     int j,		/* { dg-warning "parameter 'j' set but not used" } */
+     int k,		/* { dg-warning "parameter 'k' set but not used" } */
+     int l,		/* { dg-warning "parameter 'l' set but not used" } */
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-3.c b/gcc/testsuite/c-c++-common/Wunused-parm-3.c
new file mode 100644
index 0000000..2978cd4
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-3.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=3" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,		/* { dg-warning "parameter 'f' set but not used" } */
+     int g,		/* { dg-warning "parameter 'g' set but not used" } */
+     int h,		/* { dg-warning "parameter 'h' set but not used" } */
+     int i,		/* { dg-warning "parameter 'i' set but not used" } */
+     int j,		/* { dg-warning "parameter 'j' set but not used" } */
+     int k,		/* { dg-warning "parameter 'k' set but not used" } */
+     int l,		/* { dg-warning "parameter 'l' set but not used" } */
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-4.c b/gcc/testsuite/c-c++-common/Wunused-parm-4.c
new file mode 100644
index 0000000..063b40f
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-4.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=2" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,
+     int g,
+     int h,
+     int i,
+     int j,
+     int k,
+     int l,
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-5.c b/gcc/testsuite/c-c++-common/Wunused-parm-5.c
new file mode 100644
index 0000000..1c80a82
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-5.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=1" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,
+     int c,
+     int d,
+     int e,
+     int f,
+     int g,
+     int h,
+     int i,
+     int j,
+     int k,
+     int l,
+     int m)
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-6.c b/gcc/testsuite/c-c++-common/Wunused-parm-6.c
new file mode 100644
index 0000000..ee328bd
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-6.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=0" } */
+
+void baz (int);
+
+void
+foo (int a,
+     int b,
+     int c,
+     int d,
+     int e,
+     int f,
+     int g,
+     int h,
+     int i,
+     int j,
+     int k,
+     int l,
+     int m)
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-19.c b/gcc/testsuite/c-c++-common/Wunused-var-19.c
new file mode 100644
index 0000000..32c47e6
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-19.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;		/* { dg-warning "variable 'f' set but not used" } */
+  f += 2;
+  int g = 0;		/* { dg-warning "variable 'g' set but not used" } */
+  g |= 2;
+  int h = 0;		/* { dg-warning "variable 'h' set but not used" } */
+  h -= 2;
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  i &= 2;
+  int j = 0;		/* { dg-warning "variable 'j' set but not used" } */
+  j ^= 2;
+  int k = 0;		/* { dg-warning "variable 'k' set but not used" } */
+  k *= 2;
+  int l = 0;		/* { dg-warning "variable 'l' set but not used" } */
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-20.c b/gcc/testsuite/c-c++-common/Wunused-var-20.c
new file mode 100644
index 0000000..e25b26b
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-20.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;		/* { dg-warning "variable 'f' set but not used" } */
+  f += 2;
+  int g = 0;		/* { dg-warning "variable 'g' set but not used" } */
+  g |= 2;
+  int h = 0;		/* { dg-warning "variable 'h' set but not used" } */
+  h -= 2;
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  i &= 2;
+  int j = 0;		/* { dg-warning "variable 'j' set but not used" } */
+  j ^= 2;
+  int k = 0;		/* { dg-warning "variable 'k' set but not used" } */
+  k *= 2;
+  int l = 0;		/* { dg-warning "variable 'l' set but not used" } */
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-21.c b/gcc/testsuite/c-c++-common/Wunused-var-21.c
new file mode 100644
index 0000000..0732d98
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-21.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=3" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;		/* { dg-warning "variable 'f' set but not used" } */
+  f += 2;
+  int g = 0;		/* { dg-warning "variable 'g' set but not used" } */
+  g |= 2;
+  int h = 0;		/* { dg-warning "variable 'h' set but not used" } */
+  h -= 2;
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  i &= 2;
+  int j = 0;		/* { dg-warning "variable 'j' set but not used" } */
+  j ^= 2;
+  int k = 0;		/* { dg-warning "variable 'k' set but not used" } */
+  k *= 2;
+  int l = 0;		/* { dg-warning "variable 'l' set but not used" } */
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-22.c b/gcc/testsuite/c-c++-common/Wunused-var-22.c
new file mode 100644
index 0000000..84f57c5
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-22.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=2" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;
+  f += 2;
+  int g = 0;
+  g |= 2;
+  int h = 0;
+  h -= 2;
+  int i = 0;
+  i &= 2;
+  int j = 0;
+  j ^= 2;
+  int k = 0;
+  k *= 2;
+  int l = 0;
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-23.c b/gcc/testsuite/c-c++-common/Wunused-var-23.c
new file mode 100644
index 0000000..b74c3f4
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-23.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=1" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;
+  ++b;
+  int c = 0;
+  c++;
+  int d = 0;
+  --d;
+  int e = 0;
+  e--;
+  int f = 0;
+  f += 2;
+  int g = 0;
+  g |= 2;
+  int h = 0;
+  h -= 2;
+  int i = 0;
+  i &= 2;
+  int j = 0;
+  j ^= 2;
+  int k = 0;
+  k *= 2;
+  int l = 0;
+  l %= 2;
+  int m = 0;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-24.c b/gcc/testsuite/c-c++-common/Wunused-var-24.c
new file mode 100644
index 0000000..a59f50a
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-24.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=0" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;
+  a = 1;
+  int b = 0;
+  ++b;
+  int c = 0;
+  c++;
+  int d = 0;
+  --d;
+  int e = 0;
+  e--;
+  int f = 0;
+  f += 2;
+  int g = 0;
+  g |= 2;
+  int h = 0;
+  h -= 2;
+  int i = 0;
+  i &= 2;
+  int j = 0;
+  j ^= 2;
+  int k = 0;
+  k *= 2;
+  int l = 0;
+  l %= 2;
+  int m = 0;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-7.c b/gcc/testsuite/c-c++-common/Wunused-var-7.c
index 7419643..ea6babd 100644
--- a/gcc/testsuite/c-c++-common/Wunused-var-7.c
+++ b/gcc/testsuite/c-c++-common/Wunused-var-7.c
@@ -24,7 +24,7 @@ foo (void)
 void
 bar (void)
 {
-  int a;
+  int a;		/* { dg-warning "set but not used" } */
   int b;
   int c;		/* { dg-warning "set but not used" } */
   a = 1;
@@ -36,7 +36,7 @@ bar (void)
 void
 baz (void)
 {
-  int a;
+  int a;		/* { dg-warning "set but not used" } */
   int b;
   int c;
   int d;
diff --git a/gcc/testsuite/c-c++-common/attr-warn-unused-result-2.c b/gcc/testsuite/c-c++-common/attr-warn-unused-result-2.c
new file mode 100644
index 0000000..f35198d
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/attr-warn-unused-result-2.c
@@ -0,0 +1,16 @@
+/* PR c/82134 */
+/* { dg-do compile } */
+/* { dg-additional-options -Wno-c++-compat { target c } } */
+
+struct S {};
+
+__attribute__((warn_unused_result)) struct S foo();
+
+void use_s(struct S);
+
+void
+test (void)
+{
+  struct S s = foo(); /* { dg-bogus "ignoring return value of" } */
+  use_s(foo()); /* { dg-bogus "ignoring return value of" } */
+}
diff --git a/gcc/testsuite/c-c++-common/cpp/va-opt-6.c b/gcc/testsuite/c-c++-common/cpp/va-opt-6.c
index 8a7761b..9a92431 100644
--- a/gcc/testsuite/c-c++-common/cpp/va-opt-6.c
+++ b/gcc/testsuite/c-c++-common/cpp/va-opt-6.c
@@ -3,15 +3,15 @@
 /* { dg-options "-std=c++20" { target c++ } } */
 
 #define a ""
-#define b(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting \"a\" and \"\"\"\" does not give a valid preprocessing token" } */
-#define c(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting \"a\" and \"\"1\"\" does not give a valid preprocessing token" } */
+#define b(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting 'a' and '\"\"' does not give a valid preprocessing token" } */
+#define c(...) a ## #__VA_OPT__(1)	/* { dg-error "pasting 'a' and '\"1\"' does not give a valid preprocessing token" } */
 #define d(...) #__VA_OPT__(1) ## !
 #define e(...) #__VA_OPT__(1) ## !
 #define f(...) #__VA_OPT__(. ## !)
 #define g(...) #__VA_OPT__(. ## !)
 b()
 c(1)
-d(   )		/* { dg-error "pasting \"\"\"\" and \"!\" does not give a valid preprocessing token" } */
-e(  1 )		/* { dg-error "pasting \"\"1\"\" and \"!\" does not give a valid preprocessing token" } */
+d(   )		/* { dg-error "pasting '\"\"' and '!' does not give a valid preprocessing token" } */
+e(  1 )		/* { dg-error "pasting '\"1\"' and '!' does not give a valid preprocessing token" } */
 f()
-g(0)		/* { dg-error "pasting \".\" and \"!\" does not give a valid preprocessing token" } */
+g(0)		/* { dg-error "pasting '.' and '!' does not give a valid preprocessing token" } */
diff --git a/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c b/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c
index 4b17143..6e2c1a8 100644
--- a/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c
+++ b/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c
@@ -25,8 +25,8 @@ int f()
 
 /* { dg-final { scan-tree-dump-not "abort" "optimized" } }  */
 
-/* { dg-final { scan-tree-dump-not "omp_get_num_devices;" "optimized" { target { ! offloading_enabled } } } }  */
+/* { dg-final { scan-tree-dump-not "omp_get_num_devices" "optimized" { target { ! offloading_enabled } } } }  */
 /* { dg-final { scan-tree-dump "return 0;" "optimized" { target { ! offloading_enabled } } } }  */
 
-/* { dg-final { scan-tree-dump-times "omp_get_num_devices;" 1 "optimized" { target offloading_enabled } } }  */
+/* { dg-final { scan-tree-dump-times "omp_get_num_devices" 1 "optimized" { target offloading_enabled } } }  */
 /* { dg-final { scan-tree-dump "_1 = __builtin_omp_get_num_devices \\(\\);\[\\r\\n\]+\[ \]+return _1;" "optimized" { target offloading_enabled } } }  */
diff --git a/gcc/testsuite/c-c++-common/musttail32.c b/gcc/testsuite/c-c++-common/musttail32.c
new file mode 100644
index 0000000..f1b7052
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/musttail32.c
@@ -0,0 +1,23 @@
+/* PR ipa/121023 */
+/* { dg-do compile { target musttail } } */
+/* { dg-options "-O2" } */
+
+struct S { int a, b; };
+
+[[gnu::noipa]] int
+foo (struct S x, int y, int z)
+{
+  return x.a + y + z;
+}
+
+[[gnu::noinline]] static int
+bar (struct S x, int y, int z)
+{
+  [[gnu::musttail]] return foo ((struct S) { x.a, 0 }, y, 1);
+}
+
+int
+baz (int x)
+{
+  return bar ((struct S) { 1, 2 }, x, 2) + bar ((struct S) { 2, 3 }, x + 1, 2);
+}
diff --git a/gcc/testsuite/c-c++-common/pr121159.c b/gcc/testsuite/c-c++-common/pr121159.c
new file mode 100644
index 0000000..c8c5d67
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/pr121159.c
@@ -0,0 +1,17 @@
+/* PR middle-end/121159 */
+/* { dg-do compile { target musttail } } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times "foo \\\(\[^\n\r]*\\\); \\\[tail call\\\] \\\[must tail call\\\]" 1 "optimized" } } */
+		
+[[noreturn, gnu::noipa]] void
+foo (void)
+{
+  for (;;)
+    ;
+}
+
+void
+bar (void)
+{
+  [[gnu::musttail]] return foo ();
+}
diff --git a/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob b/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob
index 7843d3d..f344a84 100644
--- a/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob
+++ b/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob
@@ -1,7 +1,7 @@
        *> { dg-do run }
        *> { dg-options "-static" }
-       *> { dg-output-file "group2/_-static__compilation.out" }
-
+       *> { dg-prune-output {warning} }
+       *> { dg-output {hello, world} }
        IDENTIFICATION DIVISION.
        PROGRAM-ID. prog.
        PROCEDURE DIVISION.
diff --git a/gcc/testsuite/cobol.dg/group2/_-static__compilation.out b/gcc/testsuite/cobol.dg/group2/_-static__compilation.out
deleted file mode 100644
index ae0e511..0000000
--- a/gcc/testsuite/cobol.dg/group2/_-static__compilation.out
+++ /dev/null
@@ -1,2 +0,0 @@
-hello, world
-
diff --git a/gcc/testsuite/g++.dg/DRs/dr1709.C b/gcc/testsuite/g++.dg/DRs/dr1709.C
new file mode 100644
index 0000000..d3854d8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/DRs/dr1709.C
@@ -0,0 +1,18 @@
+// DR 1709 - Stringizing raw string literals containing newline
+// { dg-do run { target c++26 } }
+
+#define A(a) #a
+const char *a = A(a\f\\b"c");
+const char *b = A(R"abc(a\b
+
+)abc");
+
+int
+main ()
+{
+  if (a[1] != '\f' || a[2] != '\\' || a[4] != '"' || a[6] != '"')
+    __builtin_abort ();
+  if (b[1] != '"' || b[7] != '\\' || b[9] != '\n' || b[10] != '\n'
+      || b[11] != ')' || b[15] != '"')
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/g++.dg/DRs/dr2578.C b/gcc/testsuite/g++.dg/DRs/dr2578.C
new file mode 100644
index 0000000..0dce23b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/DRs/dr2578.C
@@ -0,0 +1,10 @@
+// DR 2578 - Undefined behavior when creating an invalid string literal via stringizing
+// { dg-do preprocess }
+// { dg-options "-pedantic-errors" }
+
+#define A(a) #a
+#define B(a) A(a)
+#define C \\
+
+const char *x = B(C);	// { dg-warning "invalid string literal, ignoring final '\\\\'" "" { target c++23_down } }
+// { dg-error "invalid string literal, ignoring final '\\\\'" "" { target c++26 }  .-1 }
diff --git a/gcc/testsuite/g++.dg/DRs/dr2579.C b/gcc/testsuite/g++.dg/DRs/dr2579.C
new file mode 100644
index 0000000..dda3643
--- /dev/null
+++ b/gcc/testsuite/g++.dg/DRs/dr2579.C
@@ -0,0 +1,9 @@
+// DR 2579 - Undefined behavior when token pasting does not create a preprocessing token
+// { dg-do preprocess }
+// { dg-options "-pedantic-errors" }
+
+#define A(a, b) a ## b
+A(5,6)
+A(-,32)			// { dg-error "pasting '-' and '32' does not give a valid preprocessing token" }
+A("","")		// { dg-error "pasting '\"\"' and '\"\"' does not give a valid preprocessing token" }
+A(\,u0393)
diff --git a/gcc/testsuite/g++.dg/DRs/dr2580.C b/gcc/testsuite/g++.dg/DRs/dr2580.C
new file mode 100644
index 0000000..462f300
--- /dev/null
+++ b/gcc/testsuite/g++.dg/DRs/dr2580.C
@@ -0,0 +1,87 @@
+// DR 2580 - Undefined behavior with #line
+// { dg-do preprocess }
+
+#line 630 "foobar.h"
+#line 6 "dr2580.C"
+#line 0				// { dg-error "line number out of range" }
+#line 8
+#line 2147483648		// { dg-error "line number out of range" }
+#line 10
+#line 4294967295		// { dg-error "line number out of range" }
+#line 12
+#line 12345678912345		// { dg-error "line number out of range" }
+#line 14
+#line 15 ""
+#line 16 "foobar.h"
+#line 17 "/a/b/c/baz.h"
+#line 18 "dr2580.C"
+#line 0 "dr2580.C"		// { dg-error "line number out of range" }
+#line 20
+#line 2147483648 "dr2580.C"	// { dg-error "line number out of range" }
+#line 22
+#line 4294967295 "dr2580.C"	// { dg-error "line number out of range" }
+#line 24
+#line 12345678912345 "dr2580.C"	// { dg-error "line number out of range" }
+#line 26
+#line 27 1			// { dg-error "'1' is not a valid filename" }
+#line 28
+#line 29 foo bar baz		// { dg-error "'foo' is not a valid filename" }
+#line 30
+#line 31 "dr2580.C" 1		// { dg-error "extra tokens at end of '#line' directive" }
+#line 32
+#line 33 "dr2580.C" foo bar baz	// { dg-error "extra tokens at end of '#line' directive" }
+#define A 35
+#line A
+#define B 0
+#line B				// { dg-error "line number out of range" }
+#line 38
+#define C 2147483648
+#line C				// { dg-error "line number out of range" }
+#line 41
+#define D 4294967295
+#line D				// { dg-error "line number out of range" }
+#line 44
+#define E 12345678912345
+#line E				// { dg-error "line number out of range" }
+#line 47
+#define F 49 "dr2580.C"
+#line F
+#define G 0 "dr2580.C"
+#line G				// { dg-error "line number out of range" }
+#line 52 "dr2580.C"
+#define H 2147483647 "dr2580.C"
+#line H				// { dg-error "line number out of range" "" { target c++98_only } }
+#line 55
+#define I 2147483648 "dr2580.C"
+#line I				// { dg-error "line number out of range" }
+#line 58
+#define J 4294967295 "dr2580.C"
+#line J				// { dg-error "line number out of range" }
+#line 61
+#define K 12345678912345 "dr2580.C"
+#line K				// { dg-error "line number out of range" }
+#line 64
+#define M 7 0
+#line M				// { dg-error "'0' is not a valid filename" }
+#line 67
+#define N 90 foo bar baz
+#line N				// { dg-error "'foo' is not a valid filename" }
+#line 70
+#define O 75 "dr2580.C" 2
+#line O				// { dg-error "extra tokens at end of '#line' directive" }
+#line 73
+#define P 90 "dr2580.C" foo bar baz
+#line P				// { dg-error "extra tokens at end of '#line' directive" }
+#line 76
+#line -5			// { dg-error "'-' after '#line' is not a positive integer" }
+#line 78
+#line -7 "dr2580.C"		// { dg-error "'-' after '#line' is not a positive integer" }
+#line 80
+#line 32767
+#line 82
+#line 32768			// { dg-error "line number out of range" "" { target c++98_only } }
+#line 84
+#line 32767 "dr2580.C"
+#line 86
+#line 32768 "dr2580.C"		// { dg-error "line number out of range" "" { target c++98_only } }
+#line 88
diff --git a/gcc/testsuite/g++.dg/abi/regparm1.C b/gcc/testsuite/g++.dg/abi/regparm1.C
index c471046..3aae3dd 100644
--- a/gcc/testsuite/g++.dg/abi/regparm1.C
+++ b/gcc/testsuite/g++.dg/abi/regparm1.C
@@ -1,5 +1,5 @@
 // PR c++/29911 (9381)
-// { dg-do run { target i?86-*-* x86_64-*-* } }
+// { dg-do run { target { { i?86-*-* x86_64-*-* } && ia32 } } }
 // { dg-require-effective-target c++11 }
 
 extern "C" int printf(const char *, ...);
diff --git a/gcc/testsuite/g++.dg/concepts/auto7a.C b/gcc/testsuite/g++.dg/concepts/auto7a.C
index 88868f4..f36038d 100644
--- a/gcc/testsuite/g++.dg/concepts/auto7a.C
+++ b/gcc/testsuite/g++.dg/concepts/auto7a.C
@@ -2,6 +2,7 @@
 
 template <class T> struct A { };
 void f(A<auto> a) { }		// { dg-error "auto. in template argument" }
+// { dg-message "in parameter declaration" "" { target c++17_down } .-1 }
 int main()
 {
   f(A<int>());
diff --git a/gcc/testsuite/g++.dg/concepts/auto7b.C b/gcc/testsuite/g++.dg/concepts/auto7b.C
new file mode 100644
index 0000000..874192c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/auto7b.C
@@ -0,0 +1,10 @@
+// PR c++/120917
+// { dg-do compile { target c++14 } }
+// { dg-additional-options "-fconcepts -Wno-abbreviated-auto-in-template-arg" }
+
+template <class T> struct A { };
+void f(A<auto> a) { }
+int main()
+{
+  f(A<int>());
+}
diff --git a/gcc/testsuite/g++.dg/concepts/auto7c.C b/gcc/testsuite/g++.dg/concepts/auto7c.C
new file mode 100644
index 0000000..5b16027
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/auto7c.C
@@ -0,0 +1,12 @@
+// { dg-do compile { target c++17 } }
+// { dg-additional-options -fconcepts }
+
+template <class T>
+concept True = true;
+
+template <class T> struct A { };
+void f(A<True auto> a) { } // { dg-error "use of .auto. in template argument" }
+int main()
+{
+  f(A<int>());
+}
diff --git a/gcc/testsuite/g++.dg/concepts/pr67249.C b/gcc/testsuite/g++.dg/concepts/pr67249.C
index e0f8d5a..e97183d 100644
--- a/gcc/testsuite/g++.dg/concepts/pr67249.C
+++ b/gcc/testsuite/g++.dg/concepts/pr67249.C
@@ -4,4 +4,4 @@
 template<class T> concept C1 = true;
 template<class A, class B> struct Pair {};
 // We used to test "Pair<auto, C1 >".
-void f(Pair<C1 auto, C1 auto>);
+void f(Pair<C1 auto, C1 auto>);	// { dg-error "auto" }
diff --git a/gcc/testsuite/g++.dg/concepts/pr67249a.C b/gcc/testsuite/g++.dg/concepts/pr67249a.C
new file mode 100644
index 0000000..cb5d90e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/pr67249a.C
@@ -0,0 +1,7 @@
+// { dg-do compile { target c++17 } }
+// { dg-options "-fconcepts -Wno-abbreviated-auto-in-template-arg" }
+
+template<class T> concept C1 = true;
+template<class A, class B> struct Pair {};
+// We used to test "Pair<auto, C1 >".
+void f(Pair<C1 auto, C1 auto>);
diff --git a/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-1.C b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-1.C
new file mode 100644
index 0000000..16bfef1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-1.C
@@ -0,0 +1,33 @@
+// PR c++/120243
+// { dg-do run }
+
+#include <coroutine>
+
+struct coro {
+     struct promise_type {
+        promise_type() = default;
+
+        std::suspend_never initial_suspend() const noexcept { return {}; }
+        std::suspend_never final_suspend() const noexcept { return {}; }
+
+        void unhandled_exception() { throw; }
+
+        coro get_return_object() { return {}; }
+        void return_void() {}
+
+    };
+};
+
+int main() {
+    auto c = []() -> coro {
+        throw "hello";
+        __builtin_abort();
+        co_return;
+    };
+    try {
+        c();
+    }
+    catch(...) {
+        __builtin_printf("Caught!\n");
+    }
+}
diff --git a/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-2.C b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-2.C
new file mode 100644
index 0000000..614c4ec
--- /dev/null
+++ b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-2.C
@@ -0,0 +1,34 @@
+// PR c++/120243
+// { dg-do run }
+
+#include <coroutine>
+
+struct coro {
+     struct promise_type {
+        promise_type() = default;
+
+        std::suspend_always initial_suspend() const noexcept { return {}; }
+        std::suspend_always final_suspend() const noexcept { return {}; }
+
+        void unhandled_exception() { throw; }
+
+        coro get_return_object() { return {std::coroutine_handle<promise_type>::from_promise(*this)}; }
+        void return_void() {}
+    };
+
+    std::coroutine_handle<promise_type> h;
+};
+
+int main() {
+    auto c = []() -> coro {
+        throw "hello";
+        __builtin_abort();
+        co_return;
+    };
+    try {
+        c().h.resume();
+    }
+    catch(...) {
+        __builtin_printf("Caught!\n");
+    }
+}
diff --git a/gcc/testsuite/g++.dg/coroutines/torture/pr121219.C b/gcc/testsuite/g++.dg/coroutines/torture/pr121219.C
new file mode 100644
index 0000000..d1e7cb1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/coroutines/torture/pr121219.C
@@ -0,0 +1,149 @@
+// PR c++/121219
+// { dg-do run }
+
+#include <coroutine>
+#ifdef OUTPUT
+#include <iostream>
+#endif
+#include <stdexcept>
+
+struct Task {
+    struct promise_type;
+    using handle_type = std::coroutine_handle<promise_type>;
+
+    struct promise_type {
+        Task* task_;
+        int result_;
+
+        static void* operator new(std::size_t size) noexcept {
+            void* p = ::operator new(size, std::nothrow);
+#ifdef OUTPUT
+            std::cerr << "operator new (no arg) " << size << " -> " << p << std::endl;  
+#endif
+            return p;
+        }
+        static void operator delete(void* ptr) noexcept {
+            return ::operator delete(ptr, std::nothrow);
+        }
+#if 1 // change to 0 to fix crash
+        static Task get_return_object_on_allocation_failure() noexcept {
+#ifdef OUTPUT
+            std::cerr << "get_return_object_on_allocation_failure" << std::endl;
+#endif
+            return Task(nullptr);
+        }
+#endif
+
+        auto get_return_object() { 
+#ifdef OUTPUT
+            std::cerr << "get_return_object" << std::endl;
+#endif
+            return Task{handle_type::from_promise(*this)}; 
+        }
+
+        auto initial_suspend() { 
+#ifdef OUTPUT
+            std::cerr << "initial_suspend" << std::endl;
+#endif
+            return std::suspend_always{}; 
+        }
+
+        auto final_suspend() noexcept { 
+#ifdef OUTPUT
+            std::cerr << "final_suspend" << std::endl;
+#endif
+            return std::suspend_never{};  // Coroutine auto-destructs
+        }
+
+        ~promise_type() {
+            if (task_) {
+#ifdef OUTPUT
+                std::cerr << "promise_type destructor: Clearing Task handle" << std::endl;
+#endif
+                task_->h_ = nullptr;
+            }
+        }
+
+        void unhandled_exception() { 
+#ifdef OUTPUT
+            std::cerr << "unhandled_exception" << std::endl;
+#endif
+            std::terminate(); 
+        }
+
+        void return_value(int value) { 
+#ifdef OUTPUT
+            std::cerr << "return_value: " << value << std::endl;
+#endif
+            result_ = value;
+            if (task_) {
+                task_->result_ = value;
+                task_->completed_ = true;
+            }
+        }
+    };
+
+    handle_type h_;
+    int result_;
+    bool completed_ = false;
+
+    Task(handle_type h) : h_(h) {
+#ifdef OUTPUT
+        std::cerr << "Task constructor" << std::endl;
+#endif
+        if (h_) {
+            h_.promise().task_ = this;  // Link promise to Task
+        }
+    }
+
+    ~Task() { 
+#ifdef OUTPUT
+        std::cerr << "~Task destructor" << std::endl;
+#endif
+        // Only destroy handle if still valid (coroutine not completed)
+        if (h_) {
+#ifdef OUTPUT
+            std::cerr << "Destroying coroutine handle" << std::endl;
+#endif
+            h_.destroy(); 
+        }
+    }
+
+    bool done() const { 
+        return completed_ || !h_ || h_.done(); 
+    }
+
+    void resume() { 
+#ifdef OUTPUT
+        std::cerr << "Resuming task" << std::endl;
+#endif
+        if (h_) h_.resume(); 
+    }
+
+    int result() const {
+        if (!done()) throw std::runtime_error("Result not available");
+        return result_;
+    }
+};
+
+Task my_coroutine() {
+#ifdef OUTPUT
+    std::cerr << "Inside my_coroutine" << std::endl;
+#endif
+   co_return 42;
+}
+
+int main() {
+    auto task = my_coroutine();
+    while (!task.done()) {
+#ifdef OUTPUT
+        std::cerr << "Resuming task in main" << std::endl;
+#endif
+        task.resume();
+    }
+#ifdef OUTPUT
+    std::cerr << "Task completed in main, printing result" << std::endl;
+#endif
+    if (task.result() != 42)
+      __builtin_abort ();
+}
diff --git a/gcc/testsuite/g++.dg/cpp/if-comma-1.C b/gcc/testsuite/g++.dg/cpp/if-comma-1.C
new file mode 100644
index 0000000..0daaff9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp/if-comma-1.C
@@ -0,0 +1,42 @@
+// PR c++/120778
+// { dg-do preprocess }
+// { dg-options "-pedantic-errors" }
+
+#if (1, 2)
+#define M1 1
+#else
+#error
+#endif
+#if 1 ? 2, 3 : 4
+#define M2 2
+#else
+#error
+#endif
+#if 0 ? 2, 0 : 1
+#define M3 3
+#else
+#error
+#endif
+#if 0 || (1, 2)
+#define M4 4
+#else
+#error
+#endif
+#if 1 || (1, 2)
+#define M5 5
+#else
+#error
+#endif
+#if (1, 2) && 1
+#define M6 6
+#else
+#error
+#endif
+#if 1 && (1, 2)
+#define M7 7
+#else
+#error
+#endif
+#if M1 + M2 + M3 + M4 + M5 + M6 + M7 != 28
+#error
+#endif
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-array29.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-array29.C
new file mode 100644
index 0000000..714d050
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-array29.C
@@ -0,0 +1,13 @@
+// PR c++/87097
+// { dg-do compile { target c++11 } }
+
+struct A {
+  constexpr A() : data() {}
+  struct X { int n; };
+  X data[2];
+};
+
+static_assert((A(), true), "");
+static_assert(A().data[0].n == 0, "");
+static_assert(A().data[1].n == 0, "");
+constexpr A x;
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-array30.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-array30.C
new file mode 100644
index 0000000..3f72407
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-array30.C
@@ -0,0 +1,22 @@
+// PR c++/120800
+// { dg-do compile { target c++11 } }
+
+template<typename T>
+struct Container
+{
+ T m_data[1] {};
+};
+
+class Element
+{
+private:
+ Element() = default;
+
+private:
+ bool m_bool1 { false };
+ bool m_bool2;
+
+ friend struct Container<Element>;
+};
+
+Container<Element> element;
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C
index b6a5323..c473257 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C
@@ -9,4 +9,6 @@ struct A
 
 constexpr int ellipsis(...) { return 1; }
 
-static_assert(ellipsis(A().empty()), "Error"); // { dg-error "non-constant condition|empty" }
+static_assert(ellipsis(A().empty()), "Error"); // { dg-error "non-constant condition" }
+// { dg-error "call to non-'constexpr' function 'bool A::empty\\\(\\\)'" "" { target c++23_down } .-1 }
+// { dg-error "temporary of non-literal type 'A' in a constant expression" "" { target c++26 } .-2 }
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C
index 9c49fa4..9abf4a3 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C
@@ -2,8 +2,8 @@
 // Explicit { dg-require-effective-target exceptions_enabled } to avoid verify compiler messages FAILs for '-fno-exceptions'.
 
 constexpr int may_throw(bool decide) {
-	return decide ? 42 : throw -1; // { dg-error "throw" }
+	return decide ? 42 : throw -1; // { dg-error "throw" "" { target c++23_down } }
 }
 
-constexpr int x = may_throw(false); // { dg-message "may_throw" }
-constexpr int y = may_throw(true);
+constexpr int x = may_throw(false); // { dg-message "may_throw" "" { target c++23_down } }
+constexpr int y = may_throw(true); // { dg-error "uncaught exception '-1'" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp0x/duplicate1.C b/gcc/testsuite/g++.dg/cpp0x/duplicate1.C
index 1545e1c..4e85edc 100644
--- a/gcc/testsuite/g++.dg/cpp0x/duplicate1.C
+++ b/gcc/testsuite/g++.dg/cpp0x/duplicate1.C
@@ -6,7 +6,7 @@ struct A
   virtual void foo() const;
 };
 
-struct B final final : A  /* { dg-error "duplicate virt-specifier" }
+struct B final final : A  /* { dg-error "duplicate 'final' specifier" }
   { dg-begin-multiline-output "" }
  struct B final final : A
                 ^~~~~
diff --git a/gcc/testsuite/g++.dg/cpp0x/final1.C b/gcc/testsuite/g++.dg/cpp0x/final1.C
new file mode 100644
index 0000000..1d64095
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/final1.C
@@ -0,0 +1,11 @@
+// PR c++/120628
+// { dg-do compile { target c++98_only } }
+
+namespace A {
+  struct B {};
+  struct B final = {};
+}
+namespace C {
+  struct D { D (int, int); };
+  struct D final (42, 0);
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/final2.C b/gcc/testsuite/g++.dg/cpp0x/final2.C
new file mode 100644
index 0000000..d8d5866
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/final2.C
@@ -0,0 +1,26 @@
+// PR c++/120628
+// { dg-do compile }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++14 } }
+
+namespace U {
+  struct A {};
+  struct A final = {};
+}
+namespace V {
+  template <int N>
+  struct B {};
+  template <int N>
+  struct B<N> final = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
+}
+struct C {
+  struct D {};
+  static D foo ();
+  struct D final = foo ();	// { dg-warning "non-static data member initializers only available with" "" { target c++98_only } }
+};
+namespace W {
+  struct E { struct F {}; };
+  struct E::F final = {};
+}
+template <int N>
+struct V::B<N> final = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3.C
new file mode 100644
index 0000000..6525213
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3.C
@@ -0,0 +1,13 @@
+// PR c++/120716
+// { dg-do compile { target c++11 } }
+
+struct A { int *r; };
+
+void
+foo ()
+{
+  static int i;
+  constexpr A a = { &i };
+  static_assert (a.r == &i, "");
+  [&] { static_assert (a.r != nullptr, ""); } ();
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3a.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3a.C
new file mode 100644
index 0000000..398e079
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3a.C
@@ -0,0 +1,12 @@
+// PR c++/120716
+// { dg-do compile { target c++11 } }
+
+struct A { int *const &r; };
+
+void
+foo (int x)
+{
+  constexpr A a = { &x };	// { dg-error "constant" }
+  static_assert (a.r == &x, "");	// { dg-error "non-constant" }
+  [&] { static_assert (a.r != nullptr, ""); } (); // { dg-error "non-constant" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/override2.C b/gcc/testsuite/g++.dg/cpp0x/override2.C
index ab4dec4..d7f542e 100644
--- a/gcc/testsuite/g++.dg/cpp0x/override2.C
+++ b/gcc/testsuite/g++.dg/cpp0x/override2.C
@@ -23,9 +23,9 @@ struct D5 : B3<D5> {};
 
 struct D6 : B4<D6> {}; // { dg-error "cannot derive from 'final' base" }
 
-struct B6 final final {}; // { dg-error "duplicate virt-specifier" }
+struct B6 final final {}; // { dg-error "duplicate 'final' specifier" }
 
-struct B7 override {}; // { dg-error "cannot specify 'override' for a class" }
+struct B7 override {}; // { dg-error "variable 'B7 override' has initializer but incomplete type" }
 
 namespace N
 {
@@ -45,7 +45,7 @@ int main()
   struct B2 final; // { dg-error "redeclaration" }
   struct B2 override; // { dg-message "previously declared here" }
   struct B2 final {}; // { dg-error "redefinition" }
-  struct B2 override {}; // { dg-error "cannot specify 'override' for a class" }
+  struct B2 override {}; // { dg-error "redeclaration of 'main\\\(\\\)::B2 override'" }
   B2 override{}; // { dg-error "redeclaration" }
   struct foo final {}; // { dg-message "previous definition" }
   struct foo final {}; // { dg-error "redefinition" }
diff --git a/gcc/testsuite/g++.dg/cpp0x/override5.C b/gcc/testsuite/g++.dg/cpp0x/override5.C
new file mode 100644
index 0000000..3382c14
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/override5.C
@@ -0,0 +1,26 @@
+// PR c++/120569
+// { dg-do compile }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++14 } }
+
+namespace U {
+  struct A {};
+  struct A override {};		// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+}
+namespace V {
+  template <int N>
+  struct B {};
+  template <int N>
+  struct B<N> override {};	// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+}				// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+struct C {
+  struct D {};
+  struct D override {};		// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+};				// { dg-warning "non-static data member initializers only available with" "" { target c++98_only } .-1 }
+namespace W {
+  struct E { struct F {}; };
+  struct E::F override {};	// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+}
+template <int N>
+struct V::B<N> override {};	// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+				// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp0x/override6.C b/gcc/testsuite/g++.dg/cpp0x/override6.C
new file mode 100644
index 0000000..601e91d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/override6.C
@@ -0,0 +1,26 @@
+// PR c++/120628
+// { dg-do compile }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++14 } }
+
+namespace U {
+  struct A {};
+  struct A override = {};
+}
+namespace V {
+  template <int N>
+  struct B {};
+  template <int N>
+  struct B<N> override = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
+}
+struct C {
+  struct D {};
+  static D foo ();
+  struct D override = foo ();	// { dg-warning "non-static data member initializers only available with" "" { target c++98_only } }
+};
+namespace W {
+  struct E { struct F {}; };
+  struct E::F override = {};
+}
+template <int N>
+struct V::B<N> override = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for40.C b/gcc/testsuite/g++.dg/cpp0x/range-for40.C
new file mode 100644
index 0000000..dea4a2a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for40.C
@@ -0,0 +1,41 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+
+int z[64];
+
+void
+foo ()
+{
+  for (static auto a : z)		// { dg-error "for-range-declaration cannot be 'static'" }
+    ;
+  for (thread_local auto a : z)		// { dg-error "for-range-declaration cannot be 'thread_local'" }
+    ;
+  for (__thread auto a : z)		// { dg-error "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-error "function-scope 'a' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+  for (register auto a : z)		// { dg-error "for-range-declaration cannot be 'register'" }
+    ;					// { dg-error "does not allow 'register' storage class specifier" "" { target c++17 } .-1 }
+  for (extern auto a : z)		// { dg-error "for-range-declaration cannot be 'extern'" }
+    ;					// { dg-error "'a' has both 'extern' and initializer" "" { target *-*-* } .-1 }
+  for (mutable auto a : z)		// { dg-error "non-member 'a' cannot be declared 'mutable'" }
+    ;
+  for (virtual auto a : z)		// { dg-error "'virtual' outside class declaration" }
+    ;
+  for (explicit auto a : z)		// { dg-error "'explicit' outside class declaration" }
+    ;
+  for (friend auto a : z)		// { dg-error "'friend' used outside of class" }
+    ;
+  for (typedef auto a : z)		// { dg-error "typedef declared 'auto'" }
+    ;					// { dg-error "typedef 'a' is initialized \\\(use 'decltype' instead\\\)" "" { target *-*-* } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto a : z)		// { dg-error "a variable cannot be declared 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto a : z)		// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto a : z)		// { dg-error "'inline' specifier invalid for variable 'a' declared at block scope" }
+    ;
+  for (struct S { int a; } a : z)	// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "conversion from 'int' to non-scalar type 'foo\\\(\\\)::S' requested" "" { target *-*-* } .-1 }
+  for (enum E { E0 } a : z)		// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "invalid conversion from 'int' to 'foo\\\(\\\)::E'" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for41.C b/gcc/testsuite/g++.dg/cpp0x/range-for41.C
new file mode 100644
index 0000000..d690365
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for41.C
@@ -0,0 +1,42 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+int z[64];
+
+void
+foo ()
+{
+  for (static auto a : z)		// { dg-warning "for-range-declaration cannot be 'static'" }
+    ;
+  for (thread_local auto a : z)		// { dg-warning "for-range-declaration cannot be 'thread_local'" }
+    ;
+  for (__thread auto a : z)		// { dg-warning "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-warning "function-scope 'a' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+  for (register auto a : z)		// { dg-warning "for-range-declaration cannot be 'register'" }
+    ;					// { dg-warning "does not allow 'register' storage class specifier" "" { target c++17 } .-1 }
+  for (extern auto a : z)		// { dg-warning "for-range-declaration cannot be 'extern'" }
+    ;					// { dg-error "'a' has both 'extern' and initializer" "" { target *-*-* } .-1 }
+  for (mutable auto a : z)		// { dg-error "non-member 'a' cannot be declared 'mutable'" }
+    ;
+  for (virtual auto a : z)		// { dg-error "'virtual' outside class declaration" }
+    ;
+  for (explicit auto a : z)		// { dg-error "'explicit' outside class declaration" }
+    ;
+  for (friend auto a : z)		// { dg-error "'friend' used outside of class" }
+    ;
+  for (typedef auto a : z)		// { dg-error "typedef declared 'auto'" }
+    ;					// { dg-error "typedef 'a' is initialized \\\(use 'decltype' instead\\\)" "" { target *-*-* } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto a : z)		// { dg-error "a variable cannot be declared 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto a : z)		// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto a : z)		// { dg-error "'inline' specifier invalid for variable 'a' declared at block scope" }
+    ;
+  for (struct S { int a; } a : z)	// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "conversion from 'int' to non-scalar type 'foo\\\(\\\)::S' requested" "" { target *-*-* } .-1 }
+  for (enum E { E0 } a : z)		// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "invalid conversion from 'int' to 'foo\\\(\\\)::E'" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for42.C b/gcc/testsuite/g++.dg/cpp0x/range-for42.C
new file mode 100644
index 0000000..a5d94fc
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for42.C
@@ -0,0 +1,41 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+
+struct S { int y; } z[64];
+
+void
+foo ()
+{
+  for (static auto [ a ] : z)		// { dg-error "for-range-declaration cannot be 'static'" }
+    ;					// { dg-error "structured binding declaration can be 'static' only in" "" { target c++17_down } .-1 }
+					// { dg-error "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (thread_local auto [ a ] : z)	// { dg-error "for-range-declaration cannot be 'thread_local'" }
+    ;					// { dg-error "structured binding declaration can be 'thread_local' only in" "" { target c++17_down } .-1 }
+					// { dg-error "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (__thread auto [ a ] : z)		// { dg-error "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-error "function-scope 'structured binding' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+					// { dg-error "structured binding declaration can be '__thread' only in" "" { target c++17_down } .-2 }
+					// { dg-error "structured bindings only available with" "" { target c++14_down } .-3 }
+  for (register auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'register'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (extern auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'extern'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (mutable auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'mutable'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (virtual auto [ a ] : z)		// { dg-error "'virtual' outside class declaration" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (explicit auto [ a ] : z)		// { dg-error "'explicit' outside class declaration" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (friend auto [ a ] : z)		// { dg-error "'friend' used outside of class" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (typedef auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'typedef'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto [ a ] : z)	// { dg-error "structured binding declaration cannot be 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto [ a ] : z)	// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'inline'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for43.C b/gcc/testsuite/g++.dg/cpp0x/range-for43.C
new file mode 100644
index 0000000..77060e3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for43.C
@@ -0,0 +1,42 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+struct S { int y; } z[64];
+
+void
+foo ()
+{
+  for (static auto [ a ] : z)		// { dg-warning "for-range-declaration cannot be 'static'" }
+    ;					// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-1 }
+					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (thread_local auto [ a ] : z)	// { dg-warning "for-range-declaration cannot be 'thread_local'" }
+    ;					// { dg-warning "structured binding declaration can be 'thread_local' only in" "" { target c++17_down } .-1 }
+					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (__thread auto [ a ] : z)		// { dg-warning "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-warning "function-scope 'structured binding' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+ 					// { dg-warning "structured binding declaration can be '__thread' only in" "" { target c++17_down } .-2 }
+					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-3 }
+  for (register auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'register'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (extern auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'extern'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (mutable auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'mutable'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (virtual auto [ a ] : z)		// { dg-error "'virtual' outside class declaration" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (explicit auto [ a ] : z)		// { dg-error "'explicit' outside class declaration" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (friend auto [ a ] : z)		// { dg-error "'friend' used outside of class" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (typedef auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'typedef'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto [ a ] : z)	// { dg-error "structured binding declaration cannot be 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto [ a ] : z)	// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'inline'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/this1.C b/gcc/testsuite/g++.dg/cpp0x/this1.C
index 486e045..e70cd1a 100644
--- a/gcc/testsuite/g++.dg/cpp0x/this1.C
+++ b/gcc/testsuite/g++.dg/cpp0x/this1.C
@@ -8,10 +8,10 @@ struct S1 {
   void m3 () noexcept(noexcept(this->a)) { }
   void m4 () noexcept(noexcept(this)) { }
 
-  static auto m5 () -> decltype(this->a) { return 0; } // { dg-error ".this. may not be used in this context" }
-  static auto m6 () -> decltype(this) { return 0; } // { dg-error ".this. may not be used in this context" }
-  static void m7 () noexcept(noexcept(this->a)) { } // { dg-error ".this. may not be used in this context" }
-  static void m8 () noexcept(noexcept(this)) { } // { dg-error ".this. may not be used in this context" }
+  static auto m5 () -> decltype(this->a) { return 0; } // { dg-error ".this." }
+  static auto m6 () -> decltype(this) { return 0; } // { dg-error ".this." }
+  static void m7 () noexcept(noexcept(this->a)) { } // { dg-error ".this." }
+  static void m8 () noexcept(noexcept(this)) { } // { dg-error ".this." }
 };
 
 template <typename T>
@@ -41,6 +41,6 @@ test ()
 }
 
 struct S5 {
-  friend auto bar() -> decltype(this); // { dg-error ".this. may not be used in this context" }
+  friend auto bar() -> decltype(this); // { dg-error ".this." }
   auto bar2() -> decltype(this);
 };
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C
index f7439d8..18efd23 100644
--- a/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C
@@ -12,7 +12,7 @@ void
 f2 ()
 { 
   for (;;)
-    constexpr bool b = ({ break; false; }) && false;	// { dg-error "is not a constant expression" }
+    constexpr bool b = ({ break; false; }) && false;	// { dg-error "'break' outside of a loop or 'switch'" }
 }
 
 constexpr bool
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C
index b5fa6ca..75e2fb8 100644
--- a/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C
@@ -27,8 +27,8 @@ constexpr int fun(int n) {
   case 0:
     return 1;
   default:
-    throw; // { dg-error "not a constant expression" }
-  }
+    throw; // { dg-error "not a constant expression" "" { target c++23_down } }
+  } // { dg-error "'void __cxa_rethrow\\\(\\\)' called with no caught exceptions active" "" { target c++26 } .-1 }
 }
 
 static_assert(fun(0), "");
diff --git a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic-a.C b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic-a.C
new file mode 100644
index 0000000..557ecb9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic-a.C
@@ -0,0 +1,15 @@
+// Basic generic lambda test
+// { dg-do run { target c++14 } }
+// { dg-additional-options -Wno-abbreviated-auto-in-template-arg }
+
+template <typename T, typename U> struct pair {};
+template <typename... T> struct tuple {};
+
+int main()
+{
+  auto a = [] (auto, pair<auto,auto> v) { return sizeof (v); };
+  auto b = [] (auto, pair<pair<auto,auto>,auto>... v) { return sizeof... (v); };
+
+  a(1, pair<int, float>());
+  b(2, pair<pair<short,char>, double>(), pair<pair<float,long>, int>());
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C
index 6d2d250..6ef8652 100644
--- a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C
+++ b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C
@@ -1,13 +1,13 @@
 // Basic generic lambda test
-// { dg-do run { target c++14 } }
+// { dg-do compile { target c++14 } }
 
 template <typename T, typename U> struct pair {};
 template <typename... T> struct tuple {};
 
 int main()
 {
-  auto a = [] (auto, pair<auto,auto> v) { return sizeof (v); };
-  auto b = [] (auto, pair<pair<auto,auto>,auto>... v) { return sizeof... (v); };
+  auto a = [] (auto, pair<auto,auto> v) { return sizeof (v); }; // { dg-error "auto" }
+  auto b = [] (auto, pair<pair<auto,auto>,auto>... v) { return sizeof... (v); }; // { dg-error "auto" }
 
   a(1, pair<int, float>());
   b(2, pair<pair<short,char>, double>(), pair<pair<float,long>, int>());
diff --git a/gcc/testsuite/g++.dg/cpp1y/pr85076.C b/gcc/testsuite/g++.dg/cpp1y/pr85076.C
index 6d54dea..b68143b 100644
--- a/gcc/testsuite/g++.dg/cpp1y/pr85076.C
+++ b/gcc/testsuite/g++.dg/cpp1y/pr85076.C
@@ -3,4 +3,4 @@
 
 template<typename> struct A*;	// { dg-error "expected unqualified-id before" }
 
-auto a = [](A<auto>) {};	// { dg-error "is not a template|has incomplete type" }
+auto a = [](A<auto>) {};	// { dg-error "is not a template|has incomplete type|auto" }
diff --git a/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C b/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C
index 35beb27..2c69b3b 100644
--- a/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C
+++ b/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C
@@ -18,7 +18,7 @@ struct H { short size () const { return 0; }
 	   constexpr const char *data () const { return ""; } };
 struct I { constexpr signed char size () const { return 0; }
 	   const char *data () const { return ""; } };
-struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" }
+struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_down } }
 	   constexpr const char *data () const { return ""; };
 	   constexpr J (int x) : j (x) {}
 	   int j; };
@@ -114,6 +114,7 @@ foo ()
 
   asm ((J (0)));
   asm ("" :: (J (1)) (1));	// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
   asm ((M {}));
 #if __cpp_constexpr_dynamic_alloc >= 201907L
   asm ((N {}));			// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++20 } }
@@ -188,6 +189,7 @@ bar ()
 
   asm ((J (0)));
   asm ("" :: (J (1)) (1));	// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
   asm ((M {}));
 #if __cpp_constexpr_dynamic_alloc >= 201907L
   asm ((N {}));			// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++20 } }
@@ -272,7 +274,7 @@ namespace NN
 #if __cplusplus >= 201402L
   struct J {
     static constexpr int size () { return 0; }
-    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++14 } }
+    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target { c++14 && c++23_down } } }
 #endif
 #if __cpp_if_consteval >= 202106L
   struct K {
@@ -284,12 +286,12 @@ namespace NN
     static constexpr const char *data () { if consteval { return "test"; } else { throw 1; } }
   };
   struct M {
-    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
     static constexpr const char *data () { return "test"; }
   };
   struct N {
     static constexpr int size () { return 4; }
-    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
   };
 #endif
   struct O { constexpr int operator () () const { return 12; } };
@@ -318,12 +320,15 @@ namespace NN
     asm ((I {}));
 #if __cplusplus >= 201402L
     asm ((J {}));		// { dg-error "constexpr string 'data\\\(\\\)' must be a core constant expression" "" { target c++14 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
 #if __cpp_if_consteval >= 202106L
     asm ((K {}));
     asm ((L {}));
     asm ((M {}));		// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
     asm ((N {}));		// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
     asm ((Q {}));
     asm ((R {}));
@@ -348,12 +353,15 @@ namespace NN
     asm ((I {}));
 #if __cplusplus >= 201402L
     asm ((J {}));		// { dg-error "constexpr string 'data\\\(\\\)' must be a core constant expression" "" { target c++14 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
 #if __cpp_if_consteval >= 202106L
     asm ((K {}));
     asm ((L {}));
     asm ((M {}));		// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
     asm ((N {}));		// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
     asm ((Q {}));
     asm ((R {}));
diff --git a/gcc/testsuite/g++.dg/cpp1z/decomp3.C b/gcc/testsuite/g++.dg/cpp1z/decomp3.C
index a9b23b0..202edc3 100644
--- a/gcc/testsuite/g++.dg/cpp1z/decomp3.C
+++ b/gcc/testsuite/g++.dg/cpp1z/decomp3.C
@@ -19,7 +19,8 @@ test (A &b, B c)
 					// { dg-error "expected primary-expression before 'decltype'" "" { target c++11_down } .-2 }
   auto & & && & [ m, n, o ] = b;	// { dg-error "multiple ref-qualifiers" }
 					// { dg-warning "structured bindings only available with '-std=c..17' or '-std=gnu..17'" "" { target c++14_down } .-1 }
-  constexpr auto [ p ] = c;		// { dg-error "structured binding declaration cannot be 'constexpr'" }
+  constexpr B c2 = { 42 };
+  constexpr auto [ p ] = c2;		// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } }
 					// { dg-warning "structured bindings only available with '-std=c..17' or '-std=gnu..17'" "" { target c++14_down } .-1 }
   friend auto [ q ] = c;		// { dg-error "'friend' used outside of class" }
 					// { dg-warning "structured bindings only available with '-std=c..17' or '-std=gnu..17'" "" { target c++14_down } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp1z/nontype8.C b/gcc/testsuite/g++.dg/cpp1z/nontype8.C
new file mode 100644
index 0000000..b81e85b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/nontype8.C
@@ -0,0 +1,12 @@
+// Test that the diagnostic mentions lack of constexpr
+// { dg-do compile { target c++17 } }
+
+template <auto f> void g() {}
+void x()
+{
+  using fp = void (*)();
+  fp f = nullptr;		// { dg-message "constexpr" }
+  g<f>();			// { dg-error "" }
+  int *p = nullptr;		// { dg-message "constexpr" }
+  g<p>();			// { dg-error "" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp23/explicit-obj-lambda18.C b/gcc/testsuite/g++.dg/cpp23/explicit-obj-lambda18.C
new file mode 100644
index 0000000..d54a93d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/explicit-obj-lambda18.C
@@ -0,0 +1,12 @@
+// PR c++/114632
+// { dg-do compile { target c++23 } }
+
+struct S {};
+
+auto lambda = [](this auto& self, const int x) /* -> void */ {};
+
+int main()
+{
+    void (*func)(S&, int) = lambda; // { dg-error "" }
+    return 0;
+}
diff --git a/gcc/testsuite/g++.dg/cpp23/static-operator-call7.C b/gcc/testsuite/g++.dg/cpp23/static-operator-call7.C
new file mode 100644
index 0000000..7c381e6
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/static-operator-call7.C
@@ -0,0 +1,12 @@
+// PR c++/114632
+// { dg-do compile { target c++23 } }
+
+struct S {};
+
+auto lambda = [](auto, const int x) static /* -> void */ {};
+
+int main()
+{
+    void (*func)(int, int) = lambda;
+    return 0;
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block1.C b/gcc/testsuite/g++.dg/cpp26/consteval-block1.C
new file mode 100644
index 0000000..9e2cf22
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block1.C
@@ -0,0 +1,82 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+constexpr int fn () { return 42; }
+struct M {
+  static consteval void foo () {}
+};
+
+consteval { }
+consteval { fn (); }
+consteval { M::foo (); }
+consteval { auto x = fn (); return; }
+consteval {
+  [](int i) { return i; }(5);
+}
+auto lam = [] { };
+consteval { lam (); }
+
+struct S {
+  consteval { }
+};
+
+struct S2 {
+  consteval { fn(); }
+};
+
+class C {
+  consteval { }
+};
+
+class C2 {
+  consteval { M::foo (); }
+};
+
+union U {
+  consteval { }
+};
+
+template<typename>
+struct TS {
+  consteval { }
+};
+
+template<typename... Ts>
+struct TS2 {
+  consteval {
+    (Ts::foo (), ...);
+  }
+};
+
+TS2<M> ts2;
+
+void
+g ()
+{
+  consteval { }
+}
+
+template<typename>
+void
+tg ()
+{
+  consteval { }
+}
+
+void die ();
+constexpr int
+bar (int i)
+{
+  if (i != 42)
+    die ();
+  return 0;
+}
+
+void
+foo ()
+{
+  constexpr int r = 42;
+  consteval {
+    bar (r);
+  }
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block2.C b/gcc/testsuite/g++.dg/cpp26/consteval-block2.C
new file mode 100644
index 0000000..895fcb6
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block2.C
@@ -0,0 +1,49 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+void fn ();
+
+consteval { fn (); }	    // { dg-error "call to non-.constexpr. function" }
+consteval { return 42; }    // { dg-error "return-statement with a value" }
+
+struct S {
+  consteval {
+    fn ();		    // { dg-error "call to non-.constexpr. function" }
+  }
+  consteval {
+    return 42;		    // { dg-error "return-statement with a value" }
+  }
+};
+
+template<typename T>
+constexpr void foo (T t) { return t; }	// { dg-error "return-statement with a value" }
+
+template<int N>
+struct R {
+  consteval { foo (N); }
+};
+
+R<1> r;
+
+template<typename T>
+constexpr void foo2 (T t) { return t; }	// { dg-error "return-statement with a value" }
+
+template<int N>
+void
+f ()
+{
+  consteval { foo2 (1); }
+}
+
+constexpr int bar (int) { return 0; }
+
+void
+g ()
+{
+  f<1>();
+
+  int r = 42;
+  consteval {
+    bar (r);	      // { dg-error ".r. is not captured" }
+  }
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block3.C b/gcc/testsuite/g++.dg/cpp26/consteval-block3.C
new file mode 100644
index 0000000..c1221c3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block3.C
@@ -0,0 +1,41 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+// Test that we actually evaluate the consteval block.
+
+void bar () { }
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    bar ();			// { dg-error "call to non-.constexpr. function" }
+}
+
+template<int N>
+struct S {
+  consteval { fn<N>(); }	// { dg-error "called in a constant expression" }
+};
+
+S<1> s;
+
+template<int N>
+constexpr void
+fn2 ()
+{
+  if (N > 0)
+    bar ();			// { dg-error "call to non-.constexpr. function" }
+}
+
+template<int N>
+void
+g ()
+{
+  consteval { fn2<N>(); }	// { dg-error "called in a constant expression" }
+}
+
+void
+f ()
+{
+  g<1>();
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block4.C b/gcc/testsuite/g++.dg/cpp26/consteval-block4.C
new file mode 100644
index 0000000..be95e17
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block4.C
@@ -0,0 +1,41 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+// Test that we actually evaluate the consteval block.
+
+void bar () { }
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    bar ();
+}
+
+template<int N>
+struct S {
+  consteval { fn<N>(); }
+};
+
+S<0> s;
+
+template<int N>
+constexpr void
+fn2 ()
+{
+  if (N > 0)
+    bar ();
+}
+
+template<int N>
+void
+g ()
+{
+  consteval { fn2<N>(); }
+}
+
+void
+f ()
+{
+  g<0>();
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block5.C b/gcc/testsuite/g++.dg/cpp26/consteval-block5.C
new file mode 100644
index 0000000..462cebe
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block5.C
@@ -0,0 +1,70 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+void bar () { }
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    bar ();
+}
+
+template<typename>
+struct S {
+  consteval { fn<1>(); }
+};
+
+template<>
+struct S<int> {
+  consteval { fn<0>(); }
+};
+
+S<int> s1;
+
+template<typename T>
+struct S<T*> {
+  consteval { fn<0>(); }
+};
+
+S<int *> s2;
+
+template<typename T, int N>
+struct W {
+  consteval { T t; fn<N - 1>(); }
+};
+
+template<typename T>
+struct W<T, 0> {
+  consteval { T t; fn<0>(); }
+};
+
+template<>
+struct W<char, 0> {
+  consteval { fn<0>(); }
+};
+
+W<int, 0> w1;
+W<int, 1> w2;
+W<char, 0> w3;
+
+template<typename>
+void
+f ()
+{
+  consteval { fn<1>(); }
+}
+
+template<>
+void
+f<int> ()
+{
+  consteval { fn<0>(); }
+}
+
+void
+g ()
+{
+  f<int> ();
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block6.C b/gcc/testsuite/g++.dg/cpp26/consteval-block6.C
new file mode 100644
index 0000000..ca90b3e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block6.C
@@ -0,0 +1,108 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+void die () {}
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    die ();
+}
+
+template<int N>
+void
+fn2 ()
+{
+  struct S {
+    consteval {
+      fn<N>();
+    }
+  };
+}
+
+template<int N>
+struct A {
+  struct B {
+    consteval {
+      fn<N>();
+    }
+  };
+  template<int M>
+  struct C {
+    consteval {
+      fn<N + M>();
+    }
+  };
+};
+
+template<int N>
+struct D {
+  constexpr static int i = 0;
+  struct E {
+    consteval {
+      fn<i>();
+    }
+  };
+};
+
+A<0>::B b;
+A<0>::C<0> c;
+D<0>::E e;
+
+void
+f ()
+{
+  fn2<0>();
+}
+
+static constexpr int j = 0;
+const int x = 0;
+
+consteval {
+  fn<j>();
+  consteval {
+    fn<j + j>();
+    consteval {
+      fn<j + j + j>();
+      consteval {
+	fn<j + j + x>();
+	consteval {
+	  fn<j + x>();
+	}
+      }
+    }
+  }
+}
+
+struct R { constexpr R() {} };
+
+template<int N>
+constexpr auto X = N;
+
+consteval {
+  R{};
+  constexpr auto x = 0;
+  fn<x>();
+  fn<X<0>>();
+  if consteval
+    {
+      fn<j>();
+    }
+  else
+    {
+      die ();
+    }
+}
+
+template<typename T>
+struct G {
+  consteval {
+    using U = T[3];
+    U arr{};
+    int i = arr[2];
+  }
+};
+
+G<int> g;
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block7.C b/gcc/testsuite/g++.dg/cpp26/consteval-block7.C
new file mode 100644
index 0000000..231682f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block7.C
@@ -0,0 +1,12 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+consteval {
+  template <class T> // { dg-error "template declaration cannot appear at block scope" }
+  struct X { };
+
+  template <class T> // { dg-error "template declaration cannot appear at block scope" }
+  concept C = true;
+
+  return; // OK
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block8.C b/gcc/testsuite/g++.dg/cpp26/consteval-block8.C
new file mode 100644
index 0000000..ad164fd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block8.C
@@ -0,0 +1,38 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+/* __func__ won't be set.  Make sure we warn.  */
+consteval { __func__; }	  // { dg-error "outside of function scope" }
+consteval { { __func__; } }	  // { dg-error "outside of function scope" }
+consteval { []() mutable consteval -> void { __func__; } (); }	// { dg-bogus "outside of function scope" }
+consteval { []() mutable consteval -> void { consteval { __func__; } } (); } // { dg-bogus "outside of function scope" }
+
+auto l = []() -> void {
+  consteval { __func__; } // { dg-bogus "outside of function scope" }
+};
+
+struct F {
+  consteval { __func__; } // { dg-error "outside of function scope" }
+};
+template<typename>
+struct TF {
+  consteval { __func__; } // { dg-error "outside of function scope" }
+};
+
+void
+g ()
+{
+  consteval { __func__; } // { dg-bogus "outside of function scope" }
+  // Not a consteval-block-declaration.
+  []() mutable consteval -> void { __func__; } (); // { dg-bogus "outside of function scope" }
+}
+
+template<typename>
+void
+f ()
+{
+  consteval { __func__; } // { dg-bogus "outside of function scope" }
+  { consteval { __func__; } } // { dg-bogus "outside of function scope" }
+  __func__; // { dg-bogus "outside of function scope" }
+  []() mutable consteval -> void { __func__; } (); // { dg-bogus "outside of function scope" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh1.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh1.C
new file mode 100644
index 0000000..9eed3aa
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh1.C
@@ -0,0 +1,140 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+  constexpr S () : s (new int (0)) {}
+  constexpr S (int x) : s (new int (x)) {}
+  constexpr S (const S &x) : s (new int (*x.s)) {}
+  constexpr ~S () { delete s; }
+  int *s;
+};
+struct T : public S {
+  constexpr T () : S () {}
+  constexpr T (int x) : S (x) {}
+  constexpr T (const T &x) : S (*x.s) {}
+  constexpr ~T () {}
+};
+struct U : public T {
+  constexpr U () : T () {}
+  constexpr U (int x) : T (x) {}
+  constexpr U (const U &x) : T (*x.s) {}
+  constexpr ~U () {}
+};
+struct V : public T {
+  constexpr V () : T () {}
+  constexpr V (int x) : T (x) {}
+  constexpr V (const U &x) : T (*x.s) {}
+  constexpr ~V () {}
+};
+
+template <typename X>
+constexpr int
+foo (X x)
+{
+  try { throw x; }
+  catch (int &a) { return 42 + a; }
+  catch (const unsigned b) { return 43 + b; }
+  catch (const long &c) { return 44 + c; }
+  catch (bool d) { return 45 + d; }
+  catch (const U &e) { return 46 + *e.s; }
+  catch (const T &f) { return 47 + *f.s; }
+  catch (S g) { return 48 + *g.s; }
+  catch (int *const &h) { return 49; }
+  catch (long long *) { return 50; }
+  catch (const S *const &) { return 51; }
+  catch (...) { return 52; }
+}
+
+template <typename X>
+constexpr int
+bar (const X &x)
+{
+  throw x;
+}
+
+template <typename X>
+constexpr int
+baz (const X &x)
+{
+  try
+    {
+      try { bar (x); }
+      catch (int &a) { a += 80; throw; }
+      catch (long b) { b += 80; throw; }
+      catch (U &c) { c.s[0] += 82; throw; }
+      catch (V d) { d.s[0] += 83; throw; }
+    }
+  catch (int a) { return 42 + a; }
+  catch (const long &b) { return 43 + b; }
+  catch (S &c) { return 44 + c.s[0]; }
+  catch (long long d) { return 45 + d; }
+  catch (...) { return -1; }
+}
+
+constexpr int
+qux (int x, bool y = true)
+{
+  try
+    {
+      switch (x)
+	{
+	case 0: throw 42; break;
+	case 1: x = y ? throw 43 : 5; break;
+	case 2: x = -(throw 44, 6); break;
+	case 3: x = x + (throw 45, 7); break;
+	case 4: x = (throw 46, 8) + x; break;
+	case 5: x = (throw 47, y) ? 4 : 5; break;
+	case 6: x += (throw 48, y); break;
+	case 7: x = (double) (throw 49, y); break;
+	case 8: x = foo ((throw 50, x)); break;
+	default: break;
+	}
+    }
+  catch (int a) { return a; }
+  return -1;
+}
+
+constexpr int
+corge ()
+{
+  try { throw 0; }
+  catch (int *const &h) { return 49; }
+  catch (long long *) { return 50; }
+  catch (const S *const &) { return 51; }
+  catch (...) { return 52; }
+}
+
+static_assert (foo (12) == 54);
+static_assert (foo (12U) == 55);
+static_assert (foo (12L) == 56);
+static_assert (foo (false) == 45);
+static_assert (foo (true) == 46);
+static_assert (foo (U (12)) == 58);
+static_assert (foo (T (20)) == 67);
+static_assert (foo (S (30)) == 78);
+static_assert (foo (nullptr) == 49);
+static_assert (foo ((int *)nullptr) == 49);
+static_assert (foo ((long long *)nullptr) == 50);
+static_assert (foo ((const S *)nullptr) == 51);
+static_assert (foo ((const T *)nullptr) == 51);
+static_assert (foo ((const U *)nullptr) == 51);
+static_assert (foo (12ULL) == 52);
+static_assert (baz (5) == 127);
+static_assert (baz (6L) == 49);
+static_assert (baz (U (25)) == 151);
+static_assert (baz (V (26)) == 70);
+static_assert (baz (T (27)) == 71);
+static_assert (baz (S (28)) == 72);
+static_assert (baz (7LL) == 52);
+static_assert (baz (8ULL) == -1);
+static_assert (qux (0) == 42);
+static_assert (qux (1) == 43);
+static_assert (qux (2) == 44);
+static_assert (qux (3) == 45);
+static_assert (qux (4) == 46);
+static_assert (qux (5) == 47);
+static_assert (qux (6) == 48);
+static_assert (qux (7) == 49);
+static_assert (qux (8) == 50);
+static_assert (corge () == 52);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh10.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh10.C
new file mode 100644
index 0000000..a86cc4d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh10.C
@@ -0,0 +1,110 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+};
+struct T {
+  constexpr ~T () noexcept (false) { throw S {}; }
+};
+struct U {
+  int u;
+};
+struct V {
+  int v;
+  constexpr V (int x)
+  try : v { x }
+  {
+    if (v > 42)
+      throw U { 42 };
+  }
+  catch (U &u)
+  {
+    --u.u;
+  }
+};
+struct W {
+  constexpr ~W () { ++w; }
+  int &w;
+};
+struct X : public V {
+  constexpr X (int x)
+  try : V(x)
+  {
+  }
+  catch (U &u)
+  {
+    --u.u;
+  }
+};
+
+constexpr int
+foo (bool x)
+{
+  try
+    {
+      T t;				// { dg-error "'std::terminate' called after throwing an exception '42'" }
+      if (x)				// { dg-message "destructor exited with an exception" "" { target *-*-* } .-1 }
+	throw 42;
+      return 10;
+    }
+  catch (S)
+    {
+      return 11;
+    }
+}
+
+constexpr int
+bar ()
+{
+  V v { 42 };
+  try
+    {
+      V w { 43 };
+    }
+  catch (const U &u)
+    {
+      if (u.u == 41)
+	return 44;
+    }
+  return -1;
+}
+
+constexpr int
+baz ()
+{
+  int i = 42;
+  try
+    {
+      W w { i };
+      throw S ();
+    }
+  catch (...)
+    {
+      if (i == 43)
+	return 42;
+    }
+  return -1;
+}
+
+constexpr int
+qux ()
+{
+  X v { 42 };
+  try
+    {
+      X w { 43 };
+    }
+  catch (const U &u)
+    {
+      if (u.u == 40)
+	return 48;
+    }
+  return -1;
+}
+
+static_assert (foo (false) == 11);
+constexpr int a = foo (true);		// { dg-message "in 'constexpr' expansion of" }
+static_assert (bar () == 44);
+static_assert (baz () == 42);
+static_assert (qux () == 48);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh11.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh11.C
new file mode 100644
index 0000000..287e066
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh11.C
@@ -0,0 +1,69 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct A {
+  explicit constexpr A (int x) noexcept : a (x) {}
+  constexpr virtual int foo () const noexcept { return a; }
+  constexpr virtual ~A () {}
+  int a;
+};
+struct B : public A {
+  explicit constexpr B (int x) noexcept : A (x) {}
+  constexpr int foo () const noexcept override { return a | 0x10; }
+};
+struct C : public A {
+  explicit constexpr C (int x) noexcept : A (x) {}
+};
+struct D : public A {
+  explicit constexpr D (int x) noexcept : A (x) {}
+};
+struct E {
+  constexpr E () noexcept : e (0) {}
+  explicit constexpr E (int x) noexcept : e (x) {}
+  int e;
+};
+struct F : public E, public B {
+  explicit constexpr F (int x) noexcept : B (x) {}
+};
+struct G : public E, public C {
+  explicit constexpr G (int x) noexcept : C (x) {}
+};
+struct H : public E, public D {
+  explicit constexpr H (int x) noexcept : D (x) {}
+};
+
+consteval int
+bar (void (*fn) ())
+{
+  try
+    {
+      fn ();
+    }
+  catch (C &a)
+    {
+      return a.foo () | 0x20;
+    }
+  catch (const C &b)		// { dg-warning "exception of type 'C' will be caught by earlier handler" }
+    {
+      return b.foo () | 0x60;
+    }
+  catch (A &c)
+    {
+      return c.foo ();
+    }
+  catch (const A &d)		// { dg-warning "exception of type 'A' will be caught by earlier handler" }
+    {
+      return d.foo () | 0x40;
+    }
+  return -1;
+}
+
+static_assert (bar ([] { throw A { 1 }; }) == 1);
+static_assert (bar ([] { throw B { 2 }; }) == 0x12);
+static_assert (bar ([] { throw C { 3 }; }) == 0x23);
+static_assert (bar ([] { throw D { 4 }; }) == 4);
+constexpr int a = bar ([] { throw E { 5 }; });	// { dg-error "uncaught exception 'E\\\{5\\\}'" }
+static_assert (bar ([] { throw F { 6 }; }) == 0x16);
+static_assert (bar ([] { throw G { 7 }; }) == 0x27);
+static_assert (bar ([] { throw H { 8 }; }) == 8);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh12.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh12.C
new file mode 100644
index 0000000..4a85f15
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh12.C
@@ -0,0 +1,74 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct A {
+  explicit constexpr A (int x) noexcept : a (x) {}
+  constexpr virtual int foo () const noexcept { return a; }
+  constexpr virtual ~A () {}
+  int a;
+};
+struct B : public A {
+  explicit constexpr B (int x) noexcept : A (x) {}
+  constexpr int foo () const noexcept override { return a | 0x10; }
+};
+struct C : public A {
+  explicit constexpr C (int x) noexcept : A (x) {}
+};
+struct D : public A {
+  explicit constexpr D (int x) noexcept : A (x) {}
+};
+struct E {
+  constexpr E () noexcept : e (0) {}
+  explicit constexpr E (int x) noexcept : e (x) {}
+  int e;
+};
+struct F : public E, public B {
+  explicit constexpr F (int x) noexcept : B (x) {}
+};
+struct G : public E, public C {
+  explicit constexpr G (int x) noexcept : C (x) {}
+};
+struct H : public E, public D {
+  explicit constexpr H (int x) noexcept : D (x) {}
+};
+
+consteval int
+bar (void (*fn) ())
+{
+  int r = 0;
+  try
+    {
+      fn ();
+    }
+  catch (C *a)
+    {
+      r = a->foo () | 0x20;
+      delete a;
+    }
+  catch (const C *b)
+    {
+      r = b->foo () | 0x60;
+      delete b;
+    }
+  catch (A *c)
+    {
+      r = c->foo ();
+      delete c;
+    }
+  catch (const A *d)
+    {
+      r = d->foo () | 0x40;
+      delete d;
+    }
+  return r;
+}
+
+static_assert (bar ([] { throw new A { 1 }; }) == 1);
+static_assert (bar ([] { throw new B { 2 }; }) == 0x12);
+static_assert (bar ([] { throw new C { 3 }; }) == 0x23);
+static_assert (bar ([] { throw new D { 4 }; }) == 4);
+constexpr int a = bar ([] { throw new E { 5 }; });	// { dg-error "uncaught exception of type 'E\\\*'" }
+static_assert (bar ([] { throw new F { 6 }; }) == 0x16);
+static_assert (bar ([] { throw new G { 7 }; }) == 0x27);
+static_assert (bar ([] { throw new H { 8 }; }) == 8);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh13.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh13.C
new file mode 100644
index 0000000..d62771c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh13.C
@@ -0,0 +1,36 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+template <typename T>
+consteval T
+foo (T x)
+{
+  try
+    {
+      throw &x;
+    }
+  catch (void *ptr)		// { dg-message "for type 'void\\\*'" }
+    {
+      return *static_cast<T *> (ptr) | 0x10;
+    }
+  catch (const void *ptr)	// { dg-message "for type 'const void\\\*'" }
+    {
+      return *static_cast<const T *> (ptr) | 0x20;
+    }
+  catch (T *ptr)		// { dg-warning "exception of type 'T\\\*' will be caught by earlier handler" }
+    {				// { dg-warning "exception of type 'int\\\*' will be caught by earlier handler" "" { target *-*-* } .-1 }
+      return *ptr | 0x30;	// { dg-warning "exception of type 'long long unsigned int\\\*' will be caught by earlier handler" "" { target *-*-* } .-2 }
+    }
+  catch (const T *ptr)
+    {
+      return *ptr | 0x40;
+    }
+  catch (...)
+    {
+      return -1;
+    }
+}
+
+static_assert (foo (1) == 0x11);
+static_assert (foo (2ULL) == 0x12ULL);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh14.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh14.C
new file mode 100644
index 0000000..3e52f2b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh14.C
@@ -0,0 +1,42 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+template <typename T>
+constexpr T
+foo (T x, auto... y)
+{
+  const T z[] = { x, y... };
+  try
+    {
+      throw z;
+    }
+  catch (const T (&a)[4])
+    {
+      return T ();
+    }
+  catch (const T *b)
+    {
+      return b[0];
+    }
+  catch (...)
+    {
+      return T ();
+    }
+}
+
+void
+bar ()
+{
+}
+
+void
+baz ()
+{
+}
+
+static_assert (foo (42, 43, 44, 45, 46) == 42);
+static_assert (foo (43U, 44U, 45U, 46U) == 43U);
+static_assert (foo (44LL, 45LL) == 44LL);
+static_assert (foo (bar, baz, bar, baz) == bar);
+static_assert (foo (baz, bar) == baz);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh15.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh15.C
new file mode 100644
index 0000000..3dea461
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh15.C
@@ -0,0 +1,39 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+int
+foo (int x, int y)
+{
+  return x + y;
+}
+
+constexpr int
+bar (int x)
+{
+  if (x < 0)
+    throw x;
+  return x;
+}
+
+constexpr int
+baz (int x, int y)
+{
+  return foo (bar (x), bar (y));
+}
+
+constexpr int
+qux (int x, int y)
+{
+  try
+    {
+      return baz (x, y);
+    }
+  catch (int)
+    {
+      return 42;
+    }
+}
+
+static_assert (qux (12, -1) == 42);
+static_assert (qux (-7, 12) == 42);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh2.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh2.C
new file mode 100644
index 0000000..8bd6b6f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh2.C
@@ -0,0 +1,112 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+  constexpr S () : s (0) {}
+  constexpr S (int x) : s (x) { if (x == 42) throw 42; }
+  constexpr S (const S &x) : s (x.s) {}
+  constexpr ~S () noexcept (false) { if (s == 41) throw 41; }
+  constexpr const char *what () const noexcept { return "S"; }
+  int s;
+};
+struct T : public S {
+  constexpr T () {}
+  constexpr T (int x) : S (x) {}
+  constexpr T (const T &x) : S (x.s) {}
+  constexpr ~T () {}
+  constexpr const char *what () const noexcept { return "T"; }
+};
+struct U {
+  constexpr U () : u (0) {}
+  constexpr U (int x) : u (x) {}
+  constexpr U (const S &x) : u (0) {}
+  constexpr U (const U &x) : u (x.u) { if (u == 42) throw 43; }
+  constexpr ~U () {}
+  constexpr const char *what () const noexcept { return "U"; }
+  int u;
+};
+
+constexpr int
+foo (int x)
+{
+  if (x == 1)
+    throw 43;
+  return x;
+}
+
+constexpr int
+bar (int x) noexcept	// { dg-error "'std::terminate' called" }
+{			// { dg-message "uncaught exception exited from 'noexcept' function 'constexpr int bar\\\(int\\\)'" "" { target *-*-* } .-1 }
+  return foo (x);
+}
+
+constexpr int
+baz (int x)
+{
+  switch (x)
+    {
+    case 0: throw 1; break;
+    case 1: try { x = bar (x); } catch (...) {} break;		// { dg-message "in 'constexpr' expansion of" }
+    case 2: throw S (2); break;
+    case 3: try { throw S (42); } catch (int a) { if (a != 42) throw -1; } break;
+    case 4: try { S s (41); throw 2; } catch (...) {} break;	// { dg-error "'std::terminate' called" }
+    case 5: return 5;						// { dg-message "destructor exited with an exception" "" { target *-*-* } .-1 }
+    case 6:
+      try
+	{
+	  throw S (5);
+	}
+      catch (const T &) {}
+      catch (int) {}
+      catch (const bool &) {}
+      catch (const T **const &) {}
+      break;
+    case 7: try { constexpr int y = foo (2); } catch (...) {} break;
+    case 8:
+      try
+	{
+	  try
+	    {
+	      throw U ();
+	    }
+	  catch (U &u)
+	    {
+	      u.u = 42;
+	      throw;
+	    }
+	}
+      catch (U u)					// { dg-error "'std::terminate' called" }
+	{						// { dg-message "constructor exited with another exception while entering handler" "" { target *-*-* } .-1 }
+	}
+      break;
+    case 9:
+      try
+	{
+	  throw U (S (41));				// { dg-error "'std::terminate' called" }
+	}						// { dg-message "destructor exited with an exception" "" { target *-*-* } .-1 }
+      catch (...)
+	{
+	}
+      break;
+    }
+  return -1;
+}
+
+constexpr int
+qux (int x)
+{
+  try { constexpr int y = foo (1); } catch (...) {}	// { dg-error "uncaught exception" }
+  return 0;
+}
+
+constexpr int a = baz (0);	// { dg-error "uncaught exception" }
+constexpr int b = baz (1);	// { dg-message "in 'constexpr' expansion of" }
+constexpr int c = baz (2);	// { dg-error "uncaught exception" }
+constexpr int d = baz (3);
+constexpr int e = baz (4);	// { dg-message "in 'constexpr' expansion of" }
+constexpr int f = baz (5);
+constexpr int g = baz (6);	// { dg-error "uncaught exception" }
+constexpr int h = baz (7);
+constexpr int i = baz (8);	// { dg-message "in 'constexpr' expansion of" }
+constexpr int j = baz (9);	// { dg-message "in 'constexpr' expansion of" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh3.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh3.C
new file mode 100644
index 0000000..f844d11
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh3.C
@@ -0,0 +1,442 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+#include <exception>
+#include <new>
+#include <typeinfo>
+
+constexpr std::exception a;
+constexpr const char *b = a.what ();
+constexpr std::bad_exception c;
+constexpr const char *d = c.what ();
+constexpr std::bad_alloc e;
+constexpr const char *f = e.what ();
+constexpr std::bad_array_new_length g;
+constexpr const char *h = g.what ();
+constexpr std::bad_cast i;
+constexpr const char *j = i.what ();
+constexpr std::bad_typeid k;
+constexpr const char *l = k.what ();
+constexpr std::exception_ptr m = nullptr;
+static_assert (m == nullptr);
+constexpr std::exception_ptr n = std::current_exception ();
+static_assert (n == nullptr);
+constexpr std::exception_ptr o;
+static_assert (o == nullptr);
+constexpr std::nested_exception p;
+static_assert (p.nested_ptr () == nullptr);
+
+struct A { virtual ~A () {} };
+struct B { virtual void b (); };
+struct C { virtual void c (); };
+struct D : private B { virtual void d (); };
+struct E { virtual void e (); };
+struct F : D, E, private C { virtual void f (); };
+struct G { constexpr G () { if (std::uncaught_exceptions () != 0) asm (""); } };
+struct H { constexpr H () : h (0) {} constexpr ~H () { if (std::uncaught_exceptions () != h) asm (""); } int h; };
+struct I : std::nested_exception { };
+struct J { virtual ~J () noexcept = default; };
+struct K final { };
+struct L : J, std::nested_exception { };
+struct M { };
+struct N : I, L { };
+struct O : private std::nested_exception { };
+
+constexpr int
+foo (int x)
+{
+  if (std::uncaught_exceptions () != 0)
+    return -1;
+  switch (x)
+    {
+    case 0:
+      try
+	{
+	  const std::type_info &s = typeid (*(A *) 0);
+	  return -1;
+	}
+      catch (const std::bad_typeid &x)
+	{
+	  if (std::uncaught_exceptions () != 0)
+	    return -1;
+	  const char *p = x.what ();
+	  return 1;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 1:
+      try
+	{
+	  static constexpr F f;
+	  D &d = dynamic_cast<D &>((B &) f);
+	  return -1;
+	}
+      catch (std::bad_cast x)
+	{
+	  const char *p = x.what ();
+	  return 2;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 2:
+      try
+	{
+	  H h;
+	  h.h = 1;
+	  if (std::current_exception () != nullptr)
+	    return -1;
+	  throw G ();
+	}
+      catch (const G &g)
+	{
+	  if (std::uncaught_exceptions () != 0)
+	    return -1;
+	  if (std::current_exception () == nullptr)
+	    return -1;
+	  return 3;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 3:
+      try
+	{
+	  decltype (sizeof 0) x = -64;
+	  char (*a)[2] = new char[x][2];
+	  delete[] a;
+	}
+      catch (std::bad_array_new_length x)
+	{
+	  return 4;
+	}
+      break;
+    case 4:
+      try
+	{
+	  int y = -1;
+	  int *a = new int[y];
+	  delete[] a;
+	}
+      catch (const std::bad_array_new_length &)
+	{
+	  return 5;
+	}
+      break;
+    case 5:
+      try
+	{
+	  int z = 1;
+	  int *a = new int[z]{1, 2, 3};
+	  delete[] a;
+	}
+      catch (std::bad_array_new_length &)
+	{
+	  return 6;
+	}
+      break;
+    case 6:
+      {
+	std::exception_ptr b, d;
+	if (b != nullptr || d != nullptr)
+	  return -1;
+	try
+	  {
+	    throw 1;
+	  }
+	catch (int a)
+	  {
+	    if (a != 1)
+	      return -1;
+	    b = std::current_exception ();
+	    if (b == nullptr)
+	      return -1;
+	    try
+	      {
+		throw 2L;
+	      }
+	    catch (long int c)
+	      {
+		if (c != 2L)
+		  return -1;
+		d = std::current_exception ();
+		if (d == nullptr || b == d)
+		  return -1;
+	      }
+	    if (std::current_exception () != b)
+	      return -1;
+	  }
+	if (std::current_exception () != nullptr)
+	  return -1;
+	try
+	  {
+	    std::rethrow_exception (d);
+	  }
+	catch (long int &e)
+	  {
+	    if (e != 2L)
+	      return -1;
+	    try
+	      {
+		std::rethrow_exception (b);
+	      }
+	    catch (const int &f)
+	      {
+		if (f != 1)
+		  return -1;
+		try
+		  {
+		    std::rethrow_exception (d);
+		  }
+		catch (const long int g)
+		  {
+		    if (g != 2L)
+		      return -1;
+		    try
+		      {
+			std::rethrow_exception (b);
+		      }
+		    catch (int h)
+		      {
+			if (h != 1)
+			  return -1;
+			std::exception_ptr i (b);
+			std::exception_ptr j;
+			if (j != nullptr || i == nullptr || i != b || bool (j))
+			  return -1;
+			j = i;
+			if (j != b || !bool (j))
+			  return -1;
+			j = nullptr;
+			std::swap (i, j);
+			if (j == nullptr || j != b || i != nullptr)
+			  return -1;
+		      }
+		  }
+	      }
+	  }
+	return 7;
+      }
+    case 7:
+      {
+	std::exception_ptr a = std::make_exception_ptr (42);
+	std::exception_ptr b = std::make_exception_ptr (std::exception ());
+	std::exception_ptr c
+	  = std::make_exception_ptr (std::bad_array_new_length ());
+	try
+	  {
+	    std::rethrow_exception (a);
+	  }
+	catch (int d)
+	  {
+	    if (d != 42)
+	      return -1;
+	    try
+	      {
+		std::rethrow_exception (b);
+	      }
+	    catch (const std::exception &e)
+	      {
+		const char *f = e.what ();
+		try
+		  {
+		    std::rethrow_exception (c);
+		  }
+		catch (const std::bad_alloc &g)
+		  {
+		    try
+		      {
+			throw;
+		      }
+		    catch (const std::bad_array_new_length &h)
+		      {
+			const char *i = h.what ();
+			const char *j = g.what ();
+		      }
+		  }
+	      }
+	  }
+	return 8;
+      }
+    case 8:
+      {
+	std::nested_exception a;
+	if (a.nested_ptr () != nullptr)
+	  return -1;
+	try
+	  {
+	    std::nested_exception b;
+	    if (b.nested_ptr () != nullptr)
+	      return -1;
+	    throw 42;
+	  }
+	catch (...)
+	  {
+	    std::nested_exception c;
+	    if (c.nested_ptr () != std::current_exception ())
+	      return -1;
+	    std::nested_exception d = c;
+	    if (d.nested_ptr () != c.nested_ptr ())
+	      return -1;
+	    c = d;
+	    try
+	      {
+		c.rethrow_nested ();
+	      }
+	    catch (const int &e)
+	      {
+		if (e != 42)
+		  return -1;
+	      }
+	  }
+	return 9;
+      }
+    case 9:
+      try
+	{
+	  std::throw_with_nested (I ());
+	}
+      catch (const std::nested_exception &a)
+	{
+	  if (a.nested_ptr () != nullptr)
+	    return -1;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const I &)
+	    {
+	      return 10;
+	    }
+	}
+      return -1;
+    case 10:
+      try
+	{
+	  std::throw_with_nested (J ());
+	}
+      catch (const std::nested_exception &a)
+	{
+	  if (a.nested_ptr () != nullptr)
+	    return -1;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const J &)
+	    {
+	      return 11;
+	    }
+	}
+      return -1;
+    case 11:
+      try
+	{
+	  std::throw_with_nested (K ());
+	}
+      catch (const std::nested_exception &)
+	{
+	  return -1;
+	}
+      catch (const K &)
+	{
+	  return 12;
+	}
+      return -1;
+    case 12:
+      try
+	{
+	  throw 42;
+	}
+      catch (...)
+	{
+	  I a;
+	  try
+	    {
+	      std::rethrow_if_nested (a);
+	    }
+	  catch (const int &b)
+	    {
+	      if (b == 42)
+		return 13;
+	    }
+	}
+      return -1;
+    case 13:
+      try
+	{
+	  throw J ();
+	}
+      catch (const J &a)
+	{
+	  std::rethrow_if_nested (a);
+	  return 14;
+	}
+      return -1;
+    case 14:
+      try
+	{
+	  throw 42;
+	}
+      catch (...)
+	{
+	  try
+	    {
+	      throw L ();
+	    }
+	  catch (const J &a)
+	    {
+	      try
+		{
+		  std::rethrow_if_nested (a);
+		}
+	      catch (const int &b)
+		{
+		  if (b == 42)
+		    return 15;
+		}
+	    }
+	}
+      return -1;
+    case 15:
+      {
+	std::rethrow_if_nested (1);
+	M m;
+	std::rethrow_if_nested (m);
+	N n;
+	std::rethrow_if_nested (n);
+	O o;
+	std::rethrow_if_nested (o);
+	return 16;
+      }
+    default:
+      break;
+    }
+  return -1;
+}
+
+static_assert (foo (0) == 1);
+static_assert (foo (1) == 2);
+static_assert (foo (2) == 3);
+static_assert (foo (3) == 4);
+static_assert (foo (4) == 5);
+static_assert (foo (5) == 6);
+static_assert (foo (6) == 7);
+static_assert (foo (7) == 8);
+static_assert (foo (8) == 9);
+static_assert (foo (9) == 10);
+static_assert (foo (10) == 11);
+static_assert (foo (11) == 12);
+static_assert (foo (12) == 13);
+static_assert (foo (13) == 14);
+static_assert (foo (14) == 15);
+static_assert (foo (15) == 16);
+static_assert (std::uncaught_exceptions () == 0);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh4.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh4.C
new file mode 100644
index 0000000..24118ca
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh4.C
@@ -0,0 +1,72 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct A { virtual ~A () {} };
+struct B { virtual void b (); };
+struct C { virtual void c (); };
+struct D : private B { virtual void d (); };
+struct E { virtual void e (); };
+struct F : D, E, private C { virtual void f (); };
+
+constexpr int
+foo (int x)
+{
+  switch (x)
+    {
+    case 1:
+      try
+	{
+	  static constexpr F f;
+	  D &d = dynamic_cast<D &>((B &) f);	// { dg-error "called without 'std::bad_cast' being defined" }
+	  return -1;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 3:
+      try
+	{
+	  decltype (sizeof 0) x = -64;
+	  char (*a)[2] = new char[x][2];	// { dg-error "called without 'std::bad_array_new_length' being defined" }
+	  delete[] a;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 4:
+      try
+	{
+	  int y = -1;
+	  int *a = new int[y];			// { dg-error "called without 'std::bad_array_new_length' being defined" }
+	  delete[] a;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 5:
+      try
+	{
+	  int z = 1;
+	  int *a = new int[z]{1, 2, 3};		// { dg-error "called without 'std::bad_array_new_length' being defined" }
+	  delete[] a;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    }
+  return -1;
+}
+
+constexpr int a = foo (1);			// { dg-message "in 'constexpr' expansion of" }
+constexpr int b = foo (3);			// { dg-message "in 'constexpr' expansion of" }
+constexpr int c = foo (4);			// { dg-message "in 'constexpr' expansion of" }
+constexpr int d = foo (5);			// { dg-message "in 'constexpr' expansion of" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh5.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh5.C
new file mode 100644
index 0000000..512aa34
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh5.C
@@ -0,0 +1,55 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+constexpr void
+foo ()
+{
+  throw 1;
+}
+
+void
+bar ()
+{
+}
+
+constexpr void
+baz ()
+{
+  foo ();
+  bar ();
+}
+
+constexpr void
+qux ()
+{
+  if consteval {
+    throw 2;
+  }
+  bar ();
+}
+
+constexpr bool
+corge ()
+{
+  try
+    {
+      baz ();
+    }
+  catch (int a)
+    {
+      if (a != 1)
+	return false;
+      try
+	{
+	  qux ();
+	}
+      catch (int b)
+	{
+	  return b == 2;
+	}
+    }
+  return false;
+}
+
+static_assert (corge ());
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh6.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh6.C
new file mode 100644
index 0000000..6fd9462
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh6.C
@@ -0,0 +1,134 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+  constexpr S () : s (0) {}
+  constexpr S (int x) : s (x) {}
+  constexpr S (const S &x) : s (x.s) {}
+  constexpr ~S () {}
+  int s;
+};
+struct T {
+  constexpr T () : t (0) {}
+  constexpr T (int x) : t (x) {}
+  constexpr T (const T &x) : t (x.t) {}
+  constexpr ~T () {}
+  int t;
+};
+struct U : public S, public T {
+  constexpr U () : S (0), T (0) {}
+  constexpr U (int x, int y) : S (x), T (y) {}
+  constexpr U (const U &x) : S (x.s), T (x.t) {}
+  constexpr ~U () {}
+};
+
+constexpr bool
+foo ()
+{
+  try
+    {
+      throw U (1, 2);
+    }
+  catch (const U &x)
+    {
+      if (x.s != 1 || x.t != 2)
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const S &y)
+	{
+	  if (y.s != 1)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const T &z)
+	    {
+	      if (z.t != 2)
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+bar ()
+{
+  try
+    {
+      throw U (1, 2);
+    }
+  catch (U &x)
+    {
+      if (x.s != 1 || x.t != 2)
+	return false;
+      try
+	{
+	  x.s = 3;
+	  x.t = 4;
+	  throw;
+	}
+      catch (S &y)
+	{
+	  if (y.s != 3)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (T &z)
+	    {
+	      if (z.t != 4)
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+baz ()
+{
+  try
+    {
+      throw U (1, 2);
+    }
+  catch (U x)
+    {
+      if (x.s != 1 || x.t != 2)
+	return false;
+      try
+	{
+	  x.s = 3;
+	  x.t = 4;
+	  throw;
+	}
+      catch (S y)
+	{
+	  if (y.s != 1)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (T z)
+	    {
+	      if (z.t != 2)
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+static_assert (foo ());
+static_assert (bar ());
+static_assert (baz ());
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh7.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh7.C
new file mode 100644
index 0000000..6bdf0c3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh7.C
@@ -0,0 +1,151 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+constexpr char p[] = "hello";
+constexpr const char *q[] = { &p[0], &p[3] };
+constexpr const char *const *r = &q[0];
+const char *s[] = { &p[0], &p[3] };
+constexpr const char **t = &s[0];
+
+constexpr bool
+foo ()
+{
+  try
+    {
+      throw t;
+    }
+  catch (const char **const &x)
+    {
+      if (x != t)
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const char **&y)
+	{
+	  if (y != t)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const char **z)
+	    {
+	      if (z != t)
+		return false;
+	      try
+		{
+		  throw;
+		}
+	      catch (const char *const *const &v)
+		{
+		  if (v != (const char *const *) t)
+		    return false;
+		  try
+		    {
+		      throw;
+		    }
+		  catch (const char *const *w)
+		    {
+		      if (w != (const char *const *) t)
+			return false;
+		      return true;
+		    }
+		}
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+bar ()
+{
+  try
+    {
+      throw nullptr;
+    }
+  catch (const char **const &x)
+    {
+      if (x != nullptr)
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const char **&y)
+	{
+	  if (y != nullptr)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const char **z)
+	    {
+	      if (z != nullptr)
+		return false;
+	      try
+		{
+		  throw;
+		}
+	      catch (const char *const *const &v)
+		{
+		  if (v != nullptr)
+		    return false;
+		  try
+		    {
+		      throw;
+		    }
+		  catch (const char *const *w)
+		    {
+		      if (w != nullptr)
+			return false;
+		      return true;
+		    }
+		}
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+baz ()
+{
+  try
+    {
+      throw r;
+    }
+  catch (const char *const *const &x)
+    {
+      if (x != r || **x != 'h')
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const char *const *&y)
+	{
+	  if (y != r || **y != 'h')
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const char *const *z)
+	    {
+	      if (z != r || **z != 'h')
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+static_assert (foo ());
+static_assert (bar ());
+static_assert (baz ());
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh8.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh8.C
new file mode 100644
index 0000000..2560364
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh8.C
@@ -0,0 +1,36 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+#include <exception>
+
+constexpr std::exception_ptr
+foo ()
+{
+  try
+    {
+      throw 42;
+    }
+  catch (...)
+    {
+      return std::current_exception ();
+    }
+}
+
+constexpr bool
+bar ()
+{
+  try
+    {
+      std::rethrow_exception (foo ());
+    }
+  catch (const int &a)
+    {
+       return a == 42;
+    }
+  return false;
+}
+
+static_assert (bar ());
+constexpr std::exception_ptr a = foo ();				// { dg-error "is not a constant expression because it refers to exception object allocated with '__cxa_allocate_exception'" }
+constexpr std::exception_ptr b = std::make_exception_ptr (42ULL);	// { dg-error "is not a constant expression because it refers to exception object allocated with '__cxa_allocate_exception'" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh9.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh9.C
new file mode 100644
index 0000000..fd39fe0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh9.C
@@ -0,0 +1,127 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+namespace std
+{
+  struct exception
+  {
+    constexpr exception () noexcept { }
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr exception (exception &&) = default;
+    constexpr exception &operator= (exception &&) = default;
+    constexpr virtual const char *what () const noexcept
+    { return "std::exception"; }
+  };
+}
+
+struct S : public std::exception {
+  constexpr S () : s (0) {}
+  constexpr S (int x) : s (x) {}
+  constexpr S (const S &x) : s (x.s) {}
+  constexpr virtual ~S () {}
+  constexpr virtual const char *what () noexcept { return "this is S"; }
+  int s;
+};
+struct T : public std::exception {
+  constexpr T () : s (new char[1]), t (0) { s[0] = '\0'; }
+  constexpr T (const char *p, int q) : s (new char[q + 1]), t (q)
+  {
+    for (int i = 0; i <= t; ++i)
+      s[i] = p[i];
+  }
+  constexpr T (const T &x) : s (new char[x.t + 1]), t (x.t)
+  {
+    for (int i = 0; i <= t; ++i)
+      s[i] = x.s[i];
+  }
+  constexpr virtual ~T () { delete[] s; }
+  constexpr virtual const char *what () noexcept { return s; }
+  char *s;
+  int t;
+};
+struct U {
+  constexpr U () : x (0), y (0), z (0) {}
+  constexpr U (int a, long b, unsigned long long c) : x (a), y (b), z (c) {}
+  constexpr U (const U &u) = default;
+  int x;
+  long y;
+  unsigned long long z;
+};
+struct V {
+  constexpr V () : v (0) {}
+  constexpr V (int x) : v (x) {}
+  constexpr V (const V &x) : v (x.v) {}
+  constexpr virtual ~V () {}
+  constexpr virtual const char *what () noexcept { return "this is V"; }
+  int v;
+};
+
+constexpr int
+foo (int x)
+{
+  if (x == 1)
+    throw S (42);
+  else if (x == 2)
+    throw T ("hello, world", sizeof ("hello, world") - 1);
+  else if (x == 3)
+    throw U (1, -2L, 42ULL);
+  else if (x == 4)
+    throw 42;
+  else if (x == 5)
+    throw 1.0;
+  else if (x == 6)
+    throw V (42);
+  else
+    return 42;
+}
+
+constexpr int
+bar (int x) noexcept
+// { dg-error "'std::terminate' called after throwing an exception of type 'S'; 'what\\\(\\\)': 'this is S'" "" { target *-*-* } .-1 }
+// { dg-message "uncaught exception exited from 'noexcept' function 'constexpr int bar\\\(int\\\)'" "" { target *-*-* } .-2 }
+// { dg-error "'std::terminate' called after throwing an exception of type 'T'; 'what\\\(\\\)': 'hello, world'" "" { target *-*-* } .-3 }
+// { dg-error "'std::terminate' called after throwing an exception 'U\\\{1, -2, 42\\\}'" "" { target *-*-* } .-4 }
+// { dg-error "'std::terminate' called after throwing an exception '42'" "" { target *-*-* } .-5 }
+// { dg-error "'std::terminate' called after throwing an exception '1\\\.0e\\\+0'" "" { target *-*-* } .-6 }
+// { dg-error "'std::terminate' called after throwing an exception 'V\\\{\[^\n\r]*42\\\}" "" { target *-*-* } .-7 }
+{
+  return foo (x);
+}
+
+constexpr int
+baz (int x)
+{
+  try
+    {
+      return foo (x);
+    }
+  catch (...)
+    {
+      return -1;
+    }
+}
+
+static_assert (bar (0) == 42);
+constexpr int a = bar (1);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int b = bar (2);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int c = bar (3);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int d = bar (4);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int e = bar (5);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int f = bar (6);		// { dg-message "in 'constexpr' expansion of" }
+static_assert (baz (0) == 42);
+static_assert (baz (1) == -1);
+static_assert (baz (2) == -1);
+static_assert (baz (3) == -1);
+static_assert (baz (4) == -1);
+static_assert (baz (5) == -1);
+static_assert (baz (6) == -1);
+static_assert (foo (0) == 42);
+constexpr int g = foo (1);		// { dg-error "uncaught exception of type 'S'; 'what\\\(\\\)': 'this is S'" }
+constexpr int h = foo (2);		// { dg-error "uncaught exception of type 'T'; 'what\\\(\\\)': 'hello, world'" }
+constexpr int i = foo (3);		// { dg-error "uncaught exception 'U\\\{1, -2, 42\\\}'" }
+constexpr int j = foo (4);		// { dg-error "uncaught exception '42'" }
+constexpr int k = foo (5);		// { dg-error "uncaught exception '1\\\.0e\\\+0'" }
+constexpr int l = foo (6);		// { dg-error "uncaught exception 'V\\\{\[^\n\r]*42\\\}'" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-new4.C b/gcc/testsuite/g++.dg/cpp26/constexpr-new4.C
new file mode 100644
index 0000000..12d8a46
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-new4.C
@@ -0,0 +1,21 @@
+// PR c++/121068
+// { dg-do compile { target c++26 } }
+
+constexpr void *operator new (__SIZE_TYPE__, void *p) { return p; }
+constexpr void *operator new[] (__SIZE_TYPE__, void *p) { return p; }
+
+consteval int
+foo()
+{
+    using T = int;
+    union { T arr[3]; };
+    new(arr) T[3]; // makes arr active
+    for (int i = 0; i < 3; ++i)
+      arr[i].~T();
+
+    new (arr + 2) T{10}; // A
+
+    return 1;
+};
+
+constexpr int g = foo();
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp22.C b/gcc/testsuite/g++.dg/cpp26/decomp22.C
new file mode 100644
index 0000000..576a93b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp22.C
@@ -0,0 +1,66 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+namespace std {
+  template <typename T> struct tuple_size;
+  template <int, typename> struct tuple_element;
+}
+
+struct A {
+  int i, j;
+  template <int I> constexpr const int &get () const { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <const A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const A> { using type = const int; };
+
+constexpr struct B {
+  int i, j;
+  long long k, l;
+} a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+
+constexpr auto [ b, c, d ] = a;				// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto [ e, f, g ] = a;				// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto [ h, i, j, k ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto [ l, m, n, o ] = a[2];			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ p, q, r ] = a;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ s, t, u ] = a;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ v, w, x, y ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ aa, ab, ac, ad ] = a[2];		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+static_assert (b.i == 1 && b.l == 4 && c.j == 6 && c.k == 7 && d.i == 9 && d.k == 11, "");
+static_assert (h == 5 && i == 6 && j == 7 && k == 8, "");
+static_assert (p.i == 1 && p.l == 4 && q.j == 6 && q.k == 7 && r.i == 9 && r.k == 11, "");
+static_assert (&p.i == &a[0].i && &p.l == &a[0].l && &q.j == &a[1].j, "");
+static_assert (&q.k == &a[1].k && &r.i == &a[2].i && &r.k == &a[2].k, "");
+static_assert (v == 5 && w == 6 && x == 7 && y == 8, "");
+static_assert (&v == &a[1].i && &w == &a[1].j && &x == &a[1].k && &y == &a[1].l, "");
+
+constexpr A z = { 42, -42 };
+constexpr auto [ ae, af, ag ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit const auto [ ah, ai, aj ] = z;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ ak, al, am ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ an, ao, ap ] = z;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+static_assert (ae == 42 && af == -42 && ag == 42, "");
+static_assert (&af == &ae + 1 && &ag == &ae, "");
+static_assert (&ae != &z.i && &af != &z.j && &ag != &z.i, "");
+static_assert (ak == 42 && al == -42 && am == 42, "");
+static_assert (&ak == &z.i && &al == &z.j && &am == &z.i, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp23.C b/gcc/testsuite/g++.dg/cpp26/decomp23.C
new file mode 100644
index 0000000..ad2f7e0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp23.C
@@ -0,0 +1,77 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+namespace std {
+  template <typename T> struct tuple_size;
+  template <int, typename> struct tuple_element;
+}
+
+struct A {
+  int i, j;
+  template <int I> constexpr const int &get () const { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <const A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const A> { using type = const int; };
+
+struct B {
+  int i, j;
+  long long k, l;
+};
+
+void
+foo ()
+{
+  static constexpr B a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+  static constexpr auto [ b, c, d ] = a;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto [ e, f, g ] = a;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto [ h, i, j, k ] = a[1];		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto [ l, m, n, o ] = a[2];		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto & [ p, q, r ] = a;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto & [ s, t, u ] = a;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto & [ v, w, x, y ] = a[1];	// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto & [ aa, ab, ac, ad ] = a[2];	// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static_assert (b.i == 1 && b.l == 4 && c.j == 6 && c.k == 7 && d.i == 9 && d.k == 11, "");
+  static_assert (h == 5 && i == 6 && j == 7 && k == 8, "");
+  static_assert (p.i == 1 && p.l == 4 && q.j == 6 && q.k == 7 && r.i == 9 && r.k == 11, "");
+  static_assert (&p.i == &a[0].i && &p.l == &a[0].l && &q.j == &a[1].j, "");
+  static_assert (&q.k == &a[1].k && &r.i == &a[2].i && &r.k == &a[2].k, "");
+  static_assert (v == 5 && w == 6 && x == 7 && y == 8, "");
+  static_assert (&v == &a[1].i && &w == &a[1].j && &x == &a[1].k && &y == &a[1].l, "");
+
+  static constexpr A z = { 42, -42 };
+  static constexpr auto [ ae, af, ag ] = z;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit const auto [ ah, ai, aj ] = z;	// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto & [ ak, al, am ] = z;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto & [ an, ao, ap ] = z;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static_assert (ae == 42 && af == -42 && ag == 42, "");
+  static_assert (&af == &ae + 1 && &ag == &ae, "");
+  static_assert (&ae != &z.i && &af != &z.j && &ag != &z.i, "");
+  static_assert (ak == 42 && al == -42 && am == 42, "");
+  static_assert (&ak == &z.i && &al == &z.j && &am == &z.i, "");
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp24.C b/gcc/testsuite/g++.dg/cpp26/decomp24.C
new file mode 100644
index 0000000..5da1321
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp24.C
@@ -0,0 +1,20 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+struct B {
+  int i, j;
+  long long k, l;
+};
+
+void
+foo ()
+{
+  constexpr B a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+  constexpr auto [ b, c, d ] = a;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+  constexpr auto [ h, i, j, k ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+  static_assert (b.i == 1 && b.l == 4 && c.j == 6 && c.k == 7 && d.i == 9 && d.k == 11, "");
+  static_assert (h == 5 && i == 6 && j == 7 && k == 8, "");
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp25.C b/gcc/testsuite/g++.dg/cpp26/decomp25.C
new file mode 100644
index 0000000..f395685
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp25.C
@@ -0,0 +1,119 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "-fno-implicit-constexpr" }
+
+namespace std {
+  template <typename T> struct tuple_size;
+  template <int, typename> struct tuple_element;
+}
+
+struct A {
+  int i, j;
+  template <int I> int &get () { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, A> { using type = int; };
+template <> struct std::tuple_size <const A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const A> { using type = int; };
+
+struct B {
+  int i, j;
+  long long k, l;
+} a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };	// { dg-message "'a' was not declared 'constexpr'" "" }
+
+struct C {
+  int i, j;
+  template <int I> const int &get () const { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <const C> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const C> { using type = const int; };
+
+constexpr auto [ b, c, d ] = a;				// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto [ e, f, g ] = a;				// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable '<structured bindings>' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target c++20 } .-2 }
+#endif
+constexpr auto [ h, i, j, k ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto [ l, m, n, o ] = a[2];			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable '<structured bindings>' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target c++20 } .-2 }
+#endif
+constexpr auto & [ p, q, r ] = a;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ s, t, u ] = a;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ v, w, x, y ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ aa, ab, ac, ad ] = a[2];		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+
+A z = { 42, -42 };					// { dg-message "'z' was not declared 'constexpr'" "" }
+constexpr auto [ ae, af, ag ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "the value of 'z' is not usable in a constant expression" "" { target *-*-* } .-2 }
+							// { dg-error "passing 'const A' as 'this' argument discards qualifiers" "" { target *-*-* } .-3 }
+							// { dg-error "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target *-*-* } .-4 }
+#if __cpp_constinit >= 201907
+constinit const auto [ ah, ai, aj ] = z;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable '<structured bindings>' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "the value of 'z' is not usable in a constant expression" "" { target c++20 } .-2 }
+							// { dg-error "passing 'const A' as 'this' argument discards qualifiers" "" { target c++20 } .-3 }
+							// { dg-error "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target c++20 } .-4 }
+							// { dg-error "'constinit' variable 'ah' does not have a constant initializer" "" { target c++20 } .-5 }
+							// { dg-error "'constinit' variable 'ai' does not have a constant initializer" "" { target c++20 } .-6 }
+							// { dg-error "'constinit' variable 'aj' does not have a constant initializer" "" { target c++20 } .-7 }
+#endif
+constexpr auto & [ ak, al, am ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto & [ an, ao, ap ] = z;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable 'an' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "'constinit' variable 'ao' does not have a constant initializer" "" { target c++20 } .-2 }
+							// { dg-error "'constinit' variable 'ap' does not have a constant initializer" "" { target c++20 } .-3 }
+							// { dg-message "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target c++20 } .-4 }
+#endif
+
+constexpr C zz = { 42, -42 };
+constexpr auto [ aq, ar, as ] = zz;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit const auto [ at, au, av ] = zz;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target c++20 } .-1 }
+							// { dg-error "'constinit' variable 'at' does not have a constant initializer" "" { target c++20 } .-2 }
+							// { dg-error "'constinit' variable 'au' does not have a constant initializer" "" { target c++20 } .-3 }
+							// { dg-error "'constinit' variable 'av' does not have a constant initializer" "" { target c++20 } .-4 }
+#endif
+constexpr auto & [ aw, ax, ay ] = zz;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto & [ az, ba, bb ] = zz;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable 'az' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "'constinit' variable 'ba' does not have a constant initializer" "" { target c++20 } .-2 }
+							// { dg-error "'constinit' variable 'bb' does not have a constant initializer" "" { target c++20 } .-3 }
+							// { dg-message "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target c++20 } .-4 }
+#endif
+
+void
+foo ()
+{
+#if __cpp_constinit >= 201907
+  constexpr B a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+  constinit auto [ b, c, d ] = a;			// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+  constinit auto & [ e, f, g ] = a;			// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+  constinit auto [ h, i, j, k ] = a[1];			// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+  constinit auto & [ l, m, n, o ] = a[2];		// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+#endif
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp9.C b/gcc/testsuite/g++.dg/cpp26/decomp9.C
index 5629c4c..ee18b60 100644
--- a/gcc/testsuite/g++.dg/cpp26/decomp9.C
+++ b/gcc/testsuite/g++.dg/cpp26/decomp9.C
@@ -63,6 +63,7 @@ foo (const S &&s)
   if (static auto [i, j, k] = t)		// { dg-warning "structured bindings in conditions only available with" "" { target c++23_down } }
     ;						// { dg-error "'static' invalid in condition" "" { target *-*-* } .-1 }
 						// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
-  if (constexpr auto [i, j, k] = t)		// { dg-warning "structured bindings in conditions only available with" "" { target c++23_down } }
-    ;						// { dg-error "structured binding declaration cannot be 'constexpr'" "" { target *-*-* } .-1 }
+  constexpr T t2 = { 1, 2, 3 };
+  if (constexpr auto [i, j, k] = t2)		// { dg-warning "structured bindings in conditions only available with" "" { target c++23_down } }
+    ;						// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
 }
diff --git a/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C b/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C
index e4ffc35..cfc5f61 100644
--- a/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C
+++ b/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C
@@ -624,19 +624,31 @@
 #endif
 
 #ifndef __cpp_pack_indexing
-# error "__cpp_pack_indexing"
+#  error "__cpp_pack_indexing"
 #elif __cpp_pack_indexing != 202311
 #  error "__cpp_pack_indexing != 202311"
 #endif
 
 #ifndef __cpp_pp_embed
-# error "__cpp_pp_embed"
+#  error "__cpp_pp_embed"
 #elif __cpp_pp_embed != 202502
 #  error "__cpp_pp_embed != 202502"
 #endif
 
 #ifndef __cpp_constexpr_virtual_inheritance
-# error "__cpp_constexpr_virtual_inheritance"
+#  error "__cpp_constexpr_virtual_inheritance"
 #elif __cpp_constexpr_virtual_inheritance != 202506
 #  error "__cpp_constexpr_virtual_inheritance != 202506"
 #endif
+
+#ifndef __cpp_constexpr_exceptions
+#  error "__cpp_constexpr_exceptions"
+#elif __cpp_constexpr_exceptions != 202411
+#  error "__cpp_constexpr_exceptions != 202411"
+#endif
+
+#ifndef __cpp_trivial_relocatability
+#  error "__cpp_trivial_relocatability"
+#elif __cpp_trivial_relocatability != 202502
+#  error "__cpp_trivial_relocatability != 202502"
+#endif
diff --git a/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C b/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C
index 0830ce8..9b56e84 100644
--- a/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C
+++ b/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C
@@ -70,7 +70,7 @@ foo ()
       ++_;
     }
     {
-      static int _ = 3;
+      static int _ = 3;		// { dg-warning "variable '_' set but not used" }
       ++_;
     }
     {
diff --git a/gcc/testsuite/g++.dg/cpp26/static_assert1.C b/gcc/testsuite/g++.dg/cpp26/static_assert1.C
index 1d0e6f2..a6eab3c 100644
--- a/gcc/testsuite/g++.dg/cpp26/static_assert1.C
+++ b/gcc/testsuite/g++.dg/cpp26/static_assert1.C
@@ -51,7 +51,7 @@ static_assert (true, H {});	// { dg-warning "'static_assert' with non-string mes
 struct I { constexpr signed char size () const { return 0; }
 	   const char *data () const { return ""; } };
 static_assert (true, I {});	// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_down } }
-struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" }
+struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_down } }
 	   constexpr const char *data () const { return ""; };
 	   constexpr J (int x) : j (x) {}
 	   int j; };
@@ -60,6 +60,7 @@ static_assert (false, J (0));	// { dg-warning "'static_assert' with non-string m
 				// { dg-error "static assertion failed" "" { target *-*-* } .-1 }
 static_assert (false, J (1));	// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_down } }
 				// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target *-*-* } .-1 }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
 struct K { constexpr operator int () { return 4; } };
 struct L { constexpr operator const char * () { return "test"; } };
 struct M { constexpr K size () const { return {}; }
@@ -261,10 +262,11 @@ namespace NN
 #if __cplusplus >= 201402L
   struct J {
     static constexpr int size () { return 0; }
-    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++14 } }
+    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target { c++14 && c++23_down } } }
   static_assert (true, J{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target { c++14 && c++23_down } } }
   static_assert (false, J{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target { c++14 && c++23_down } } }
 					// { dg-error "constexpr string 'data\\\(\\\)' must be a core constant expression" "" { target c++14 } .-1 }
+					// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
 #endif
 #if __cpp_if_consteval >= 202106L
   struct K {
@@ -282,19 +284,21 @@ namespace NN
   static_assert (false, L{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
 					// { dg-error "static assertion failed: test" "" { target c++23 } .-1 }
   struct M {
-    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
     static constexpr const char *data () { return "test"; }
   };
   static_assert (true, M{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
   static_assert (false, M{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
 					// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target c++23 } .-1 }
+					// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
   struct N {
     static constexpr int size () { return 4; }
-    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
   };
   static_assert (true, N{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
   static_assert (false, N{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
 					// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++23 } .-1 }
+					// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
 #endif
   struct O { constexpr int operator () () const { return 12; } };
   struct P { constexpr const char *operator () () const { return "another test"; } };
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable1.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable1.C
new file mode 100644
index 0000000..29ba907
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable1.C
@@ -0,0 +1,137 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A {};
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+
+struct B {
+  B ();
+  ~B ();
+  B (const B &);
+  B (B &&);
+  B &operator= (const B &);
+  B &operator= (B &&);
+};
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+struct C {
+  C (C &&) = delete;
+  C &operator= (C &&) = delete;
+  C () = default;
+};
+
+// Note, P2786R13 says it is trivially relocatable, but I think
+// it isn't default-movable because overload resolution in both
+// cases selects a deleted special member fn.
+static_assert (!std::is_trivially_relocatable_v <C>, "");
+static_assert (!std::is_replaceable_v <C>, "");
+
+struct D : A {};
+
+static_assert (std::is_trivially_relocatable_v <D>, "");
+static_assert (std::is_replaceable_v <D>, "");
+
+struct E : virtual A {};
+
+static_assert (!std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_replaceable_v <E>, "");
+
+struct F trivially_relocatable_if_eligible : virtual A {};
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+struct G { B data; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_replaceable_v <G>, "");
+
+struct H { ~H () = default; };
+
+static_assert (std::is_trivially_relocatable_v <H>, "");
+static_assert (std::is_replaceable_v <H>, "");
+
+struct I { ~I (); };
+I::~I () = default;
+
+static_assert (!std::is_trivially_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+struct J { virtual ~J () = default; };
+
+// Note, P2786R13 says otherwise for both, but that looks like
+// a bug in the paper, it otherwise says that polymorphic types
+// can be both trivially relocatable and replaceable.
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (std::is_replaceable_v <J>, "");
+
+struct K { ~K () = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L { L (L &&) = default; };
+
+// Note, P2786R13 says otherwise for both, but that looks like
+// a bug in the paper to me.  While move ctor is trivial here,
+// copy assignment operator is implicitly declared as deleted
+// and move assignent operator is not declared.
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M { M (M &&); };
+M::M (M &&) = default;
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N { N (N &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable10.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable10.C
new file mode 100644
index 0000000..2ed1d5e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable10.C
@@ -0,0 +1,135 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A replaceable_if_eligible { A (A &&); A &operator= (A &&); ~A (); int a; };
+
+static_assert (std::is_replaceable_v <A>, "");
+static_assert (!std::is_replaceable_v <const A>, "");
+static_assert (!std::is_replaceable_v <A volatile>, "");
+static_assert (!std::is_replaceable_v <const A volatile>, "");
+
+struct B { B (B &&); B &operator= (B &&); ~B (); int a; };
+
+static_assert (!std::is_replaceable_v <B>, "");
+
+struct C replaceable_if_eligible : public A { C (C &&); C &operator= (C &&); ~C (); int a; };
+
+static_assert (std::is_replaceable_v <C>, "");
+
+struct D replaceable_if_eligible : public B { D (D &&); D &operator= (D &&); ~D (); int a; };
+
+static_assert (!std::is_replaceable_v <D>, "");
+
+struct E replaceable_if_eligible { E (E &&); E &operator= (E &&); ~E (); A a; };
+
+static_assert (std::is_replaceable_v <E>, "");
+
+struct F replaceable_if_eligible { F (F &&); F &operator= (F &&); ~F (); B a; };
+
+static_assert (!std::is_replaceable_v <F>, "");
+
+struct G replaceable_if_eligible { G (G &&); G &operator= (G &&); ~G () = delete; int a; };
+
+static_assert (!std::is_replaceable_v <G>, "");
+
+struct H replaceable_if_eligible : virtual A { H (H &&); H &operator= (H &&); ~H (); int a; };
+
+static_assert (std::is_replaceable_v <H>, "");
+
+struct I replaceable_if_eligible { I (I &&) = delete; I &operator= (I &&); ~I (); int a; };
+
+static_assert (!std::is_replaceable_v <I>, "");
+
+struct J replaceable_if_eligible { J (J &&); J &operator= (J &&) = delete; ~J (); int a; };
+
+static_assert (!std::is_replaceable_v <J>, "");
+
+struct K replaceable_if_eligible { K (const K &) = delete; K &operator= (K &&); ~K (); int a; };
+
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L replaceable_if_eligible { L (L &&); L &operator= (const L &) = delete; ~L (); int a; };
+
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M replaceable_if_eligible { M (); private: M (M &&); M &operator= (M &&); ~M (); int a; };
+
+static_assert (std::is_replaceable_v <M>, "");
+
+struct N replaceable_if_eligible { N (N &&); N &operator= (N &&); ~N (); const A a; };
+
+static_assert (!std::is_replaceable_v <N>, "");
+
+struct O replaceable_if_eligible { O (O &&); O &operator= (O &&); ~O (); volatile A a; };
+
+static_assert (!std::is_replaceable_v <O>, "");
+
+struct P replaceable_if_eligible { P (P &&); P &operator= (P &&); ~P (); const volatile A a; };
+
+static_assert (!std::is_replaceable_v <P>, "");
+
+struct Q replaceable_if_eligible { Q (Q &&); Q &operator= (Q &&); ~Q (); union { A a; int b; char c; }; };
+
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct R replaceable_if_eligible { R (R &&); R &operator= (R &&); ~R (); union { int a; B b; short c; }; };
+
+static_assert (!std::is_replaceable_v <R>, "");
+
+struct S replaceable_if_eligible { S (S &&); S &operator= (S &&); ~S (); union { int a; const int b; short c; }; };
+
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T replaceable_if_eligible { T (T &&); T &operator= (T &&); ~T () = default; int a; };
+
+static_assert (std::is_replaceable_v <T>, "");
+
+struct U replaceable_if_eligible { U (U &&); U &operator= (U &&); ~U (); union { T a; int b; char c; }; };
+
+static_assert (!std::is_replaceable_v <U>, "");
+
+struct V replaceable_if_eligible { V (V &&); V &operator= (V &&); ~V (); union { unsigned long long a; int b; char c; }; };
+
+static_assert (std::is_replaceable_v <V>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable11.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable11.C
new file mode 100644
index 0000000..71fc6f1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable11.C
@@ -0,0 +1,134 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A { A (A &&) = default; A &operator= (A &&) = default; ~A () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+
+struct B { B (B &&); B &operator= (B &&) = default; ~B () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+union C { int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_replaceable_v <C>, "");
+
+union D { int a; A b; B c; };
+
+static_assert (!std::is_trivially_relocatable_v <D>, "");
+static_assert (!std::is_replaceable_v <D>, "");
+
+union E { E (); int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_replaceable_v <E>, "");
+
+union F { F () = default; int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+union G { G (const G &); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_replaceable_v <G>, "");
+
+union H { H (const H &) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+union I { I (I &&); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+union J { J (J &&) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <J>, "");
+static_assert (!std::is_replaceable_v <J>, "");
+
+union K { K &operator= (const K &); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+union L { L &operator= (const L &) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+union M { M &operator= (M &&); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+union N { N &operator= (N &&) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
+
+union O { ~O (); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <O>, "");
+static_assert (!std::is_replaceable_v <O>, "");
+
+union P { ~P () = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <P>, "");
+static_assert (!std::is_replaceable_v <P>, "");
+
+union Q { int a; const A b; };
+
+static_assert (std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+union S { volatile int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable2.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable2.C
new file mode 100644
index 0000000..b740061
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable2.C
@@ -0,0 +1,204 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+class A {};
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+static_assert (std::is_trivially_relocatable <A>::value, "");
+static_assert (std::is_replaceable <A>::value, "");
+
+struct B { ~B (); };
+static B z;
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+static_assert (!std::is_trivially_relocatable <B>::value, "");
+static_assert (!std::is_replaceable <B>::value, "");
+
+class C trivially_relocatable_if_eligible {};
+
+static_assert (std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_replaceable_v <C>, "");
+
+class D trivially_relocatable_if_eligible : A {};
+
+static_assert (std::is_trivially_relocatable_v <D>, "");
+static_assert (std::is_replaceable_v <D>, "");
+
+class E trivially_relocatable_if_eligible {
+  int a;
+  void *b;
+  int c[3];
+  A d[3];
+  B &e = z;
+};
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (!std::is_replaceable_v <E>, "");
+
+class F trivially_relocatable_if_eligible : A {};
+
+static_assert (std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+class G trivially_relocatable_if_eligible : virtual A {};
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (std::is_replaceable_v <G>, "");
+
+class H trivially_relocatable_if_eligible : B {};
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+class I trivially_relocatable_if_eligible { I (I &&); };
+
+static_assert (std::is_trivially_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+class J trivially_relocatable_if_eligible { ~J (); };
+
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (!std::is_replaceable_v <J>, "");
+
+class K trivially_relocatable_if_eligible {
+  B a;
+  B b[1];
+  const B c;
+  const B d[1];
+};
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+class L trivially_relocatable_if_eligible: virtual A, B { B a; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+static_assert (!std::is_trivially_relocatable_v <void>, "");
+static_assert (!std::is_trivially_relocatable_v <const void>, "");
+static_assert (std::is_trivially_relocatable_v <int>, "");
+static_assert (std::is_trivially_relocatable_v <const int>, "");
+static_assert (std::is_trivially_relocatable_v <char>, "");
+static_assert (std::is_trivially_relocatable_v <char const volatile>, "");
+static_assert (std::is_trivially_relocatable_v <unsigned long long>, "");
+static_assert (std::is_trivially_relocatable_v <void *>, "");
+static_assert (std::is_trivially_relocatable_v <const int *>, "");
+static_assert (!std::is_trivially_relocatable_v <int &>, "");
+static_assert (!std::is_trivially_relocatable_v <A &>, "");
+static_assert (std::is_trivially_relocatable_v <const A>, "");
+static_assert (std::is_trivially_relocatable_v <A [1]>, "");
+static_assert (std::is_trivially_relocatable_v <A []>, "");
+static_assert (!std::is_replaceable_v <void>, "");
+static_assert (!std::is_replaceable_v <const void>, "");
+static_assert (std::is_replaceable_v <int>, "");
+static_assert (!std::is_replaceable_v <const int>, "");
+static_assert (std::is_replaceable_v <char>, "");
+static_assert (!std::is_replaceable_v <char const volatile>, "");
+static_assert (std::is_replaceable_v <unsigned long long>, "");
+static_assert (std::is_replaceable_v <void *>, "");
+static_assert (std::is_replaceable_v <const int *>, "");
+static_assert (!std::is_replaceable_v <int &>, "");
+static_assert (!std::is_replaceable_v <A &>, "");
+static_assert (!std::is_replaceable_v <const A>, "");
+static_assert (std::is_replaceable_v <A [1]>, "");
+static_assert (std::is_replaceable_v <A []>, "");
+
+struct M { const int i; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N trivially_relocatable_if_eligible { const int i; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
+
+struct O { ~O (); };
+
+static_assert (!std::is_trivially_relocatable_v <O>, "");
+static_assert (!std::is_replaceable_v <O>, "");
+
+struct P { ~P () = default; };
+
+static_assert (std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_replaceable_v <P>, "");
+
+struct Q { Q (Q &&); Q (const Q &) = default; };
+
+static_assert (!std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct R { R (R &&); };
+
+static_assert (!std::is_trivially_relocatable_v <R>, "");
+static_assert (!std::is_replaceable_v <R>, "");
+
+struct S { S (S &&) = default; };
+
+static_assert (!std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T { T (T &&) = default; T &operator= (T &&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <T>, "");
+static_assert (std::is_replaceable_v <T>, "");
+
+struct U { U (const U &); };
+
+static_assert (!std::is_trivially_relocatable_v <U>, "");
+static_assert (!std::is_replaceable_v <U>, "");
+
+struct V { V (const V&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <V>, "");
+static_assert (std::is_replaceable_v <V>, "");
+
+struct W { W (W &&) = delete; W (const W &) = default; };
+
+static_assert (!std::is_trivially_relocatable_v <W>, "");
+static_assert (!std::is_replaceable_v <W>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable3.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable3.C
new file mode 100644
index 0000000..312c11b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable3.C
@@ -0,0 +1,213 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+class A {};
+
+struct B { ~B (); };
+
+class C trivially_relocatable_if_eligible { C (C &&); };
+
+template <typename T>
+class D trivially_relocatable_if_eligible : T {};
+D<A> a;
+D<B> b;
+
+static_assert (std::is_trivially_relocatable_v <D<A>>, "");
+static_assert (!std::is_trivially_relocatable_v <D<B>>, "");
+static_assert (std::is_replaceable_v <D<A>>, "");
+static_assert (!std::is_replaceable_v <D<B>>, "");
+
+struct E { E (E &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <E>, "");
+static_assert (!std::is_replaceable_v <E>, "");
+
+struct F { F (const F &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (!std::is_replaceable_v <F>, "");
+
+struct G { G &operator= (G &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_replaceable_v <G>, "");
+
+struct H { ~H () = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+union U { C u; };
+
+static_assert (std::is_trivially_relocatable_v <U>, "");
+static_assert (!std::is_replaceable_v <U>, "");
+
+template <typename T>
+struct I { int s; T t; };
+
+static_assert (std::is_trivially_relocatable_v <I<int>>, "");
+static_assert (std::is_trivially_relocatable_v <I<volatile int>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<const int>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<const int &>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<int &>>, "");
+static_assert (std::is_trivially_relocatable_v <I<int [2]>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<const int [2]>>, "");
+static_assert (std::is_trivially_relocatable_v <I<int []>>, "");
+static_assert (std::is_replaceable_v <I<int>>, "");
+static_assert (!std::is_replaceable_v <I<volatile int>>, "");
+static_assert (!std::is_replaceable_v <I<const int>>, "");
+static_assert (!std::is_replaceable_v <I<const int &>>, "");
+static_assert (!std::is_replaceable_v <I<int &>>, "");
+static_assert (std::is_replaceable_v <I<int [2]>>, "");
+static_assert (!std::is_replaceable_v <I<const int [2]>>, "");
+
+template <typename T>
+struct J trivially_relocatable_if_eligible { int s; T t; };
+
+static_assert (std::is_trivially_relocatable_v <J<int>>, "");
+static_assert (std::is_trivially_relocatable_v <J<volatile int>>, "");
+static_assert (std::is_trivially_relocatable_v <J<const int>>, "");
+static_assert (std::is_trivially_relocatable_v <J<const int &>>, "");
+static_assert (std::is_trivially_relocatable_v <J<int &>>, "");
+static_assert (std::is_trivially_relocatable_v <J<int [2]>>, "");
+static_assert (std::is_trivially_relocatable_v <J<const int [2]>>, "");
+static_assert (std::is_trivially_relocatable_v <J<int []>>, "");
+static_assert (std::is_replaceable_v <J<int>>, "");
+static_assert (!std::is_replaceable_v <J<volatile int>>, "");
+static_assert (!std::is_replaceable_v <J<const int>>, "");
+static_assert (!std::is_replaceable_v <J<const int &>>, "");
+static_assert (!std::is_replaceable_v <J<int &>>, "");
+static_assert (std::is_replaceable_v <J<int [2]>>, "");
+static_assert (!std::is_replaceable_v <J<const int [2]>>, "");
+static_assert (std::is_replaceable_v <J<int []>>, "");
+
+struct K { K (K &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L { L (const L &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M { M &operator= (M &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N { N (N &&) = default; N &operator= (N &&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (std::is_replaceable_v <N>, "");
+
+struct O {
+  O (const O &) = default;
+  O (O &&) = default;
+  O &operator= (O &&) = default;
+};
+
+static_assert (std::is_trivially_relocatable_v <O>, "");
+static_assert (std::is_replaceable_v <O>, "");
+
+struct P { P (P &&) = default; P &operator= (P &&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_replaceable_v <P>, "");
+
+struct Q { Q (Q &&) {} };
+
+static_assert (!std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct R { R (const R &) {} };
+
+static_assert (!std::is_trivially_relocatable_v <R>, "");
+static_assert (!std::is_replaceable_v <R>, "");
+
+struct S { S &operator= (const S &) { return *this; }; };
+
+static_assert (!std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T {};
+
+static_assert (std::is_trivially_relocatable_v <T>, "");
+static_assert (std::is_replaceable_v <T>, "");
+
+struct V replaceable_if_eligible {};
+
+static_assert (std::is_trivially_relocatable_v <V>, "");
+static_assert (std::is_replaceable_v <V>, "");
+
+struct W { template <typename U> W (const U &) = delete; };
+
+static_assert (std::is_trivially_relocatable_v <W>, "");
+static_assert (std::is_replaceable_v <W>, "");
+
+template <typename T>
+struct X : T {};
+
+static_assert (!std::is_trivially_relocatable_v <X<Q>>, "");
+static_assert (!std::is_replaceable_v <X<Q>>, "");
+
+template <typename T>
+struct Y : virtual T {};
+
+static_assert (!std::is_trivially_relocatable_v <Y<I<int>>>, "");
+static_assert (!std::is_trivially_relocatable_v <Y<I<const int>>>, "");
+static_assert (!std::is_trivially_relocatable_v <Y<Q>>, "");
+static_assert (std::is_replaceable_v <Y<I<int>>>, "");
+static_assert (!std::is_replaceable_v <Y<I<const int>>>, "");
+static_assert (!std::is_replaceable_v <Y<Q>>, "");
+
+struct Z {
+  virtual ~Z () = default;
+  Z (Z &&) = default;
+  Z &operator= (Z &&) = default;
+};
+
+static_assert (std::is_trivially_relocatable_v <Z>, "");
+static_assert (std::is_replaceable_v <Z>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable4.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable4.C
new file mode 100644
index 0000000..10aafc1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable4.C
@@ -0,0 +1,128 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A replaceable_if_eligible {
+  ~A () = delete;
+  A (A &&) = default;
+  A &operator= (A &&) = default;
+};
+
+static_assert (!std::is_trivially_relocatable_v <A>, "");
+static_assert (!std::is_replaceable_v <A>, "");
+
+struct B replaceable_if_eligible { B (const B &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+template <typename T>
+struct C replaceable_if_eligible : virtual T {};
+
+static_assert (!std::is_trivially_relocatable_v <C<A>>, "");
+static_assert (!std::is_trivially_relocatable_v <C<B>>, "");
+static_assert (!std::is_replaceable_v <C<A>>, "");
+static_assert (!std::is_replaceable_v <C<B>>, "");
+
+template <typename T>
+struct D { int s; T t; };
+
+static_assert (!std::is_trivially_relocatable_v <C<D<int>>>, "");
+static_assert (std::is_replaceable_v <C<D<int>>>, "");
+
+struct E trivially_relocatable_if_eligible replaceable_if_eligible {
+  E (E &&);
+  E &operator= (E &&) = default;
+};
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_replaceable_v <E>, "");
+
+struct F trivially_relocatable_if_eligible replaceable_if_eligible {
+  F (F &&) = default;
+  F &operator= (F &&);
+};
+
+static_assert (std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+struct G replaceable_if_eligible { G (G const &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <G>, "");
+static_assert (std::is_replaceable_v <G>, "");
+
+struct H { H (H const &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <H>, "");
+static_assert (std::is_replaceable_v <H>, "");
+
+struct I replaceable_if_eligible { I &operator= (const I &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <I>, "");
+static_assert (std::is_replaceable_v <I>, "");
+
+struct J { J &operator= (J const &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (std::is_replaceable_v <J>, "");
+
+struct K { K (const K &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L { L (L&&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M { M operator= (M); };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N { N operator= (N &&); };
+
+static_assert (!std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable5.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable5.C
new file mode 100644
index 0000000..0416137
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable5.C
@@ -0,0 +1,77 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-additional-options "-Wc++26-compat" }
+
+struct A __trivially_relocatable_if_eligible { A (const A &&); };
+struct B __replaceable_if_eligible { B (const B &&); B &operator= (B &&); };
+struct C __replaceable_if_eligible __final __trivially_relocatable_if_eligible { C (const C &&); C &operator= (C &&); };
+#if __cpp_trivial_relocatability >= 202502L
+struct D trivially_relocatable_if_eligible { D (const D &&); };
+struct E replaceable_if_eligible { E (const E &&); E &operator= (E &&); };
+struct F trivially_relocatable_if_eligible replaceable_if_eligible final { F (const F &&); F &operator= (F &&); };
+#else
+struct D trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+// { dg-error "variable 'D trivially_relocatable_if_eligible' has initializer but incomplete type" "" { target c++23_down } .-1 }
+struct E replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+// { dg-error "variable 'E replaceable_if_eligible' has initializer but incomplete type" "" { target c++23_down } .-1 }
+struct F trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target c++23_down } .-1 }
+#endif
+#if __cplusplus <= 202302L
+struct G {};
+struct G trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+struct H {};
+struct H replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+struct I {};
+struct I trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+#endif							// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target c++23_down } .-1 }
+struct J {};
+struct J __trivially_relocatable_if_eligible {};	// { dg-error "redefinition of 'struct J'" }
+struct K {};
+struct K __replaceable_if_eligible {};			// { dg-error "redefinition of 'struct K'" }
+struct L {};
+struct L __trivially_relocatable_if_eligible __replaceable_if_eligible {};	// { dg-error "redefinition of 'struct L'" }
+struct M __trivially_relocatable_if_eligible __trivially_relocatable_if_eligible {}; // { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" }
+struct N __replaceable_if_eligible __replaceable_if_eligible {};		// { dg-error "duplicate '__replaceable_if_eligible' specifier" }
+struct O __trivially_relocatable_if_eligible __replaceable_if_eligible __replaceable_if_eligible __trivially_relocatable_if_eligible final final {};
+// { dg-error "duplicate '__replaceable_if_eligible' specifier" "" { target *-*-* } .-1 }
+// { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" "" { target *-*-* } .-2 }
+// { dg-error "duplicate 'final' specifier" "" { target *-*-* } .-3 }
+#if __cpp_trivial_relocatability >= 202502L
+struct P trivially_relocatable_if_eligible trivially_relocatable_if_eligible {}; // { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } }
+struct Q replaceable_if_eligible replaceable_if_eligible {};			// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } }
+struct R trivially_relocatable_if_eligible replaceable_if_eligible replaceable_if_eligible trivially_relocatable_if_eligible final final {};
+// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } .-1 }
+// { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } .-2 }
+// { dg-error "duplicate 'final' specifier" "" { target c++26 } .-3 }
+struct S trivially_relocatable_if_eligible __trivially_relocatable_if_eligible {}; // { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" "" { target c++26 } }
+struct T replaceable_if_eligible __replaceable_if_eligible {};			// { dg-error "duplicate '__replaceable_if_eligible' specifier" "" { target c++26 } }
+struct U trivially_relocatable_if_eligible replaceable_if_eligible __replaceable_if_eligible __trivially_relocatable_if_eligible final __final {};
+// { dg-error "duplicate '__replaceable_if_eligible' specifier" "" { target c++26 } .-1 }
+// { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" "" { target c++26 } .-2 }
+// { dg-error "duplicate '__final' specifier" "" { target c++26 } .-3 }
+struct V __trivially_relocatable_if_eligible trivially_relocatable_if_eligible {}; // { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } }
+struct W __replaceable_if_eligible replaceable_if_eligible {};			// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } }
+struct X __trivially_relocatable_if_eligible __replaceable_if_eligible replaceable_if_eligible trivially_relocatable_if_eligible __final final {};
+// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } .-1 }
+// { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } .-2 }
+// { dg-error "duplicate 'final' specifier" "" { target c++26 } .-3 }
+#else
+struct Y {};
+Y foo ();
+struct Y trivially_relocatable_if_eligible = foo ();    // { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+struct Z {};
+Z bar ();
+struct Z replaceable_if_eligible = bar ();		// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+#endif
+
+static_assert (__builtin_is_trivially_relocatable (A), "");
+static_assert (__builtin_is_replaceable (B), "");
+static_assert (__builtin_is_trivially_relocatable (C), "");
+static_assert (__builtin_is_replaceable (C), "");
+#if __cpp_trivial_relocatability >= 202502L
+static_assert (__builtin_is_trivially_relocatable (D), "");
+static_assert (__builtin_is_replaceable (E), "");
+static_assert (__builtin_is_trivially_relocatable (F), "");
+static_assert (__builtin_is_replaceable (F), "");
+#endif
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable6.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable6.C
new file mode 100644
index 0000000..ffcf12b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable6.C
@@ -0,0 +1,30 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++98_only } }
+// { dg-additional-options "-Wc++26-compat" }
+
+struct A __trivially_relocatable_if_eligible {};
+struct B __replaceable_if_eligible {};
+struct C __replaceable_if_eligible __final __trivially_relocatable_if_eligible {};
+struct D trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "variable 'D trivially_relocatable_if_eligible' has initializer but incomplete type" "" { target *-*-* } .-1 }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-2 }
+struct E replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" }
+// { dg-error "variable 'E replaceable_if_eligible' has initializer but incomplete type" "" { target *-*-* } .-1 }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-2 }
+struct F trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target *-*-* } .-1 }
+struct G {};
+struct G trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-1 }
+struct H {};
+struct H replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-1 }
+struct I {};
+struct I trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target *-*-* } .-1 }
+struct J {};
+J foo ();
+struct J trivially_relocatable_if_eligible = foo ();	// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+struct K {};
+K bar ();
+struct K replaceable_if_eligible = bar ();		// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" }
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable7.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable7.C
new file mode 100644
index 0000000..608c245
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable7.C
@@ -0,0 +1,33 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+							// { dg-error "invalid use of incomplete type 'struct A'" "" { target *-*-* } .-1 }
+
+template <typename T>
+inline constexpr bool is_nothrow_relocatable_v		// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_nothrow_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+							// { dg-error "invalid use of incomplete type 'struct A'" "" { target *-*-* } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}							// { dg-error "invalid use of incomplete type 'struct A'" "" { target *-*-* } .-1 }
+
+struct A;						// { dg-message "forward declaration of 'struct A'" }
+
+auto a = std::is_trivially_relocatable_v <A>;		// { dg-message "required from here" }
+auto b = std::is_nothrow_relocatable_v <A>;		// { dg-message "required from here" }
+auto c = std::is_replaceable_v <A>;			// { dg-message "required from here" }
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable8.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable8.C
new file mode 100644
index 0000000..5f8390d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable8.C
@@ -0,0 +1,190 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_nothrow_relocatable
+  : public integral_constant <bool, __builtin_is_nothrow_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_nothrow_relocatable_v		// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_nothrow_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A { A (A &&) = default; A &operator= (A &&) = default; ~A () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_nothrow_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+
+struct B { B (B &&); B &operator= (B &&) = default; ~B () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_nothrow_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+struct C { C (C &&) = default; C &operator= (C &&); ~C () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_nothrow_relocatable_v <C>, "");
+static_assert (!std::is_replaceable_v <C>, "");
+
+struct D { D (D &&) = delete; D &operator= (D &&) = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <D>, "");
+static_assert (!std::is_nothrow_relocatable_v <D>, "");
+static_assert (!std::is_replaceable_v <D>, "");
+
+struct E { E (E &&) = default; E &operator= (E &&) = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_nothrow_relocatable_v <E>, "");
+static_assert (!std::is_replaceable_v <E>, "");
+
+struct F { F (F &&) = default; F &operator= (F &&) = default; ~F () = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (!std::is_nothrow_relocatable_v <F>, "");
+static_assert (!std::is_replaceable_v <F>, "");
+
+struct G { G (const G &) = default; G &operator= (const G &) = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <G>, "");
+static_assert (std::is_nothrow_relocatable_v <G>, "");
+static_assert (std::is_replaceable_v <G>, "");
+
+struct H { H (const H &); H &operator= (const H &) = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_nothrow_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+struct I { I (const I &) = default; I &operator= (const I &); ~I () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <I>, "");
+static_assert (std::is_nothrow_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+struct J { J (const J &) = delete; J &operator= (const J &) = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <J>, "");
+static_assert (!std::is_nothrow_relocatable_v <J>, "");
+static_assert (!std::is_replaceable_v <J>, "");
+
+struct K { K (const K &) = default; K &operator= (const K &) = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (std::is_nothrow_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct M;
+struct L { L (L &&) = default; L (M &&); L &operator= (L &&) = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <L>, "");
+static_assert (std::is_nothrow_relocatable_v <L>, "");
+static_assert (std::is_replaceable_v <L>, "");
+
+struct M : public L { using L::L; M (const M &); M &operator= (M &&) = default; int b; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_nothrow_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct O;
+struct N { N (N &&) = default; N &operator= (N &&) = default; N &operator= (O &&); int a; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (std::is_nothrow_relocatable_v <N>, "");
+static_assert (std::is_replaceable_v <N>, "");
+
+struct O : public N { using N::operator=; O (O &&) = default; int b; };
+
+static_assert (!std::is_trivially_relocatable_v <O>, "");
+static_assert (std::is_nothrow_relocatable_v <O>, "");
+static_assert (!std::is_replaceable_v <O>, "");
+
+struct Q;
+struct P { template <typename T> P (T &&) {} };
+
+static_assert (std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_nothrow_relocatable_v <P>, "");
+static_assert (std::is_replaceable_v <P>, "");
+
+struct Q : public P { using P::P; Q (const Q &); };
+
+static_assert (!std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_nothrow_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct S;
+struct R { R (const R &) = default; R (const M &); R &operator= (R &&) = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <R>, "");
+static_assert (std::is_nothrow_relocatable_v <R>, "");
+static_assert (std::is_replaceable_v <R>, "");
+
+struct S : public R { using R::R; S &operator= (S &&) = default; int b; };
+
+static_assert (!std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_nothrow_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T { T (T &&) = default; T &operator= (T &&) = default; ~T (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <T>, "");
+static_assert (std::is_nothrow_relocatable_v <T>, "");
+static_assert (!std::is_replaceable_v <T>, "");
+
+struct U { U (const U &) = default; U &operator= (const U &) = default; ~U (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <U>, "");
+static_assert (std::is_nothrow_relocatable_v <U>, "");
+static_assert (!std::is_replaceable_v <U>, "");
+
+struct V { public: V (); private: V (V &&) = default; V &operator= (V &&) = default; ~V () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <V>, "");
+static_assert (std::is_nothrow_relocatable_v <V>, "");
+static_assert (std::is_replaceable_v <V>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable9.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable9.C
new file mode 100644
index 0000000..1c45b53
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable9.C
@@ -0,0 +1,134 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_nothrow_relocatable
+  : public integral_constant <bool, __builtin_is_nothrow_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_nothrow_relocatable_v		// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_nothrow_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A trivially_relocatable_if_eligible { A (A &&); A &operator= (A &&); ~A (); int a; };
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_nothrow_relocatable_v <A>, "");
+
+struct B { B (B &&); B &operator= (B &&); ~B (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_nothrow_relocatable_v <B>, "");
+
+struct C trivially_relocatable_if_eligible : public A { C (C &&); C &operator= (C &&); ~C (); int a; };
+
+static_assert (std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_nothrow_relocatable_v <C>, "");
+
+struct D trivially_relocatable_if_eligible : public B { D (D &&); D &operator= (D &&); ~D (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <D>, "");
+static_assert (!std::is_nothrow_relocatable_v <D>, "");
+
+struct E trivially_relocatable_if_eligible { E (E &&); E &operator= (E &&); ~E (); A a; };
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_nothrow_relocatable_v <E>, "");
+
+struct F trivially_relocatable_if_eligible { F (F &&) noexcept; F &operator= (F &&); ~F (); B a; };
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_nothrow_relocatable_v <F>, "");
+
+struct G trivially_relocatable_if_eligible { G (G &&); G &operator= (G &&); ~G () = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_nothrow_relocatable_v <G>, "");
+
+struct H trivially_relocatable_if_eligible : virtual A { H (H &&); H &operator= (H &&); ~H (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_nothrow_relocatable_v <H>, "");
+
+struct I trivially_relocatable_if_eligible { I (I &&); I &operator= (I &&); ~I (); A &a; int &b; };
+
+static_assert (std::is_trivially_relocatable_v <I>, "");
+static_assert (std::is_nothrow_relocatable_v <I>, "");
+
+struct J trivially_relocatable_if_eligible { J (J &&); J &operator= (J &&); ~J (); B &a; int &b; };
+
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (std::is_nothrow_relocatable_v <J>, "");
+
+struct K trivially_relocatable_if_eligible { K (K &&) noexcept; K &operator= (K &&); ~K (); union { A a; int b; char c; }; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (std::is_nothrow_relocatable_v <K>, "");
+
+struct L trivially_relocatable_if_eligible { L (L &&); L &operator= (L &&); ~L (); union { int a; B b; short c; }; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_nothrow_relocatable_v <L>, "");
+
+struct M trivially_relocatable_if_eligible { M (M &&); M &operator= (M &&); ~M () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <M>, "");
+static_assert (std::is_nothrow_relocatable_v <M>, "");
+
+struct N trivially_relocatable_if_eligible { N (N &&); N &operator= (N &&); ~N (); union { M a; int b; char c; }; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (std::is_nothrow_relocatable_v <N>, "");
+
+struct O trivially_relocatable_if_eligible { O (O &&); O &operator= (O &&); ~O (); union { unsigned long long a; int b; char c; }; };
+
+static_assert (std::is_trivially_relocatable_v <O>, "");
+static_assert (std::is_nothrow_relocatable_v <O>, "");
+
+struct P { P (P &&) noexcept; P &operator= (P &&); ~P (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_nothrow_relocatable_v <P>, "");
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C
index a31750e..baddc1c 100644
--- a/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C
@@ -7,4 +7,4 @@ concept C = true;
 template <class T>
 struct A {};
 
-void f(A<C<int> auto >) {} 
+void f(A<C<int> auto >) {} 	// { dg-error "auto" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210a.C b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210a.C
new file mode 100644
index 0000000..1d63a84
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210a.C
@@ -0,0 +1,11 @@
+// PR c++/67210
+// { dg-do compile { target c++20 } }
+// { dg-additional-options -Wno-abbreviated-auto-in-template-arg }
+
+template <class T, class U>
+concept C = true;
+
+template <class T>
+struct A {};
+
+void f(A<C<int> auto >) {} 
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C b/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C
index 3e87da4..90d859a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C
@@ -1,49 +1,58 @@
 // PR c++/100474
 // { dg-do compile { target c++20 } }
 
-struct S { S() = delete; S(const S&); };
+struct S { S() = delete; S(const S&); };  // { dg-line S }
 
 template<class T>
 concept Aggregate = __is_aggregate(T);
-// { dg-message "'S' is not an aggregate" "" { target *-*-* } .-1  }
+// { dg-message "'S' is not an aggregate" "" { target *-*-* } S }
 
 template<class T>
 concept TriviallyCopyable = __is_trivially_copyable(T);
-// { dg-message "'S' is not trivially copyable" "" { target *-*-* } .-1  }
+// { dg-message "'S' is not trivially copyable" "" { target *-*-* } S }
 
 template<class T, class U>
 concept Assignable = __is_assignable(T, U);
-// { dg-message "'S' is not assignable from 'int'" "" { target *-*-* } .-1  }
+// { dg-message "'S' is not assignable from 'int', because" "" { target *-*-* } .-1  }
+// { dg-error "no match for 'operator='" "" { target *-*-* } .-2 }
 
 template<class T, class U>
 concept TriviallyAssignable = __is_trivially_assignable(T, U);
 // { dg-message "'S' is not trivially assignable from 'int'" "" { target *-*-* } .-1  }
+// { dg-error "no match for 'operator='" "" { target *-*-* } .-2 }
 
 template<class T, class U>
 concept NothrowAssignable = __is_nothrow_assignable(T, U);
 // { dg-message "'S' is not nothrow assignable from 'int'" "" { target *-*-* } .-1  }
+// { dg-error "no match for 'operator='" "" { target *-*-* } .-2 }
 
 template<class T, class... Args>
 concept Constructible = __is_constructible(T, Args...);
 // { dg-message "'S' is not default constructible" "" { target *-*-* } .-1  }
-// { dg-message "'S' is not constructible from 'int'" "" { target *-*-* } .-2  }
-// { dg-message "'S' is not constructible from 'int, char'" "" { target *-*-* } .-3  }
+// { dg-error "use of deleted function 'S::S\\(\\)'" "" { target *-*-* } .-2 }
+// { dg-message "'S' is not constructible from 'int'" "" { target *-*-* } .-3  }
+// { dg-message "'S' is not constructible from 'int, char'" "" { target *-*-* } .-4  }
+// { dg-error "no matching function for call to 'S::S" "" { target *-*-* } .-5 }
 
 template<class T, class... Args>
 concept TriviallyConstructible = __is_trivially_constructible(T, Args...);
 // { dg-message "'S' is not trivially default constructible" "" { target *-*-* } .-1  }
-// { dg-message "'S' is not trivially constructible from 'int'" "" { target *-*-* } .-2  }
-// { dg-message "'S' is not trivially constructible from 'int, char'" "" { target *-*-* } .-3  }
+// { dg-error "use of deleted function 'S::S\\(\\)'" "" { target *-*-* } .-2 }
+// { dg-message "'S' is not trivially constructible from 'int'" "" { target *-*-* } .-3  }
+// { dg-message "'S' is not trivially constructible from 'int, char'" "" { target *-*-* } .-4  }
+// { dg-error "no matching function for call to 'S::S" "" { target *-*-* } .-5 }
 
 template<class T, class... Args>
 concept NothrowConstructible = __is_nothrow_constructible(T, Args...);
 // { dg-message "'S' is not nothrow default constructible" "" { target *-*-* } .-1  }
-// { dg-message "'S' is not nothrow constructible from 'int'" "" { target *-*-* } .-2  }
-// { dg-message "'S' is not nothrow constructible from 'int, char'" "" { target *-*-* } .-3  }
+// { dg-error "use of deleted function 'S::S\\(\\)'" "" { target *-*-* } .-2 }
+// { dg-message "'S' is not nothrow constructible from 'int'" "" { target *-*-* } .-3  }
+// { dg-message "'S' is not nothrow constructible from 'int, char'" "" { target *-*-* } .-4  }
+// { dg-error "no matching function for call to 'S::S" "" { target *-*-* } .-5 }
 
 template<class T>
 concept UniqueObjReps = __has_unique_object_representations(T);
-// { dg-message "'S' does not have unique object representations" "" { target *-*-* } .-1  }
+// { dg-message "'S' does not have unique object representations" "" { target *-*-* } S }
 
 static_assert(Aggregate<S>); // { dg-error "assert" }
 static_assert(TriviallyCopyable<S>); // { dg-error "assert" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-traits4.C b/gcc/testsuite/g++.dg/cpp2a/concepts-traits4.C
new file mode 100644
index 0000000..caad816
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-traits4.C
@@ -0,0 +1,77 @@
+// PR c++/117294
+// { dg-do compile { target c++20 } }
+// { dg-additional-options "-fconcepts-diagnostics-depth=2" }
+
+template <typename T> struct norm
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> constexpr bool norm_v = __is_constructible(T);
+
+template <typename T> struct part
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> struct part<T*>
+  { static constexpr bool value = false; };
+template <typename T> struct part<const T>
+  { static constexpr bool value = __is_same(T, void); };
+template <typename T> constexpr bool part_v = __is_constructible(T);
+template <typename T> constexpr bool part_v<T*> = false;
+template <typename T> constexpr bool part_v<const T> = __is_same(T, void);
+
+template <typename T> struct expl
+  { static constexpr bool value = __is_constructible(T); };
+template <> struct expl<int*>
+  { static constexpr bool value = false; };
+template <> struct expl<const int>
+  { static constexpr bool value = __is_same(int, void); };
+template <typename T> constexpr bool expl_v = __is_constructible(T);
+template <> constexpr bool expl_v<int*> = false;
+template <> constexpr bool expl_v<const int> = __is_same(int, void);
+
+template <typename T> concept test_norm = norm<T>::value;  // { dg-line norm }
+template <typename T> concept test_part = part<T>::value;  // { dg-line part }
+template <typename T> concept test_expl = expl<T>::value;  // { dg-line expl }
+template <typename T> concept test_norm_v = norm_v<T>;  // { dg-line norm_v }
+template <typename T> concept test_part_v = part_v<T>;  // { dg-line part_v }
+template <typename T> concept test_expl_v = expl_v<T>;  // { dg-line expl_v }
+
+static_assert(test_norm<void>);  // { dg-error "assert" }
+static_assert(test_part<void>);  // { dg-error "assert" }
+static_assert(test_expl<void>);  // { dg-error "assert" }
+static_assert(test_norm_v<void>);  // { dg-error "assert" }
+static_assert(test_part_v<void>);  // { dg-error "assert" }
+static_assert(test_expl_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } norm }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } part }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } expl }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } norm_v }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } part_v }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } expl_v }
+// { dg-prune-output "'void' is incomplete" }
+
+static_assert(test_part<int*>);  // { dg-error "assert" }
+static_assert(test_expl<int*>);  // { dg-error "assert" }
+static_assert(test_part_v<int*>);  // { dg-error "assert" }
+static_assert(test_expl_v<int*>);  // { dg-error "assert" }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } part }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } expl }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } part_v }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } expl_v }
+
+static_assert(test_part<const int>);  // { dg-error "assert" }
+static_assert(test_part_v<const int>);  // { dg-error "assert" }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } part }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } part_v }
+
+struct S { S(int); };
+static_assert(requires { requires test_norm<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_part<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_expl<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_norm_v<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_part_v<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_expl_v<S>; });  // { dg-error "assert" }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } norm }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } part }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } expl }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } norm_v }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } part_v }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } expl_v }
+// { dg-prune-output "no matching function for call" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-using5.C b/gcc/testsuite/g++.dg/cpp2a/concepts-using5.C
new file mode 100644
index 0000000..d42b8a0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-using5.C
@@ -0,0 +1,19 @@
+// PR c++/121351
+// { dg-do compile { target c++20 } }
+
+template<class T> concept C = true;
+
+template<class T>
+struct A {
+  template<class U> void f(U) requires C<T> = delete; // #1
+};
+
+struct B : A<int> {
+  using A::f;
+  template<class U> void f(U) requires C<int>; // #2
+};
+
+int main() {
+  B b;
+  b.f(42); // OK, #2 corresponds to and therefore hides #1
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-using5a.C b/gcc/testsuite/g++.dg/cpp2a/concepts-using5a.C
new file mode 100644
index 0000000..5d31948
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-using5a.C
@@ -0,0 +1,20 @@
+// PR c++/121351
+// { dg-do compile { target c++20 } }
+// A version of concepts-using5a.C where B instead of A is a template.
+
+template<class T> concept C = true;
+
+struct A {
+  template<class U> void f(U) requires C<int> = delete; // #1
+};
+
+template<class T>
+struct B : A {
+  using A::f;
+  template<class U> void f(U) requires C<T>; // #2
+};
+
+int main() {
+  B<int> b;
+  b.f(42); // OK, #2 corresponds to and therefore hides #1
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-using6.C b/gcc/testsuite/g++.dg/cpp2a/concepts-using6.C
new file mode 100644
index 0000000..a40519a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-using6.C
@@ -0,0 +1,20 @@
+// PR c++/121351
+// { dg-do compile { target c++20 } }
+
+template<class T> concept C = true;
+
+template<class T>
+struct A {
+  template<class U> void f(U) requires C<T>; // #1
+};
+
+template<class T>
+struct B : A<T*> {
+  using A<T*>::f;
+  template<class U> void f(U) requires C<T>; // #2
+};
+
+int main() {
+  B<int> b;
+  b.f(42); // { dg-error "ambiguous" } #1 and #2 don't correspond
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C b/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C
index ca92351..17a391b 100644
--- a/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C
@@ -4,13 +4,13 @@
 
 struct fixed_string {
   consteval int size(int n) const {
-    if (n < 0) throw; // { dg-error "not a constant" }
-    return n;
+    if (n < 0) throw; // { dg-error "not a constant" "" { target c++23_down } }
+    return n; // { dg-error "'void __cxa_rethrow\\\(\\\)' called with no caught exceptions active" "" { target c++26 } .-1 }
   }
 
   static consteval int size_static(int n) {
-    if (n < 0) throw; // { dg-error "not a constant" }
-    return n;
+    if (n < 0) throw; // { dg-error "not a constant" "" { target c++23_down } }
+    return n; // { dg-error "'void __cxa_rethrow\\\(\\\)' called with no caught exceptions active" "" { target c++26 } .-1 }
   }
 
   consteval void operator()() const { }
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval34.C b/gcc/testsuite/g++.dg/cpp2a/consteval34.C
index 7562f40..73bcf77 100644
--- a/gcc/testsuite/g++.dg/cpp2a/consteval34.C
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval34.C
@@ -1,42 +1,51 @@
 // { dg-do compile { target c++20 } }
 // Explicit { dg-require-effective-target exceptions_enabled } to avoid verify compiler messages FAILs for '-fno-exceptions'.
 
-consteval int bar (int i) { if (i != 1) throw 1; return 0; }	// { dg-error "is not a constant expression" }
+consteval int bar (int i) { if (i != 1) throw 1; return 0; }	// { dg-error "is not a constant expression" "" { target c++23_down } }
 
 constexpr int
 foo (bool b)
 {
-  return b ? bar (3) : 2; // { dg-message "in .constexpr. expansion" }
+  return b ? bar (3) : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-}
+} // { dg-error "uncaught exception '1'" "" { target c++26 } }
 
 static_assert (foo (false) == 2);
 
-__extension__ constexpr int g1 = false ?: bar (2); // { dg-message "in .constexpr. expansion" }
-__extension__ constexpr int g2 = false ?: (1 + bar (2)); // { dg-message "in .constexpr. expansion" }
+__extension__ constexpr int g1 = false ?: bar (2); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
+__extension__ constexpr int g2 = false ?: (1 + bar (2)); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 __extension__ constexpr int g3 = true ?: bar (2);
 __extension__ constexpr int g4 = true ?: (1 + bar (2));
-constexpr int g5 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" }
-constexpr int g6 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" }
+constexpr int g5 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
+constexpr int g6 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 
 void
 g ()
 {
-  __extension__ int a1[bar(3)]; // { dg-message "in .constexpr. expansion" }
+  __extension__ int a1[bar(3)]; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
   int a2[sizeof (bar(3))];
 
-  int a3 = false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" }
+  int a3 = false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  a3 += false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  a3 += false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
 
-  __extension__ int a4 = false ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" }
+  __extension__ int a4 = false ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  __extension__ int a5 = true ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  __extension__ int a5 = true ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  int a6 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  int a6 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  int a7 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  int a7 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-}
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+} // { dg-error "uncaught exception '1'" "" { target c++26 } }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C
index 9ee93c3..12a071a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C
@@ -8,6 +8,22 @@
 // a pointer to or object of the constructor or destructor's own class or one
 // of its bases, the dynamic_cast results in undefined behavior.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 struct V {
   virtual void f();
 };
@@ -19,7 +35,7 @@ struct B : V {
 };
 
 struct D : A, B {
-  constexpr D() : B((A*)this, this) { } // { dg-message "in 'constexpr' expansion of" }
+  constexpr D() : B((A*)this, this) { } // { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
 };
 
 constexpr B::B(V* v, A* a)
@@ -29,8 +45,9 @@ constexpr B::B(V* v, A* a)
   if (b != nullptr)
     __builtin_abort ();
 
-  B& br = dynamic_cast<B&>(*v); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .B." "" { target *-*-* } .-1 }
+  B& br = dynamic_cast<B&>(*v); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .B." "" { target c++23_down } .-1 }
 }
 
-constexpr D d; // { dg-message "in 'constexpr' expansion of" }
+constexpr D d; // { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+// { dg-error "uncaught exception" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C
index cc1cada..f966db4 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C
@@ -4,6 +4,22 @@
 // Adopted from g++.old-deja/g++.other/dyncast1.C.
 // But use reference dynamic_cast.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // 1. downcast
 // 1.1. single inheritance case
 
@@ -21,27 +37,35 @@ class CCC : protected B {};
 class DDD : protected CCC {};
 
 constexpr D d;
-constexpr bool b01 = (dynamic_cast<D&> ((A&)d), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .D." "" { target *-*-* } .-1 }
-constexpr bool b02 = (dynamic_cast<D&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target *-*-* } .-1 }
+constexpr bool b01 = (dynamic_cast<D&> ((A&)d), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .D." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b02 = (dynamic_cast<D&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&d == &dynamic_cast<const D&> ((C&)d));
-constexpr bool b03 = (dynamic_cast<C&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target *-*-* } .-1 }
+constexpr bool b03 = (dynamic_cast<C&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 constexpr DD dd;
-constexpr bool b04 = (dynamic_cast<DD&> ((A&)dd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DD." "" { target *-*-* } .-1 }
-constexpr bool b05 = (dynamic_cast<DD&> ((B&)dd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DD." "" { target *-*-* } .-1 }
+constexpr bool b04 = (dynamic_cast<DD&> ((A&)dd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b05 = (dynamic_cast<DD&> ((B&)dd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 constexpr DDD ddd;
-constexpr bool b06 = (dynamic_cast<DDD&> ((A&)ddd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DDD." "" { target *-*-* } .-1 }
-constexpr bool b07 = (dynamic_cast<DDD&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target *-*-* } .-1 }
-constexpr bool b08 = (dynamic_cast<CCC&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target *-*-* } .-1 }
+constexpr bool b06 = (dynamic_cast<DDD&> ((A&)ddd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DDD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b07 = (dynamic_cast<DDD&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b08 = (dynamic_cast<CCC&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // 1.2. multiple inheritance case
 // 1.2.1. all bases are public
@@ -50,19 +74,23 @@ struct E : D, CC {};
 struct EE : CC, D {}; //Will search in reverse order.
 
 constexpr E e;
-constexpr bool b09 = (dynamic_cast<E&> ((A&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .E." "" { target *-*-* } .-1 }
-constexpr bool b10 = (dynamic_cast<E&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .E." "" { target *-*-* } .-1 }
+constexpr bool b09 = (dynamic_cast<E&> ((A&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .E." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b10 = (dynamic_cast<E&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .E." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&e == &dynamic_cast<E&> ((C&)(D&)e));
 static_assert (&e == &dynamic_cast<E&> ((B&)(CC&)e));
 static_assert (&(CC&)e == &dynamic_cast<CC&> ((B&)(CC&)e));
 
 constexpr EE ee;
-constexpr bool b11 = (dynamic_cast<EE&> ((A&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .EE." "" { target *-*-* } .-1 }
-constexpr bool b12 = (dynamic_cast<EE&> ((B&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .EE." "" { target *-*-* } .-1 }
+constexpr bool b11 = (dynamic_cast<EE&> ((A&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .EE." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b12 = (dynamic_cast<EE&> ((B&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .EE." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&ee == &dynamic_cast<EE&> ((C&)(D&)ee));
 static_assert (&ee == &dynamic_cast<EE&> ((B&)(CC&)ee));
 static_assert (&(CC&)ee == &dynamic_cast<CC&> ((B&)(CC&)ee));
@@ -78,14 +106,17 @@ constexpr X x;
 static_assert (&x == &dynamic_cast<X&>((B&)(CC&)(E&)x));
 
 constexpr XX xx;
-constexpr bool b13 = (dynamic_cast<XX&>((B&)(CC&)(E&)xx), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .XX." "" { target *-*-* } .-1 }
+constexpr bool b13 = (dynamic_cast<XX&>((B&)(CC&)(E&)xx), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .XX." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 constexpr Y y;
-constexpr bool b14 = (dynamic_cast<Y&>((B&)y), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .Y." "" { target *-*-* } .-1 }
-constexpr bool b15 = (dynamic_cast<Y&>((A&)(B&)y), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .Y." "" { target *-*-* } .-1 }
+constexpr bool b14 = (dynamic_cast<Y&>((B&)y), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .Y." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b15 = (dynamic_cast<Y&>((A&)(B&)y), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .Y." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // 2. crosscast
 
@@ -93,13 +124,16 @@ struct J { virtual void j(); };
 struct K : CC, private J {}; 
 class KK : J, CC{};
 		
-constexpr bool b16 = (dynamic_cast<CC&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .CC." "" { target *-*-* } .-1 }
+constexpr bool b16 = (dynamic_cast<CC&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .CC." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&(CC&)e == &dynamic_cast<CC&> ((C&)(D&)e));
 
 constexpr K k;
-constexpr bool b17 = (dynamic_cast<J&> ((B&)k), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .K. of its operand does not have an unambiguous public base class .J." "" { target *-*-* } .-1 }
+constexpr bool b17 = (dynamic_cast<J&> ((B&)k), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .K. of its operand does not have an unambiguous public base class .J." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 constexpr KK kk;
-constexpr bool b18 = (dynamic_cast<J&> ((CC&)kk), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const CC. of its operand is a non-public base class of dynamic type .KK." "" { target *-*-* } .-1 }
+constexpr bool b18 = (dynamic_cast<J&> ((CC&)kk), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const CC. of its operand is a non-public base class of dynamic type .KK." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C
index 25d98c2..112ff8a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C
@@ -2,6 +2,22 @@
 // { dg-do compile { target c++20 } }
 // Here 'b' doesn't point/refer to a public base of Derived.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 struct Base {
     constexpr virtual ~Base(){}
 };
@@ -11,12 +27,12 @@ struct Derived: Base {
 };
 
 constexpr const Derived& cast(const Base& b) {
-    return dynamic_cast<const Derived&>(b); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .const Base. of its operand does not have a base class of type .Derived." "" { target *-*-* } .-1 }
+    return dynamic_cast<const Derived&>(b); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .const Base. of its operand does not have a base class of type .Derived." "" { target c++23_down } .-1 }
 }
 
 auto test() {
     static constexpr Base b;
-    constexpr auto res = cast(b);
+    constexpr auto res = cast(b);	// { dg-error "uncaught exception" "" { target c++26 } }
     return res;
 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic19.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic19.C
new file mode 100644
index 0000000..27d167c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic19.C
@@ -0,0 +1,10 @@
+// PR c++/120620
+// { dg-do compile }
+
+#include <cxxabi.h>
+
+struct A* a;
+
+void f() {
+  void* const p = abi::__dynamic_cast(&a, 0, 0, 42);
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic1a.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic1a.C
new file mode 100644
index 0000000..4077a47
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic1a.C
@@ -0,0 +1,6 @@
+// Test that including <cxxabi.h>, whence the actual abi:__dynamic_cast
+// is declared, doesn't affect our constexpr dynamic_cast handling.
+// { dg-do compile { target c++20 } }
+
+#include <cxxabi.h>
+#include "constexpr-dynamic1.C"
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C
index da647bf..a237134 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C
@@ -4,6 +4,22 @@
 
 // From clang's constant-expression-cxx2a.cpp.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 struct A2 { virtual void a2(); };
 struct A : A2 { virtual void a(); };
 struct B : A {}; 
@@ -26,31 +42,36 @@ static_assert(dynamic_cast<const A*>(static_cast<const C2*>(&g)) == nullptr);
 static_assert(g.f == (void*)(F*)&g);
 static_assert(dynamic_cast<const void*>(static_cast<const D*>(&g)) == &g);
 
-constexpr int d_a = (dynamic_cast<const A&>(static_cast<const D&>(g)), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target *-*-* } .-1 }
-
+constexpr int d_a = (dynamic_cast<const A&>(static_cast<const D&>(g)), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 // Can navigate from A2 to its A...
 static_assert(&dynamic_cast<A&>((A2&)(B&)g) == &(A&)(B&)g);
 // ... and from B to its A ...
 static_assert(&dynamic_cast<A&>((B&)g) == &(A&)(B&)g);
 // ... but not from D.
-static_assert(&dynamic_cast<A&>((D&)g) == &(A&)(B&)g); // { dg-error "non-constant condition for static assertion|reference .dynamic_cast. failed" }
-// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target *-*-* } .-1 }
-
+static_assert(&dynamic_cast<A&>((D&)g) == &(A&)(B&)g); // { dg-error "non-constant condition for static assertion" }
+// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } .-2 }
+// { dg-error "uncaught exception" "" { target c++26 } .-3 }
 // Can cast from A2 to sibling class D.
 static_assert(&dynamic_cast<D&>((A2&)(B&)g) == &(D&)g);
 
 // Cannot cast from private base E to derived class F.
-constexpr int e_f = (dynamic_cast<F&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target *-*-* } .-1 }
+constexpr int e_f = (dynamic_cast<F&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // Cannot cast from B to private sibling E.
-constexpr int b_e = (dynamic_cast<E&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .E." "" { target *-*-* } .-1 }
+constexpr int b_e = (dynamic_cast<E&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .E." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 struct Unrelated { virtual void unrelated(); };
 
-constexpr int b_unrelated = (dynamic_cast<Unrelated&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .Unrelated." "" { target *-*-* } .-1 }
-constexpr int e_unrelated = (dynamic_cast<Unrelated&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target *-*-* } .-1 }
+constexpr int b_unrelated = (dynamic_cast<Unrelated&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .Unrelated." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr int e_unrelated = (dynamic_cast<Unrelated&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C
index d67c307..4434a1a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Private base.
 
 struct P1 { virtual void p1(); };
@@ -12,14 +28,16 @@ struct A : B, C, private P2 { virtual void a(); };
 constexpr A a;
 
 // P1 is a non-public base of A.
-constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // Don't error here.
 static_assert (dynamic_cast<B*>((P1*)&a) == nullptr);
 
-constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<C*>((P1*)&a) == nullptr);
 static_assert (dynamic_cast<C*>((P2*)&a) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C
index bc3efd0..d504efd 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Protected base.
 
 struct P1 { virtual void p1(); };
@@ -12,14 +28,16 @@ struct A : B, C, protected P2 { virtual void a(); };
 constexpr A a;
 
 // P1 is a non-public base of A.
-constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // Don't error here.
 static_assert (dynamic_cast<B*>((P1*)&a) == nullptr);
 
-constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<C*>((P1*)&a) == nullptr);
 static_assert (dynamic_cast<C*>((P2*)&a) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C
index 1958cae4..2fc8242 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Unrelated type.
 
 struct B { virtual void b(); };
@@ -12,12 +28,15 @@ constexpr A a;
 
 struct U { virtual void u(); };
 
-constexpr bool b1 = (dynamic_cast<U&>((B&)a), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .U." "" { target *-*-* } .-1 }
-constexpr bool b2 = (dynamic_cast<U&>((P1&)a), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
-constexpr bool b3 = (dynamic_cast<U&>((P2&)a), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<U&>((B&)a), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .U." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b2 = (dynamic_cast<U&>((P1&)a), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b3 = (dynamic_cast<U&>((P2&)a), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<U*>((B*)&a) == nullptr);
 static_assert (dynamic_cast<U*>((P1*)&a) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C
index 7d42ffa..3a50dde 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Ambiguous base.
 
 struct A { virtual void a(); };
@@ -11,7 +27,8 @@ struct E : B, C, D { virtual void d(); };
 
 constexpr E e;
 
-constexpr bool b1 = (dynamic_cast<A&>((D&)e), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message ".A. is an ambiguous base class of dynamic type .E. of its operand" "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<A&>((D&)e), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message ".A. is an ambiguous base class of dynamic type .E. of its operand" "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<A*>((D*)&e) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-new27.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-new27.C
new file mode 100644
index 0000000..a26fc7e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-new27.C
@@ -0,0 +1,41 @@
+// { dg-do compile { target c++20 } }
+
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_alloc : public exception {
+    constexpr virtual ~bad_alloc () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_alloc"; }
+  };
+  struct bad_array_new_length : public bad_alloc {
+    constexpr virtual ~bad_array_new_length () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_array_new_length"; }
+  };
+}
+#endif
+
+constexpr int
+foo (__SIZE_TYPE__ x, int y, int z)
+{
+  char (*a)[2] = new char[x][2];	// { dg-error "call to non-'constexpr' function 'void __cxa_throw_bad_array_new_length\\\(\\\)'" "" { target c++23_down } }
+  delete[] a;				// { dg-message "declared here" "" { target c++23_down } .-1 }
+  int *b = new int[y];			// { dg-error "call to non-'constexpr' function 'void __cxa_throw_bad_array_new_length\\\(\\\)'" "" { target c++23_down } }
+  delete[] b;
+  int *c = new int[z]{1, 2, 3};		// { dg-error "call to non-'constexpr' function 'void __cxa_throw_bad_array_new_length\\\(\\\)'" "" { target c++23_down } }
+  delete[] c;
+  return 0;
+}
+
+constexpr int a = foo (16, 2, 3);
+constexpr int b = foo (-64, 2, 3);	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+					// { dg-error "uncaught exception" "" { target c++26 } .-1 }
+constexpr int c = foo (16, -1, 3);	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+					// { dg-error "uncaught exception" "" { target c++26 } .-1 }
+constexpr int d = foo (16, 2, 1);	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+					// { dg-error "uncaught exception" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-typeid5.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-typeid5.C
new file mode 100644
index 0000000..567383d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-typeid5.C
@@ -0,0 +1,20 @@
+// PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
+// { dg-do compile { target c++17 } }
+
+#include <typeinfo>
+
+template <class T>
+constexpr bool foo ()
+{
+   bool r = false;
+   const std::type_info &s = typeid( (r = true), *(T *) 0);	// { dg-error "call to non-'constexpr' function 'void __cxa_bad_typeid\\\(\\\)'" "" { target c++23_down } }
+   return r;							// { dg-message "declared here" "" { target c++23_down } .-1 }
+}
+
+struct A {};
+struct B { virtual ~B () {} };
+
+static_assert (!foo <int> ());
+static_assert (!foo <A> ());
+constexpr bool a = foo <B> ();	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+// { dg-error "uncaught exception" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C
index 00bda53..ab8c979 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C
@@ -45,9 +45,9 @@ constexpr int test5() {
   union {
     int data[1];
   } u;
-  std::construct_at(u.data, 0);  // { dg-message "in .constexpr. expansion" }
+  std::construct_at(u.data, 0);  // { dg-bogus "in .constexpr. expansion" }
   return 0;
 }
-constexpr int x5 = test5();  // { dg-message "in .constexpr. expansion" }
+constexpr int x5 = test5();  // { dg-bogus "in .constexpr. expansion" }
 
 // { dg-error "accessing (uninitialized member|.* member instead of)" "" { target *-*-* } 0 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-union9.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-union9.C
new file mode 100644
index 0000000..7db1030
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-union9.C
@@ -0,0 +1,33 @@
+// PR c++/120577
+// { dg-do compile { target c++20 } }
+
+template <class _Tp> struct optional {
+  union {
+    _Tp __val_;
+  };
+  template <class... _Args>
+  constexpr optional(_Args... __args)
+      : __val_(__args...) {}
+};
+template <class _Tp, class... _Args>
+constexpr optional<_Tp> make_optional(_Args... __args) {
+  return optional<_Tp>(__args...);
+}
+
+struct __non_trivial_if {
+  constexpr __non_trivial_if() {}
+};
+struct allocator : __non_trivial_if {};
+struct __padding {};
+struct __short {
+  [[__no_unique_address__]] __padding __padding_;
+  int __data_;
+};
+struct basic_string {
+  union {
+    __short __s;
+  };
+  [[__no_unique_address__]] allocator __alloc_;
+  constexpr basic_string(int, int) {}
+};
+auto opt = make_optional<basic_string>(4, 'X');
diff --git a/gcc/testsuite/g++.dg/cpp2a/lambda-targ16.C b/gcc/testsuite/g++.dg/cpp2a/lambda-targ16.C
new file mode 100644
index 0000000..11f2337
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/lambda-targ16.C
@@ -0,0 +1,29 @@
+// PR c++/120748
+// From Clang cxx20-lambda-decltype-this.cpp.
+// { dg-do compile { target c++20 } }
+
+namespace PR45881 {
+struct A {
+    void f();
+};
+int id(A*);
+void A::f() {
+    auto z = [*this](auto z2, decltype(z2(this)) z3){};
+    z(id,3);
+}
+
+struct B {
+    void f();
+};
+void B::f() {
+    auto z = []<typename TT, typename TTT=decltype(TT()(this))>(){return 0;};
+    z.template operator()<int(*)(B*)>();
+}
+struct C {
+    void f();
+};
+void C::f() {
+    auto z = []<typename TT, decltype(TT()(this)) n>(){return 0;};
+    z.template operator()<int(*)(C*), 8>();
+}
+} // namespace PR45881
diff --git a/gcc/testsuite/g++.dg/cpp2a/lambda-targ17.C b/gcc/testsuite/g++.dg/cpp2a/lambda-targ17.C
new file mode 100644
index 0000000..84955ae
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/lambda-targ17.C
@@ -0,0 +1,8 @@
+// PR c++/12012
+// { dg-do compile { target c++20 } }
+
+template<auto> int x;
+
+int main() {
+	x<[](auto) {}>;
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/lambda-uneval28.C b/gcc/testsuite/g++.dg/cpp2a/lambda-uneval28.C
new file mode 100644
index 0000000..4b1bfc8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/lambda-uneval28.C
@@ -0,0 +1,10 @@
+// PR c++/121008
+// { dg-do compile { target c++20 } }
+
+struct A {
+  void f()
+    noexcept(noexcept([this]() noexcept(noexcept(this)) {}))
+  {}
+};
+
+int main() {}
diff --git a/gcc/testsuite/g++.dg/diagnostic/static_assert5.C b/gcc/testsuite/g++.dg/diagnostic/static_assert5.C
new file mode 100644
index 0000000..16681b2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/diagnostic/static_assert5.C
@@ -0,0 +1,70 @@
+// PR c++/117294
+// { dg-do compile { target c++14 } }
+
+template <typename T> struct norm
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> constexpr bool norm_v = __is_constructible(T);
+
+template <typename T> struct part
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> struct part<T*>
+  { static constexpr bool value = false; };
+template <typename T> struct part<const T>
+  { static constexpr bool value = __is_same(T, void); };
+template <typename T> constexpr bool part_v = __is_constructible(T);
+template <typename T> constexpr bool part_v<T*> = false;
+template <typename T> constexpr bool part_v<const T> = __is_same(T, void);
+
+template <typename T> struct expl
+  { static constexpr bool value = __is_constructible(T); };
+template <> struct expl<int*>
+  { static constexpr bool value = false; };
+template <> struct expl<const int>
+  { static constexpr bool value = __is_same(int, void); };
+template <typename T> constexpr bool expl_v = __is_constructible(T);
+template <> constexpr bool expl_v<int*> = false;
+template <> constexpr bool expl_v<const int> = __is_same(int, void);
+
+// === Primary template can give customised diagnostics when using traits
+static_assert(norm<void>::value);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(part<void>::value);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(expl<void>::value);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(norm_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(part_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(expl_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+
+// { dg-prune-output "'void' is incomplete" }
+
+
+// === Specialisations don't customise just because primary template had trait
+static_assert(part<int*>::value);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+static_assert(expl<int*>::value);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+static_assert(part_v<int*>);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+static_assert(expl_v<int*>);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+
+
+// === But partial specialisations actually using a trait can customise
+static_assert(part<const int>::value);  // { dg-error "assert" }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } .-1 }
+static_assert(part_v<const int>);  // { dg-error "assert" }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } .-1 }
+
+
+// === For these cases, we no longer know that the error was caused by the trait
+// === because it's been folded away before we process the failure.
+static_assert(expl<const int>::value);  // { dg-error "assert" }
+// { dg-bogus "because" "" { target *-*-* } .-1 }
+static_assert(expl_v<const int>);  // { dg-error "assert" }
+// { dg-bogus "because" "" { target *-*-* } .-1 }
+static_assert(__is_constructible(void));  // { dg-error "assert" }
+// { dg-bogus "because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/has_virtual_destructor2.C b/gcc/testsuite/g++.dg/ext/has_virtual_destructor2.C
new file mode 100644
index 0000000..14eea80
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/has_virtual_destructor2.C
@@ -0,0 +1,27 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T> struct has_virtual_destructor {
+  static constexpr bool value = __has_virtual_destructor(T);
+};
+
+static_assert(has_virtual_destructor<int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' does not have a virtual destructor" "" { target *-*-* } .-1 }
+
+struct A {};  // { dg-message "'A' does not have a virtual destructor" }
+static_assert(has_virtual_destructor<A>::value, "");  // { dg-error "assert" }
+
+struct B {
+  ~B();  // { dg-message "'B' does not have a virtual destructor" }
+};
+static_assert(has_virtual_destructor<B>::value, "");  // { dg-error "assert" }
+
+struct C {  // { dg-bogus "" }
+  virtual ~C();  // { dg-bogus "" }
+};
+static_assert(has_virtual_destructor<C[5]>::value, "");  // { dg-error "assert" }
+// { dg-message "'C \\\[5\\\]' does not have a virtual destructor" "" { target *-*-* } .-1 }
+
+union U {  // { dg-message "'U' does not have a virtual destructor" }
+  ~U();
+};
+static_assert(has_virtual_destructor<U>::value, "");  // { dg-error "assert" }
diff --git a/gcc/testsuite/g++.dg/ext/is_assignable2.C b/gcc/testsuite/g++.dg/ext/is_assignable2.C
new file mode 100644
index 0000000..b346d7b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_assignable2.C
@@ -0,0 +1,36 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T>
+struct is_copy_assignable {
+  static constexpr bool value = __is_assignable(T&, const T&);
+};
+
+static_assert(is_copy_assignable<const int>::value, "");  // { dg-error "assert" }
+// { dg-error "assignment to read-only type 'const int'" "" { target *-*-* } .-1 }
+
+struct A {
+  void operator=(A) = delete;  // { dg-message "declared here" }
+};
+static_assert(is_copy_assignable<A>::value, "");  // { dg-error "assert" }
+// { dg-message "is not assignable" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
+
+template <typename T>
+struct is_nothrow_copy_assignable {
+  static constexpr bool value = __is_nothrow_assignable(T&, const T&);
+};
+struct B {
+  void operator=(const B&);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_copy_assignable<B>::value, "");  // { dg-error "assert" }
+// { dg-message "is not nothrow assignable" "" { target *-*-* } .-1 }
+
+template <typename T>
+struct is_trivially_copy_assignable {
+  static constexpr bool value = __is_trivially_assignable(T&, const T&);
+};
+struct C {
+  void operator=(const C&);  // { dg-message "non-trivial" }
+};
+static_assert(is_trivially_copy_assignable<C>::value, "");  // { dg-error "assert" }
+// { dg-message "is not trivially assignable" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_constructible9.C b/gcc/testsuite/g++.dg/ext/is_constructible9.C
new file mode 100644
index 0000000..5448878
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_constructible9.C
@@ -0,0 +1,60 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T, typename... Args>
+struct is_constructible {
+  static constexpr bool value = __is_constructible(T, Args...);
+};
+
+static_assert(is_constructible<void>::value, "");  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible, because" "" { target *-*-* } .-1 }
+// { dg-error "'void' is incomplete" "" { target *-*-* } .-2 }
+
+static_assert(is_constructible<int&, const int&>::value, "");  // { dg-error "assert" }
+// { dg-message "'int&' is not constructible from 'const int&', because" "" { target *-*-* } .-1 }
+// { dg-error "discards qualifiers" "" { target *-*-* } .-2 }
+
+static_assert(is_constructible<int, int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not constructible from 'int, int', because" "" { target *-*-* } .-1 }
+// { dg-error "too many initializers for non-class type 'int'" "" { target *-*-* } .-2 }
+
+struct A {
+  A(int);  // { dg-message "candidate" }
+};
+static_assert(is_constructible<A, int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'A' is not constructible from 'int, int', because" "" { target *-*-* } .-1 }
+// { dg-error "no matching function for call to" "" { target *-*-* } .-2 }
+
+template <typename T, typename... Args>
+struct is_nothrow_constructible {
+  static constexpr bool value = __is_nothrow_constructible(T, Args...);
+};
+
+struct B {
+  B(int);  // { dg-message "candidate" }
+};
+static_assert(is_nothrow_constructible<B>::value, "");  // { dg-error "assert" }
+// { dg-message "'B' is not nothrow default constructible, because" "" { target *-*-* } .-1 }
+// { dg-error "no matching function for call to" "" { target *-*-* } .-2 }
+
+struct C {
+  C(int);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_constructible<C, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'C' is not nothrow constructible from 'int', because" "" { target *-*-* } .-1 }
+
+template <typename T, typename... Args>
+struct is_trivially_constructible {
+  static constexpr bool value = __is_trivially_constructible(T, Args...);
+};
+
+struct D {
+  D();  // { dg-message "non-trivial" }
+};
+static_assert(is_trivially_constructible<D>::value, "");  // { dg-error "assert" }
+// { dg-message "'D' is not trivially default constructible, because" "" { target *-*-* } .-1 }
+
+struct E {
+  operator int();  // { dg-message "non-trivial" }
+};
+static_assert(is_trivially_constructible<int, E>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not trivially constructible from 'E', because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_convertible7.C b/gcc/testsuite/g++.dg/ext/is_convertible7.C
new file mode 100644
index 0000000..b38fc04
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_convertible7.C
@@ -0,0 +1,29 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T, typename U>
+struct is_convertible {
+  static constexpr bool value = __is_convertible(T, U);
+};
+
+static_assert(is_convertible<int*, int>::value, "");  // { dg-error "assert" }
+// { dg-error "invalid conversion" "" { target *-*-* } .-1 }
+
+static_assert(is_convertible<int(), double (*)()>::value, "");  // { dg-error "assert" }
+// { dg-error "invalid conversion" "" { target *-*-* } .-1 }
+
+struct A {
+  explicit A(int);
+};
+static_assert(is_convertible<int, A>::value, "");  // { dg-error "assert" }
+// { dg-error "could not convert 'int' to 'A'" "" { target *-*-* } .-1 }
+
+template <typename T, typename U>
+struct is_nothrow_convertible {
+  static constexpr bool value = __is_nothrow_convertible(T, U);
+};
+
+struct B {
+  B(int);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_convertible<int, B>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not nothrow convertible from 'B', because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_destructible3.C b/gcc/testsuite/g++.dg/ext/is_destructible3.C
new file mode 100644
index 0000000..a8501d6
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_destructible3.C
@@ -0,0 +1,65 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T>
+struct is_destructible {
+  static constexpr bool value = __is_destructible(T);
+};
+
+static_assert(is_destructible<void>::value, "");  // { dg-error "assert" }
+// { dg-message "'void' is not destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "'void' is incomplete" "" { target *-*-* } .-2 }
+
+struct A {
+  ~A() = delete;  // { dg-message "declared here" }
+};
+static_assert(is_destructible<A>::value, "");  // { dg-error "assert" }
+// { dg-message "'A' is not destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
+
+struct B {
+private:
+  ~B();  // { dg-message "declared private here" }
+};
+static_assert(is_destructible<B>::value, "");  // { dg-error "assert" }
+// { dg-message "'B' is not destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
+
+template <typename T>
+struct is_nothrow_destructible {
+  static constexpr bool value = __is_nothrow_destructible(T);
+};
+
+struct C {
+  ~C() noexcept(false);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_destructible<C>::value, "");  // { dg-error "assert" }
+// { dg-message "'C' is not nothrow destructible, because" "" { target *-*-* } .-1 }
+
+struct D {
+private:
+  ~D() {}  // { dg-message "declared private here" }
+};
+static_assert(is_nothrow_destructible<D>::value, "");  // { dg-error "assert" }
+// { dg-message "'D' is not nothrow destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
+
+template <typename T>
+struct is_trivially_destructible {
+  static constexpr bool value = __is_trivially_destructible(T);
+};
+
+struct E {
+  ~E();
+};
+struct F { E d; };  // { dg-message "non-trivial" }
+static_assert(is_trivially_destructible<F>::value, "");  // { dg-error "assert" }
+// { dg-message "'F' is not trivially destructible, because" "" { target *-*-* } .-1 }
+
+struct G {
+private:
+  ~G();  // { dg-message "declared private here" }
+};
+struct H { G g; };  // { dg-error "private within this context" }
+static_assert(is_trivially_destructible<H>::value, "");  // { dg-error "assert" }
+// { dg-message "'H' is not trivially destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/g++.dg/ext/is_invocable5.C b/gcc/testsuite/g++.dg/ext/is_invocable5.C
new file mode 100644
index 0000000..460eed5
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_invocable5.C
@@ -0,0 +1,15 @@
+// PR c++/121055
+// { dg-do compile { target c++11 } }
+// { dg-skip-if "requires hosted libstdc++ for functional function" { ! hostedlib } }
+
+#include <functional>
+
+#define SA(X) static_assert((X),#X)
+
+struct F;
+
+SA( __is_invocable(void (F::*)() &, std::reference_wrapper<F>) );
+SA( ! __is_invocable(void (F::*)() &&, std::reference_wrapper<F>) );
+
+SA( __is_invocable(void (F::*)(int) &, std::reference_wrapper<F>, int) );
+SA( ! __is_invocable(void (F::*)(int) &&, std::reference_wrapper<F>, int) );
diff --git a/gcc/testsuite/g++.dg/ext/is_invocable6.C b/gcc/testsuite/g++.dg/ext/is_invocable6.C
new file mode 100644
index 0000000..64c5c76
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_invocable6.C
@@ -0,0 +1,45 @@
+// { dg-do compile { target c++11 } }
+
+template <typename F, typename... Args>
+struct is_invocable {
+  static constexpr bool value = __is_invocable(F, Args...);
+};
+
+static_assert(is_invocable<int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not invocable, because" "" { target *-*-* } .-1 }
+// { dg-error "'int' cannot be used as a function" "" { target *-*-* } .-2 }
+
+static_assert(is_invocable<void(*)(), int>::value, "");  // { dg-error "assert" }
+// { dg-message "'void \[^'\]*' is not invocable by 'int', because" "" { target *-*-* } .-1 }
+// { dg-error "too many arguments" "" { target *-*-* } .-2 }
+
+static_assert(is_invocable<void(void*), void() const>::value, "");  // { dg-error "assert" }
+// { dg-message "'void.void..' is not invocable by 'void.. const', because" "" { target *-*-* } .-1 }
+// { dg-error "qualified function type" "" { target *-*-* } .-2 }
+
+struct A {};
+static_assert(is_invocable<const A&&, int, double>::value, "");  // { dg-error "assert" }
+// { dg-message "'const A&&' is not invocable by 'int, double', because" "" { target *-*-* } .-1 }
+// { dg-error "no match for call to " "" { target *-*-* } .-2 }
+
+struct B {
+  void operator()() = delete;  // { dg-message "declared here" }
+};
+static_assert(is_invocable<B>::value, "");  // { dg-error "assert" }
+// { dg-message "'B' is not invocable, because" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
+
+template <typename F, typename... Args>
+struct is_nothrow_invocable {
+  static constexpr bool value = __is_nothrow_invocable(F, Args...);
+};
+
+static_assert(is_nothrow_invocable<void(*)()>::value, "");  // { dg-error "assert" }
+// { dg-message "'void \[^'\]*' is not nothrow invocable, because" "" { target *-*-* } .-1 }
+// { dg-message "'void \[^'\]*' is not 'noexcept'" "" { target *-*-* } .-2 }
+
+struct C {
+  int operator()(int, double) const;  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_invocable<const C&, int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'const C&' is not nothrow invocable by 'int, int', because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_invocable7.C b/gcc/testsuite/g++.dg/ext/is_invocable7.C
new file mode 100644
index 0000000..5c852fc
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_invocable7.C
@@ -0,0 +1,21 @@
+// PR c++/121291
+// { dg-do compile { target c++17 } }
+
+template <typename T>
+constexpr bool is_invocable = __is_invocable(T);
+
+template <typename T>
+constexpr bool is_nothrow_invocable = __is_nothrow_invocable(T);
+
+struct S {
+private:
+  int operator()() noexcept;  // { dg-message "here" }
+};
+
+static_assert(is_invocable<S>);  // { dg-error "assert" }
+// { dg-message "not invocable" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
+
+static_assert(is_nothrow_invocable<S>);  // { dg-error "assert" }
+// { dg-message "not nothrow invocable" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/g++.dg/ext/is_nothrow_convertible5.C b/gcc/testsuite/g++.dg/ext/is_nothrow_convertible5.C
new file mode 100644
index 0000000..0ce8fb8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_nothrow_convertible5.C
@@ -0,0 +1,15 @@
+// PR c++/121291
+// { dg-do compile { target c++17 } }
+
+template <typename T, typename U>
+constexpr bool is_nothrow_convertible = __is_nothrow_convertible(T, U);
+
+struct A {};
+struct B {
+private:
+  operator A() noexcept;  // { dg-message "here" }
+};
+
+static_assert(is_nothrow_convertible<B, A>);  // { dg-error "assert" }
+// { dg-message "not nothrow convertible" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/g++.dg/ext/is_virtual_base_of_diagnostic2.C b/gcc/testsuite/g++.dg/ext/is_virtual_base_of_diagnostic2.C
new file mode 100644
index 0000000..ac28121
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_virtual_base_of_diagnostic2.C
@@ -0,0 +1,13 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T, typename U>
+struct is_virtual_base_of {
+  static constexpr bool value = __builtin_is_virtual_base_of(T, U);
+};
+
+static_assert(is_virtual_base_of<int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not a virtual base of 'int'" "" { target *-*-* } .-1 }
+
+struct A {};  // { dg-message "'A' is not a virtual base of 'B'" }
+struct B : A {};  // { dg-message "declared here" }
+static_assert(is_virtual_base_of<A, B>::value, "");  // { dg-error "assert" }
diff --git a/gcc/testsuite/g++.dg/lookup/operator-8.C b/gcc/testsuite/g++.dg/lookup/operator-8.C
index 64d8a97..32d432d 100644
--- a/gcc/testsuite/g++.dg/lookup/operator-8.C
+++ b/gcc/testsuite/g++.dg/lookup/operator-8.C
@@ -16,11 +16,16 @@ struct A {
 template<class T>
 void f() {
   A a;
-  (void)(a != 0, 0 != a); // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a < 0, 0 < a);   // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a <= 0, 0 <= a); // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a > 0, 0 > a);   // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a >= 0, 0 >= a); // { dg-bogus "deleted" "" { xfail *-*-* } }
+  (void)(a != 0);
+  (void)(0 != a);
+  (void)(a < 0);
+  (void)(0 < a);
+  (void)(a <= 0);
+  (void)(0 <= a);
+  (void)(a > 0);
+  (void)(0 > a);
+  (void)(a >= 0);
+  (void)(0 >= a);
 }
 
 // These later-declared namespace-scope overloads shouldn't be considered
@@ -31,4 +36,10 @@ bool operator<=(A, int) = delete;
 bool operator>(A, int) = delete;
 bool operator>=(A, int) = delete;
 
+bool operator!=(int, A) = delete;
+bool operator<(int, A) = delete;
+bool operator<=(int, A) = delete;
+bool operator>(int, A) = delete;
+bool operator>=(int, A) = delete;
+
 template void f<int>();
diff --git a/gcc/testsuite/g++.dg/lto/pr114790_0.C b/gcc/testsuite/g++.dg/lto/pr114790_0.C
new file mode 100644
index 0000000..eed112d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/lto/pr114790_0.C
@@ -0,0 +1,16 @@
+// { dg-lto-do link }
+// { dg-lto-options { { -w -flto -g -flto-partition=1to1 -O2 -shared -fPIC -fvisibility=hidden} } }
+// { dg-require-effective-target fpic }
+// { dg-require-effective-target shared }
+struct APITracerContext {
+  virtual ~APITracerContext() = default;
+  virtual void releaseActivetracersList() = 0;
+};
+struct APITracerContextImp : APITracerContext {
+  ~APITracerContextImp() override;
+  void releaseActivetracersList() override;
+};
+struct APITracerContextImp globalAPITracerContextImp;
+struct APITracerContextImp *pGlobalAPITracerContextImp = &globalAPITracerContextImp;
+APITracerContextImp::~APITracerContextImp() {}
+
diff --git a/gcc/testsuite/g++.dg/lto/pr114790_1.C b/gcc/testsuite/g++.dg/lto/pr114790_1.C
new file mode 100644
index 0000000..511fae4
--- /dev/null
+++ b/gcc/testsuite/g++.dg/lto/pr114790_1.C
@@ -0,0 +1,15 @@
+struct APITracerContext {
+  virtual void releaseActivetracersList() = 0;
+};
+extern struct APITracerContextImp *pGlobalAPITracerContextImp;
+struct APITracerContextImp : APITracerContext { void releaseActivetracersList();};
+int g();
+inline int
+apiTracerWrapperImp(  ) {
+  for (int i = 0; i < g(); i++) 
+  pGlobalAPITracerContextImp->releaseActivetracersList();
+}
+__attribute__((visibility("default"))) int
+zeCommandListAppendMemoryCopyTracing() {
+  return apiTracerWrapperImp(      );
+}
diff --git a/gcc/testsuite/g++.dg/missing-return.C b/gcc/testsuite/g++.dg/missing-return.C
index 5f8e2cc..f6934b0 100644
--- a/gcc/testsuite/g++.dg/missing-return.C
+++ b/gcc/testsuite/g++.dg/missing-return.C
@@ -5,4 +5,6 @@ int foo(int a)
 {
 } /* { dg-warning "no return statement" } */
 
-/* { dg-final { scan-tree-dump "__builtin_unreachable" "optimized" } } */
+/* For targets without traps, it will be an infinite loop */
+/* { dg-final { scan-tree-dump "__builtin_unreachable" "optimized" { target trap } } } */
+/* { dg-final { scan-tree-dump "goto <" "optimized" { target { ! trap } } } } */
diff --git a/gcc/testsuite/g++.dg/modules/class-11_a.H b/gcc/testsuite/g++.dg/modules/class-11_a.H
new file mode 100644
index 0000000..799dbdd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/class-11_a.H
@@ -0,0 +1,36 @@
+// Check for some additional lang_type flags that we'd missed.
+// { dg-additional-options "-fmodule-header -fabi-version=21 -Wabi=15" }
+// { dg-module-cmi {} }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+struct A trivially_relocatable_if_eligible { A(A&&); };
+struct B replaceable_if_eligible { B(B&&); B& operator=(B&&); };
+struct C {};
+static_assert(__builtin_is_trivially_relocatable(C) && __builtin_is_replaceable(C), "");
+
+
+struct pr106381 {
+  long l;
+  char c = -1;
+};
+struct L1 : pr106381 {
+  char x;  // { dg-warning "offset" "" { target c++14 } }
+};
+static_assert(sizeof(L1) == sizeof(pr106381), "");
+
+
+struct pr120012 {
+  pr120012(const pr120012&) = default;
+  pr120012(pr120012&&) = default;
+  pr120012& operator=(pr120012&&) = default;
+  unsigned int a;
+  unsigned char b;
+};
+struct L2 : pr120012 {
+  unsigned char y;  // { dg-warning "offset" "" { target c++20 } }
+};
+static_assert(sizeof(L2) > sizeof(pr120012), "");
diff --git a/gcc/testsuite/g++.dg/modules/class-11_b.C b/gcc/testsuite/g++.dg/modules/class-11_b.C
new file mode 100644
index 0000000..2450a45
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/class-11_b.C
@@ -0,0 +1,15 @@
+// { dg-additional-options "-fmodules -fabi-version=21 -Wabi=15" }
+
+import "class-11_a.H";
+
+static_assert(__builtin_is_trivially_relocatable(A), "");
+static_assert(__builtin_is_replaceable(B), "");
+static_assert(__builtin_is_trivially_relocatable(C) && __builtin_is_replaceable(C), "");
+
+struct M1 : pr106381 {
+  char x;  // { dg-warning "offset" "" { target c++14 } }
+};
+
+struct M2 : pr120012 {
+  unsigned char y;  // { dg-warning "offset" "" { target c++20 } }
+};
diff --git a/gcc/testsuite/g++.dg/modules/cpp-21.C b/gcc/testsuite/g++.dg/modules/cpp-21.C
new file mode 100644
index 0000000..fdd0492
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/cpp-21.C
@@ -0,0 +1,8 @@
+// PR c++/120845
+// { dg-do compile }
+// { dg-additional-options "-fmodules" }
+
+export module pr120485
+    [[foobarbaz]];
+// { dg-error "expected ';' before end of line" "" { target *-*-* } .-2 }
+// { dg-warning "attribute ignored" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/g++.dg/modules/internal-14_a.C b/gcc/testsuite/g++.dg/modules/internal-14_a.C
new file mode 100644
index 0000000..07eb965
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/internal-14_a.C
@@ -0,0 +1,17 @@
+// PR c++/120412
+// { dg-additional-options "-fmodules -std=c++20 -Wtemplate-names-tu-local" }
+// { dg-module-cmi m:part }
+
+export module m:part;
+
+export template <typename F>
+auto fun1(F) {
+  return true;
+}
+
+using Dodgy = decltype([]{});
+
+export template <typename T>
+auto fun2(T&&) {  // { dg-warning "TU-local" }
+  return fun1(Dodgy{});
+}
diff --git a/gcc/testsuite/g++.dg/modules/internal-14_b.C b/gcc/testsuite/g++.dg/modules/internal-14_b.C
new file mode 100644
index 0000000..ad3b09d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/internal-14_b.C
@@ -0,0 +1,6 @@
+// PR c++/120412
+// { dg-additional-options "-fmodules -std=c++20 -Wtemplate-names-tu-local" }
+// { dg-module-cmi m }
+
+export module m;
+export import :part;
diff --git a/gcc/testsuite/g++.dg/modules/internal-14_c.C b/gcc/testsuite/g++.dg/modules/internal-14_c.C
new file mode 100644
index 0000000..4f8e785c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/internal-14_c.C
@@ -0,0 +1,9 @@
+// PR c++/120412
+// { dg-additional-options "-fmodules -std=c++20" }
+
+import m;
+
+int main() {
+  // { dg-error "instantiation exposes TU-local entity '(fun1|Dodgy)'" "" { target *-*-* } 0 }
+  fun2(123);  // { dg-message "required from here" }
+}
diff --git a/gcc/testsuite/g++.dg/modules/merge-19.h b/gcc/testsuite/g++.dg/modules/merge-19.h
new file mode 100644
index 0000000..c3faadc
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/merge-19.h
@@ -0,0 +1,21 @@
+// PR c++/121238
+
+inline void inc(const char*& __first) {
+  ++__first;
+}
+
+template <typename = void>
+bool parse_integer(const char *first) {
+  const char *start = first;
+  inc(first);
+  return first != start;
+}
+template bool parse_integer<void>(const char*);
+
+
+struct S { ~S() {} int x; };
+template <typename = void>
+bool take_by_invisiref(S s) {
+  return s.x == 5;
+}
+template bool take_by_invisiref<void>(S);
diff --git a/gcc/testsuite/g++.dg/modules/merge-19_a.H b/gcc/testsuite/g++.dg/modules/merge-19_a.H
new file mode 100644
index 0000000..149a447
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/merge-19_a.H
@@ -0,0 +1,5 @@
+// PR c++/121238
+// { dg-additional-options "-fmodule-header" }
+// { dg-module-cmi {} }
+
+#include "merge-19.h"
diff --git a/gcc/testsuite/g++.dg/modules/merge-19_b.C b/gcc/testsuite/g++.dg/modules/merge-19_b.C
new file mode 100644
index 0000000..345e7fe
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/merge-19_b.C
@@ -0,0 +1,16 @@
+// PR c++/121238
+// { dg-module-do run }
+// { dg-additional-options "-fmodules -fno-module-lazy" }
+
+#include "merge-19.h"
+import "merge-19_a.H";
+
+int main() {
+  const char fmt[] = "5";
+  if (!parse_integer<void>(fmt))
+    __builtin_abort();
+
+  S s{ 5 };
+  if (!take_by_invisiref(s))
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/g++.dg/modules/pr108080.H b/gcc/testsuite/g++.dg/modules/pr108080.H
new file mode 100644
index 0000000..b05d957
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/pr108080.H
@@ -0,0 +1,5 @@
+// PR c++/108080
+// { dg-additional-options "-fmodules" }
+// Give a diagnostic message rather than a crash for unsupported features.
+
+[[gnu::optimize("-O3")]] void foo();  // { dg-warning "optimize" }
diff --git a/gcc/testsuite/g++.dg/parse/pr120940.C b/gcc/testsuite/g++.dg/parse/pr120940.C
new file mode 100644
index 0000000..5da36b2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/pr120940.C
@@ -0,0 +1,18 @@
+// PR c++/120940
+// { dg-do run }
+
+int a[8] = { 1, 2, 3, 4, 5, 6, 7, 8 };
+int b[8] = { 9, 10, 11, 12, 13, 14, 15, 16 };
+
+__attribute__((noipa)) int
+foo (int x, int y)
+{
+  return (x ? a : b)[y];
+}
+
+int
+main ()
+{
+  if (foo (1, 4) != 5 || foo (0, 6) != 15)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/g++.dg/parse/template32.C b/gcc/testsuite/g++.dg/parse/template32.C
new file mode 100644
index 0000000..b090f40
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/template32.C
@@ -0,0 +1,13 @@
+// PR c++/119838
+// { dg-do compile { target c++11 } }
+
+template<typename T>
+struct S { using U = T; static const int x = 0; };
+void
+g ()
+{
+  ::S<int>::U a;
+  ::template S<int>::U b;
+  auto c = ::S<int>::x;
+  auto d = ::template S<int>::x;
+}
diff --git a/gcc/testsuite/g++.dg/parse/union1.C b/gcc/testsuite/g++.dg/parse/union1.C
new file mode 100644
index 0000000..d567ea3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union1.C
@@ -0,0 +1,19 @@
+// PR c++/83469
+// { dg-do compile }
+
+struct S {
+  union U { int m; };
+};
+
+template <typename T>
+void
+f ()
+{
+  union T::U u;
+}
+
+int
+main()
+{
+  f<S>();
+}
diff --git a/gcc/testsuite/g++.dg/parse/union2.C b/gcc/testsuite/g++.dg/parse/union2.C
new file mode 100644
index 0000000..cdb1392
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union2.C
@@ -0,0 +1,19 @@
+// PR c++/83469
+// { dg-do compile }
+
+struct S {
+  union U { int m; };
+};
+
+template <typename T>
+void
+f ()
+{
+  struct T::U u;  // { dg-error "not a non-union class type" }
+}
+
+int
+main()
+{
+  f<S>();
+}
diff --git a/gcc/testsuite/g++.dg/parse/union3.C b/gcc/testsuite/g++.dg/parse/union3.C
new file mode 100644
index 0000000..61552a4
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union3.C
@@ -0,0 +1,19 @@
+// PR c++/83469
+// { dg-do compile }
+
+struct S {
+  struct C { int m; };
+};
+
+template <typename T>
+void
+f ()
+{
+  union T::C u; // { dg-error "not a union type" }
+}
+
+int
+main()
+{
+  f<S>();
+}
diff --git a/gcc/testsuite/g++.dg/parse/union4.C b/gcc/testsuite/g++.dg/parse/union4.C
new file mode 100644
index 0000000..709f6a4
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union4.C
@@ -0,0 +1,12 @@
+// PR c++/93809
+// { dg-do compile }
+
+class C { };
+enum E { };
+struct S { };
+union U { };
+
+typedef typename ::C C2;
+typedef typename ::E E2;
+typedef typename ::S S2;
+typedef typename ::U U2;
diff --git a/gcc/testsuite/g++.dg/parse/union5.C b/gcc/testsuite/g++.dg/parse/union5.C
new file mode 100644
index 0000000..18238dd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union5.C
@@ -0,0 +1,5 @@
+// PR c++/93809
+// { dg-do compile { target c++11 } }
+
+union U {};
+auto var = new (typename ::U);
diff --git a/gcc/testsuite/g++.dg/parse/union6.C b/gcc/testsuite/g++.dg/parse/union6.C
new file mode 100644
index 0000000..61b9568
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union6.C
@@ -0,0 +1,5 @@
+// PR c++/93809
+// { dg-do compile }
+
+typedef union{} U;
+typename ::U foo () { return U();  }
diff --git a/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C b/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C
index 3e93126..71ae8f2 100644
--- a/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C
+++ b/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C
@@ -1,4 +1,4 @@
-/* Verify colorization of the labels in diagnostic-show-locus.c
+/* Verify colorization of the labels in diagnostics/source-printing.cc
    for template comparisons.
    Doing so requires a plugin; see the comments in the plugin for the
    rationale.  */
diff --git a/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc b/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc
index 6151b6e..5d185ff 100644
--- a/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc
+++ b/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc
@@ -21,7 +21,8 @@
 int plugin_is_GPL_compatible;
 
 void
-noop_text_starter_fn (diagnostic_text_output_format &, const diagnostic_info *)
+noop_text_starter_fn (diagnostics::text_sink &,
+		      const diagnostics::diagnostic_info *)
 {
 }
 
@@ -32,7 +33,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  diagnostic_text_starter (global_dc) = noop_text_starter_fn;
+  diagnostics::text_starter (global_dc) = noop_text_starter_fn;
 
   return 0;
 }
diff --git a/gcc/testsuite/g++.dg/tc1/dr49.C b/gcc/testsuite/g++.dg/tc1/dr49.C
index 753d96b..6ddea6b 100644
--- a/gcc/testsuite/g++.dg/tc1/dr49.C
+++ b/gcc/testsuite/g++.dg/tc1/dr49.C
@@ -10,8 +10,8 @@ template struct R<&p>; // OK
 template struct S<&p>; // OK due to parameter adjustment
 
 int *ptr;
-template struct R<ptr>; // { dg-error "argument" }
-template struct S<ptr>; // { dg-error "argument" }
+template struct R<ptr>; // { dg-error "template argument|constant expression" }
+template struct S<ptr>; // { dg-error "template argument|constant expression" }
 
 int v[5];
 template struct R<v>; // OK due to implicit argument conversion
diff --git a/gcc/testsuite/g++.dg/template/error45.C b/gcc/testsuite/g++.dg/template/error45.C
index 064554d..f4c6560 100644
--- a/gcc/testsuite/g++.dg/template/error45.C
+++ b/gcc/testsuite/g++.dg/template/error45.C
@@ -11,7 +11,7 @@ struct enable_if< true, T >
 
 template < typename T >
 struct enable_if< true, T >::type
-f( T x );			// { dg-error "not a class type" }
+f( T x );			// { dg-error "not a non-union class type" }
 
 void
 g( void )
diff --git a/gcc/testsuite/g++.dg/template/func2.C b/gcc/testsuite/g++.dg/template/func2.C
index 0116f23..360f430 100644
--- a/gcc/testsuite/g++.dg/template/func2.C
+++ b/gcc/testsuite/g++.dg/template/func2.C
@@ -4,8 +4,7 @@ typedef void (*fptr)();
 fptr zeroptr = 0;
 template<typename T, fptr F> struct foo { };
 template<typename T> struct foo<T,zeroptr> { };
-// { dg-error "not a valid template argument" "not valid" { target *-*-* } .-1 } 
-// { dg-message "must be the address" "must be the address " { target *-*-* } .-2 }
+// { dg-error "template argument|constant expression" "not valid" { target *-*-* } .-1 } 
 
 // The rest is needed to trigger the ICE in 4.0 to 4.3:
 void f() { }
diff --git a/gcc/testsuite/g++.dg/template/permissive-error3.C b/gcc/testsuite/g++.dg/template/permissive-error3.C
new file mode 100644
index 0000000..988b7fa
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/permissive-error3.C
@@ -0,0 +1,12 @@
+// PR c++/120575
+// { dg-additional-options -Wno-template-body }
+
+template< int >
+struct T {};
+
+template< int >
+struct S {
+
+  operator typename T< oops >::anything () {};
+
+};
diff --git a/gcc/testsuite/g++.dg/torture/pr120119-1.C b/gcc/testsuite/g++.dg/torture/pr120119-1.C
new file mode 100644
index 0000000..1206feb
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr120119-1.C
@@ -0,0 +1,15 @@
+// { dg-do compile }
+// { dg-additional-options "-mcpu=cortex-a57" { target aarch64*-*-* } }
+
+// PR target/120119
+
+struct a {
+  float operator()(int b, int c) { return d[c * 4 + b]; }
+  float *d;
+};
+float e(float *);
+auto f(a b) {
+  float g[]{b(1, 1), b(2, 1), b(3, 1), b(1, 2), b(2, 2), b(3, 2), b(1, 3),
+            b(2, 3), b(3, 3), b(3, 2), b(1, 3), b(2, 3), b(3, 3)};
+  return b.d[0] * e(g);
+}
diff --git a/gcc/testsuite/g++.dg/tree-prof/eh1.C b/gcc/testsuite/g++.dg/tree-prof/eh1.C
new file mode 100644
index 0000000..10a3596
--- /dev/null
+++ b/gcc/testsuite/g++.dg/tree-prof/eh1.C
@@ -0,0 +1,34 @@
+/* { dg-options "-O3 -fdump-ipa-profile-details -fno-inline -fdump-tree-fixup_cfg3-details -fdump-tree-optimized-details" } */
+char a[10000];
+char b[10000];
+int sz = 1000;
+
+__attribute__((noipa))
+     void test2 ()
+{
+  throw (sz);
+}
+void
+test ()
+{
+  try
+  {
+    test2 ();
+  }
+  catch (int v)
+  {
+    __builtin_memcpy (b, a, v);
+  }
+}
+int
+main ()
+{
+  for (int i = 0; i < 100000; i++)
+    test ();
+}
+/* { dg-final-use-not-autofdo { scan-ipa-dump-times "Average value sum:100000000" 2 "profile" } } */
+/* 1 zero count for resx block.  */
+/* { dg-final-use-not-autofdo { scan-tree-dump-times "count: 0" 1 "fixup_cfg3" } } */
+/* 2 zero count for resx block and return block since return gets duplicated by tracer.  */
+/* { dg-final-use-not-autofdo { scan-tree-dump-times "count: 0" 2 "optimized" } } */
+/* { dg-final-use-not-autofdo { scan-tree-dump-times "Average value sum:100000000" 1 "optimized" } } */
diff --git a/gcc/testsuite/g++.dg/warn/Wduplicated-branches9.C b/gcc/testsuite/g++.dg/warn/Wduplicated-branches9.C
new file mode 100644
index 0000000..f9fafcd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wduplicated-branches9.C
@@ -0,0 +1,11 @@
+// PR c++/120940
+// { dg-do compile }
+// { dg-options "-Wduplicated-branches" }
+
+static char a[16][8], b[16][8];
+
+char *
+foo (int x, int y)
+{
+  return (x ? a : b)[y];
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C b/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C
index dd41b08..db85150 100644
--- a/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C
+++ b/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C
@@ -29,6 +29,8 @@ typedef struct diagnostic_event_id_t diagnostic_event_id_t;
 namespace pp_markup { class element; }
 typedef pp_markup::element pp_element;
 
+typedef class string_slice string_slice;
+
 #define FORMAT(kind) __attribute__ ((format (__gcc_## kind ##__, 1, 2)))
 
 void diag (const char*, ...) FORMAT (diag);
@@ -63,7 +65,7 @@ void test_diag (tree t, gimple *gc, diagnostic_event_id_t *event_id_ptr,
   diag ("%e", 42); /* { dg-warning "format" } */
 }
 
-void test_cdiag (tree t, gimple *gc)
+void test_cdiag (tree t, gimple *gc, string_slice *s)
 {
   cdiag ("%<");   /* { dg-warning "unterminated quoting directive" } */
   cdiag ("%>");   /* { dg-warning "unmatched quoting directive " } */
@@ -74,6 +76,7 @@ void test_cdiag (tree t, gimple *gc)
   cdiag ("%F", t);       /* { dg-warning ".F. conversion used unquoted" } */
   cdiag ("%G", gc);      /* { dg-warning "format" } */
   cdiag ("%K", t);       /* { dg-warning "format" } */
+  cdiag ("%B", s);
 
   cdiag ("%R");       /* { dg-warning "unmatched color reset directive" } */
   cdiag ("%r", "");   /* { dg-warning "unterminated color directive" } */
@@ -90,6 +93,7 @@ void test_cdiag (tree t, gimple *gc)
   cdiag ("%<%F%>", t);
   cdiag ("%<%G%>", gc);  /* { dg-warning "format" } */
   cdiag ("%<%K%>", t);   /* { dg-warning "format" } */
+  cdiag ("%<%B%>", s);
 
   cdiag ("%<%R%>");      /* { dg-warning "unmatched color reset directive" } */
   cdiag ("%<%r%>", "");  /* { dg-warning "unterminated color directive" } */
@@ -101,9 +105,10 @@ void test_cdiag (tree t, gimple *gc)
   cdiag ("%<%qD%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   cdiag ("%<%qE%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   cdiag ("%<%qT%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
+  cdiag ("%<%qB%>", s);  /* { dg-warning ".q. flag used within a quoted sequence" } */
 }
 
-void test_tdiag (tree t, gimple *gc)
+void test_tdiag (tree t, gimple *gc, string_slice *s)
 {
   tdiag ("%<");       /* { dg-warning "unterminated quoting directive" } */
   tdiag ("%>");       /* { dg-warning "unmatched quoting directive " } */
@@ -113,6 +118,7 @@ void test_tdiag (tree t, gimple *gc)
   tdiag ("%E", t);
   tdiag ("%G", gc);     /* { dg-warning "format" } */
   tdiag ("%K", t);      /* { dg-warning "format" } */
+  tdiag ("%B", s);
 
   tdiag ("%R");          /* { dg-warning "unmatched color reset directive" } */
   tdiag ("%r", "");   /* { dg-warning "unterminated color directive" } */
@@ -138,9 +144,10 @@ void test_tdiag (tree t, gimple *gc)
   tdiag ("%<%qD%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   tdiag ("%<%qE%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   tdiag ("%<%qT%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
+  tdiag ("%<%qB%>", s);  /* { dg-warning ".q. flag used within a quoted sequence" } */
 }
 
-void test_cxxdiag (tree t, gimple *gc)
+void test_cxxdiag (tree t, gimple *gc, string_slice *s)
 {
   cxxdiag ("%A", t);     /* { dg-warning ".A. conversion used unquoted" } */
   cxxdiag ("%D", t);     /* { dg-warning ".D. conversion used unquoted" } */
@@ -148,6 +155,7 @@ void test_cxxdiag (tree t, gimple *gc)
   cxxdiag ("%F", t);     /* { dg-warning ".F. conversion used unquoted" } */
   cxxdiag ("%G", gc);    /* { dg-warning "format" } */
   cxxdiag ("%K", t);     /* { dg-warning "format" } */
+  cxxdiag ("%B", s);
 
   cxxdiag ("%R");        /* { dg-warning "unmatched color reset directive" } */
   cxxdiag ("%r", "");    /* { dg-warning "unterminated color directive" } */
@@ -172,9 +180,10 @@ void test_cxxdiag (tree t, gimple *gc)
   cxxdiag ("%<%T%>", t);
   cxxdiag ("%<%V%>", t);
   cxxdiag ("%<%X%>", t);
+  cxxdiag ("%<%B%>", s);
 }
 
-void test_dump (tree t, gimple *stmt, cgraph_node *node)
+void test_dump (tree t, gimple *stmt, cgraph_node *node, string_slice *s)
 {
   dump ("%<");   /* { dg-warning "unterminated quoting directive" } */
   dump ("%>");   /* { dg-warning "unmatched quoting directive " } */
@@ -197,4 +206,5 @@ void test_dump (tree t, gimple *stmt, cgraph_node *node)
   dump ("%C", node);
   dump ("%f", 1.0);
   dump ("%4.2f", 1.0); /* { dg-warning "format" } */
+  dump ("%B", s);
 }
diff --git a/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C b/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
index bac2b68..a21e864 100644
--- a/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
+++ b/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
@@ -31,7 +31,7 @@ void warn_placement_new ()
 
 void warn_placement_array_new ()
 {
-  void *p = malloc (sizeof (int));
+  void *p = malloc (sizeof (int) * 2);
   int *q = new (p) int[2];
   delete q;                   // { dg-warning "-Wmismatched-new-delete" }
 }
diff --git a/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C b/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C
index 0eeee34..dcccdca 100644
--- a/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C
+++ b/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C
@@ -28,7 +28,7 @@ struct N::S s3;       // { dg-warning "-Wredundant-tags" }
 
 N::U u1;
 typename N::U u2;     // { dg-bogus "-Wredundant-tags" }
-                      // { dg-bogus "'class' tag used in naming 'union N::U" "pr93809" { xfail *-*-*} .-1 }
+                      // { dg-bogus "'class' tag used in naming 'union N::U" "pr93809" { target *-*-*} .-1 }
 union N::U u3;        // { dg-warning "-Wredundant-tags" }
 
 
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-parm-12.C b/gcc/testsuite/g++.dg/warn/Wunused-parm-12.C
new file mode 100644
index 0000000..03029f9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-parm-12.C
@@ -0,0 +1,59 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-parameter" }
+
+void baz (int);
+
+template <int N>
+void
+foo (int a,		// { dg-warning "parameter 'a' set but not used" }
+     int b,		// { dg-warning "parameter 'b' set but not used" }
+     int c,		// { dg-warning "parameter 'c' set but not used" }
+     int d,		// { dg-warning "parameter 'd' set but not used" }
+     int e,		// { dg-warning "parameter 'e' set but not used" }
+     int f,		// { dg-warning "parameter 'f' set but not used" }
+     int g,		// { dg-warning "parameter 'g' set but not used" }
+     int h,		// { dg-warning "parameter 'h' set but not used" }
+     int i,		// { dg-warning "parameter 'i' set but not used" }
+     int j,		// { dg-warning "parameter 'j' set but not used" }
+     int k,		// { dg-warning "parameter 'k' set but not used" }
+     int l,		// { dg-warning "parameter 'l' set but not used" }
+     int m)		// { dg-warning "parameter 'm' set but not used" }
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <int N>
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <0> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+  bar <0> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-parm-13.C b/gcc/testsuite/g++.dg/warn/Wunused-parm-13.C
new file mode 100644
index 0000000..f2d357f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-parm-13.C
@@ -0,0 +1,59 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-parameter" }
+
+void baz (int);
+
+template <typename T>
+void
+foo (T a,		// { dg-warning "parameter 'a' set but not used" }
+     T b,		// { dg-warning "parameter 'b' set but not used" }
+     T c,		// { dg-warning "parameter 'c' set but not used" }
+     T d,		// { dg-warning "parameter 'd' set but not used" }
+     T e,		// { dg-warning "parameter 'e' set but not used" }
+     T f,		// { dg-warning "parameter 'f' set but not used" }
+     T g,		// { dg-warning "parameter 'g' set but not used" }
+     T h,		// { dg-warning "parameter 'h' set but not used" }
+     T i,		// { dg-warning "parameter 'i' set but not used" }
+     T j,		// { dg-warning "parameter 'j' set but not used" }
+     T k,		// { dg-warning "parameter 'k' set but not used" }
+     T l,		// { dg-warning "parameter 'l' set but not used" }
+     T m)		// { dg-warning "parameter 'm' set but not used" }
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (T n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <typename T>
+T
+bar (T a, T b, T c, T d, T e, T f, T g, T h, T i, T j,
+     T k, T l, T m, T n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <int> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+  bar <int> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-var-2.C b/gcc/testsuite/g++.dg/warn/Wunused-var-2.C
index 0b21ef1..869065f 100644
--- a/gcc/testsuite/g++.dg/warn/Wunused-var-2.C
+++ b/gcc/testsuite/g++.dg/warn/Wunused-var-2.C
@@ -18,9 +18,9 @@ f1 ()
 }
 
 void
-f2 (int x)
+f2 (int x)	// { dg-warning "parameter 'x' set but not used" }
 {
-  int a = 0;
+  int a = 0;	// { dg-warning "variable 'a' set but not used" }
   x++;
   ++a;
 }
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-var-40.C b/gcc/testsuite/g++.dg/warn/Wunused-var-40.C
new file mode 100644
index 0000000..9351367
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-var-40.C
@@ -0,0 +1,69 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-variable" }
+
+void baz (int);
+
+template <int N>
+void
+foo (void)
+{
+  int a = 0;		// { dg-warning "variable 'a' set but not used" }
+  a = 1;
+  int b = 0;		// { dg-warning "variable 'b' set but not used" }
+  ++b;
+  int c = 0;		// { dg-warning "variable 'c' set but not used" }
+  c++;
+  int d = 0;		// { dg-warning "variable 'd' set but not used" }
+  --d;
+  int e = 0;		// { dg-warning "variable 'e' set but not used" }
+  e--;
+  int f = 0;		// { dg-warning "variable 'f' set but not used" }
+  f += 2;
+  int g = 0;		// { dg-warning "variable 'g' set but not used" }
+  g |= 2;
+  int h = 0;		// { dg-warning "variable 'h' set but not used" }
+  h -= 2;
+  int i = 0;		// { dg-warning "variable 'i' set but not used" }
+  i &= 2;
+  int j = 0;		// { dg-warning "variable 'j' set but not used" }
+  j ^= 2;
+  int k = 0;		// { dg-warning "variable 'k' set but not used" }
+  k *= 2;
+  int l = 0;		// { dg-warning "variable 'l' set but not used" }
+  l %= 2;
+  int m = 0;		// { dg-warning "variable 'm' set but not used" }
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <int N>
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <0> ();
+  bar <0> ();
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-var-41.C b/gcc/testsuite/g++.dg/warn/Wunused-var-41.C
new file mode 100644
index 0000000..ff981ee
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-var-41.C
@@ -0,0 +1,69 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-variable" }
+
+void baz (int);
+
+template <typename T>
+void
+foo (void)
+{
+  T a = 0;		// { dg-warning "variable 'a' set but not used" }
+  a = 1;
+  T b = 0;		// { dg-warning "variable 'b' set but not used" }
+  ++b;
+  T c = 0;		// { dg-warning "variable 'c' set but not used" }
+  c++;
+  T d = 0;		// { dg-warning "variable 'd' set but not used" }
+  --d;
+  T e = 0;		// { dg-warning "variable 'e' set but not used" }
+  e--;
+  T f = 0;		// { dg-warning "variable 'f' set but not used" }
+  f += 2;
+  T g = 0;		// { dg-warning "variable 'g' set but not used" }
+  g |= 2;
+  T h = 0;		// { dg-warning "variable 'h' set but not used" }
+  h -= 2;
+  T i = 0;		// { dg-warning "variable 'i' set but not used" }
+  i &= 2;
+  T j = 0;		// { dg-warning "variable 'j' set but not used" }
+  j ^= 2;
+  T k = 0;		// { dg-warning "variable 'k' set but not used" }
+  k *= 2;
+  T l = 0;		// { dg-warning "variable 'l' set but not used" }
+  l %= 2;
+  T m = 0;		// { dg-warning "variable 'm' set but not used" }
+  for (T n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <typename T>
+T
+bar (void)
+{
+  T a = 0;
+  T b = ++a;
+  T c = 0;
+  T d = --c;
+  T e = 0;
+  T f = e--;
+  T g = 0;
+  T h = g++;
+  T i = 0;
+  T j;
+  j = i += 42;
+  T k = 0;
+  T l;
+  l = k *= 4;
+  T m = 0;
+  T n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <int> ();
+  bar <int> ();
+}
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-1.C b/gcc/testsuite/g++.dg/warn/pr121133-1.C
new file mode 100644
index 0000000..6d6e13b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-1.C
@@ -0,0 +1,16 @@
+// PR c++/121133
+// { dg-do compile }
+// { dg-options "-std=c++98 -Wno-long-long -pedantic-errors" }
+
+__extension__ typedef long long L;
+__extension__ long long a;
+struct S {
+  __extension__ long long b;
+};
+
+void
+foo ()
+{
+  __extension__ long long c;
+  c = c + (__extension__ (long long) 1);
+}
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-2.C b/gcc/testsuite/g++.dg/warn/pr121133-2.C
new file mode 100644
index 0000000..cd97a76
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-2.C
@@ -0,0 +1,5 @@
+// PR c++/121133
+// { dg-do compile }
+// { dg-options "-std=c++98 -pedantic-errors" }
+
+#include "pr121133-1.C"
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-3.C b/gcc/testsuite/g++.dg/warn/pr121133-3.C
new file mode 100644
index 0000000..9ffd407
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-3.C
@@ -0,0 +1,5 @@
+// PR c++/121133
+// { dg-do compile { target c++11 } }
+// { dg-options "-pedantic-errors" }
+
+#include "pr121133-1.C"
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-4.C b/gcc/testsuite/g++.dg/warn/pr121133-4.C
new file mode 100644
index 0000000..76885ba
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-4.C
@@ -0,0 +1,5 @@
+// PR c++/121133
+// { dg-do compile { target c++11 } }
+// { dg-options "-pedantic-errors -Wlong-long" }
+
+#include "pr121133-1.C"
diff --git a/gcc/testsuite/g++.target/aarch64/mv-cpu-features.C b/gcc/testsuite/g++.target/aarch64/mv-cpu-features.C
new file mode 100644
index 0000000..ad6accd
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/mv-cpu-features.C
@@ -0,0 +1,82 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <cstdint>
+#include <sys/auxv.h>
+
+__attribute__((target_version ("default")))
+int foo ()
+{
+  return 0;
+}
+
+__attribute__((target_version ("rng")))
+int foo ()
+{
+  return 1;
+}
+
+__attribute__((target_version ("lse")))
+int foo ()
+{
+  return 2;
+}
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+  uint64_t hwcap3;
+  uint64_t hwcap4;
+} ifunc_arg_t;
+
+int impl ()
+{
+  return 0;
+}
+
+#ifndef _IFUNC_ARG_HWCAP
+#define _IFUNC_ARG_HWCAP (1ULL << 62)
+#endif
+
+extern "C" void
+__init_cpu_features_resolver (unsigned long hwcap, const ifunc_arg_t *arg);
+
+extern "C" void *
+fun_resolver (uint64_t a0, const ifunc_arg_t *a1)
+{
+  ifunc_arg_t arg = {};
+  arg.size = sizeof (ifunc_arg_t);
+  /* These flags determine that the implementation of foo ()
+     that returns 2 will be selected.  */
+  arg.hwcap = HWCAP_ATOMICS;
+  arg.hwcap2 = HWCAP2_RNG;
+  __init_cpu_features_resolver (arg.hwcap | _IFUNC_ARG_HWCAP, &arg);
+  return (void *)(uintptr_t)impl;
+}
+
+extern "C" int fun (void) __attribute__((ifunc ("fun_resolver")));
+
+/* In this test we expect that the manual resolver for the fun ()
+   function will be executed before the automatic resolver for the
+   FMV function foo ().  This is because resolvers from the same TU
+   are executed according to the offset of corresponding relocations.
+
+   Automatic resolver is generated in a dedicated section while the
+   manually written resolver will be put in the .text section which
+   will come first.
+
+   The manual resolver above calls __init_cpu_features_resolver()
+   supplying synthetic ifunc_arg_t fields that will determine the
+   choice for the FMV implementation.
+   */
+
+int main ()
+{
+  int res = fun ();
+  if (res == 0 && foo () == 2)
+    return 0;
+  return 1;
+}
diff --git a/gcc/testsuite/g++.target/aarch64/pr119498.C b/gcc/testsuite/g++.target/aarch64/pr119498.C
new file mode 100644
index 0000000..03f1659
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/pr119498.C
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-require-ifunc "" } */
+/* { dg-options "-O0" } */
+/* { dg-additional-options "-Wno-experimental-fmv-target" } */
+
+__attribute__ ((target_version ("default"))) int
+foo ();
+
+__attribute__ ((target_version ("default"))) int
+foo () { return 1; } /* { dg-message "old declaration" } */
+
+__attribute__ ((target_version ("dotprod"))) float
+foo () { return 3; } /* { dg-error "ambiguating new declaration" } */
+
+__attribute__ ((target_version ("sve"))) int
+foo2 () { return 1; } /* { dg-message "old declaration" } */
+
+__attribute__ ((target_version ("dotprod"))) float
+foo2 () { return 3; } /* { dg-error "ambiguating new declaration of" } */
diff --git a/gcc/testsuite/g++.target/aarch64/sme/sme_throw_1.C b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_1.C
new file mode 100644
index 0000000..76f1e8b
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_1.C
@@ -0,0 +1,55 @@
+/* { dg-do run { target { aarch64*-linux-gnu* && aarch64_sme_hw } } } */
+
+#include <signal.h>
+#include <arm_sme.h>
+
+static bool caught;
+
+[[gnu::noipa]] void thrower(int)
+{
+  throw 1;
+}
+
+[[gnu::noipa]] void bar()
+{
+  *(volatile int *)0 = 0;
+}
+
+[[gnu::noipa]] void foo()
+{
+  try
+    {
+      bar();
+    }
+  catch (int)
+    {
+      caught = true;
+    }
+}
+
+__arm_new("za") __arm_locally_streaming void sme_user()
+{
+  svbool_t all = svptrue_b8();
+  for (unsigned int i = 0; i < svcntb(); ++i)
+    {
+      svint8_t expected = svindex_s8(i + 1, i);
+      svwrite_hor_za8_m(0, i, all, expected);
+    }
+  foo();
+  for (unsigned int i = 0; i < svcntb(); ++i)
+    {
+      svint8_t expected = svindex_s8(i + 1, i);
+      svint8_t actual = svread_hor_za8_m(svdup_s8(0), all, 0, i);
+      if (svptest_any(all, svcmpne(all, expected, actual)))
+	__builtin_abort();
+    }
+  if (!caught)
+    __builtin_abort();
+}
+
+int main()
+{
+  signal(SIGSEGV, thrower);
+  sme_user();
+  return 0;
+}
diff --git a/gcc/testsuite/g++.target/aarch64/sme/sme_throw_2.C b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_2.C
new file mode 100644
index 0000000..db3197c
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_2.C
@@ -0,0 +1,4 @@
+/* { dg-do run { target { aarch64*-linux-gnu* && aarch64_sme_hw } } } */
+/* { dg-options "-O2" } */
+
+#include "sme_throw_1.C"
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_1.C
new file mode 100644
index 0000000..9d6342b
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_1.C
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -msve-vector-bits=2048" } */
+
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+#define ADD(a, b) a + b
+#define SUB(a, b) a - b
+#define MUL(a, b) a * b
+#define MAX(a, b) (a > b) ? a : b
+#define MIN(a, b) (a > b) ? b : a
+
+#define TEST_OP(TYPE, OP)                                        \
+  TYPE test_##TYPE##_##OP (TYPE a, TYPE b) { return OP (a, b); } \
+
+#define TEST_ALL(TYPE, SIZE)                                        \
+  typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE)));       \
+  TEST_OP (TYPE##SIZE, ADD)                                         \
+  TEST_OP (TYPE##SIZE, SUB)                                         \
+  TEST_OP (TYPE##SIZE, MUL)                                         \
+  TEST_OP (TYPE##SIZE, MIN)                                         \
+  TEST_OP (TYPE##SIZE, MAX)
+
+TEST_ALL (__bf16, 64)
+
+TEST_ALL (__bf16, 128)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, p[0-7]/m. z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_2.C
new file mode 100644
index 0000000..63de293
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_2.C
@@ -0,0 +1,15 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -ffinite-math-only -fno-signed-zeros -fno-trapping-math -msve-vector-bits=2048 " } */
+
+#include "unpacked_binary_bf16_1.C"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C
new file mode 100644
index 0000000..560d874
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C
@@ -0,0 +1,46 @@
+/* { dg-do compile }*/
+/* { dg-options "-O -ffinite-math-only -fno-signed-zeros -fno-trapping-math -msve-vector-bits=2048 " } */
+
+#include <stdint.h>
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+#define ADD(a, b) a + b
+#define SUB(a, b) a - b
+#define MUL(a, b) a * b
+#define MAX(a, b) (a > b) ? a : b
+#define MIN(a, b) (a > b) ? b : a
+
+#define COND_OP(OP, TYPE, PRED_TYPE, ARG2, MERGE)                                  \
+  TYPE test_##OP##_##TYPE##_##ARG2##_##MERGE (TYPE a, TYPE b, TYPE c, PRED_TYPE p) \
+  {return p ? OP (a, ARG2) : MERGE; }
+
+#define TEST_OP(OP, TYPE, PRED_TYPE, T)     \
+  T (OP, TYPE, PRED_TYPE, b, a)             \
+  T (OP, TYPE, PRED_TYPE, b, b)             \
+  T (OP, TYPE, PRED_TYPE, b, c)
+
+#define TEST_ALL(TYPE, PRED_TYPE, T) \
+  TEST_OP (ADD, TYPE, PRED_TYPE, T)  \
+  TEST_OP (SUB, TYPE, PRED_TYPE, T)  \
+  TEST_OP (MUL, TYPE, PRED_TYPE, T)  \
+  TEST_OP (MAX, TYPE, PRED_TYPE, T)  \
+  TEST_OP (MIN, TYPE, PRED_TYPE, T)
+
+#define TEST(TYPE, PTYPE, SIZE)                                   \
+  typedef TYPE TYPE##SIZE __attribute__ ((vector_size (SIZE)));   \
+  typedef PTYPE PTYPE##SIZE __attribute__ ((vector_size (SIZE))); \
+  TEST_ALL (TYPE##SIZE, PTYPE##SIZE, COND_OP)
+
+TEST (__bf16, uint16_t, 128)
+
+TEST (__bf16, uint16_t, 64)
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+// There's no BFSUBR.
+/* { dg-final { scan-assembler-times {\tsel\t} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C
new file mode 100644
index 0000000..02880ef
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C
@@ -0,0 +1,18 @@
+/* { dg-do compile }*/
+/* { dg-options "-O -ffinite-math-only -fno-signed-zeros -msve-vector-bits=2048 " } */
+
+#include "unpacked_cond_binary_bf16_1.C"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 15 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 15 } } */
+/* { dg-final { scan-assembler-times {\tand} 30 } } */
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+// There's no BFSUBR.
+/* { dg-final { scan-assembler-times {\tsel\t} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C
new file mode 100644
index 0000000..95cd698
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C
@@ -0,0 +1,35 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2  -fno-trapping-math -msve-vector-bits=2048 " } */
+
+#include <stdint.h>
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+#define COND_BFMLA(TYPE, PRED_TYPE, MERGE)                               \
+  TYPE test_bfmla_##TYPE##_##MERGE (TYPE a, TYPE b, TYPE c, PRED_TYPE p) \
+  {return p ? a * b + c : MERGE; }
+
+#define COND_BFMLS(TYPE, PRED_TYPE, MERGE)                               \
+  TYPE test_bfmls_##TYPE##_##MERGE (TYPE a, TYPE b, TYPE c, PRED_TYPE p) \
+  {return p ? a * -b + c : MERGE; }
+
+#define TEST_OP(TYPE, PRED_TYPE, T) \
+  T (TYPE, PRED_TYPE, c)            \
+  T (TYPE, PRED_TYPE, 0)
+
+#define TEST(TYPE, PTYPE, SIZE)                                   \
+  typedef TYPE TYPE##SIZE __attribute__ ((vector_size (SIZE)));   \
+  typedef PTYPE PTYPE##SIZE __attribute__ ((vector_size (SIZE))); \
+  TEST_OP (TYPE##SIZE, PTYPE##SIZE, COND_BFMLA)                   \
+  TEST_OP (TYPE##SIZE, PTYPE##SIZE, COND_BFMLS)
+
+TEST (__bf16, uint16_t, 128)
+
+TEST (__bf16, uint16_t, 64)
+
+/* { dg-final { scan-assembler-times {\tptrue} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C
new file mode 100644
index 0000000..c0d7c50
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=2048" } */
+
+#include "unpacked_cond_ternary_bf16_1.C"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tand} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_1.C
new file mode 100644
index 0000000..19bfe95
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_1.C
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=2048" } */
+
+#define BFMLA(TYPE)                               \
+  TYPE test_bfmla_##TYPE (TYPE a, TYPE b, TYPE c) \
+  { return a * b + c; }
+
+#define BFMLS(TYPE)                               \
+  TYPE test_bfmls_##TYPE (TYPE a, TYPE b, TYPE c) \
+  { return a * -b + c; }
+
+#define TEST_TYPE(TYPE, SIZE)                                 \
+  typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
+  BFMLA (TYPE##SIZE)                                          \
+  BFMLS (TYPE##SIZE)
+
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+TEST_TYPE (__bf16,  128)
+
+TEST_TYPE (__bf16,  64)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_2.C
new file mode 100644
index 0000000..ef37400
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_2.C
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=2048 -fno-trapping-math" } */
+
+#include "unpacked_ternary_bf16_1.C"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.c-torture/compile/pr120807.c b/gcc/testsuite/gcc.c-torture/compile/pr120807.c
new file mode 100644
index 0000000..9b37e60
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/compile/pr120807.c
@@ -0,0 +1,20 @@
+typedef __UINT8_TYPE__ uint8_t;
+typedef __UINT32_TYPE__ uint32_t;
+
+typedef struct
+{
+  uint32_t dword[2];
+  uint8_t byte[8];
+} reg64_t;
+reg64_t TestF20F_opgd, TestF20F_oped;
+
+void
+TestF20F ()
+{
+  TestF20F_opgd.dword[0] ^= TestF20F_oped.byte[0];
+  for (int i = 0; i < 8; i++)
+    if (TestF20F_opgd.dword[0] & 1)
+      TestF20F_opgd.dword[0] = TestF20F_opgd.dword[0] >> 1 ^ (uint32_t)2197175160UL;
+    else
+      TestF20F_opgd.dword[0] = TestF20F_opgd.dword[0] >> 1;
+}
diff --git a/gcc/testsuite/gcc.dg/20021014-1.c b/gcc/testsuite/gcc.dg/20021014-1.c
index e43f7b2..ee5d459 100644
--- a/gcc/testsuite/gcc.dg/20021014-1.c
+++ b/gcc/testsuite/gcc.dg/20021014-1.c
@@ -2,6 +2,7 @@
 /* { dg-require-profiling "-p" } */
 /* { dg-options "-O2 -p" } */
 /* { dg-options "-O2 -p -static" { target hppa*-*-hpux* } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 /* { dg-error "profiler" "No profiler support" { target xstormy16-*-* } 0 } */
 /* { dg-message "" "consider using `-pg' instead of `-p' with gprof(1)" { target *-*-freebsd* } 0 } */
 
diff --git a/gcc/testsuite/gcc.dg/Warray-parameter-11.c b/gcc/testsuite/gcc.dg/Warray-parameter-11.c
index 8ca1b55..e05835c 100644
--- a/gcc/testsuite/gcc.dg/Warray-parameter-11.c
+++ b/gcc/testsuite/gcc.dg/Warray-parameter-11.c
@@ -9,7 +9,7 @@ typedef __INTPTR_TYPE__ intptr_t;
 void f0 (double[!copysign (~2, 3)]);
 
 void f1 (double[!copysign (~2, 3)]);
-void f1 (double[1]);                    // { dg-warning "-Warray-parameter" }
+void f1 (double[1]);                    // { dg-warning "-Wvla-parameter" }
 
 void f2 (int[(int)+1.0]);
 void f2 (int[(int)+1.1]);
@@ -21,4 +21,4 @@ extern struct S *sp;
 
 void f3 (int[(intptr_t)((char*)sp->a - (char*)sp)]);
 void f3 (int[(intptr_t)((char*)&sp->a[0] - (char*)sp)]);
-void f3 (int[(intptr_t)((char*)&sp->a[1] - (char*)sp)]);   // { dg-warning "-Warray-parameter" }
+void f3 (int[(intptr_t)((char*)&sp->a[1] - (char*)sp)]);   // { dg-warning "-Wvla-parameter" }
diff --git a/gcc/testsuite/gcc.dg/Warray-parameter.c b/gcc/testsuite/gcc.dg/Warray-parameter.c
index 6c5195a..31879a8 100644
--- a/gcc/testsuite/gcc.dg/Warray-parameter.c
+++ b/gcc/testsuite/gcc.dg/Warray-parameter.c
@@ -118,8 +118,7 @@ typedef int IA2[2];
 typedef int IA3[3];
 
 // The message should differentiate between the [] form and *.
-void f1IAx_A1 (IAx);            // { dg-message "previously declared as 'int\\\[]'" "pr?????" { xfail *-*-* } }
-                                // { dg-message "previously declared as 'int *\\\*'" "note" { target *-*-* } .-1 }
+void f1IAx_A1 (IAx);            // { dg-message "previously declared as 'int\\\[]'" }
 void f1IAx_A1 (IA1);            // { dg-message "argument 1 of type 'int\\\[1]' with mismatched bound" }
 
 void f1IA1_A2 (IA1);            // { dg-message "previously declared as 'int\\\[1]'" }
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py
index d2967d4..d92af83 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py
@@ -6,7 +6,7 @@ import pytest
 def sarif():
     return sarif_from_env()
 
-def test_xml_state(sarif):
+def test_state_graph(sarif):
     result = get_result_by_index(sarif, 0)
 
     assert result['level'] == 'warning'
@@ -17,16 +17,49 @@ def test_xml_state(sarif):
 
     # Event "(1)": "entry to 'test'" (index == 0)
     assert events[0]['location']['message']['text'] == "entry to 'test'"
-    state0 = get_xml_state(events, 0)
-    memory_regions = state0.find('memory-regions')
-    assert memory_regions is not None
-    stack = memory_regions.find('stack')
-    assert stack is not None
-    frame = stack.find('stack-frame')
-    assert frame.get('function') == 'test'
+    state0 = get_state_graph(events, 0)
+
+    stack = state0['nodes'][0]
+    assert stack['id'] == 'stack'
+    assert get_state_node_kind(stack) == 'stack'
+
+    frame = stack['children'][0]
+    assert frame['id'].startswith('frame-region-')
+    assert get_state_node_kind(frame) == 'stack-frame'
+    assert get_state_node_attr(frame, 'function') == 'test'
+    assert frame['location']['logicalLocations'][0]['fullyQualifiedName'] == 'test'
 
     # Final event:
     assert events[-1]['location']['message']['text'].startswith("use after 'free' of ")
-    state = get_xml_state(events, -1)
-    # TODO
+    state = get_state_graph(events, -1)
+
+    stack = state['nodes'][0]
+    assert stack['id'] == 'stack'
+    assert get_state_node_kind(stack) == 'stack'
+
+    frame = stack['children'][0]
+    assert frame['id'].startswith('frame-region-')
+    assert get_state_node_kind(frame) == 'stack-frame'
+    assert get_state_node_attr(frame, 'function') == 'test'
+    assert frame['location']['logicalLocations'][0]['fullyQualifiedName'] == 'test'
+
+    heap = state['nodes'][1]
+    assert heap['id'] == 'heap'
+    assert get_state_node_kind(heap) == 'heap'
+
+    assert len(heap['children']) == 3
+    heap_buffer0 = heap['children'][0]
+    assert heap_buffer0['id'].startswith('heap-allocated-region-')
+    assert get_state_node_kind(heap_buffer0) == 'dynalloc-buffer'
+
+    globals_ = state['nodes'][2]
+    assert globals_['id'] == 'globals'
+    assert get_state_node_kind(globals_) == 'globals'
 
+    first = globals_['children'][0]
+    assert first['id'].startswith('decl-region-')
+    assert get_state_node_kind(first) == 'variable'
+    assert get_state_node_name(first) == 'first'
+    assert get_state_node_type(first) == 'struct node *'
+    
+    assert len(state['edges']) == 3
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c
index 3d853d2..3b35cfa 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c
@@ -1,4 +1,4 @@
-/* { dg-additional-options "-fdiagnostics-add-output=sarif:xml-state=yes" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:state-graphs=yes" } */
 
 #include "analyzer-decls.h"
 
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c
index 28cf580..b981cf9 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c
@@ -1,4 +1,4 @@
-/* { dg-additional-options "-fdiagnostics-add-output=sarif:xml-state=yes" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:state-graphs=yes" } */
 
 #include "analyzer-decls.h"
 
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py
index 484da96..3a2c6f8 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py
@@ -1,5 +1,3 @@
-import xml.etree.ElementTree as ET
-
 from sarif import *
 
 import pytest
@@ -8,7 +6,7 @@ import pytest
 def sarif():
     return sarif_from_env()
 
-def test_nested_types_in_xml_state(sarif):
+def test_nested_types_in_state_graph(sarif):
     result = get_result_by_index(sarif, 0)
 
     assert result['level'] == 'note'
@@ -16,16 +14,17 @@ def test_nested_types_in_xml_state(sarif):
     events = result["codeFlows"][0]["threadFlows"][0]['locations']
 
     assert events[0]['location']['message']['text'] == 'here'
-    state = get_xml_state(events, 0)
-
-    memory_regions = state.find('memory-regions')
-    assert memory_regions is not None
+    state = get_state_graph(events, 0)
 
-    stack = memory_regions.find('stack')
-    assert stack is not None
+    stack = state['nodes'][0]
+    assert stack['id'] == 'stack'
+    assert get_state_node_kind(stack) == 'stack'
 
-    frame = stack.find('stack-frame')
-    assert frame.get('function') == 'test'
+    frame = stack['children'][0]
+    assert frame['id'].startswith('frame-region-')
+    assert get_state_node_kind(frame) == 'stack-frame'
+    assert get_state_node_attr(frame, 'function') == 'test'
+    assert frame['location']['logicalLocations'][0]['fullyQualifiedName'] == 'test'
 
     # We have:
     #   baz_arr[1].m_bars[1].m_foos[2].m_ints[1] = 42;
@@ -34,40 +33,57 @@ def test_nested_types_in_xml_state(sarif):
     # representation to nested elements and fields.
 
     # "baz_arr":
-    baz_arr = frame.find("variable[@name='baz_arr']")
-    assert baz_arr.get('type') == 'struct baz[2]'
+    baz_arr = frame['children'][0]
+    assert get_state_node_kind(baz_arr) == 'variable'
+    assert get_state_node_type(baz_arr) == 'struct baz[2]'
+
+    assert len(baz_arr['children']) == 2
+
+    bindings = baz_arr['children'][0]
+    assert bindings['id'].startswith('concrete-bindings-')
+    assert get_state_node_kind(bindings) == 'other'
+    assert get_state_node_value(bindings['children'][0]) == '(int)42'
 
     # "baz_arr[1]":
-    baz_arr_1 = baz_arr.find("element[@index='1']")
-    assert baz_arr_1.get('type') == 'struct baz'
+    baz_arr_1 = baz_arr['children'][1]
+    assert get_state_node_type(baz_arr_1) == 'struct baz'
+    assert get_state_node_kind(baz_arr_1) == 'element'
+    assert get_state_node_attr(baz_arr_1, 'index') == '1'
 
-    assert baz_arr.find("element[@index='0']") is None
+    assert len(baz_arr_1['children']) == 1
 
     # "baz_arr[1].m_bars":
-    baz_arr_1_m_bars = baz_arr_1.find("field[@name='m_bars']")
-    assert baz_arr_1_m_bars.get('type') == 'struct bar[2]'
+    baz_arr_1_m_bars = baz_arr_1['children'][0]
+    assert get_state_node_name(baz_arr_1_m_bars) == 'm_bars'
+    assert get_state_node_type(baz_arr_1_m_bars) == 'struct bar[2]'
 
     # "baz_arr[1].m_bars[1]"
-    baz_arr_1_m_bars_1 = baz_arr_1_m_bars.find("element[@index='1']")
-    assert baz_arr_1_m_bars_1.get('type') == 'struct bar'
+    baz_arr_1_m_bars_1 = baz_arr_1_m_bars['children'][0]
+    assert get_state_node_type(baz_arr_1_m_bars_1) == 'struct bar'
+    assert get_state_node_kind(baz_arr_1_m_bars_1) == 'element'
+    assert get_state_node_attr(baz_arr_1_m_bars_1, 'index') == '1'
 
     # "baz_arr[1].m_bars[1].m_foos"
-    baz_arr_1_m_bars_1_m_foos = baz_arr_1_m_bars_1.find("field[@name='m_foos']")
-    assert baz_arr_1_m_bars_1_m_foos.get('type') == 'struct foo[3]'
+    baz_arr_1_m_bars_1_m_foos = baz_arr_1_m_bars_1['children'][0]
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos) == 'field'
+    assert get_state_node_name(baz_arr_1_m_bars_1_m_foos) == 'm_foos'
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos) == 'struct foo[3]'
 
     # "baz_arr[1].m_bars[1].m_foos[2]"
-    baz_arr_1_m_bars_1_m_foos_2 = baz_arr_1_m_bars_1_m_foos.find("element[@index='2']")
-    assert baz_arr_1_m_bars_1_m_foos_2.get('type') == 'struct foo'
-    
+    baz_arr_1_m_bars_1_m_foos_2 = baz_arr_1_m_bars_1_m_foos['children'][0]
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos_2) == 'struct foo'
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos_2) == 'element'
+    assert get_state_node_attr(baz_arr_1_m_bars_1_m_foos_2, 'index') == '2'
+
     # "baz_arr[1].m_bars[1].m_foos[2].m_ints"
-    baz_arr_1_m_bars_1_m_foos_2_m_ints = baz_arr_1_m_bars_1_m_foos_2.find('field[@name="m_ints"]')
-    assert baz_arr_1_m_bars_1_m_foos_2_m_ints.get('type') == 'int[4]'
-    
-    # "baz_arr[1].m_bars[1].m_foos[2].m_ints[1]"
-    baz_arr_1_m_bars_1_m_foos_2_m_ints_1 = baz_arr_1_m_bars_1_m_foos_2_m_ints.find('element[@index="1"]')
-    assert baz_arr_1_m_bars_1_m_foos_2_m_ints_1.get('type') == 'int'
+    baz_arr_1_m_bars_1_m_foos_2_m_ints = baz_arr_1_m_bars_1_m_foos_2['children'][0]
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos_2_m_ints) == 'field'
+    assert get_state_node_name(baz_arr_1_m_bars_1_m_foos_2_m_ints) == 'm_ints'
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos_2_m_ints) == 'int[4]'
 
-    value = baz_arr_1_m_bars_1_m_foos_2_m_ints_1.find('value-of-region')
-    constant = value.find('constant')
-    assert constant.get('value') == '42'
-    assert constant.get('type') == 'int'
+    # "baz_arr[1].m_bars[1].m_foos[2].m_ints[1]"
+    baz_arr_1_m_bars_1_m_foos_2_m_ints_1 = baz_arr_1_m_bars_1_m_foos_2_m_ints['children'][0]
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos_2_m_ints_1) == 'int'
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos_2_m_ints_1) == 'element'
+    assert get_state_node_attr(baz_arr_1_m_bars_1_m_foos_2_m_ints_1, 'index') == '1'
+    assert get_state_node_value(baz_arr_1_m_bars_1_m_foos_2_m_ints_1) == '(int)42'
diff --git a/gcc/testsuite/gcc.dg/aru-2.c b/gcc/testsuite/gcc.dg/aru-2.c
index 054223c..61898de 100644
--- a/gcc/testsuite/gcc.dg/aru-2.c
+++ b/gcc/testsuite/gcc.dg/aru-2.c
@@ -1,6 +1,7 @@
 /* { dg-do run } */
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 
 static int __attribute__((noinline))
 bar (int x)
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-1.c b/gcc/testsuite/gcc.dg/asm-hard-reg-1.c
new file mode 100644
index 0000000..6a5a9ad
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-1.c
@@ -0,0 +1,85 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+#if defined (__aarch64__)
+# define GPR "{x4}"
+/* { dg-final { scan-assembler-times "foo\tx4" 8 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\tr4" 8 { target { arm*-*-* } } } } */
+#elif defined (__i386__)
+# define GPR "{ecx}"
+/* { dg-final { scan-assembler-times "foo\t%cl" 2 { target { i?86-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%cx" 2 { target { i?86-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%ecx" 4 { target { i?86-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR "{r5}"
+/* { dg-final { scan-assembler-times "foo\t5" 8 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR "{t5}"
+/* { dg-final { scan-assembler-times "foo\tt5" 8 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\t%r4" 8 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR "{rcx}"
+/* { dg-final { scan-assembler-times "foo\t%cl" 2 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%cx" 2 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%ecx" 2 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%rcx" 2 { target { x86_64-*-* } } } } */
+#endif
+
+char
+test_char (char x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+char
+test_char_from_mem (char *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
+
+short
+test_short (short x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+short
+test_short_from_mem (short *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
+
+int
+test_int (int x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+int
+test_int_from_mem (int *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
+
+long
+test_long (long x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+long
+test_long_from_mem (long *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-2.c b/gcc/testsuite/gcc.dg/asm-hard-reg-2.c
new file mode 100644
index 0000000..7dabf96
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-2.c
@@ -0,0 +1,33 @@
+/* { dg-do compile { target aarch64*-*-* powerpc64*-*-* riscv64-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-std=c99" } we need long long */
+
+#if defined (__aarch64__)
+# define GPR "{x4}"
+/* { dg-final { scan-assembler-times "foo\tx4" 2 { target { aarch64*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR "{r5}"
+/* { dg-final { scan-assembler-times "foo\t5" 2 { target { powerpc64*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR "{t5}"
+/* { dg-final { scan-assembler-times "foo\tt5" 2 { target { riscv64-*-* } } } } */
+#elif defined (__s390__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\t%r4" 2 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR "{rcx}"
+/* { dg-final { scan-assembler-times "foo\t%rcx" 2 { target { x86_64-*-* } } } } */
+#endif
+
+long long
+test_longlong (long long x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+long long
+test_longlong_from_mem (long long *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-3.c b/gcc/testsuite/gcc.dg/asm-hard-reg-3.c
new file mode 100644
index 0000000..fa4472a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-3.c
@@ -0,0 +1,25 @@
+/* { dg-do compile { target { { aarch64*-*-* powerpc64*-*-* riscv64-*-* s390*-*-* x86_64-*-* } && int128 } } } */
+/* { dg-options "-O2" } get rid of -ansi since we use __int128 */
+
+#if defined (__aarch64__)
+# define REG "{x4}"
+/* { dg-final { scan-assembler-times "foo\tx4" 1 { target { aarch64*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define REG "{r5}"
+/* { dg-final { scan-assembler-times "foo\t5" 1 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define REG "{t5}"
+/* { dg-final { scan-assembler-times "foo\tt5" 1 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define REG "{r4}"
+/* { dg-final { scan-assembler-times "foo\t%r4" 1 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define REG "{xmm0}"
+/* { dg-final { scan-assembler-times "foo\t%xmm0" 1 { target { x86_64-*-* } } } } */
+#endif
+
+void
+test (void)
+{
+  __asm__ ("foo\t%0" :: REG ((__int128) 42));
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-4.c b/gcc/testsuite/gcc.dg/asm-hard-reg-4.c
new file mode 100644
index 0000000..0134bf0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-4.c
@@ -0,0 +1,50 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+#if defined (__aarch64__)
+# define FPR "{d5}"
+/* { dg-final { scan-assembler-times "foo\tv5" 4 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define FPR "{d5}"
+/* { dg-additional-options "-march=armv7-a+fp -mfloat-abi=hard" { target arm*-*-* } } */
+/* { dg-final { scan-assembler-times "foo\ts10" 4 { target { arm*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define FPR "{5}"
+/* { dg-final { scan-assembler-times "foo\t5" 4 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define FPR "{fa5}"
+/* { dg-final { scan-assembler-times "foo\tfa5" 4 { target { rsicv*-*-* } } } } */
+#elif defined (__s390__)
+# define FPR "{f5}"
+/* { dg-final { scan-assembler-times "foo\t%f5" 4 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define FPR "{xmm5}"
+/* { dg-final { scan-assembler-times "foo\t%xmm5" 4 { target { x86_64-*-* } } } } */
+#endif
+
+float
+test_float (float x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (x));
+  return x;
+}
+
+float
+test_float_from_mem (float *x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (*x));
+  return *x;
+}
+
+double
+test_double (double x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (x));
+  return x;
+}
+
+double
+test_double_from_mem (double *x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-5.c b/gcc/testsuite/gcc.dg/asm-hard-reg-5.c
new file mode 100644
index 0000000..a9e25ce
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-5.c
@@ -0,0 +1,36 @@
+/* { dg-do compile { target aarch64*-*-* powerpc64*-*-* riscv64-*-* s390*-*-* x86_64-*-* } } */
+
+typedef int V __attribute__ ((vector_size (4 * sizeof (int))));
+
+#if defined (__aarch64__)
+# define VR "{v20}"
+/* { dg-final { scan-assembler-times "foo\tv20" 2 { target { aarch64*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define VR "{v5}"
+/* { dg-final { scan-assembler-times "foo\t5" 2 { target { powerpc64*-*-* } } } } */
+#elif defined (__riscv)
+# define VR "{v5}"
+/* { dg-additional-options "-march=rv64imv" { target riscv64-*-* } } */
+/* { dg-final { scan-assembler-times "foo\tv5" 2 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define VR "{v5}"
+/* { dg-require-effective-target s390_mvx { target s390*-*-* } } */
+/* { dg-final { scan-assembler-times "foo\t%v5" 2 { target s390*-*-* } } } */
+#elif defined (__x86_64__)
+# define VR "{xmm9}"
+/* { dg-final { scan-assembler-times "foo\t%xmm9" 2 { target { x86_64-*-* } } } } */
+#endif
+
+V
+test (V x)
+{
+  __asm__ ("foo\t%0" : "+"VR (x));
+  return x;
+}
+
+V
+test_from_mem (V *x)
+{
+  __asm__ ("foo\t%0" : "+"VR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-6.c b/gcc/testsuite/gcc.dg/asm-hard-reg-6.c
new file mode 100644
index 0000000..d9b7fae
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-6.c
@@ -0,0 +1,60 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-O2" } */
+
+/* Test multiple alternatives.  */
+
+#if defined (__aarch64__)
+# define GPR1 "{x1}"
+# define GPR2 "{x2}"
+# define GPR3 "{x3}"
+/* { dg-final { scan-assembler-times "foo\tx1,x3" 1 { target { aarch64*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\tx2,\\\[x1\\\]" 1 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define GPR1 "{r1}"
+# define GPR2 "{r2}"
+# define GPR3 "{r3}"
+/* { dg-final { scan-assembler-times "foo\tr1,r3" 1 { target { arm*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\tr2,\\\[r1\\\]" 1 { target { arm*-*-* } } } } */
+#elif defined (__i386__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+# define GPR3 "{ecx}"
+/* { dg-final { scan-assembler-times "foo\t4\\(%esp\\),%ecx" 1 { target { i?86-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t%ebx,\\(%eax\\)" 1 { target { i?86-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR1 "{r4}"
+# define GPR2 "{r5}"
+# define GPR3 "{r6}"
+/* { dg-final { scan-assembler-times "foo\t4,6" 1 { target { powerpc*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t5,0\\(4\\)" 1 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR1 "{t1}"
+# define GPR2 "{t2}"
+# define GPR3 "{t3}"
+/* { dg-final { scan-assembler-times "foo\tt1,t3" 1 { target { riscv*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\tt2,0\\(a1\\)" 1 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define GPR1 "{r0}"
+# define GPR2 "{r1}"
+# define GPR3 "{r2}"
+/* { dg-final { scan-assembler-times "foo\t%r0,%r2" 1 { target { s390*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t%r1,0\\(%r3\\)" 1 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+# define GPR3 "{rcx}"
+/* { dg-final { scan-assembler-times "foo\t%eax,%rcx" 1 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t%ebx,\\(%rsi\\)" 1 { target { x86_64-*-* } } } } */
+#endif
+
+void
+test_reg_reg (int x, long long *y)
+{
+  __asm__ ("foo\t%0,%1" :: GPR1"m,"GPR2 (x), GPR3",m" (y));
+}
+
+void
+test_reg_mem (int x, long long *y)
+{
+  __asm__ ("bar\t%0,%1" :: GPR1"m,"GPR2 (x), GPR3",m" (*y));
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-7.c b/gcc/testsuite/gcc.dg/asm-hard-reg-7.c
new file mode 100644
index 0000000..761a6b7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-7.c
@@ -0,0 +1,41 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-O2" } */
+
+/* Test multiple alternatives.  */
+
+#if defined (__aarch64__)
+# define GPR "{x1}"
+/* { dg-final { scan-assembler-times "foo\tx1,x1" 2 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define GPR "{r1}"
+/* { dg-final { scan-assembler-times "foo\tr1,r1" 2 { target { arm*-*-* } } } } */
+#elif defined (__i386__)
+# define GPR "{eax}"
+/* { dg-final { scan-assembler-times "foo\t%eax,%eax" 2 { target { i?86-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\t4,4" 2 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR "{t1}"
+/* { dg-final { scan-assembler-times "foo\tt1,t1" 2 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define GPR "{r0}"
+/* { dg-final { scan-assembler-times "foo\t%r0,%r0" 2 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR "{eax}"
+/* { dg-final { scan-assembler-times "foo\t%eax,%eax" 2 { target { x86_64-*-* } } } } */
+#endif
+
+int
+test_1 (int x)
+{
+  __asm__ ("foo\t%0,%0" : "+"GPR (x));
+  return x;
+}
+
+int
+test_2 (int x, int y)
+{
+  __asm__ ("foo\t%0,%1" : "="GPR (x) : GPR (y));
+  return x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-8.c b/gcc/testsuite/gcc.dg/asm-hard-reg-8.c
new file mode 100644
index 0000000..cda5e3e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-8.c
@@ -0,0 +1,49 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+/* Due to hard register constraints, X must be copied.  */
+
+#if defined (__aarch64__)
+# define GPR1 "{x1}"
+# define GPR2 "{x2}"
+#elif defined (__arm__)
+# define GPR1 "{r1}"
+# define GPR2 "{r2}"
+#elif defined (__i386__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR1 "{r4}"
+# define GPR2 "{r5}"
+#elif defined (__riscv)
+# define GPR1 "{t1}"
+# define GPR2 "{t2}"
+#elif defined (__s390__)
+# define GPR1 "{r0}"
+# define GPR2 "{r1}"
+#elif defined (__x86_64__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+#endif
+
+#define TEST(T) \
+int \
+test_##T (T x) \
+{ \
+  int out; \
+  __asm__ ("foo" : "=r" (out) : GPR1 (x), GPR2 (x)); \
+  return out; \
+}
+
+TEST(char)
+TEST(short)
+TEST(int)
+TEST(long)
+
+int
+test_subreg (long x)
+{
+  int out;
+  short subreg_x = x;
+  __asm__ ("foo" : "=r" (out) : GPR1 (x), GPR2 (subreg_x));
+  return out;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-1.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-1.c
new file mode 100644
index 0000000..0d7c2f2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-1.c
@@ -0,0 +1,83 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+#if defined (__aarch64__)
+# define GPR1_RAW "x0"
+# define GPR2 "{x1}"
+# define GPR3 "{x2}"
+# define INVALID_GPR_A "{x31}"
+#elif defined (__arm__)
+# define GPR1_RAW "r0"
+# define GPR2 "{r1}"
+# define GPR3 "{r2}"
+# define INVALID_GPR_A "{r16}"
+#elif defined (__i386__)
+# define GPR1_RAW "%eax"
+# define GPR2 "{%ebx}"
+# define GPR3 "{%edx}"
+# define INVALID_GPR_A "{%eex}"
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR1_RAW "r4"
+# define GPR2 "{r5}"
+# define GPR3 "{r6}"
+# define INVALID_GPR_A "{r33}"
+#elif defined (__riscv)
+# define GPR1_RAW "t4"
+# define GPR2 "{t5}"
+# define GPR3 "{t6}"
+# define INVALID_GPR_A "{t7}"
+#elif defined (__s390__)
+# define GPR1_RAW "r4"
+# define GPR2 "{r5}"
+# define GPR3 "{r6}"
+# define INVALID_GPR_A "{r17}"
+#elif defined (__x86_64__)
+# define GPR1_RAW "rax"
+# define GPR2 "{rbx}"
+# define GPR3 "{rcx}"
+# define INVALID_GPR_A "{rex}"
+#endif
+
+#define GPR1 "{"GPR1_RAW"}"
+#define INVALID_GPR_B "{"GPR1_RAW
+
+struct { int a[128]; } s = {0};
+
+void
+test (void)
+{
+  int x, y;
+  register int gpr1 __asm__ (GPR1_RAW) = 0;
+
+  __asm__ ("" :: "{}" (42)); /* { dg-error "invalid input constraint: \{\}" } */
+  __asm__ ("" :: INVALID_GPR_A (42)); /* { dg-error "invalid input constraint" } */
+  __asm__ ("" :: INVALID_GPR_B (42)); /* { dg-error "invalid input constraint" } */
+
+  __asm__ ("" :: GPR1 (s)); /* { dg-error "data type isn't suitable for register .* of operand 0" } */
+
+  __asm__ ("" :: "r" (gpr1), GPR1 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1 (42), "r" (gpr1)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1 (42), GPR1 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1","GPR2 (42), GPR2","GPR3 (42));
+  __asm__ ("" :: GPR1","GPR2 (42), GPR3","GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1","GPR2 (42), GPR1","GPR3 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1 GPR2 (42), GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" : "+"GPR1 (x), "="GPR1 (y)); /* { dg-error "multiple outputs to hard register" } */
+  __asm__ ("" : "="GPR1 (y) : GPR1 (42), "0" (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" : "+"GPR1 (x) : GPR1 (42)); /* { dg-error "multiple inputs to hard register" } */
+
+  __asm__ ("" : "="GPR1 (gpr1));
+  __asm__ ("" : "="GPR2 (gpr1)); /* { dg-error "constraint and register 'asm' for output operand 0 are unsatisfiable" } */
+  __asm__ ("" :: GPR2 (gpr1)); /* { dg-error "constraint and register 'asm' for input operand 0 are unsatisfiable" } */
+  __asm__ ("" : "="GPR1 (x) : "0" (gpr1));
+  __asm__ ("" : "="GPR1 GPR2 (x) : "0" (gpr1)); /* { dg-error "constraint and register 'asm' for input operand 0 are unsatisfiable" } */
+
+  __asm__ ("" : "=&"GPR1 (x) : "0" (gpr1));
+  __asm__ ("" : "=&"GPR1 (x) : "0" (42));
+  __asm__ ("" : "=&"GPR2","GPR1 (x) : "r,"GPR1 (42));
+  __asm__ ("" : "="GPR2",&"GPR1 (x) : "r,"GPR1 (42)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&"GPR1 (x) : GPR1 (42)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&"GPR2","GPR1 (x) : "r,r" (gpr1));
+  __asm__ ("" : "="GPR2",&"GPR1 (x) : "r,r" (gpr1)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&r" (gpr1) : GPR1 (42)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&"GPR1 (x),  "=r" (y) : "1" (gpr1)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-2.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-2.c
new file mode 100644
index 0000000..d0d5cfe
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile { target { { aarch64*-*-* s390x-*-* } && int128 } } } */
+/* { dg-options "-O2" } get rid of -ansi since we use __int128 */
+
+/* Test register pairs.  */
+
+#if defined (__aarch64__)
+# define GPR1 "{x4}"
+# define GPR2_RAW "x5"
+#elif defined (__s390__)
+# define GPR1 "{r4}"
+# define GPR2_RAW "r5"
+#endif
+
+#define GPR2 "{"GPR2_RAW"}"
+
+void
+test (void)
+{
+  __asm__ ("" :: GPR1 ((__int128) 42));
+  __asm__ ("" :: GPR2 ((__int128) 42)); /* { dg-error "register .* for operand 0 isn't suitable for data type" } */
+  __asm__ ("" :: GPR1 ((__int128) 42), GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+
+  __int128 x;
+  __asm__ ("" : "="GPR1 (x) :: GPR2_RAW); /* { dg-error "hard register constraint for output 0 conflicts with 'asm' clobber list" } */
+  __asm__ ("" : "=r" (x) : GPR1 (x) : GPR2_RAW); /* { dg-error "hard register constraint for input 0 conflicts with 'asm' clobber list" } */
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-3.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-3.c
new file mode 100644
index 0000000..17b2317
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-3.c
@@ -0,0 +1,27 @@
+/* { dg-do compile { target arm-*-* s390-*-* } } */
+/* { dg-options "-std=c99" } we need long long */
+/* { dg-additional-options "-march=armv7-a" { target arm-*-* } } */
+
+/* Test register pairs.  */
+
+#if defined (__arm__)
+# define GPR1 "{r4}"
+# define GPR2_RAW "r5"
+#elif defined (__s390__)
+# define GPR1 "{r4}"
+# define GPR2_RAW "r5"
+#endif
+
+#define GPR2 "{"GPR2_RAW"}"
+
+void
+test (void)
+{
+  __asm__ ("" :: GPR1 (42ll));
+  __asm__ ("" :: GPR2 (42ll)); /* { dg-error "register .* for operand 0 isn't suitable for data type" } */
+  __asm__ ("" :: GPR1 (42ll), GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+
+  long long x;
+  __asm__ ("" : "="GPR1 (x) :: GPR2_RAW); /* { dg-error "hard register constraint for output 0 conflicts with 'asm' clobber list" } */
+  __asm__ ("" : "=r" (x) : GPR1 (x) : GPR2_RAW); /* { dg-error "hard register constraint for input 0 conflicts with 'asm' clobber list" } */
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-4.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-4.c
new file mode 100644
index 0000000..465f24b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-4.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+
+/* Verify output operands.  */
+
+int
+test (void)
+{
+  int x;
+  register int y __asm__ ("0");
+
+  /* Preserve status quo and don't error out.  */
+  __asm__ ("" : "=r" (x), "=r" (x));
+
+  /* Be more strict for hard register constraints and error out.  */
+  __asm__ ("" : "={0}" (x), "={1}" (x)); /* { dg-error "multiple outputs to lvalue 'x'" } */
+
+  /* Still error out in case of a mixture.  */
+  __asm__ ("" : "=r" (x), "={1}" (x)); /* { dg-error "multiple outputs to lvalue 'x'" } */
+
+  return x + y;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-5.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-5.c
new file mode 100644
index 0000000..85398f0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-5.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+
+/* Test clobbers.
+   See asm-hard-reg-error-{2,3}.c for tests involving register pairs.  */
+
+int
+test (void)
+{
+  int x, y;
+  __asm__ ("" : "={0}" (x), "={1}" (y) : : "1"); /* { dg-error "hard register constraint for output 1 conflicts with 'asm' clobber list" } */
+  __asm__ ("" : "={0}" (x) : "{0}" (y), "{1}" (y) : "1"); /* { dg-error "hard register constraint for input 1 conflicts with 'asm' clobber list" } */
+  return x + y;
+}
diff --git a/gcc/testsuite/gcc.dg/bitint-124.c b/gcc/testsuite/gcc.dg/bitint-124.c
new file mode 100644
index 0000000..160a1e3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/bitint-124.c
@@ -0,0 +1,30 @@
+/* PR tree-optimization/121131 */
+/* { dg-do run { target bitint } } */
+/* { dg-options "-O2" } */
+
+#if __BITINT_MAXWIDTH__ >= 156
+struct A { _BitInt(156) b : 135; };
+
+static inline _BitInt(156)
+foo (struct A *x)
+{
+  return x[1].b;
+}
+
+__attribute__((noipa)) _BitInt(156)
+bar (void)
+{
+  struct A a[] = { 1, 1, -13055525270329736316393717310914023773847wb,
+		   1, 1, 1, 1, 1, 1, 1, 1, 1 };
+  return foo (&a[1]);
+}
+#endif
+
+int
+main ()
+{
+#if __BITINT_MAXWIDTH__ >= 156
+  if (bar () != -13055525270329736316393717310914023773847wb)
+    __builtin_abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.dg/bitintext.h b/gcc/testsuite/gcc.dg/bitintext.h
index 99fedb3..d5f2689d 100644
--- a/gcc/testsuite/gcc.dg/bitintext.h
+++ b/gcc/testsuite/gcc.dg/bitintext.h
@@ -4,6 +4,24 @@ do_copy (void *p, const void *q, __SIZE_TYPE__ r)
   __builtin_memcpy (p, q, r);
 }
 
+/* Obtain the value of N from a _BitInt(N)-typed expression X
+   at compile time.  */
+#define S(x) \
+  ((typeof (x)) -1 < 0                                                   \
+   ? __builtin_clrsbg (__builtin_choose_expr ((typeof (x)) -1 < 0,       \
+					      (typeof (x)) -1, -1)) + 1  \
+   : __builtin_popcountg (__builtin_choose_expr ((typeof (x)) -1 < 0,    \
+						 0U, (typeof (x)) -1)))
+
+#define CEIL(x,y) (((x) + (y) - 1) / (y))
+
+/* Promote a _BitInt type to include its padding bits.  */
+#if defined (__s390x__) || defined(__arm__)
+#define PROMOTED_SIZE(x) sizeof (x)
+#elif defined(__loongarch__)
+#define PROMOTED_SIZE(x) (sizeof (x) > 8 ? CEIL (S (x), 64) * 8 : sizeof (x))
+#endif
+
 /* Macro to test whether (on targets where psABI requires it) _BitInt
    with padding bits have those filled with sign or zero extension.  */
 #if defined(__s390x__) || defined(__arm__) || defined(__loongarch__)
@@ -11,14 +29,14 @@ do_copy (void *p, const void *q, __SIZE_TYPE__ r)
   do {								\
     if ((typeof (x)) -1 < 0)					\
       {								\
-	_BitInt(sizeof (x) * __CHAR_BIT__) __x;			\
+	_BitInt(PROMOTED_SIZE (x) * __CHAR_BIT__) __x;		\
 	do_copy (&__x, &(x), sizeof (__x));			\
 	if (__x != (x))						\
 	  __builtin_abort ();					\
       }								\
     else							\
       {								\
-	unsigned _BitInt(sizeof (x) * __CHAR_BIT__) __x;	\
+	unsigned _BitInt(PROMOTED_SIZE (x) * __CHAR_BIT__) __x;	\
 	do_copy (&__x, &(x), sizeof (__x));			\
 	if (__x != (x))						\
 	  __builtin_abort ();					\
diff --git a/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c b/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c
index 0d6593e..12e6c29 100644
--- a/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c
+++ b/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c
@@ -207,7 +207,7 @@ test5 (size_t sz)
 }
 
 int
-main (size_t sz)
+main (void)
 {
   test1 (sizeof (struct container));
   test1 (sizeof (struct container) - sizeof (int));
diff --git a/gcc/testsuite/gcc.dg/c23-attr-syntax-6.c b/gcc/testsuite/gcc.dg/c23-attr-syntax-6.c
index f8c5b0f..4e6b80b 100644
--- a/gcc/testsuite/gcc.dg/c23-attr-syntax-6.c
+++ b/gcc/testsuite/gcc.dg/c23-attr-syntax-6.c
@@ -45,7 +45,7 @@ typedef int [[__extension__ __extension__]] b2; /* { dg-error {'extension' attri
 typedef int [[__extension__ unknown_attribute]] b3; /* { dg-error {'unknown_attribute' attribute ignored} } */
 typedef int [[__extension__ gnu:vector_size(4)]] b4; /* { dg-error {expected '\]' before ':'} } */
 /* { dg-error {'gnu' attribute ignored} "" { target *-*-* } .-1 } */
-typedef int [[__extension__ gnu JOIN2(:,:) vector_size (4)]] b5; /* { dg-error {pasting ":" and ":" does not give a valid preprocessing token} } */
+typedef int [[__extension__ gnu JOIN2(:,:) vector_size (4)]] b5; /* { dg-error {pasting ':' and ':' does not give a valid preprocessing token} } */
 /* { dg-error {expected '\]' before ':'} "" { target *-*-* } .-1 } */
 /* { dg-error {'gnu' attribute ignored} "" { target *-*-* } .-2 } */
 typedef int [[__extension__ gnu : : vector_size (4)]] b6; /* { dg-error {expected '\]' before ':'} } */
@@ -81,7 +81,7 @@ typedef int [[gnu :: vector_size (4)]] b18; /* { dg-error {attributes before C23
 typedef int [[gnu FOO vector_size (4)]] b19; /* { dg-error {attributes before C23} } */
 typedef int [[gnu :: vector_size (sizeof (void (*)(...)))]] b20; /* { dg-error {attributes before C23} } */
 /* { dg-error {requires a named argument before} "" { target *-*-* } .-1 } */
-typedef int [[gnu JOIN2(:,:) vector_size (4)]] b21; /* { dg-error {pasting ":" and ":" does not give a valid preprocessing token} } */
+typedef int [[gnu JOIN2(:,:) vector_size (4)]] b21; /* { dg-error {pasting ':' and ':' does not give a valid preprocessing token} } */
 /* { dg-error {expected '\]' before ':'} "" { target *-*-* } .-1 } */
 /* { dg-error {'gnu' attribute ignored} "" { target *-*-* } .-2 } */
 /* { dg-error {attributes before C23} "" { target *-*-* } .-3 } */
diff --git a/gcc/testsuite/gcc.dg/cpp/paste12-2.c b/gcc/testsuite/gcc.dg/cpp/paste12-2.c
index 6e2e4f1..f46645a 100644
--- a/gcc/testsuite/gcc.dg/cpp/paste12-2.c
+++ b/gcc/testsuite/gcc.dg/cpp/paste12-2.c
@@ -6,6 +6,6 @@
 /* Test correct diagnostics when pasting in #include.
    Source: PR preprocessor/6780.  */
 
-#define inc2(a,b) <##a.b>  /* { dg-error "pasting \"<\" and \"stdio\" does not" } */
+#define inc2(a,b) <##a.b>  /* { dg-error "pasting '<' and 'stdio' does not" } */
 #define INC(X) inc2(X,h)
 #include INC(stdio)
diff --git a/gcc/testsuite/gcc.dg/cpp/paste12.c b/gcc/testsuite/gcc.dg/cpp/paste12.c
index 3e0f7b9..f6b3696 100644
--- a/gcc/testsuite/gcc.dg/cpp/paste12.c
+++ b/gcc/testsuite/gcc.dg/cpp/paste12.c
@@ -8,4 +8,4 @@
 
 #define inc2(a,b) <##a.b>
 #define INC(X) inc2(X,h)
-#include INC(stdio) /* { dg-error "pasting \"<\" and \"stdio\" does not" } */
+#include INC(stdio) /* { dg-error "pasting '<' and 'stdio' does not" } */
diff --git a/gcc/testsuite/gcc.dg/cpp/paste14-2.c b/gcc/testsuite/gcc.dg/cpp/paste14-2.c
index 3b23ada..bb51999 100644
--- a/gcc/testsuite/gcc.dg/cpp/paste14-2.c
+++ b/gcc/testsuite/gcc.dg/cpp/paste14-2.c
@@ -4,8 +4,8 @@
    { dg-do preprocess }
 */
 
-#define foo - ## >> /* { dg-error "pasting \"-\" and \">>\"" } */
+#define foo - ## >> /* { dg-error "pasting '-' and '>>'" } */
 foo
-#define bar = ## == /* { dg-error "pasting \"=\" and \"==\"" } */
+#define bar = ## == /* { dg-error "pasting '=' and '=='" } */
 bar
 
diff --git a/gcc/testsuite/gcc.dg/cpp/paste14.c b/gcc/testsuite/gcc.dg/cpp/paste14.c
index 043d5e5..d60b328 100644
--- a/gcc/testsuite/gcc.dg/cpp/paste14.c
+++ b/gcc/testsuite/gcc.dg/cpp/paste14.c
@@ -5,6 +5,6 @@
 */
 
 #define foo - ## >>
-foo		/* { dg-error "pasting \"-\" and \">>\"" } */
+foo		/* { dg-error "pasting '-' and '>>'" } */
 #define bar = ## ==
-bar		/* { dg-error "pasting \"=\" and \"==\"" } */
+bar		/* { dg-error "pasting '=' and '=='" } */
diff --git a/gcc/testsuite/gcc.dg/crc-non-cst-poly-1.c b/gcc/testsuite/gcc.dg/crc-non-cst-poly-1.c
new file mode 100644
index 0000000..0c3d905
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/crc-non-cst-poly-1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "" } */
+
+/* PR middle-end/120709 */
+/* Make sure we don't ICE on a non-constant poly argument. */
+
+
+typedef unsigned char uint8_t;
+uint8_t crc8_data8(uint8_t crc, uint8_t data, uint8_t polynomial) {
+  return __builtin_rev_crc32_data8 (crc, data, polynomial); /* { dg-error "must be a constant" } */
+}
diff --git a/gcc/testsuite/gcc.dg/darwin-minversion-link.c b/gcc/testsuite/gcc.dg/darwin-minversion-link.c
index af712a1b..55f7c7e 100644
--- a/gcc/testsuite/gcc.dg/darwin-minversion-link.c
+++ b/gcc/testsuite/gcc.dg/darwin-minversion-link.c
@@ -20,6 +20,7 @@
 /* { dg-additional-options "-mmacosx-version-min=013.000.00 -DCHECK=130000" { target *-*-darwin22* } } */
 /* { dg-additional-options "-mmacosx-version-min=014.000.00 -DCHECK=140000" { target *-*-darwin23* } } */
 /* { dg-additional-options "-mmacosx-version-min=015.000.00 -DCHECK=150000" { target *-*-darwin24* } } */
+/* { dg-additional-options "-mmacosx-version-min=026.000.00 -DCHECK=260000" { target *-*-darwin25* } } */
 
 int
 main ()
diff --git a/gcc/testsuite/gcc.dg/flex-array-counted-by-pr121000.c b/gcc/testsuite/gcc.dg/flex-array-counted-by-pr121000.c
new file mode 100644
index 0000000..5b9a2c6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/flex-array-counted-by-pr121000.c
@@ -0,0 +1,43 @@
+/* PR middle-end/121000 */
+/* { dg-do run } */
+/* { dg-options "-O" } */
+
+#include "builtin-object-size-common.h"
+
+/* The parameter m must be const qualified to avoid the m is 
+   marked as TREE_SIDE_EFFECTS in IR.
+   The __builtin_dynamic_object_size will be folded as -1 by
+   fold_builtin_object_size when m is NOT const qualified.  */
+
+void
+foo (int n, const int m)
+{
+  typedef int A[m];
+  struct S { int n, m; A a[2]; A b[] __attribute__((counted_by (n))); } *p;
+  p = __builtin_malloc (sizeof (struct S) + sizeof (A) * n);
+  p->n = n;
+  p->m = m;
+  EXPECT (__builtin_dynamic_object_size (p->b, 1), sizeof (A) * n);
+}
+
+/* The parameter m1, m2 must be const qualified to avoid the m is 
+   marked as TREE_SIDE_EFFECTS in IR.
+   The __builtin_dynamic_object_size will be folded as -1 by
+   fold_builtin_object_size when m1 or m2 is NOT const qualified.  */
+void
+foo_1 (int n, const int m1, const int m2)
+{
+  typedef int A[m1][m2];
+  struct S { int n; A a[2]; A b[] __attribute__((counted_by (n))); } *p;
+  p = __builtin_malloc (sizeof (struct S) + sizeof (A) * n);
+  p->n = n;
+  EXPECT (__builtin_dynamic_object_size (p->b, 1), sizeof (A) * n);
+}
+
+int
+main ()
+{
+  foo (2, 10);
+  foo_1 (2, 10, 20);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/flex-array-counted-by.c b/gcc/testsuite/gcc.dg/flex-array-counted-by.c
index 4fa91ff..16eb2c6 100644
--- a/gcc/testsuite/gcc.dg/flex-array-counted-by.c
+++ b/gcc/testsuite/gcc.dg/flex-array-counted-by.c
@@ -10,7 +10,7 @@ int x __attribute ((counted_by (size))); /* { dg-error "attribute is not allowed
 
 struct trailing {
   int count;
-  int field __attribute ((counted_by (count))); /* { dg-error "attribute is not allowed for a non-array or non-pointer field" } */
+  int field __attribute ((counted_by (count))); /* { dg-error "attribute is not allowed for a non-array field" } */
 };
 
 struct trailing_1 {
diff --git a/gcc/testsuite/gcc.dg/guality/guality.h b/gcc/testsuite/gcc.dg/guality/guality.h
index d41327c..48b59d2e 100644
--- a/gcc/testsuite/gcc.dg/guality/guality.h
+++ b/gcc/testsuite/gcc.dg/guality/guality.h
@@ -204,9 +204,10 @@ int volatile guality_attached;
    of this wrapping, guality_main may not have an empty argument
    list.  */
 
-extern int guality_main (int argc, char *argv[]);
+extern int __attribute__((noipa))
+guality_main (int argc, char *argv[]);
 
-static void __attribute__((noinline))
+static void __attribute__((noipa))
 guality_check (const char *name, gualchk_t value, int unknown_ok);
 
 /* Set things up, run guality_main, then print a summary and quit.  */
diff --git a/gcc/testsuite/gcc.dg/memchr-3.c b/gcc/testsuite/gcc.dg/memchr-3.c
index 9a35735..9caa2ac 100644
--- a/gcc/testsuite/gcc.dg/memchr-3.c
+++ b/gcc/testsuite/gcc.dg/memchr-3.c
@@ -17,9 +17,10 @@ struct SX
 const struct SX sx = { 0x1221 };
 const char sx_rep[] = { };
 
-void test_find (void)
+int test_find (void)
 {
   int n = 0, nb = (const char*)&sx.a - (const char*)&sx;
   const char *p = (const char*)&sx, *q = sx_rep;
   n += p + 1 == memchr (p, q[1], nb);
+  return n;
 }
diff --git a/gcc/testsuite/gcc.dg/nest.c b/gcc/testsuite/gcc.dg/nest.c
index 5734c11..2dce65e 100644
--- a/gcc/testsuite/gcc.dg/nest.c
+++ b/gcc/testsuite/gcc.dg/nest.c
@@ -3,6 +3,7 @@
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
 /* { dg-options "-O2 -pg -static" { target hppa*-*-hpux* } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 /* { dg-error "profiler" "No profiler support" { target xstormy16-*-* } 0 } */
 
 extern void abort (void);
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc
index 1fe5b5c..01ab766 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc
@@ -23,8 +23,8 @@
 #include "target.h"
 #include "fold-const.h"
 #include "tree-pretty-print.h"
-#include "diagnostic-color.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/color.h"
+#include "diagnostics/metadata.h"
 #include "tristate.h"
 #include "bitmap.h"
 #include "selftest.h"
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc
index fa2f2fa..c101c45 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc
@@ -16,7 +16,7 @@
 #include "gimple.h"
 #include "gimple-iterator.h"
 #include "gimple-walk.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "analyzer/common.h"
 #include "analyzer/analyzer-logging.h"
 #include "json.h"
@@ -120,20 +120,22 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
+
     if (change.is_global_p ())
       {
 	if (change.m_new_state == m_sm.m_released_gil)
-	  return diagnostic_event::meaning (diagnostic_event::verb::release,
-					    diagnostic_event::noun::lock);
+	  return event::meaning (event::verb::release,
+				 event::noun::lock);
 	else if (change.m_new_state == m_sm.get_start_state ())
-	  return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					    diagnostic_event::noun::lock);
+	  return event::meaning (event::verb::acquire,
+				 event::noun::lock);
       }
-    return diagnostic_event::meaning ();
+    return event::meaning ();
   }
  protected:
   gil_diagnostic (const gil_state_machine &sm) : m_sm (sm)
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc
index 18e054b..fc282a7 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc
@@ -23,8 +23,8 @@
 #include "target.h"
 #include "fold-const.h"
 #include "tree-pretty-print.h"
-#include "diagnostic-color.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/color.h"
+#include "diagnostics/metadata.h"
 #include "tristate.h"
 #include "bitmap.h"
 #include "selftest.h"
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc
index 5a6e075..44fcf37 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc
@@ -23,8 +23,8 @@
 #include "target.h"
 #include "fold-const.h"
 #include "tree-pretty-print.h"
-#include "diagnostic-color.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/color.h"
+#include "diagnostics/metadata.h"
 #include "tristate.h"
 #include "bitmap.h"
 #include "selftest.h"
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.c
new file mode 100644
index 0000000..2256a63
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-fdiagnostics-add-output=experimental-html:javascript=no" } */
+
+extern void here (void);
+
+void test_graphs (void)
+{
+  here (); /* { dg-error "this is a placeholder error, with graphs" } */
+}
+
+/* Use a Python script to verify various properties about the generated
+   HTML file:
+   { dg-final { run-html-pytest diagnostic-test-graphs-html.c "diagnostic-test-graphs-html.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.py
new file mode 100644
index 0000000..9ff4645
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.py
@@ -0,0 +1,48 @@
+# Verify that metadata works in HTML output.
+
+from htmltest import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def html_tree():
+    return html_tree_from_env()
+
+def test_result_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    diag_list = body.find('xhtml:div', ns)
+    assert diag_list is not None
+    assert diag_list.attrib['class'] == 'gcc-diagnostic-list'
+
+    diag = diag_list.find('xhtml:div', ns)
+    assert diag is not None
+
+    message = diag.find("./xhtml:div[@class='gcc-message']", ns)
+    assert message.attrib['id'] == 'gcc-diag-0-message'
+
+    assert message[0].tag == make_tag('strong')
+    assert message[0].tail == ' this is a placeholder error, with graphs'
+
+    graph = diag.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'foo'
+
+def test_run_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    graph = body.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'Optimization Passes'
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c
new file mode 100644
index 0000000..4170f51
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-fdiagnostics-add-output=sarif" } */
+
+extern void here (void);
+
+void test_graphs (void)
+{
+  here (); /* { dg-error "this is a placeholder error, with graphs" } */
+}
+
+/* Verify that some JSON was written to a file with the expected name.  */
+/* { dg-final { verify-sarif-file } } */
+
+/* Use a Python script to verify various properties about the generated
+   .sarif file:
+   { dg-final { run-sarif-pytest diagnostic-test-graphs-sarif.c "diagnostic-test-graphs-sarif.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.py
new file mode 100644
index 0000000..4bb7535
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.py
@@ -0,0 +1,55 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_basics(sarif):
+    schema = sarif['$schema']
+    assert schema == "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json"
+
+    version = sarif['version']
+    assert version == "2.1.0"
+
+def test_result_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+    results = run['results']
+
+    assert len(results) == 1
+    
+    result = results[0]
+    assert result['level'] == 'error'
+    assert result['message']['text'] == "this is a placeholder error, with graphs"
+
+    assert len(result['graphs']) == 2
+
+    assert result['graphs'][0]['description']['text'] == 'foo'
+
+    assert len(result['graphs'][0]['nodes']) == 2
+    assert result['graphs'][0]['nodes'][0]['id'] == 'a'
+    assert result['graphs'][0]['nodes'][1]['id'] == 'b'
+    assert result['graphs'][0]['nodes'][1]['properties']['/placeholder-prefix/color'] == 'red'
+    assert len(result['graphs'][0]['nodes'][1]['children']) == 1
+    assert result['graphs'][0]['nodes'][1]['children'][0]['id'] == 'c'
+    assert result['graphs'][0]['nodes'][1]['children'][0]['label']['text'] == 'I am a node label'
+
+    assert len(result['graphs'][0]['edges']) == 1
+    result['graphs'][0]['edges'][0]['id'] == 'my-edge'
+    assert result['graphs'][0]['edges'][0]['label']['text'] == 'I am an edge label'
+    assert result['graphs'][0]['edges'][0]['sourceNodeId'] == 'a'
+    assert result['graphs'][0]['edges'][0]['targetNodeId'] == 'c'
+    
+    assert result['graphs'][1]['description']['text'] == 'bar'
+
+def test_run_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+
+    assert len(run['graphs']) == 1
+
+    assert run['graphs'][0]['description']['text'] == 'Optimization Passes'
+    assert run['graphs'][0]['nodes'][0]['id'] == 'all_lowering_passes'
+    assert run['graphs'][0]['edges'][0]['id'] == 'edge0'
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs.c
new file mode 100644
index 0000000..7973954
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+
+extern void here (void);
+
+void test_graphs (void)
+{
+  here (); /* { dg-error "this is a placeholder error, with graphs" } */
+}
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-nesting-html.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-nesting-html.c
new file mode 100644
index 0000000..8ff7b35
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-nesting-html.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-fdiagnostics-add-output=experimental-html:javascript=no" } */
+
+extern void foo (void);
+
+void test_nesting (void)
+{
+  foo (); /* { dg-error "top-level error" } */
+}
+
+/* Use a Python script to verify various properties about the generated
+   .html file:
+   { dg-final { run-html-pytest diagnostic-test-nesting-html.c "diagnostic-test-nesting-html.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-nesting-html.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-nesting-html.py
new file mode 100644
index 0000000..3899ba5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-nesting-html.py
@@ -0,0 +1,69 @@
+# Verify that nesting works in HTML output.
+
+from htmltest import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def html_tree():
+    return html_tree_from_env()
+
+def test_nesting(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    diag_list = body.find('xhtml:div', ns)
+    assert diag_list is not None
+    assert diag_list.attrib['class'] == 'gcc-diagnostic-list'
+
+    diag = diag_list.find('xhtml:div', ns)
+    assert diag is not None
+
+    message = diag.find("./xhtml:div[@class='gcc-message']", ns)
+    assert message.attrib['id'] == 'gcc-diag-0-message'
+
+    assert message[0].tag == make_tag('strong')
+    assert message[0].tail == ' top-level error'
+
+    # We expect 12 messages, with the given IDs and text:
+    for i in range(12):
+        child = diag.find(".//xhtml:div[@id='gcc-diag-%i']" % (i + 1),
+                          ns)
+        assert child is not None
+
+        message = child.find("./xhtml:div[@class='gcc-message']", ns)
+        assert message.attrib['id'] == 'gcc-diag-%i-message' % (i + 1)
+
+        if i % 4 == 0:
+            assert message.text == 'child %i' % (i / 4)
+        else:
+            assert message.text == 'grandchild %i %i' % ((i / 4), (i % 4) - 1)
+
+    # We expect the messages to be organized into nested <ul> with
+    # "nesting-level" set, all below a <ul>
+    child_ul = diag.find("./xhtml:ul[@nesting-level='1']", ns)
+    assert child_ul is not None
+    msg_id = 1
+    for i in range(3):
+        child_li = child_ul.find("./xhtml:li[@nesting-level='1'][%i]" % (i + 1), ns)
+        assert child_li is not None
+        child = child_li.find("./xhtml:div[@id='gcc-diag-%i']" % msg_id, ns)
+        assert child is not None
+        message = child.find("./xhtml:div[@class='gcc-message']", ns)
+        assert message.attrib['id'] == 'gcc-diag-%i-message' % msg_id
+        assert message.text == 'child %i' % i
+        msg_id += 1
+        grandchild_ul = child_ul.find("./xhtml:ul[@nesting-level='2'][%i]" % (i + 1), ns)
+        assert grandchild_ul is not None
+        for j in range(3):
+            grandchild_li = grandchild_ul.find("./xhtml:li[@nesting-level='2'][%i]" % (j + 1), ns)
+            assert grandchild_li is not None
+            grandchild = grandchild_li.find("./xhtml:div[@id='gcc-diag-%i']" % msg_id, ns)
+            assert grandchild is not None
+            message = grandchild.find("./xhtml:div[@class='gcc-message']", ns)
+            assert message.attrib['id'] == 'gcc-diag-%i-message' % msg_id
+            assert message.text == 'grandchild %i %i' % (i, j)
+            msg_id += 1
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py
index aca1b6c..eaca35a 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py
@@ -1,4 +1,4 @@
-# Verify that diagnostic-show-locus.cc works with HTML output.
+# Verify that diagnostics/source-printing.cc works with HTML output.
 
 from htmltest import *
 
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc
index 4ade232..48f8325 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc
@@ -27,7 +27,7 @@
 #include "plugin-version.h"
 #include "c-family/c-common.h"
 #include "diagnostic.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 #include "context.h"
 
 int plugin_is_GPL_compatible;
@@ -165,8 +165,8 @@ pass_test_groups::execute (function *fun)
    expected output.  */
 
 void
-test_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-			      const diagnostic_info *diagnostic)
+test_diagnostic_text_starter (diagnostics::text_sink &text_output,
+			      const diagnostics::diagnostic_info *diagnostic)
 {
   pp_set_prefix (text_output.get_printer (), xstrdup ("PREFIX: "));
 }
@@ -175,22 +175,22 @@ test_diagnostic_text_starter (diagnostic_text_output_format &text_output,
    expected output.  */
 
 void
-test_diagnostic_start_span_fn (const diagnostic_location_print_policy &,
-			       to_text &sink,
+test_diagnostic_start_span_fn (const diagnostics::location_print_policy &,
+			       diagnostics::to_text &sink,
 			       expanded_location)
 {
-  pretty_printer *pp = get_printer (sink);
+  pretty_printer *pp = diagnostics::get_printer (sink);
   pp_string (pp, "START_SPAN_FN: ");
   pp_newline (pp);
 }
 
-/* Custom output format subclass.  */
+/* Custom text_sink subclass.  */
 
-class test_output_format : public diagnostic_text_output_format
+class custom_test_sink : public diagnostics::text_sink
 {
  public:
-  test_output_format (diagnostic_context &context)
-  : diagnostic_text_output_format (context)
+  custom_test_sink (diagnostics::context &context)
+  : diagnostics::text_sink (context)
   {}
 
   void on_begin_group () final override
@@ -228,10 +228,9 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  diagnostic_text_starter (global_dc) = test_diagnostic_text_starter;
-  diagnostic_start_span (global_dc) = test_diagnostic_start_span_fn;
-  global_dc->set_output_format
-    (::std::make_unique<test_output_format> (*global_dc));
+  diagnostics::text_starter (global_dc) = test_diagnostic_text_starter;
+  diagnostics::start_span (global_dc) = test_diagnostic_start_span_fn;
+  global_dc->set_sink (::std::make_unique<custom_test_sink> (*global_dc));
 
   pass_info.pass = new pass_test_groups (g);
   pass_info.reference_pass_name = "*warn_function_noreturn";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc
index 44b94da..7e34a42 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc
@@ -115,7 +115,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   register_callback (plugin_name,
 		     PLUGIN_PRE_GENERICIZE,
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_graphs.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_graphs.cc
new file mode 100644
index 0000000..7398a29
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_graphs.cc
@@ -0,0 +1,283 @@
+/* This plugin exercises diagnostic graphs.
+   We emit an error with a pair of digraphs associated with it,
+   and a global digraph showing the optimization passes.  */
+
+#define INCLUDE_MAP
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "gcc-plugin.h"
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "tm.h"
+#include "tree.h"
+#include "stringpool.h"
+#include "toplev.h"
+#include "basic-block.h"
+#include "hash-table.h"
+#include "vec.h"
+#include "ggc.h"
+#include "basic-block.h"
+#include "tree-ssa-alias.h"
+#include "internal-fn.h"
+#include "gimple.h"
+#include "gimple-iterator.h"
+#include "gimple-fold.h"
+#include "tree-eh.h"
+#include "gimple-expr.h"
+#include "is-a.h"
+#include "tree.h"
+#include "tree-pass.h"
+#include "intl.h"
+#include "plugin-version.h"
+#include "diagnostic.h"
+#include "context.h"
+#include "gcc-rich-location.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/digraphs.h"
+#include "pass_manager.h"
+
+
+int plugin_is_GPL_compatible;
+
+const pass_data pass_data_test_graph_emission =
+{
+  GIMPLE_PASS, /* type */
+  "test_graph_emission", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  PROP_ssa, /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
+
+class pass_test_graph_emission : public gimple_opt_pass
+{
+public:
+  pass_test_graph_emission(gcc::context *ctxt)
+    : gimple_opt_pass(pass_data_test_graph_emission, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  bool gate (function *) { return true; }
+  virtual unsigned int execute (function *);
+
+}; // class pass_test_graph_emission
+
+/* Determine if STMT is a call with NUM_ARGS arguments to a function
+   named FUNCNAME.
+   If so, return STMT as a gcall *.  Otherwise return NULL.  */
+
+static gcall *
+check_for_named_call (gimple *stmt,
+		      const char *funcname, unsigned int num_args)
+{
+  gcc_assert (funcname);
+
+  gcall *call = dyn_cast <gcall *> (stmt);
+  if (!call)
+    return NULL;
+
+  tree fndecl = gimple_call_fndecl (call);
+  if (!fndecl)
+    return NULL;
+
+  if (strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)), funcname))
+    return NULL;
+
+  if (gimple_call_num_args (call) != num_args)
+    {
+      error_at (stmt->location, "expected number of args: %i (got %i)",
+		num_args, gimple_call_num_args (call));
+      return NULL;
+    }
+
+  return call;
+}
+
+class lazy_passes_graph : public lazily_created<diagnostics::digraphs::digraph>
+{
+public:
+  lazy_passes_graph (const ::gcc::pass_manager &pass_manager_)
+  : m_pass_manager (pass_manager_)
+  {
+  }
+
+private:
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  create_object () const final override
+  {
+    auto g = std::make_unique<diagnostics::digraphs::digraph> ();
+    g->set_description ("Optimization Passes");
+
+#define DEF_PASS_LIST(NAME)					\
+    add_top_level_pass_list (*g, #NAME, m_pass_manager.NAME);
+
+    GCC_PASS_LISTS
+
+#undef DEF_PASS_LIST
+
+    return g;
+  }
+
+  void
+  add_top_level_pass_list (diagnostics::digraphs::digraph &g,
+			   const char *pass_list_name,
+			   const opt_pass *p) const
+  {
+    gcc_assert (p);
+    auto n = std::make_unique<diagnostics::digraphs::node> (g, pass_list_name);
+    n->set_label (pass_list_name);
+    add_child_pass (g, *n, *p);
+    g.add_node (std::move (n));
+  }
+
+  diagnostics::digraphs::node &
+  add_child_pass (diagnostics::digraphs::digraph &g,
+		  diagnostics::digraphs::node &parent_node,
+		  const opt_pass &p) const
+  {
+    std::string node_label;
+    std::string node_id;
+    if (p.static_pass_number > 0 )
+      {
+	node_label = std::to_string (p.static_pass_number) + "_" + p.name;
+	node_id = node_label;
+      }
+    else
+      {
+	node_label = std::string (p.name);
+	pretty_printer pp;
+	pp_printf (&pp, "%s_%p", p.name, &p);
+	node_id = pp_formatted_text (&pp);
+      }
+    auto n
+      = std::make_unique<diagnostics::digraphs::node> (g,
+						       std::move (node_id));
+    n->set_label (node_label.c_str ());
+    diagnostics::digraphs::node &result = *n;
+    parent_node.add_child (std::move (n));
+
+    // TODO: add attrs for things like type, properties_*, etc
+
+    if (p.sub)
+      {
+	auto &other_node = add_child_pass (g, parent_node, *p.sub);
+	g.add_edge (nullptr, result, other_node, "sub");
+      }
+
+    if (p.next)
+      {
+	auto &other_node = add_child_pass (g, parent_node, *p.next);
+	g.add_edge (nullptr, result, other_node, "next");
+      }
+
+    return result;
+  }
+
+  const ::gcc::pass_manager &m_pass_manager;
+};
+
+static void
+report_diag_with_graphs (location_t loc)
+{
+  class my_lazy_digraphs : public diagnostics::metadata::lazy_digraphs
+  {
+  public:
+    using diagnostic_graph = diagnostics::digraphs::digraph;
+    using diagnostic_node = diagnostics::digraphs::node;
+    using diagnostic_edge = diagnostics::digraphs::edge;
+
+    std::unique_ptr<std::vector<std::unique_ptr<diagnostic_graph>>>
+    create_object () const final override
+    {
+      auto graphs
+	= std::make_unique<std::vector<std::unique_ptr<diagnostic_graph>>> ();
+
+      graphs->push_back (make_test_graph ("foo"));
+      graphs->push_back (make_test_graph ("bar"));
+
+      return graphs;
+    }
+
+  private:
+    std::unique_ptr<diagnostic_graph>
+    make_test_graph (const char *desc) const
+    {
+      auto g = std::make_unique<diagnostic_graph> ();
+      g->set_description (desc);
+      auto a = std::make_unique<diagnostic_node> (*g, "a");
+      auto b = std::make_unique<diagnostic_node> (*g, "b");
+#define KEY_PREFIX "/placeholder-prefix/"
+      b->set_attr (KEY_PREFIX, "color", "red");
+#undef KEY_PREFIX
+      auto c = std::make_unique<diagnostic_node> (*g, "c");
+      c->set_label ("I am a node label");
+
+      auto e = std::make_unique<diagnostic_edge> (*g, "my-edge", *a, *c);
+      e->set_label ("I am an edge label");
+      g->add_edge (std::move (e));
+
+      g->add_node (std::move (a));
+
+      b->add_child (std::move (c));
+      g->add_node (std::move (b));
+
+      return g;
+    }
+  };
+
+  gcc_rich_location rich_loc (loc);
+  diagnostics::metadata meta;
+  my_lazy_digraphs ldg;
+  meta.set_lazy_digraphs (&ldg);
+  error_meta (&rich_loc, meta,
+	      "this is a placeholder error, with graphs");
+}
+
+/* Exercise diagnostic graph emission.  */
+
+unsigned int
+pass_test_graph_emission::execute (function *fun)
+{
+  gimple_stmt_iterator gsi;
+  basic_block bb;
+
+  FOR_EACH_BB_FN (bb, fun)
+    for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+      {
+	gimple *stmt = gsi_stmt (gsi);
+
+	if (gcall *call = check_for_named_call (stmt, "here", 0))
+	  report_diag_with_graphs (gimple_location (call));
+      }
+
+  return 0;
+}
+
+int
+plugin_init (struct plugin_name_args *plugin_info,
+	     struct plugin_gcc_version *version)
+{
+  struct register_pass_info pass_info;
+  const char *plugin_name = plugin_info->base_name;
+  int argc = plugin_info->argc;
+  struct plugin_argument *argv = plugin_info->argv;
+
+  if (!plugin_default_version_check (version, &gcc_version))
+    return 1;
+
+  pass_info.pass = new pass_test_graph_emission (g);
+  pass_info.reference_pass_name = "ssa";
+  pass_info.ref_pass_instance_number = 1;
+  pass_info.pos_op = PASS_POS_INSERT_AFTER;
+  register_callback (plugin_name, PLUGIN_PASS_MANAGER_SETUP, NULL,
+		     &pass_info);
+
+  gcc_assert (::g->get_passes ());
+  global_dc->report_global_digraph (lazy_passes_graph (*::g->get_passes ()));
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc
index 7edce1f..d38538d 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc
@@ -169,7 +169,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   pass_info.pass = new pass_test_inlining (g);
   pass_info.reference_pass_name = "*warn_function_noreturn";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc
index b86a8b3..f172258 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc
@@ -1,5 +1,6 @@
-/* This plugin exercises diagnostic_metadata.  */
+/* This plugin exercises diagnostics::metadata.  */
 
+#define INCLUDE_VECTOR
 #include "gcc-plugin.h"
 #include "config.h"
 #include "system.h"
@@ -28,7 +29,7 @@
 #include "diagnostic.h"
 #include "context.h"
 #include "gcc-rich-location.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/metadata.h"
 
 int plugin_is_GPL_compatible;
 
@@ -89,7 +90,7 @@ check_for_named_call (gimple *stmt,
   return call;
 }
 
-/* Exercise diagnostic_metadata.  */
+/* Exercise diagnostics::metadata.  */
 
 unsigned int
 pass_test_metadata::execute (function *fun)
@@ -106,13 +107,13 @@ pass_test_metadata::execute (function *fun)
 	if (gcall *call = check_for_named_call (stmt, "gets", 1))
 	  {
 	    gcc_rich_location richloc (gimple_location (call));
-	    diagnostic_metadata m;
+	    diagnostics::metadata m;
 
 	    /* CWE-242: Use of Inherently Dangerous Function.  */
 	    m.add_cwe (242);
 
-	    /* Example of a diagnostic_metadata::rule.  */
-	    diagnostic_metadata::precanned_rule
+	    /* Example of a diagnostics::metadata::rule.  */
+	    diagnostics::metadata::precanned_rule
 	      test_rule ("STR34-C", "https://example.com/");
 	    m.add_rule (test_rule);
 
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc
index a7963fa..875f4a8 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc
@@ -6,6 +6,7 @@
    specific tests within the compiler's IR.  We can't use any real
    diagnostics for this, so we have to fake it, hence this plugin.  */
 
+#define INCLUDE_VECTOR
 #include "gcc-plugin.h"
 #include "config.h"
 #include "system.h"
@@ -32,8 +33,7 @@
 #include "intl.h"
 #include "plugin-version.h"
 #include "diagnostic.h"
-#include "diagnostic-path.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/metadata.h"
 #include "context.h"
 #include "print-tree.h"
 #include "gcc-rich-location.h"
@@ -223,7 +223,7 @@ class test_diagnostic_path : public simple_diagnostic_path
   diagnostic_event_id_t
   add_event_2 (event_location_t evloc, int stack_depth,
 	       const char *desc,
-	       diagnostic_thread_id_t thread_id = 0)
+	       diagnostics::paths::thread_id_t thread_id = 0)
   {
     gcc_assert (evloc.m_fun);
     return add_thread_event (thread_id, evloc.m_loc, evloc.m_fun->decl,
@@ -232,7 +232,7 @@ class test_diagnostic_path : public simple_diagnostic_path
   diagnostic_event_id_t
   add_event_2_with_event_id (event_location_t evloc, int stack_depth,
 			     const char *fmt,
-			     diagnostic_thread_id_t thread_id,
+			     diagnostics::paths::thread_id_t thread_id,
 			     diagnostic_event_id_t event_id)
   {
     gcc_assert (evloc.m_fun);
@@ -242,7 +242,7 @@ class test_diagnostic_path : public simple_diagnostic_path
   }
   void add_entry (event_location_t evloc, int stack_depth,
 		  const char *funcname,
-		  diagnostic_thread_id_t thread_id = 0)
+		  diagnostics::paths::thread_id_t thread_id = 0)
   {
     gcc_assert (evloc.m_fun);
     add_thread_event (thread_id, evloc.m_loc, evloc.m_fun->decl, stack_depth,
@@ -363,7 +363,7 @@ example_2 ()
 
       richloc.set_path (&path);
 
-      diagnostic_metadata m;
+      diagnostics::metadata m;
       m.add_cwe (415); /* CWE-415: Double Free.  */
 
       warning_meta (&richloc, m, 0,
@@ -441,7 +441,7 @@ example_3 ()
 
       richloc.set_path (&path);
 
-      diagnostic_metadata m;
+      diagnostics::metadata m;
       /* CWE-479: Signal Handler Use of a Non-reentrant Function.  */
       m.add_cwe (479);
 
@@ -502,8 +502,8 @@ example_4 ()
 
       gcc_rich_location richloc (call_to_acquire_lock_a_in_bar.m_loc);
       test_diagnostic_path path (global_dc->get_reference_printer ());
-      diagnostic_thread_id_t thread_1 = path.add_thread ("Thread 1");
-      diagnostic_thread_id_t thread_2 = path.add_thread ("Thread 2");
+      diagnostics::paths::thread_id_t thread_1 = path.add_thread ("Thread 1");
+      diagnostics::paths::thread_id_t thread_2 = path.add_thread ("Thread 2");
       path.add_entry (entry_to_foo, 0, "foo", thread_1);
       diagnostic_event_id_t event_a_acquired
 	= path.add_event_2 (call_to_acquire_lock_a_in_foo, 1,
@@ -524,7 +524,7 @@ example_4 ()
 	 thread_2, event_a_acquired);
       richloc.set_path (&path);
 
-      diagnostic_metadata m;
+      diagnostics::metadata m;
       warning_meta (&richloc, m, 0,
 		    "deadlock due to inconsistent lock acquisition order");
     }
@@ -559,7 +559,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   pass_info.pass = make_pass_test_show_path (g);
   pass_info.reference_pass_name = "whole-program";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc
index cd3834b..92839cd 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc
@@ -62,7 +62,8 @@
 #include "print-tree.h"
 #include "gcc-rich-location.h"
 #include "text-range-label.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/file-cache.h"
 
 int plugin_is_GPL_compatible;
 
@@ -125,14 +126,14 @@ static bool force_show_locus_color = false;
 /* We want to verify the colorized output of diagnostic_show_locus,
    but turning on colorization for everything confuses "dg-warning" etc.
    Hence we special-case it within this plugin by using this modified
-   version of default_diagnostic_text_finalizer, which, if "color" is
+   version of diagnostics::default_text_finalizer, which, if "color" is
    passed in as a plugin argument turns on colorization, but just
    for diagnostic_show_locus.  */
 
 static void
-custom_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-				  const diagnostic_info *diagnostic,
-				  diagnostic_t)
+custom_diagnostic_text_finalizer (diagnostics::text_sink &text_output,
+				  const diagnostics::diagnostic_info *diag,
+				  enum diagnostics::kind)
 {
   pretty_printer *const pp = text_output.get_printer ();
   bool old_show_color = pp_show_color (pp);
@@ -143,7 +144,7 @@ custom_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
   pp_newline (pp);
   diagnostic_show_locus (&text_output.get_context (),
 			 text_output.get_source_printing_options (),
-			 diagnostic->richloc, diagnostic->kind, pp);
+			 diag->m_richloc, diag->m_kind, pp);
   pp_show_color (pp) = old_show_color;
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
@@ -176,11 +177,11 @@ test_show_locus (function *fun)
   location_t fnstart = fun->function_start_locus;
   int fnstart_line = LOCATION_LINE (fnstart);
 
-  diagnostic_text_finalizer (global_dc) = custom_diagnostic_text_finalizer;
+  diagnostics::text_finalizer (global_dc) = custom_diagnostic_text_finalizer;
 
   /* Hardcode the "terminal width", to verify the behavior of
      very wide lines.  */
-  global_dc->m_source_printing.max_width = 71;
+  global_dc->get_source_printing_options ().max_width = 71;
 
   if (0 == strcmp (fnname, "test_simple"))
     {
@@ -251,7 +252,7 @@ test_show_locus (function *fun)
   if (0 == strcmp (fnname, "test_very_wide_line"))
     {
       const int line = fnstart_line + 2;
-      global_dc->m_source_printing.show_ruler_p = true;
+      global_dc->get_source_printing_options ().show_ruler_p = true;
       text_range_label label0 ("label 0");
       text_range_label label1 ("label 1");
       rich_location richloc (line_table,
@@ -263,7 +264,7 @@ test_show_locus (function *fun)
 			 &label1);
       richloc.add_fixit_replace ("bar * foo");
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.show_ruler_p = false;
+      global_dc->get_source_printing_options ().show_ruler_p = false;
     }
 
   /* Likewise, but with a secondary location that's immediately before
@@ -271,7 +272,7 @@ test_show_locus (function *fun)
   if (0 == strcmp (fnname, "test_very_wide_line_2"))
     {
       const int line = fnstart_line + 2;
-      global_dc->m_source_printing.show_ruler_p = true;
+      global_dc->get_source_printing_options ().show_ruler_p = true;
       text_range_label label0 ("label 0");
       text_range_label label1 ("label 1");
       rich_location richloc (line_table,
@@ -283,7 +284,7 @@ test_show_locus (function *fun)
       richloc.add_range (get_loc (line, 34), SHOW_RANGE_WITHOUT_CARET,
 			 &label1);
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.show_ruler_p = false;
+      global_dc->get_source_printing_options ().show_ruler_p = false;
     }
 
   /* Example of multiple carets.  */
@@ -294,11 +295,11 @@ test_show_locus (function *fun)
       location_t caret_b = get_loc (line, 11);
       rich_location richloc (line_table, caret_a);
       add_range (&richloc, caret_b, caret_b, SHOW_RANGE_WITH_CARET);
-      global_dc->m_source_printing.caret_chars[0] = 'A';
-      global_dc->m_source_printing.caret_chars[1] = 'B';
+      global_dc->get_source_printing_options ().caret_chars[0] = 'A';
+      global_dc->get_source_printing_options ().caret_chars[1] = 'B';
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.caret_chars[0] = '^';
-      global_dc->m_source_printing.caret_chars[1] = '^';
+      global_dc->get_source_printing_options ().caret_chars[0] = '^';
+      global_dc->get_source_printing_options ().caret_chars[1] = '^';
     }
 
   /* Tests of rendering fixit hints.  */
@@ -412,11 +413,11 @@ test_show_locus (function *fun)
       location_t caret_b = get_loc (line - 1, 19);
       rich_location richloc (line_table, caret_a);
       richloc.add_range (caret_b, SHOW_RANGE_WITH_CARET);
-      global_dc->m_source_printing.caret_chars[0] = '1';
-      global_dc->m_source_printing.caret_chars[1] = '2';
+      global_dc->get_source_printing_options ().caret_chars[0] = '1';
+      global_dc->get_source_printing_options ().caret_chars[1] = '2';
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.caret_chars[0] = '^';
-      global_dc->m_source_printing.caret_chars[1] = '^';
+      global_dc->get_source_printing_options ().caret_chars[0] = '^';
+      global_dc->get_source_printing_options ().caret_chars[1] = '^';
     }
 
   /* Example of using the "%q+D" format code, which as well as printing
@@ -443,8 +444,8 @@ test_show_locus (function *fun)
       rich_location richloc (line_table, loc);
       for (int line = start_line; line <= finish_line; line++)
 	{
-	  file_cache &fc = global_dc->get_file_cache ();
-	  char_span content = fc.get_source_line (file, line);
+	  diagnostics::file_cache &fc = global_dc->get_file_cache ();
+	  diagnostics::char_span content = fc.get_source_line (file, line);
 	  gcc_assert (content);
 	  /* Split line up into words.  */
 	  for (int idx = 0; idx < content.length (); idx++)
@@ -464,7 +465,8 @@ test_show_locus (function *fun)
 		      richloc.add_range (word, SHOW_RANGE_WITH_CARET, &label);
 
 		      /* Add a fixit, converting to upper case.  */
-		      char_span word_span = content.subspan (start_idx, idx - start_idx);
+		      diagnostics::char_span word_span
+			= content.subspan (start_idx, idx - start_idx);
 		      char *copy = word_span.xstrdup ();
 		      for (char *ch = copy; *ch; ch++)
 			*ch = TOUPPER (*ch);
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc
index 1b5fad2..b64d60f 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc
@@ -208,7 +208,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   pass_info.pass = new pass_test_string_literals (g);
   pass_info.reference_pass_name = "ssa";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc
index e28d697..ce2f1d3 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc
@@ -9,7 +9,7 @@
 #include "coretypes.h"
 #include "plugin-version.h"
 #include "diagnostic.h"
-#include "diagnostic-diagram.h"
+#include "diagnostics/diagram.h"
 #include "text-art/canvas.h"
 #include "text-art/table.h"
 
@@ -22,7 +22,7 @@ using namespace text_art;
 static void
 emit_canvas (const canvas &c, const char *alt_text)
 {
-  diagnostic_diagram diagram (c, alt_text);
+  diagnostics::diagram diagram (c, alt_text);
   global_dc->emit_diagram (diagram);
 }
 
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc
index fbdb2f8..bbd0faa 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc
@@ -89,7 +89,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 130;
+  global_dc->get_source_printing_options ().max_width = 130;
 
   register_callback (plugin_name,
 		     PLUGIN_PRE_GENERICIZE,
diff --git a/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc b/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc
index 7b9b8d4..67722d4 100644
--- a/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc
@@ -6,9 +6,9 @@
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 
 int plugin_is_GPL_compatible;
 
@@ -47,14 +47,15 @@ static void
 test_richloc (rich_location *richloc)
 {
   /* Run the diagnostic and fix-it printing code.  */
-  test_diagnostic_context dc;
-  diagnostic_show_locus (&dc, dc.m_source_printing,
-			 richloc, DK_ERROR, dc.get_reference_printer ());
+  diagnostics::selftest::test_context dc;
+  diagnostic_show_locus (&dc, dc.get_source_printing_options (),
+			 richloc, diagnostics::kind::error,
+			 dc.get_reference_printer ());
 
   /* Generate a diff.  */
-  edit_context ec (global_dc->get_file_cache ());
-  ec.add_fixits (richloc);
-  char *diff = ec.generate_diff (true);
+  diagnostics::changes::change_set edit (global_dc->get_file_cache ());
+  edit.add_fixits (richloc);
+  char *diff = edit.generate_diff (true);
   free (diff);
 }
 
diff --git a/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc b/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc
index f770d35..00ad870 100644
--- a/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc
@@ -7,7 +7,7 @@
 #include "coretypes.h"
 #include "spellcheck.h"
 #include "diagnostic.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 int plugin_is_GPL_compatible;
 
@@ -39,12 +39,12 @@ on_pragma_registration (void */*gcc_data*/, void */*user_data*/)
 /* We add some extra testing during diagnostics by chaining up
    to the text finalizer.  */
 
-static diagnostic_text_finalizer_fn original_text_finalizer = NULL;
+static diagnostics::text_finalizer_fn original_text_finalizer = NULL;
 
 static void
-verify_unpacked_ranges  (diagnostic_text_output_format &text_output,
-			 const diagnostic_info *diagnostic,
-			 diagnostic_t orig_diag_kind)
+verify_unpacked_ranges  (diagnostics::text_sink &text_output,
+			 const diagnostics::diagnostic_info *diagnostic,
+			 enum diagnostics::kind orig_diag_kind)
 {
   /* Verify that the locations are ad-hoc, not packed. */
   location_t loc = diagnostic_location (diagnostic);
@@ -56,9 +56,9 @@ verify_unpacked_ranges  (diagnostic_text_output_format &text_output,
 }
 
 static void
-verify_no_columns  (diagnostic_text_output_format &text_output,
-		    const diagnostic_info *diagnostic,
-		    diagnostic_t orig_diag_kind)
+verify_no_columns  (diagnostics::text_sink &text_output,
+		    const diagnostics::diagnostic_info *diagnostic,
+		    enum diagnostics::kind orig_diag_kind)
 {
   /* Verify that the locations have no columns. */
   location_t loc = diagnostic_location (diagnostic);
@@ -104,15 +104,15 @@ plugin_init (struct plugin_name_args *plugin_info,
 		     NULL); /* void *user_data */
 
   /* Hack in additional testing, based on the exact value supplied.  */
-  original_text_finalizer = diagnostic_text_finalizer (global_dc);
+  original_text_finalizer = diagnostics::text_finalizer (global_dc);
   switch (base_location)
     {
     case LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES + 1:
-      diagnostic_text_finalizer (global_dc) = verify_unpacked_ranges;
+      diagnostics::text_finalizer (global_dc) = verify_unpacked_ranges;
       break;
 
     case LINE_MAP_MAX_LOCATION_WITH_COLS + 1:
-      diagnostic_text_finalizer (global_dc) = verify_no_columns;
+      diagnostics::text_finalizer (global_dc) = verify_no_columns;
       break;
 
     default:
diff --git a/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c b/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c
index d51d15c..6f65f4a 100644
--- a/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c
+++ b/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c
@@ -55,5 +55,5 @@ volatile fn_ptr_t fn_ptr;
 void
 test_5 (void)
 {
-  fn_ptr (); /* { dg-error "cannot tail-call: " } */
+  fn_ptr ();
 }
diff --git a/gcc/testsuite/gcc.dg/plugin/plugin.exp b/gcc/testsuite/gcc.dg/plugin/plugin.exp
index d1d7f5d..3bb6063 100644
--- a/gcc/testsuite/gcc.dg/plugin/plugin.exp
+++ b/gcc/testsuite/gcc.dg/plugin/plugin.exp
@@ -107,11 +107,16 @@ set plugin_test_list [list \
 	  diagnostic-test-metadata.c \
 	  diagnostic-test-metadata-html.c \
 	  diagnostic-test-metadata-sarif.c } \
+    { diagnostic_plugin_test_graphs.cc
+	  diagnostic-test-graphs.c \
+	  diagnostic-test-graphs-html.c \
+	  diagnostic-test-graphs-sarif.c } \
     { diagnostic_plugin_test_nesting.cc \
 	  diagnostic-test-nesting-text-plain.c \
 	  diagnostic-test-nesting-text-indented.c \
 	  diagnostic-test-nesting-text-indented-show-levels.c \
 	  diagnostic-test-nesting-text-indented-unicode.c \
+	  diagnostic-test-nesting-html.c \
 	  diagnostic-test-nesting-sarif.c } \
     { diagnostic_plugin_test_paths.cc \
 	  diagnostic-test-paths-1.c \
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-1.c b/gcc/testsuite/gcc.dg/pointer-counted-by-1.c
deleted file mode 100644
index 395af34..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-1.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* More testing the correct usage of attribute counted_by for pointer field.  */
-/* { dg-do compile } */
-/* { dg-options "-O0" } */
-
-typedef struct item1 Item1;
-typedef union item2 Item2;
-
-struct pointer_array {
-  int count1;
-  Item1 *array_1 __attribute__ ((counted_by (count1))); 
-  Item2 *array_2 __attribute__ ((counted_by (count2))); 
-  int count2;
-} *pointer_data; 
-
-struct item1 {
-  int a;
-  float b[];
-};
-
-union item2 {
-  int c;
-  float d[];
-};
-
-void foo ()
-{
-  pointer_data 
-    = (struct pointer_array *) __builtin_malloc (sizeof (struct pointer_array));
-  pointer_data->array_1 /* { dg-error "attribute is not allowed for a pointer to structure or union with flexible array member" } */ 
-    = (Item1 *) __builtin_malloc (sizeof (Item1) + 3 * sizeof (float));
-  pointer_data->array_2 /* { dg-error "attribute is not allowed for a pointer to structure or union with flexible array member" } */
-    = (Item2 *) __builtin_malloc (sizeof (Item2) + 3 * sizeof (float));
-  return;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-2.c b/gcc/testsuite/gcc.dg/pointer-counted-by-2.c
deleted file mode 100644
index 1f4a278..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-2.c
+++ /dev/null
@@ -1,10 +0,0 @@
-/* Testing the correct usage of attribute counted_by for pointer: _BitInt  */
-/* { dg-do compile { target bitint } } */
-/* { dg-options "-O2 -std=c23" } */
-
-struct pointer_array {
-  _BitInt(24) count; 
-  int *array __attribute__ ((counted_by (count)));
-  int *array1 __attribute__ ((counted_by (count1)));
-  _BitInt(24) count1; 
-};
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-3.c b/gcc/testsuite/gcc.dg/pointer-counted-by-3.c
deleted file mode 100644
index 7005609..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-3.c
+++ /dev/null
@@ -1,127 +0,0 @@
- /* Testing the correct usage of attribute counted_by for pointer in c23,
-    multiple definitions of the same tag in same or different scopes.
-    { dg-do compile }
-    { dg-options "-std=c23" }
- */
-
-/* Allowed redefinitions of the same struct in the same scope, with the
-   same counted_by attribute.  */
-struct f {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); };
-struct f {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); };
-struct f {
-  int b;
-  int c;
-  int *a; }; /* { dg-error "redefinition of struct or union" } */
-
-/* Error when the counted_by attribute is defined differently.  */
-struct f {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (c))); }; /* { dg-error "redefinition of struct or union" } */
-
-struct h {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); } p;  
-
-void test (void)
-{
-  struct h {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); } x;
-
-  p = x;
-}
-
-void test1 (void)
-{
-  struct h {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (c))); } y;
-
-  p = y;   /* { dg-error "incompatible types when assigning to type" } */
-}
-
-struct nested_f {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-}; 
-
-struct nested_f {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-}; 
-
-struct nested_f {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (n)));
-};  /* { dg-error "redefinition of struct or union" } */
-
-struct nested_h {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-} nested_p; 
-
-void test_2 (void)
-{
-struct nested_h {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-} nested_x; 
-
- nested_p = nested_x;
-}
-
-void test_3 (void)
-{
-struct nested_h {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (n)));
-} nested_y; 
-
- nested_p = nested_y; /* { dg-error "incompatible types when assigning to type" } */
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-char.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-char.c
deleted file mode 100644
index c404e5b..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-char.c
+++ /dev/null
@@ -1,6 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-#define PTR_TYPE char
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-float.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-float.c
deleted file mode 100644
index 383d8fb..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-float.c
+++ /dev/null
@@ -1,6 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-#define PTR_TYPE float 
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-struct.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-struct.c
deleted file mode 100644
index 50246d2..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-struct.c
+++ /dev/null
@@ -1,10 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-struct A {
-  int a;
-  char *b;
-};
-#define PTR_TYPE struct A 
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-union.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-union.c
deleted file mode 100644
index e786d99..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-union.c
+++ /dev/null
@@ -1,10 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-union A {
-  int a;
-  float b;
-};
-#define PTR_TYPE union A 
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4.c
deleted file mode 100644
index c4b3631..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-#ifndef PTR_TYPE
-#define PTR_TYPE int
-#endif
-struct pointer_array {
-  int b;
-  PTR_TYPE *c;
-} *p_array;
-
-struct annotated {
-  PTR_TYPE *c __attribute__ ((counted_by (b)));
-  int b;
-} *p_array_annotated;
-
-struct nested_annotated {
-  PTR_TYPE *c __attribute__ ((counted_by (b)));
-  struct {
-    union {
-      int b;
-      float f;	
-    };
-    int n;
-  };
-} *p_array_nested_annotated;
-
-void __attribute__((__noinline__)) setup (int normal_count, int attr_count)
-{
-  p_array
-    = (struct pointer_array *) malloc (sizeof (struct pointer_array));
-  p_array->c = (PTR_TYPE *) malloc (sizeof (PTR_TYPE) * normal_count);
-  p_array->b = normal_count;
-
-  p_array_annotated
-    = (struct annotated *) malloc (sizeof (struct annotated));
-  p_array_annotated->c = (PTR_TYPE *) malloc (sizeof (PTR_TYPE) * attr_count);
-  p_array_annotated->b = attr_count;
-
-  p_array_nested_annotated
-    = (struct nested_annotated *) malloc (sizeof (struct nested_annotated));
-  p_array_nested_annotated->c = (PTR_TYPE *) malloc (sizeof (PTR_TYPE) * attr_count);
-  p_array_nested_annotated->b = attr_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test ()
-{
-  EXPECT(__builtin_dynamic_object_size(p_array->c, 1), -1);
-  EXPECT(__builtin_dynamic_object_size(p_array_annotated->c, 1),
-	 p_array_annotated->b * sizeof (PTR_TYPE));
-  EXPECT(__builtin_dynamic_object_size(p_array_nested_annotated->c, 1),
-	 p_array_nested_annotated->b * sizeof (PTR_TYPE));
-}
-
-void cleanup ()
-{
-  free (p_array->c); 
-  free (p_array);
-  free (p_array_annotated->c);
-  free (p_array_annotated);
-  free (p_array_nested_annotated->c);
-  free (p_array_nested_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (10,10);   
-  test ();
-  DONE ();
-  cleanup ();
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-5.c b/gcc/testsuite/gcc.dg/pointer-counted-by-5.c
deleted file mode 100644
index b43ffdf..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-5.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
- * __builtin_dynamic_object_size: when the counted_by field is negative.  */
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *array_annotated;
-
-struct nested_annotated {
-  int *c __attribute__ ((counted_by (b)));
-  struct {
-    union {
-      int b;
-      float f;	
-    };
-    int n;
-  };
-} *array_nested_annotated;
-
-void __attribute__((__noinline__)) setup (int attr_count)
-{
-  array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  array_annotated->b = attr_count;
-
-  array_nested_annotated
-    = (struct nested_annotated *)malloc (sizeof (struct nested_annotated));
-  array_nested_annotated->b = attr_count - 1;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test ()
-{
-  EXPECT(__builtin_dynamic_object_size(array_annotated->c, 1), 0);
-  EXPECT(__builtin_dynamic_object_size(array_nested_annotated->c, 1), 0);
-}
-
-void cleanup ()
-{
-  free (array_annotated);
-  free (array_nested_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (-10);   
-  test ();
-  DONE ();
-  cleanup ();
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-6.c b/gcc/testsuite/gcc.dg/pointer-counted-by-6.c
deleted file mode 100644
index 355558c..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-6.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
- * __builtin_dynamic_object_size: when the type of the pointer  
- * is casting to another type.  */
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-
-typedef unsigned short u16;
-
-struct info {
-  u16 data_len;
-  char *data __attribute__((counted_by(data_len)));
-};
-
-struct foo {
-  int a;
-  int b;
-};
-
-static __attribute__((__noinline__))
-struct info *setup ()
-{
-  struct info *p;
-  size_t bytes = 3 * sizeof(struct foo);
-
-  p = (struct info *) malloc (sizeof (struct info));
-  p->data = (char *) malloc (bytes);
-  p->data_len = bytes;
-
-  return p;
-}
-
-static void
-__attribute__((__noinline__)) report (struct info *p)
-{
-  struct foo *bar = (struct foo *)p->data;
-  EXPECT(__builtin_dynamic_object_size((char *)(bar + 1), 1),
-	 sizeof (struct foo) * 2);
-  EXPECT(__builtin_dynamic_object_size((char *)(bar + 2), 1),
-	 sizeof (struct foo));
-}
-
-void cleanup (struct info *p)
-{
-  free (p->data);
-  free (p);
-}
-
-int main(int argc, char *argv[])
-{
-  struct info *p = setup();
-  report(p);
-  cleanup (p);
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-7.c b/gcc/testsuite/gcc.dg/pointer-counted-by-7.c
deleted file mode 100644
index af1ab27..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-7.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Additional test of the attribute counted_by for pointer field and its usage
-   in __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-};
-
-struct annotated __attribute__((__noinline__)) setup (int attr_count)
-{
-  struct annotated p_array_annotated;
-  p_array_annotated.c = (int *) malloc (sizeof (int) * attr_count);
-  p_array_annotated.b = attr_count;
-
-  return p_array_annotated;
-}
-
-int main(int argc, char *argv[])
-{
-  struct annotated x = setup (10);   
-  int *p = x.c;
-  x = setup (20);
-  EXPECT(__builtin_dynamic_object_size (p, 1), 10 * sizeof (int));
-  EXPECT(__builtin_dynamic_object_size (x.c, 1), 20 * sizeof (int));
-  free (p);
-  free (x.c);
-  DONE ();
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by.c b/gcc/testsuite/gcc.dg/pointer-counted-by.c
deleted file mode 100644
index 0f18828..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by.c
+++ /dev/null
@@ -1,111 +0,0 @@
-/* Testing the correct usage of attribute counted_by for pointer field.
-   and also mixed pointer field and FMA field in the same structure.  */
-/* { dg-do compile } */
-/* { dg-options "-O0" } */
-
-int size;
-int *x __attribute__ ((counted_by (size))); /* { dg-error "attribute is not allowed for a non-field declaration" } */
-
-struct pointer_array_0 {
-  int count;
-  int array __attribute__ ((counted_by (count))); /* { dg-error "attribute is not allowed for a non-array or non-pointer field" } */
-  int other;
-};
-
-int count;
-struct pointer_array_1 {
-  int other;
-  int *array_1 __attribute__ ((counted_by (count))); /* { dg-error "attribute is not a field declaration in the same structure as" } */
-  int array_fam[] __attribute__ ((counted_by (count))); /* { dg-error "attribute is not a field declaration in the same structure as" } */
-};
-
-struct pointer_array_2 {
-  float count1;
-  float count2;
-  int *array_2 __attribute__ ((counted_by (count1))); /* { dg-error "attribute is not a field declaration with an integer type" } */
-  int array_fam[] __attribute__ ((counted_by (count2))); /* { dg-error "attribute is not a field declaration with an integer type" } */
-}; 
-
-struct pointer_array_3 {
-  int count;
-  int *array_3 __attribute__ ((counted_by (count))) __attribute__ ((counted_by (count)));
-}; 
-
-struct pointer_array_4 {
-  int count1;
-  int count2;
-  int *array_4 __attribute__ ((counted_by (count1))) __attribute__ ((counted_by (count2))); /* { dg-error "conflicts with previous declaration" } */
-  float array_fam[] __attribute__ ((counted_by (count2))) __attribute__ ((counted_by (count1))); /* { dg-error "conflicts with previous declaration" } */
-}; 
-
-struct pointer_array_5 {
-  _Bool count;
-  int *array_5 __attribute__ ((counted_by (count)));
-}; 
-
-enum week {Mon, Tue, Wed};
-struct pointer_array_6 {
-  enum week days;
-  int *array_6 __attribute__ ((counted_by (days)));
-}; 
-
-struct pointer_array_7 {
-  int count;
-  void *array_7 __attribute__ ((counted_by (count))); /* { dg-error "attribute is not allowed for a pointer to void" } */
-}; 
-
-struct pointer_array_8 {
-  int count;
-  int (*fpr)(int,int) __attribute__ ((counted_by (count))); /* { dg-error "attribute is not allowed for a pointer to function" } */
-}; 
-
-struct item1 {
-  int a;
-  float b;
-};
-
-union item2 {
-  char *a;
-  int *b; 
-};
-
-typedef struct item3 Item3;
-typedef union item4 Item4;
-
-struct item5 {
-  int a;
-  float b[];
-};
-
-/* Incomplete structure and union are allowed.  */
-struct pointer_array_9 {
-  int count1;
-  int count2;
-  int count3;
-  struct item1 *array_1 __attribute__ ((counted_by (count1)));
-  union item2 *array_2 __attribute__ ((counted_by (count2)));
-  Item3 *array_3 __attribute__ ((counted_by (count3))); 
-  Item4 *array_4 __attribute__ ((counted_by (count4))); 
-  int count4;
-  int count5;
-  /* structure with flexible array member is not allowed.  */
-  struct item5 *array_5 __attribute__ ((counted_by (count5))); /* { dg-error "attribute is not allowed for a pointer to structure or union with flexible array member" } */
-}; 
-
-struct mixed_array {
-  int count1;
-  float *array_1 __attribute__ ((counted_by (count1)));
-  float *array_2 __attribute__ ((counted_by (count1)));
-  int count2;
-  long *array_3 __attribute__ ((counted_by (count2)));
-  long array_4[] __attribute__ ((counted_by (count2)));
-};
-
-struct mixed_array_2 {
-  float *array_1 __attribute__ ((counted_by (count1)));
-  int count1;
-  float *array_2 __attribute__ ((counted_by (count1)));
-  long *array_3 __attribute__ ((counted_by (count2)));
-  int count2;
-  long array_4[] __attribute__ ((counted_by (count2)));
-};
diff --git a/gcc/testsuite/gcc.dg/pr109267-1.c b/gcc/testsuite/gcc.dg/pr109267-1.c
new file mode 100644
index 0000000..e762e59
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr109267-1.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+/* PR middle-end/109267 */
+
+int f(void)
+{
+  __builtin_unreachable();
+}
+
+/* This unreachable should be changed to be a trap. */
+
+/* { dg-final { scan-tree-dump-times "__builtin_unreachable trap \\\(" 1 "optimized" { target trap }  } } */
+/* { dg-final { scan-tree-dump-times "goto <" 1 "optimized" { target { ! trap } } } } */
+/* { dg-final { scan-tree-dump-not "__builtin_unreachable \\\(" "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/pr109267-2.c b/gcc/testsuite/gcc.dg/pr109267-2.c
new file mode 100644
index 0000000..6cd1419
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr109267-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+/* PR middle-end/109267 */
+void g(void);
+int f(int *t)
+{
+  g();
+  __builtin_unreachable();
+}
+
+/* The unreachable should stay a unreachable. */
+/* { dg-final { scan-tree-dump-not "__builtin_unreachable trap \\\(" "optimized"} } */
+/* { dg-final { scan-tree-dump-times "__builtin_unreachable \\\(" 1 "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/pr116906-1.c b/gcc/testsuite/gcc.dg/pr116906-1.c
index 7187507..ee60ad6 100644
--- a/gcc/testsuite/gcc.dg/pr116906-1.c
+++ b/gcc/testsuite/gcc.dg/pr116906-1.c
@@ -1,3 +1,4 @@
+/* { dg-do run } */
 /* { dg-require-effective-target alarm } */
 /* { dg-require-effective-target signal } */
 /* { dg-options "-O2" } */
diff --git a/gcc/testsuite/gcc.dg/pr116906-2.c b/gcc/testsuite/gcc.dg/pr116906-2.c
index 41a352b..4172ec3 100644
--- a/gcc/testsuite/gcc.dg/pr116906-2.c
+++ b/gcc/testsuite/gcc.dg/pr116906-2.c
@@ -1,3 +1,4 @@
+/* { dg-do run } */
 /* { dg-require-effective-target alarm } */
 /* { dg-require-effective-target signal } */
 /* { dg-options "-O2 -fno-tree-ch" } */
diff --git a/gcc/testsuite/gcc.dg/pr118948-1.c b/gcc/testsuite/gcc.dg/pr118948-1.c
new file mode 100644
index 0000000..2a46cf1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr118948-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "" } */
+
+/* PR c/118948 */
+
+/* Used to ICE in tree_expr_nonnegative_p after an error. */
+
+void f(void) {
+    int i; /* { dg-note "previous" } */
+    for (i = 0; i < 2; i++) ;
+    float i; /* { dg-error "conflicting types for" } */
+}
diff --git a/gcc/testsuite/gcc.dg/pr120660.c b/gcc/testsuite/gcc.dg/pr120660.c
new file mode 100644
index 0000000..6e8c5e8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr120660.c
@@ -0,0 +1,19 @@
+/* { dg-do run } */
+/* { dg-options "-O -favoid-store-forwarding" } */
+
+int c;
+
+short
+foo (short s)
+{
+  __builtin_memset (&s, c, 1);
+  return s;
+}
+
+int
+main ()
+{
+  short x = foo (0x1111);
+  if (x != 0x1100 && x != 0x0011)
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/pr121035.c b/gcc/testsuite/gcc.dg/pr121035.c
new file mode 100644
index 0000000..fc0edce
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121035.c
@@ -0,0 +1,94 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fgimple" } */
+
+int printf(const char *, ...);
+int a, b, d;
+unsigned c;
+int __GIMPLE (ssa,startwith("pre"))
+main ()
+{
+  unsigned int g;
+  int f;
+  unsigned int _1;
+  unsigned int _2;
+  int _3;
+  int _4;
+  int _5;
+  unsigned int _6;
+  unsigned int _7;
+  int _10;
+  unsigned int _11;
+  _Bool _19;
+  _Bool _20;
+  _Bool _22;
+  int _25;
+
+  __BB(2):
+  _25 = a;
+  if (_25 != 0)
+    goto __BB11;
+  else
+    goto __BB10;
+
+  __BB(11):
+  goto __BB3;
+
+  __BB(3):
+  f_26 = __PHI (__BB12: f_18, __BB11: 0);
+  g_15 = c;
+  if (f_26 != 0)
+    goto __BB4;
+  else
+    goto __BB5;
+
+  __BB(4):
+  __builtin_putchar (48);
+  goto __BB5;
+
+  __BB(5):
+  _1 = c;
+  _2 = _1 << 1;
+  _3 = a;
+  _4 = d;
+  _5 = _3 * _4;
+  if (_5 != 0)
+    goto __BB7;
+  else
+    goto __BB6;
+
+  __BB(6):
+  goto __BB7;
+
+  __BB(7):
+  _11 = __PHI (__BB5: 0u, __BB6: 4294967295u);
+  _6 = g_15 * 4294967294u;
+  _7 = _6 | _11;
+  _20 = _3 != 0;
+  _19 = _7 != 0u;
+  _22 = _19 & _20;
+  if (_22 != _Literal (_Bool) 0)
+    goto __BB9;
+  else
+    goto __BB8;
+
+  __BB(8):
+  goto __BB9;
+
+  __BB(9):
+  _10 = __PHI (__BB7: 1, __BB8: 0);
+  b = _10;
+  f_18 = (int) _1;
+  if (_3 != 0)
+    goto __BB12;
+  else
+    goto __BB10;
+
+  __BB(12):
+  goto __BB3;
+
+  __BB(10):
+  return 0;
+
+}
+
+
diff --git a/gcc/testsuite/gcc.dg/pr121202.c b/gcc/testsuite/gcc.dg/pr121202.c
new file mode 100644
index 0000000..30ecf4a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121202.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fno-tree-copy-prop" } */
+
+int a, b, c;
+int e(int f, int g) { return f >> g; }
+int h(int f) { return a > 1 ? 0 : f << a; }
+int main() {
+  while (c--)
+    b = e(h(1), a);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr121216.c b/gcc/testsuite/gcc.dg/pr121216.c
new file mode 100644
index 0000000..a695b40
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121216.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "" } */
+
+int foo (void)
+{
+    const char *key = "obscurelevelofabstraction";
+    const char reverse_key[__builtin_strlen(key)] = {'\0'}; /* { dg-error "variable-sized object may not be initialized except with an empty initializer" } */
+    return __builtin_strlen(reverse_key);
+}
diff --git a/gcc/testsuite/gcc.dg/pr121322.c b/gcc/testsuite/gcc.dg/pr121322.c
new file mode 100644
index 0000000..2fad5b5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121322.c
@@ -0,0 +1,14 @@
+/* PR middle-end/121322 */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+unsigned long long
+foo (unsigned long long *p)
+{
+  unsigned long long a = *p;
+  unsigned long long b = __builtin_bswap64 (a);
+  return ((b << 32)
+	  | ((b >> 8) & 0xff000000ULL)
+	  | ((b >> 24) & 0xff0000ULL)
+	  | ((b >> 40) & 0xff00ULL));
+}
diff --git a/gcc/testsuite/gcc.dg/pr32450.c b/gcc/testsuite/gcc.dg/pr32450.c
index 9606e30..0af262f 100644
--- a/gcc/testsuite/gcc.dg/pr32450.c
+++ b/gcc/testsuite/gcc.dg/pr32450.c
@@ -4,6 +4,7 @@
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
 /* { dg-options "-O2 -pg -mtune=core2" { target { i?86-*-* x86_64-*-* } } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 /* { dg-options "-O2 -pg -static" { target hppa*-*-hpux* } } */
 
 extern void abort (void);
diff --git a/gcc/testsuite/gcc.dg/pr43643.c b/gcc/testsuite/gcc.dg/pr43643.c
index 43896ab..41c00c8 100644
--- a/gcc/testsuite/gcc.dg/pr43643.c
+++ b/gcc/testsuite/gcc.dg/pr43643.c
@@ -4,6 +4,7 @@
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
 /* { dg-options "-O2 -pg -static" { target hppa*-*-hpux* } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 
 extern char *strdup (const char *);
 
diff --git a/gcc/testsuite/gcc.dg/pr78185.c b/gcc/testsuite/gcc.dg/pr78185.c
index ada8b1b..4c3af4f 100644
--- a/gcc/testsuite/gcc.dg/pr78185.c
+++ b/gcc/testsuite/gcc.dg/pr78185.c
@@ -1,3 +1,4 @@
+/* { dg-do run } */
 /* { dg-require-effective-target alarm } */
 /* { dg-require-effective-target signal } */
 /* { dg-options "-O" } */
diff --git a/gcc/testsuite/gcc.dg/pr87600-1.c b/gcc/testsuite/gcc.dg/pr87600-1.c
index 3517957..9d74cad 100644
--- a/gcc/testsuite/gcc.dg/pr87600-1.c
+++ b/gcc/testsuite/gcc.dg/pr87600-1.c
@@ -1,5 +1,5 @@
 /* PR rtl-optimization/87600  */
-/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* loongarch*-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
 /* { dg-options "-O2" } */
 
 #include "pr87600.h"
diff --git a/gcc/testsuite/gcc.dg/pr87600-2.c b/gcc/testsuite/gcc.dg/pr87600-2.c
index e8a9f19..822afe0 100644
--- a/gcc/testsuite/gcc.dg/pr87600-2.c
+++ b/gcc/testsuite/gcc.dg/pr87600-2.c
@@ -1,5 +1,5 @@
 /* PR rtl-optimization/87600  */
-/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* loongarch*-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
 /* { dg-options "-O2" } */
 
 #include "pr87600.h"
@@ -23,22 +23,3 @@ test1 (void)
   asm ("blah %0 %1" : "=r" (var1) : "0" (var2)); /* { dg-error "invalid hard register usage between output operand and matching constraint operand" } */
   return var1;
 }
-
-long
-test2 (void)
-{
-  register long var1 asm (REG1);
-  register long var2 asm (REG1);
-  asm ("blah %0 %1" : "=&r" (var1) : "r" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
-  return var1;
-}
-
-long
-test3 (void)
-{
-  register long var1 asm (REG1);
-  register long var2 asm (REG1);
-  long var3;
-  asm ("blah %0 %1" : "=&r" (var1), "=r" (var3) : "1" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
-  return var1 + var3;
-}
diff --git a/gcc/testsuite/gcc.dg/pr87600-3.c b/gcc/testsuite/gcc.dg/pr87600-3.c
new file mode 100644
index 0000000..4f43a5f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr87600-3.c
@@ -0,0 +1,26 @@
+/* PR rtl-optimization/87600  */
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-O2" } */
+
+#include "pr87600.h"
+
+/* The following are all invalid uses of local register variables.  */
+
+long
+test2 (void)
+{
+  register long var1 asm (REG1);
+  register long var2 asm (REG1);
+  asm ("blah %0 %1" : "=&r" (var1) : "r" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  return var1;
+}
+
+long
+test3 (void)
+{
+  register long var1 asm (REG1);
+  register long var2 asm (REG1);
+  long var3;
+  asm ("blah %0 %1" : "=&r" (var1), "=r" (var3) : "1" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  return var1 + var3;
+}
diff --git a/gcc/testsuite/gcc.dg/pr87600.h b/gcc/testsuite/gcc.dg/pr87600.h
index c89071eb..29f065e 100644
--- a/gcc/testsuite/gcc.dg/pr87600.h
+++ b/gcc/testsuite/gcc.dg/pr87600.h
@@ -7,6 +7,9 @@
 #elif defined (__i386__)
 # define REG1 "%eax"
 # define REG2 "%edx"
+#elif defined (__loongarch__)
+# define REG1 "$t0"
+# define REG2 "$t1"
 #elif defined (__powerpc__) || defined (__POWERPC__) || defined (__PPC__)
 # define REG1 "r3"
 # define REG2 "r4"
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-1.c b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-1.c
new file mode 100644
index 0000000..6f795c6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-1.c
@@ -0,0 +1,35 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-options "-O2 -msve-vector-bits=256 -mlittle-endian" } */
+
+#include <arm_sve.h>
+
+#pragma GCC target "+sve"
+
+svint64x2_t __RTL (startwith ("vregs")) foo ()
+{
+  (function "foo"
+    (insn-chain
+      (block 2
+	(edge-from entry (flags "FALLTHRU"))
+	(cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+        (cnote 2 NOTE_INSN_FUNCTION_BEG)
+	(insn 3 (set (reg:VNx4DI <0>)
+		     (const_vector:VNx4DI [(const_int 11)
+					   (const_int 12)
+					   (const_int 13)
+					   (const_int 14)
+					   (const_int 15)
+					   (const_int 16)
+					   (const_int 17)
+					   (const_int 18)])))
+	(insn 4 (set (reg:VNx4DI v0) (reg:VNx4DI <0>)))
+        (insn 5 (use (reg:VNx4DI v0)))
+	(edge-to exit (flags "FALLTHRU"))
+      ) ;; block 2
+    ) ;; insn-chain
+    (crtl (return_rtx (reg:VNx4DI v0)))
+  ) ;; function
+}
+
+/* { dg-final { scan-assembler {\tindex\tz0\.d, #11, #1\n} } } */
+/* { dg-final { scan-assembler {\tindex\tz1\.d, #15, #1\n} } } */
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-2.c b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-2.c
new file mode 100644
index 0000000..17e46cb
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-2.c
@@ -0,0 +1,35 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-options "-O2 -msve-vector-bits=256 -mlittle-endian" } */
+
+#include <arm_sve.h>
+
+#pragma GCC target "+sve"
+
+svint64x2_t __RTL (startwith ("vregs")) foo ()
+{
+  (function "foo"
+    (insn-chain
+      (block 2
+	(edge-from entry (flags "FALLTHRU"))
+	(cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+        (cnote 2 NOTE_INSN_FUNCTION_BEG)
+	(insn 3 (set (reg:VNx4DI <0>)
+		     (const_vector:VNx4DI [(const_int -16)
+					   (const_int -15)
+					   (const_int -14)
+					   (const_int -13)
+					   (const_int -12)
+					   (const_int -11)
+					   (const_int -10)
+					   (const_int -9)])))
+	(insn 4 (set (reg:VNx4DI v0) (reg:VNx4DI <0>)))
+        (insn 5 (use (reg:VNx4DI v0)))
+	(edge-to exit (flags "FALLTHRU"))
+      ) ;; block 2
+    ) ;; insn-chain
+    (crtl (return_rtx (reg:VNx4DI v0)))
+  ) ;; function
+}
+
+/* { dg-final { scan-assembler {\tindex\tz0\.d, #-16, #1\n} } } */
+/* { dg-final { scan-assembler {\tindex\tz1\.d, #-12, #1\n} } } */
diff --git a/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h b/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h
index 382ac02..f60fb33 100644
--- a/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h
+++ b/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h
@@ -1,4 +1,4 @@
-/* Generate a warning with a diagnostic_path within a header.  */
+/* Generate a warning with a diagnostic path within a header.  */
 
 void test (void *ptr)
 {
diff --git a/gcc/testsuite/gcc.dg/torture/pr120654.c b/gcc/testsuite/gcc.dg/torture/pr120654.c
index 3819b78..aacfeea 100644
--- a/gcc/testsuite/gcc.dg/torture/pr120654.c
+++ b/gcc/testsuite/gcc.dg/torture/pr120654.c
@@ -2,8 +2,6 @@
 
 int a, c, e, f, h, j;
 long g, k;
-void *malloc(long);
-void free(void *);
 int b(int m) {
   if (m || a)
     return 1;
@@ -16,9 +14,9 @@ int i() {
 }
 void n() {
   long o;
-  int *p = malloc(sizeof(int));
+  int *p = __builtin_malloc(sizeof(int));
   k = 1 % j;
   for (; i() + f + h; o++)
     if (p[d(j + 6, (int)k + 1992695866) + h + f + j + (int)k - 1 + o])
-      free(p);
+      __builtin_free(p);
 }
diff --git a/gcc/testsuite/gcc.dg/torture/pr120944.c b/gcc/testsuite/gcc.dg/torture/pr120944.c
new file mode 100644
index 0000000..92f3c77
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr120944.c
@@ -0,0 +1,34 @@
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
+/* { dg-additional-options "-fdump-tree-optimized" } */
+
+#include <stdlib.h>
+
+typedef union {
+  int u32;
+  struct
+  {
+     int A:1;
+     int B:2;
+     int C:3;
+  };
+} u_t;
+
+typedef union {
+   volatile int u[3];
+   volatile struct {
+        u_t a;
+        int b;
+        int c;
+   };
+} DATA;
+
+void foo (volatile DATA *d)
+{
+     d->a.u32 = ~0;
+     u_t u = d->a;
+     int v = u.A;
+     if (v)
+        abort();
+}
+
+/* { dg-final { scan-tree-dump-times "if \\\(" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/torture/pr120951-1.c b/gcc/testsuite/gcc.dg/torture/pr120951-1.c
new file mode 100644
index 0000000..4e2b41d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr120951-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-fnon-call-exceptions -fsignaling-nans" } */
+
+/* PR tree-optimization/120951 */
+
+/* cdce would create a trapping comparison inside a condition.
+   tests to make sure that does not happen.  */
+
+double f(double r, double i) {
+   return __builtin_fmod(r, i);
+}
+
diff --git a/gcc/testsuite/gcc.dg/torture/pr121116.c b/gcc/testsuite/gcc.dg/torture/pr121116.c
new file mode 100644
index 0000000..637324f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121116.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target bitint } } */
+
+#include <stdlib.h>
+#include <stdckdint.h>
+#include <string.h>
+typedef _BitInt(16) bit16;
+[[nodiscard]] static bit16 process_data(bit16 input) {
+    _Static_assert(sizeof(bit16) == 2, "Unexpected size of bit16");
+    return (input << 5) | (input >> 9);
+}
+int main(void) {
+    const bit16 data = 0b101'0101'0000'0000;
+    bit16 result = 0;
+    for (bit16 i = 0; i < 0b1000; ++i) {
+        result ^= process_data(data ^ i);
+    }
+    if (ckd_add(&result, result, 0x1234)) {
+        return EXIT_FAILURE;
+    }
+    return (result & 0xFF00) ? 0 : 1;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr121194.c b/gcc/testsuite/gcc.dg/torture/pr121194.c
new file mode 100644
index 0000000..20f5ff7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121194.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+
+int a, b, c, d;
+void e() {
+  int *f = &b;
+  for (a = 0; a < 8; a++) {
+    *f = 0;
+    for (c = 0; c < 2; c++)
+      *f = *f == 0;
+  }
+}
+int main() {
+  e();
+  int *g = &b;
+  *g = *g == (d == 0);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr121236-1.c b/gcc/testsuite/gcc.dg/torture/pr121236-1.c
new file mode 100644
index 0000000..2b397e3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121236-1.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* PR tree-optimization/121236 */
+
+
+unsigned func_26(short *p_27, int gg, int p) {
+  unsigned l_184 = 0;
+  unsigned m = 0;
+  for (int g_59 = 0; g_59 < 10; g_59++)
+    {
+      if (gg)
+	l_184--;
+      else
+	{
+	  m |= l_184 |= p;
+	  (l_184)--;
+	}
+    }
+ return m;
+}
+
diff --git a/gcc/testsuite/gcc.dg/torture/pr121295-1.c b/gcc/testsuite/gcc.dg/torture/pr121295-1.c
new file mode 100644
index 0000000..7825c6e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121295-1.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-additional-options " -fno-tree-copy-prop -fno-tree-pre -fno-code-hoisting" */
+
+/* PR tree-optimization/121295 */
+
+
+int a, b, c;
+int main() {
+  int *d = &a;
+  while (b)
+    b = (*d &= 10) <= 0 || (*d = c);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr121370.c b/gcc/testsuite/gcc.dg/torture/pr121370.c
new file mode 100644
index 0000000..d40f3b2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121370.c
@@ -0,0 +1,25 @@
+/* { dg-do run } */
+/* { dg-require-effective-target int32plus } */
+
+int a;
+int main()
+{
+  int c = -2147483647;
+  int d = -2147483647;
+  int e = 2147483647;
+  if (0)
+  f:
+    e = d + e - 2;
+g:
+  if (d - c - e > 0) {
+    a = -c;
+    if (a + d) {
+      d = 1;
+      goto g;
+    }
+    return 0;
+  }
+  if (e)
+    goto f;
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr121382.c b/gcc/testsuite/gcc.dg/torture/pr121382.c
new file mode 100644
index 0000000..20b49b1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121382.c
@@ -0,0 +1,23 @@
+/* { dg-do run } */
+/* { dg-require-effective-target int32plus } */
+
+int a, b, c;
+__attribute__((noipa))
+static void d(int e, int f)
+{
+  if (e != 5 && e != 2147483647)
+    __builtin_abort();
+  f = 2147483647;
+  do {
+    f = f - e;
+    if (c - 9 * f) {
+      __builtin_abort();
+    }
+  } while (c);
+}
+__attribute__((noipa))
+int main(void)
+{
+  d(2147483647, 2147483647);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c b/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c
index dd53295..79ba529 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c
@@ -1,3 +1,8 @@
+/* { dg-require-effective-target lto } */
+/* { dg-additional-sources "afdo-crossmodule-1.c" } */
+/* { dg-options "-O3 -flto -fdump-ipa-afdo_offline -fdump-tree-einline-details" } */
+/* { dg-require-profiling "-fauto-profile" } */
+
 extern int foo2 ();
 
 int bar (int (*fooptr) (int (*)()))
diff --git a/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c b/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c
index b67b3cb..ded4068 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c
@@ -1,6 +1,15 @@
 /* { dg-options "-O2 -fdump-tree-einline-details --param early-inlining-insns=1" } */
 /* { dg-require-profiling "-fauto-profile" } */ 
 volatile int a[1000];
+
+#define STR1(X) #X
+#define STR2(X) STR1(X)
+
+int reta (int i)
+asm(STR2(__USER_LABEL_PREFIX__) "renamed_reta");
+int test ()
+asm(STR2(__USER_LABEL_PREFIX__) "renamed_test");
+
 int reta (int i)
 {
 	if (a[i])
diff --git a/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c b/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c
index 43a9090..904dd0c 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c
@@ -31,4 +31,4 @@ int main()
 }
 /* We will have profiles for test2 and test2.constprop.0 that will have to be
    merged,  */
-/* { dg-final-use-autofdo { scan-ipa-dump "Merging duplicate symbol test2" "afdo_offline"} } */
+/* { dg-final-use-autofdo { scan-ipa-dump "Merging duplicate instance: test2" "afdo_offline"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/cmp-2.c b/gcc/testsuite/gcc.dg/tree-ssa/cmp-2.c
new file mode 100644
index 0000000..9b02901
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/cmp-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-tree-forwprop" } */
+
+/* PR tree-optimization/110949 */
+/* Transform `(cmp) - 1` into `-icmp`. */
+
+int f1(int a)
+{
+  int t = a == 115;
+  return t - 1;
+}
+
+/* { dg-final { scan-tree-dump " != 115" "forwprop1" } } */
+/* { dg-final { scan-tree-dump-not " == 115" "forwprop1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/cswtch-7.c b/gcc/testsuite/gcc.dg/tree-ssa/cswtch-7.c
new file mode 100644
index 0000000..7b797807
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/cswtch-7.c
@@ -0,0 +1,48 @@
+/* PR tree-optimization/120523 */
+/* PR tree-optimization/120451 */
+/* { dg-do compile { target elf } } */
+/* { dg-options "-O2" } */
+
+void foo (int, int);
+
+__attribute__((noinline, noclone)) void
+f1 (int v, int w)
+{
+  int i, j;
+  if (w)
+    {
+      i = 129;
+      j = i - 1;
+      goto lab;
+    }
+  switch (v)
+    {
+    case 170:
+      j = 7;
+      i = 27;
+      break;
+    case 171:
+      i = 8;
+      j = 122;
+      break;
+    case 172:
+      i = 21;
+      j = -19;
+      break;
+    case 173:
+      i = 18;
+      j = 17;
+      break;
+    case 174:
+      i = 33;
+      j = 55;
+      break;
+    default:
+      __builtin_abort ();
+    }
+
+ lab:
+  foo (i, j);
+}
+
+/* { dg-final { scan-assembler ".rodata.cst32" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/max-bitcmp-1.c b/gcc/testsuite/gcc.dg/tree-ssa/max-bitcmp-1.c
new file mode 100644
index 0000000..81b5a27
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/max-bitcmp-1.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-tree-forwprop -fdump-tree-optimized" } */
+
+/* PR tree-optimization/95906 */
+/* this should become MAX_EXPR<a,b> */
+
+int f2(int a, int b)
+{
+    int cmp = -(a > b);
+    return (cmp & a) | (~cmp & b);
+}
+
+/*  we should not end up with -_2 */
+/*  we should not end up and & nor a `+ -1` */
+/* In optimized we should have a max.  */
+/* { dg-final { scan-tree-dump-not " -\[a-zA-Z_\]" "forwprop1" } } */
+/* { dg-final { scan-tree-dump-not " & " "forwprop1" } } */
+/* { dg-final { scan-tree-dump-not " . -1" "forwprop1" } } */
+/* { dg-final { scan-tree-dump-times "MAX_EXPR " 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr117423.c b/gcc/testsuite/gcc.dg/tree-ssa/pr117423.c
new file mode 100644
index 0000000..a5d3b29
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr117423.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-options "-O1" } */
+
+struct s4 {
+    int _0;
+};
+struct s1 {
+    unsigned char _4;
+    long _1;
+};
+struct s2 {
+    union {
+        struct s3 {
+            unsigned char _1;
+            struct s4 _0;
+        } var_0;
+        struct s1 var_1;
+    } DATA;
+};
+int f1(int arg0) { return arg0 > 12345; }
+__attribute__((noinline))
+struct s4 f2(int arg0) {
+    struct s4 rv = {arg0};
+    return rv;
+}
+struct s2 f3(int arg0) {
+    struct s2 rv;
+    struct s1 var6 = {0};
+    struct s4 var7;
+    if (f1(arg0)) {
+        rv.DATA.var_1 = var6;
+        return rv;
+    } else {
+        rv.DATA.var_0._1 = 2;
+        var7 = f2(arg0);
+        rv.DATA.var_0._0 = var7;
+        return rv;
+    }
+}
+int main() {
+  if (f3(12345).DATA.var_0._0._0 == 12345)
+    ;
+  else
+    __builtin_abort();
+  if (f3(12344).DATA.var_0._0._0 == 12344)
+    ;
+  else
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr119085.c b/gcc/testsuite/gcc.dg/tree-ssa/pr119085.c
new file mode 100644
index 0000000..e9811ce
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr119085.c
@@ -0,0 +1,37 @@
+/* { dg-do run } */
+/* { dg-options "-O1" } */
+
+struct with_hole {
+  int x;
+  long y;
+};
+struct without_hole {
+  int x;
+  int y;
+};
+union u {
+  struct with_hole with_hole;
+  struct without_hole without_hole;
+};
+
+void __attribute__((noinline))
+test (union u *up, union u u)
+{
+  union u u2;
+  volatile int f = 0;
+  u2 = u;
+  if (f)
+    u2.with_hole = u.with_hole;
+  *up = u2;
+}
+
+int main(void)
+{
+  union u u;
+  union u u2;
+  u2.without_hole.y = -1;
+  test (&u, u2);
+  if (u.without_hole.y != -1)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr121264.c b/gcc/testsuite/gcc.dg/tree-ssa/pr121264.c
new file mode 100644
index 0000000..bd5acc0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr121264.c
@@ -0,0 +1,12 @@
+/* PR tree-optimization/121264 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump " \\\| " "optimized" } } */
+
+struct A { char b; char c[0x20000010]; } a;
+
+int
+foo ()
+{
+  return a.c[0x20000000] || a.c[1];
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c b/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c
index 9ba43be..ef35b29 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c
@@ -4,6 +4,7 @@
 int a, b, c, d[6], e = 3, f;
 
 void abort (void);
+void fn1 () __attribute__((noinline));
 void fn1 ()
 {
   for (b = 1; b < 5; b++)
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-105.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-105.c
new file mode 100644
index 0000000..61b93c0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-105.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fno-tree-sra -fdump-tree-fre1-details" } */
+
+struct s1
+{
+  int t, t1;
+};
+
+struct s3
+{
+  struct s1 t;
+};
+
+struct s2
+{
+  struct s3 t;
+};
+
+void f(int, int);
+void l();
+void g(int a, int b, int *p)
+{
+  struct s2 c;
+  {
+    struct s1 tmp = {a,b};
+    struct s3 *t = &c.t;
+    t->t = tmp;
+  }
+  f(c.t.t.t, c.t.t.t1);
+}
+
+/* { dg-final { scan-tree-dump "Replaced c.t.t.t1 with b" "fre1" } } */
+/* { dg-final { scan-tree-dump "Replaced c.t.t.t with a" "fre1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-106.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-106.c
new file mode 100644
index 0000000..6da4201
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-106.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fno-tree-sra -fdump-tree-fre1-details" } */
+
+struct s1
+{
+  int t, t1;
+};
+
+struct s3
+{
+  struct s1 t;
+};
+
+struct s2
+{
+  struct s3 t;
+};
+
+void f(int, int);
+void l();
+void g(int a, int b, int *p)
+{
+  struct s2 c;
+  {
+    struct s1 tmp = {a,b};
+    c.t.t = tmp;
+  }
+  struct s1 *t = &c.t.t;
+  f(t->t, t->t1);
+}
+
+/* { dg-final { scan-tree-dump "Replaced \[^\r\n\]*.t1 with b" "fre1" } } */
+/* { dg-final { scan-tree-dump "Replaced \[^\r\n\]*.t with a" "fre1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-sink-23.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-sink-23.c
new file mode 100644
index 0000000..f632dc8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-sink-23.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-sink1-details" } */
+
+struct S {
+    int* x;
+    int* y;
+};
+
+void __attribute__((noreturn)) bar(const struct S* s);
+
+void foo(int a, int b) {
+    struct S s;
+    s.x = &a;
+    s.y = &b;
+    if (a < b) {
+        bar(&s);
+    }
+}
+
+/* { dg-final { scan-tree-dump "Sinking.*s.y" "sink1" } } */
+/* { dg-final { scan-tree-dump "Sinking.*s.x" "sink1" } } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-2.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-2.c
deleted file mode 100644
index 0653ecc..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-2.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
-   bounds sanitizer combined with VLA.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-/* { dg-output "index 11 out of bounds for type 'int \\\[\\\*\\\]\\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*index 20 out of bounds for type 'int \\\[\\\*\\\]\\\[\\\*\\\]\\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*index 11 out of bounds for type 'int \\\[\\\*\\\]\\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*index 10 out of bounds for type 'int \\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-
-
-#include <stdlib.h>
-
-void __attribute__((__noinline__)) setup_and_test_vla (int n, int m)
-{
-  struct foo {
-    int n;
-    int (*p)[n] __attribute__((counted_by(n)));
-  } *f;
-
-  f = (struct foo *) malloc (sizeof (struct foo));
-  f->p = (int (*)[n]) malloc (m * sizeof (int[n]));
-  f->n = m;
-  f->p[m][n-1] = 1;
-  free (f->p);
-  free (f);
-  return;
-}
-
-void __attribute__((__noinline__)) setup_and_test_vla_1 (int n1, int n2, int m)
-{
-  struct foo {
-    int n;
-    int (*p)[n2][n1] __attribute__((counted_by(n)));
-  } *f;
-
-  f = (struct foo *) malloc (sizeof(struct foo));
-  f->p = (int (*)[n2][n1]) malloc (m * sizeof (int[n2][n1]));
-  f->n = m;
-  f->p[m][n2][n1] = 1;
-  free (f->p);
-  free (f);
-  return;
-}
-
-int main(int argc, char *argv[])
-{
-  setup_and_test_vla (10, 11);
-  setup_and_test_vla_1 (10, 11, 20);
-  return 0;
-}
-
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-3.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-3.c
deleted file mode 100644
index 731422d..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-3.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in bounds
-   sanitizer. when counted_by field is negative value.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *array_annotated;
-
-void __attribute__((__noinline__)) setup (int annotated_count)
-{
-  array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  array_annotated->c = (int *) malloc (sizeof (int) * 10);
-  array_annotated->b = annotated_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test (int annotated_index)
-{
-  array_annotated->c[annotated_index] = 2;
-}
-
-void cleanup ()
-{
-  free (array_annotated->c);
-  free (array_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (-3);   
-  test (2);
-  cleanup ();
-  return 0;
-}
-
-/* { dg-output "25:21: runtime error: index 2 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-4.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-4.c
deleted file mode 100644
index 52f202f..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-4.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in bounds
-   sanitizer. when counted_by field is zero value.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *array_annotated;
-
-void __attribute__((__noinline__)) setup (int annotated_count)
-{
-  array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  array_annotated->c = (int *)malloc (sizeof (int) * 10);
-  array_annotated->b = annotated_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test (int annotated_index)
-{
-  array_annotated->c[annotated_index] = 2;
-}
-
-void cleanup ()
-{
-  free (array_annotated->c);
-  free (array_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (0);   
-  test (1);
-  cleanup ();
-  return 0;
-}
-
-/* { dg-output "25:21: runtime error: index 1 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-5.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-5.c
deleted file mode 100644
index 8ad7572..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-5.c
+++ /dev/null
@@ -1,40 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
-   bounds sanitizer.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *p_array_annotated;
-
-void __attribute__((__noinline__)) setup (int annotated_count)
-{
-  p_array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  p_array_annotated->c = (int *) malloc (annotated_count *  sizeof (int));
-  p_array_annotated->b = annotated_count;
-
-  return;
-}
-
-void cleanup ()
-{
-  free (p_array_annotated->c);
-  free (p_array_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  int i;
-  setup (10);
-  for (i = 0; i < 11; i++)
-    p_array_annotated->c[i] = 2; // goes boom at i == 10
-  cleanup ();
-  return 0;
-}
-
-
-/* { dg-output "34:25: runtime error: index 10 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds.c
deleted file mode 100644
index c5a1ac5..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
-   bounds sanitizer.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct pointer_array {
-  int b;
-  int *c;
-} *p_array;
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *p_array_annotated;
-
-void __attribute__((__noinline__)) setup (int normal_count, int annotated_count)
-{
-  p_array
-    = (struct pointer_array *) malloc (sizeof (struct pointer_array));
-  p_array->c = (int *) malloc (normal_count * sizeof (int));
-  p_array->b = normal_count;
-
-  p_array_annotated
-    = (struct annotated *) malloc (sizeof (struct annotated));
-  p_array_annotated->c = (int *) malloc (annotated_count * sizeof (int));
-  p_array_annotated->b = annotated_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test (int normal_index, int annotated_index)
-{
-  p_array->c[normal_index] = 1;
-  p_array_annotated->c[annotated_index] = 2;
-}
-
-int main(int argc, char *argv[])
-{
-  setup (10, 10);   
-  test (10, 10);
-  return 0;
-}
-
-/* { dg-output "36:23: runtime error: index 10 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pr120837.c b/gcc/testsuite/gcc.dg/ubsan/pr120837.c
new file mode 100644
index 0000000..97c85c7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ubsan/pr120837.c
@@ -0,0 +1,32 @@
+/* PR c/120837 */
+/* { dg-do run } */
+/* { dg-options "-O1 -fsanitize=undefined -fno-sanitize-recover=undefined" } */
+
+[[gnu::noipa]] void
+bar (void **x, void **y)
+{
+  x[0] = 0;
+  x[1] = 0;
+  x[2] = 0;
+  y[0] = 0;
+  y[1] = 0;
+  y[2] = 0;
+  y[3] = 0;
+  y[4] = 0;
+}
+
+[[gnu::noipa]] void *
+foo (int x, int y)
+{
+  void *a[3];
+  void *b[5];
+  bar (a, b);
+  return (x > y ? b : a)[y - 1];
+}
+
+int
+main ()
+{
+  if (foo (2, 1) != 0)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.dg/uninit-pr120924.c b/gcc/testsuite/gcc.dg/uninit-pr120924.c
new file mode 100644
index 0000000..bfc8ae9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/uninit-pr120924.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wmaybe-uninitialized" } */
+
+int foo(int);
+enum {
+  BPF_TRACE_RAW_TP,
+  BPF_MODIFY_RETURN,
+  BPF_LSM_MAC,
+  BPF_TRACE_ITER,
+  BPF_LSM_CGROUP
+};
+int btf_get_kernel_prefix_kind_prefix, obj_1, attach_name___trans_tmp_1;
+char attach_name_fn_name;
+void attach_name(int attach_type)
+{
+  int mod_len;
+  char mod_name = attach_name_fn_name;
+  if (attach_name_fn_name)
+    mod_len = mod_name;
+  for (; obj_1;) {
+    if (mod_name && foo(mod_len))
+      continue;
+    switch (attach_type) {
+    case BPF_TRACE_RAW_TP:
+    case BPF_LSM_MAC:
+    case BPF_LSM_CGROUP:
+      btf_get_kernel_prefix_kind_prefix = 1;
+    case BPF_TRACE_ITER:
+      attach_name_fn_name = 2;
+    }
+    if (attach_name___trans_tmp_1)
+      return;
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/unused-9.c b/gcc/testsuite/gcc.dg/unused-9.c
index bdf36e1..ad1ad0e 100644
--- a/gcc/testsuite/gcc.dg/unused-9.c
+++ b/gcc/testsuite/gcc.dg/unused-9.c
@@ -2,12 +2,9 @@
 /* { dg-do compile } */
 /* { dg-options "-Wunused" } */
 
-
 void g(void)
 {
-  int i = 0;
-  volatile int x;
-  (x, i++);	/* { dg-bogus "set but not used" } */
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  volatile int x;	/* { dg-bogus "variable 'x' set but not used" } */
+  (x, i++);
 }
-
-
diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-39.c b/gcc/testsuite/gcc.dg/vect/bb-slp-39.c
index f05ce8f..255bb10 100644
--- a/gcc/testsuite/gcc.dg/vect/bb-slp-39.c
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-39.c
@@ -16,5 +16,4 @@ void foo (double *p)
 }
 
 /* See that we vectorize three SLP instances.  */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "slp2" { target { ! { s390*-*-* riscv*-*-* } } } } } */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 5 "slp2" { target {   s390*-*-* riscv*-*-* } } } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "slp2" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr101145.c b/gcc/testsuite/gcc.dg/vect/pr101145.c
index cd11c03..c055ae6 100644
--- a/gcc/testsuite/gcc.dg/vect/pr101145.c
+++ b/gcc/testsuite/gcc.dg/vect/pr101145.c
@@ -2,7 +2,7 @@
 /* { dg-additional-options "-O3" } */
 #include <limits.h>
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   while (n < ++l)
@@ -10,7 +10,7 @@ foo (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
 {
   while (UINT_MAX - 64 < ++l)
@@ -18,7 +18,7 @@ foo_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_2 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   l = UINT_MAX - 32;
@@ -27,7 +27,7 @@ foo_2 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_3 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   while (n <= ++l)
@@ -35,7 +35,7 @@ foo_3 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_4 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {  // infininate 
   while (0 <= ++l)
@@ -43,7 +43,7 @@ foo_4 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_5 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   //no loop
@@ -53,7 +53,7 @@ foo_5 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 bar (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   while (--l < n)
@@ -61,7 +61,7 @@ bar (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 bar_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
 {
   while (--l < 64)
@@ -69,7 +69,7 @@ bar_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 bar_2 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   l = 32;
diff --git a/gcc/testsuite/gcc.dg/vect/pr112325.c b/gcc/testsuite/gcc.dg/vect/pr112325.c
index 8689fbf..d380595 100644
--- a/gcc/testsuite/gcc.dg/vect/pr112325.c
+++ b/gcc/testsuite/gcc.dg/vect/pr112325.c
@@ -5,6 +5,7 @@
 /* { dg-additional-options "-mavx2" { target x86_64-*-* i?86-*-* } } */
 /* { dg-additional-options "--param max-completely-peeled-insns=200" { target powerpc64*-*-* } } */
 /* { dg-additional-options "-mlsx" { target loongarch64-*-* } } */
+/* { dg-additional-options "--param max-completely-peeled-insns=200 --param min-vect-loop-bound=0" { target s390*-*-* } } */
 
 typedef unsigned short ggml_fp16_t;
 static float table_f32_f16[1 << 16];
diff --git a/gcc/testsuite/gcc.dg/vect/pr116125.c b/gcc/testsuite/gcc.dg/vect/pr116125.c
index eab9efd..1b882ec 100644
--- a/gcc/testsuite/gcc.dg/vect/pr116125.c
+++ b/gcc/testsuite/gcc.dg/vect/pr116125.c
@@ -17,12 +17,12 @@ main (void)
 {
   check_vect ();
 
-  struct st a[9] = {};
+  struct st a[10] = {};
 
-  // input a = 0, 0, 0, 0, 0, 0, 0, 0, 0
+  // input a = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
   mem_overlap (&a[1], a);
 
-  // output a = 0, 1, 2, 3, 4, 5, 6, 7, 8
+  // output a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
   if (a[2].num == 2)
     return 0;
   else
diff --git a/gcc/testsuite/gcc.dg/vect/pr117888-1.c b/gcc/testsuite/gcc.dg/vect/pr117888-1.c
index 0b31fcd..884aed2 100644
--- a/gcc/testsuite/gcc.dg/vect/pr117888-1.c
+++ b/gcc/testsuite/gcc.dg/vect/pr117888-1.c
@@ -5,6 +5,7 @@
 /* { dg-additional-options "-mavx2" { target x86_64-*-* i?86-*-* } } */
 /* { dg-additional-options "--param max-completely-peeled-insns=200" { target powerpc64*-*-* } } */
 /* { dg-additional-options "-mlsx" { target loongarch64-*-* } } */
+/* { dg-additional-options "--param max-completely-peeled-insns=200 --param min-vect-loop-bound=0" { target s390*-*-* } } */
 
 typedef unsigned short ggml_fp16_t;
 static float table_f32_f16[1 << 16];
diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-1.c b/gcc/testsuite/gcc.dg/vect/pr120687-1.c
new file mode 100644
index 0000000..ce9cf63
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120687-1.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_int } */
+
+unsigned
+frd (unsigned *p, unsigned *lastone)
+{
+  unsigned sum = 0;
+  for (; p <= lastone; p += 16)
+    sum += p[0] + p[1] + p[2] + p[3] + p[4] + p[5] + p[6] + p[7]
+           + p[8] + p[9] + p[10] + p[11] + p[12] + p[13] + p[14] + p[15];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-2.c b/gcc/testsuite/gcc.dg/vect/pr120687-2.c
new file mode 100644
index 0000000..dfc6dc7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120687-2.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_float } */
+/* { dg-additional-options "-ffast-math" } */
+
+float
+frd (float *p, float *lastone)
+{
+  float sum = 0;
+  for (; p <= lastone; p += 16)
+    sum += p[0] + p[1] + p[2] + p[3] + p[4] + p[5] + p[6] + p[7]
+           + p[8] + p[9] + p[10] + p[11] + p[12] + p[13] + p[14] + p[15];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-3.c b/gcc/testsuite/gcc.dg/vect/pr120687-3.c
new file mode 100644
index 0000000..f20a66a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120687-3.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_double } */
+/* { dg-additional-options "-ffast-math" } */
+
+float
+frd (float *p, float *lastone)
+{
+  float sum = 0;
+  for (; p <= lastone; p += 2)
+    sum += p[0] + p[1];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr120817.c b/gcc/testsuite/gcc.dg/vect/pr120817.c
new file mode 100644
index 0000000..199189a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120817.c
@@ -0,0 +1,41 @@
+/* { dg-additional-options "-O1" } */
+/* { dg-require-effective-target aarch64_sve_hw { target aarch64*-*-* } } */
+/* { dg-additional-options "-march=armv8-a+sve -mtune=neoverse-n2" { target aarch64*-*-* } } */
+
+#include "tree-vect.h"
+
+typedef struct {
+    int _M_current;
+} __normal_iterator;
+
+typedef struct {
+    char _M_elems[5];
+} array_5;
+
+__normal_iterator __trans_tmp_1 = {-5};
+
+__attribute__((noipa))
+array_5 copySourceIntoTarget() {
+    array_5 target;
+    char* target_it = target._M_elems;
+
+    while (__trans_tmp_1._M_current != 0) {
+        *target_it = 1;
+        __trans_tmp_1._M_current++;
+        target_it++;
+    }
+
+    return target;
+}
+
+int main ()
+{
+  check_vect ();
+
+  array_5 res = copySourceIntoTarget();
+
+#pragma GCC novector
+  for (int i = 0; i < 5; i++)
+    if (res._M_elems[i] != 1)
+      __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.dg/vect/pr120922.c b/gcc/testsuite/gcc.dg/vect/pr120922.c
new file mode 100644
index 0000000..1a7247a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120922.c
@@ -0,0 +1,18 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-fsigned-char -fno-strict-aliasing -fwrapv" } */
+/* { dg-additional-options "-march=rv64gcv_zvl1024b -mrvv-vector-bits=zvl -mrvv-max-lmul=m8 -O3" { target { riscv_v } } } */
+
+char g;
+unsigned char h;
+int i[9][6];
+int main() {
+  int k[5];
+  if (g)
+    goto l;
+  for (; h <= 5; h++)
+    i[0][h] = *k;
+l:
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "loop vectorized" "vect" { target riscv_v } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr121034.c b/gcc/testsuite/gcc.dg/vect/pr121034.c
new file mode 100644
index 0000000..de20781
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121034.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3" } */
+
+int b, e;
+char c, d;
+unsigned g;
+int abs(int);
+void f() {
+  char *a = &d;
+  int h;
+  for (; e; e++) {
+    h = 0;
+    for (; h < 16; h++)
+      g += __builtin_abs(a[h] - c);
+    a += b;
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/vect/pr121049.c b/gcc/testsuite/gcc.dg/vect/pr121049.c
new file mode 100644
index 0000000..558c92a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121049.c
@@ -0,0 +1,25 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-march=x86-64-v4" { target avx512f_runtime } } */
+
+#include "tree-vect.h"
+
+int mon_lengths[12] = { 1, 10, 100 };
+
+__attribute__ ((noipa)) long
+transtime (int mon)
+{
+  long value = 0;
+  for (int i = 0; i < mon; ++i)
+    value += mon_lengths[i] * 2l;
+  return value;
+}
+
+int
+main ()
+{
+  check_vect ();
+  if (transtime (3) != 222)
+    __builtin_abort ();
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.dg/vect/pr121059.c b/gcc/testsuite/gcc.dg/vect/pr121059.c
new file mode 100644
index 0000000..2bbfcea
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121059.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 --param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-march=x86-64-v4" { target avx512f } } */
+
+typedef struct {
+  long left, right, top, bottom;
+} MngBox;
+typedef struct {
+  MngBox object_clip[6];
+  char exists[256], frozen[];
+} MngReadInfo;
+MngReadInfo mng_info;
+
+long ReadMNGImage_i;
+
+void ReadMNGImage(int ReadMNGImage_i)
+{
+  for (; ReadMNGImage_i < 256; ReadMNGImage_i++)
+    if (mng_info.exists[ReadMNGImage_i] && mng_info.frozen[ReadMNGImage_i])
+      mng_info.object_clip[ReadMNGImage_i].left =
+          mng_info.object_clip[ReadMNGImage_i].right =
+              mng_info.object_clip[ReadMNGImage_i].top =
+                  mng_info.object_clip[ReadMNGImage_i].bottom = 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/pr121126.c b/gcc/testsuite/gcc.dg/vect/pr121126.c
new file mode 100644
index 0000000..ae6603b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121126.c
@@ -0,0 +1,30 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=2" } */
+
+#include "tree-vect.h"
+
+unsigned char a;
+unsigned b;
+int r[11];
+static void __attribute__((noipa))
+c(int e, unsigned s[][11][11])
+{
+  for (int u = -(e ? 2000424973 : 0) - 2294542319; u < 7; u += 4)
+    for (int x = 0; x < 300000011; x += 4)
+      for (int y = 0; y < (0 < s[u][4][1]) + 11; y += 3) {
+        a = a ?: 1;
+        b = r[2];
+      }
+}
+long long ab;
+int e = 1;
+unsigned s[11][11][11];
+int main()
+{
+  check_vect ();
+  for (int t = 0; t < 11; ++t)
+    r[t] = 308100;
+  c(e,s);
+  ab = b;
+  if (ab != 308100)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.dg/vect/pr59984.c b/gcc/testsuite/gcc.dg/vect/pr59984.c
index c00c2267..8ca446e 100644
--- a/gcc/testsuite/gcc.dg/vect/pr59984.c
+++ b/gcc/testsuite/gcc.dg/vect/pr59984.c
@@ -64,3 +64,7 @@ main ()
   return 0;
 }
 
+/* { dg-final { scan-tree-dump "31:17: optimized: loop vectorized" "vect" } } */
+/* { dg-final { scan-tree-dump "37:7: optimized: loop vectorized" "vect" } } */
+/* { dg-final { scan-tree-dump "44:17: optimized: loop vectorized" "vect" } } */
+/* { dg-final { scan-tree-dump "50:7: optimized: loop vectorized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/slp-28.c b/gcc/testsuite/gcc.dg/vect/slp-28.c
index 67b7be2..1f98787 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-28.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-28.c
@@ -59,8 +59,8 @@ main1 ()
         abort ();
     }
   
-  /* Not vectorizable because of data dependencies: distance 3 is greater than 
-     the actual VF with SLP (2), but the analysis fail to detect that for now.  */
+  /* Dependence distance 3 is greater than the actual VF with SLP (2),
+     thus vectorizable.  */
   for (i = 3; i < N/4; i++)
     {
       in3[i*4] = in3[(i-3)*4] + 5;
@@ -91,7 +91,6 @@ int main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { ! vect32 } } } } */
-/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target vect32 } } } */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" { target { ! vect32 } } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" } } */
   
diff --git a/gcc/testsuite/gcc.dg/vect/vect-127.c b/gcc/testsuite/gcc.dg/vect/vect-127.c
new file mode 100644
index 0000000..8b913dc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-127.c
@@ -0,0 +1,15 @@
+// { dg-do compile }
+// { dg-require-effective-target vect_int }
+
+void foo (int *p)
+{
+  for (int i = 0; i < 1024; ++i)
+    {
+      int a0 = p[2*i + 0];
+      int a1 = p[2*i + 1];
+      p[2*i + 4] = a0;
+      p[2*i + 5] = a1;
+    }
+}
+
+/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect" { target { vect128 && vect_hw_misalign } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_137-pr121190.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_137-pr121190.c
new file mode 100644
index 0000000..e6b071c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_137-pr121190.c
@@ -0,0 +1,62 @@
+/* PR tree-optimization/121190 */
+/* { dg-options "-O3" } */
+/* { dg-additional-options "-march=znver2" { target x86_64-*-* i?86-*-* } } */
+/* { dg-require-effective-target mmap } */
+/* { dg-require-effective-target vect_early_break } */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include "tree-vect.h"
+
+#define MAX_COMPARE 5000
+
+__attribute__((noipa))
+int diff (uint64_t *restrict p, uint64_t *restrict q)
+{
+  int i = 0;
+  while (i < MAX_COMPARE) {
+    if (*(p + i) != *(q + i))
+      return i;
+    i++;
+  }
+  return -1;
+}
+
+int main ()
+{
+  check_vect ();
+
+  long pgsz = sysconf (_SC_PAGESIZE);
+  if (pgsz == -1) {
+    fprintf (stderr, "sysconf failed\n");
+    return 0;
+  }
+
+  /* Allocate 2 consecutive pages of memory and let p1 and p2 point to the
+     beginning of each.  */
+  void *mem = mmap (NULL, pgsz * 2, PROT_READ | PROT_WRITE,
+		    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (mem == MAP_FAILED) {
+    fprintf (stderr, "mmap failed\n");
+    return 0;
+  }
+  uint64_t *p1 = (uint64_t *) mem;
+  uint64_t *p2 = (uint64_t *) mem + pgsz / sizeof (uint64_t);
+
+  /* Fill the first page with zeros, except for its last 64 bits.  */
+  memset (p1, 0, pgsz);
+  *(p2 - 1) = -1;
+
+  /* Make the 2nd page not accessable.  */
+  mprotect (p2, pgsz, PROT_NONE);
+
+  /* Calls to diff should not read the 2nd page.  */
+  for (int i = 1; i <= 20; i++) {
+    if (diff (p2 - i, p1) != i - 1)
+      __builtin_abort ();
+  }
+}
+
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_138-pr121020.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_138-pr121020.c
new file mode 100644
index 0000000..8cb62bf
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_138-pr121020.c
@@ -0,0 +1,54 @@
+/* PR tree-optimization/121020 */
+/* { dg-options "-O3 --vect-cost-model=unlimited" } */
+/* { dg-additional-options "-march=znver2" { target x86_64-*-* i?86-*-* } } */
+/* { dg-require-effective-target mmap } */
+/* { dg-require-effective-target vect_early_break } */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include "tree-vect.h"
+
+__attribute__((noipa))
+bool equal (uint64_t *restrict p, uint64_t *restrict q, int length)
+{
+  for (int i = 0; i < length; i++) {
+    if (*(p + i) != *(q + i))
+      return false;
+  }
+  return true;
+}
+
+int main ()
+{
+  check_vect ();
+
+  long pgsz = sysconf (_SC_PAGESIZE);
+  if (pgsz == -1) {
+    fprintf (stderr, "sysconf failed\n");
+    return 0;
+  }
+
+  /* Allocate a whole page of memory.  */
+  void *mem = mmap (NULL, pgsz, PROT_READ | PROT_WRITE,
+		    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (mem == MAP_FAILED) {
+    fprintf (stderr, "mmap failed\n");
+    return 0;
+  }
+  uint64_t *p1 = (uint64_t *) mem;
+  uint64_t *p2 = (uint64_t *) mem + 32;
+
+  /* The first 16 elements pointed to by p1 and p2 are the same.  */
+  for (int i = 0; i < 32; i++) {
+    *(p1 + i) = 0;
+    *(p2 + i) = (i < 16 ? 0 : -1);
+  }
+
+  /* All calls to equal should return true.  */
+  for (int len = 0; len < 16; len++) {
+    if (!equal (p1 + 1, p2 + 1, len))
+      __builtin_abort();
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c
index 86a632f..6abfcd6 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c
@@ -18,4 +18,4 @@ int main1 (short X)
     }
 }
 
-/* { dg-final { scan-tree-dump "vectorized 1 loops in function" "vect" { target { ! "x86_64-*-* i?86-*-*" } } } } */
+/* { dg-final { scan-tree-dump "vectorized 1 loops in function" "vect" { target { ! "x86_64-*-* i?86-*-* arm*-*-*" } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-pr120927-2.c b/gcc/testsuite/gcc.dg/vect/vect-pr120927-2.c
new file mode 100644
index 0000000..e38cebe
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-pr120927-2.c
@@ -0,0 +1,24 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-mavx512bw -mavx512vl" { target avx512f_runtime } } */
+
+#include "tree-vect.h"
+
+static const double __attribute__((aligned(__BIGGEST_ALIGNMENT__))) a[] = { 1., 2., 3., 4., 5. };
+
+void __attribute__((noipa))
+foo (double *b, double *bp, double c, int n)
+{
+  for (int i = 0; i < n; ++i)
+    b[i] = bp[i] = a[i] * c;
+}
+
+int main()
+{
+  double b[6], bp[6];
+  b[5] = bp[5] = 13.;
+  check_vect ();
+  foo (b, bp, 3., 5);
+  if (b[5] != 13. || bp[5] != 13.)
+    abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-pr120927.c b/gcc/testsuite/gcc.dg/vect/vect-pr120927.c
new file mode 100644
index 0000000..793593f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-pr120927.c
@@ -0,0 +1,24 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-mavx512bw -mavx512vl" { target avx512f_runtime } } */
+
+#include "tree-vect.h"
+
+static const double a[] = { 1., 2., 3., 4., 5. };
+
+void __attribute__((noipa))
+foo (double *b, double *bp, double c, int n)
+{
+  for (int i = 0; i < n; ++i)
+    b[i] = bp[i] = a[i] * c;
+}
+
+int main()
+{
+  double b[6], bp[6];
+  b[5] = bp[5] = 13.;
+  check_vect ();
+  foo (b, bp, 3., 5);
+  if (b[5] != 13. || bp[5] != 13.)
+    abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c
new file mode 100644
index 0000000..7350fd9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c
@@ -0,0 +1,49 @@
+/* { dg-additional-options "-mavx2" { target avx2_runtime } } */
+
+#include "tree-vect.h"
+
+#define B 0
+#define G 1
+#define R 2
+
+int red = 153;
+int green = 66;
+int blue = 187;
+
+static void __attribute__((noipa))
+sub_left_prediction_bgr32(int *restrict dst, int *restrict src)
+{
+  for (int i = 0; i < 8; i++) {
+    int rt = src[i * 3 + R];
+    int gt = src[i * 3 + G];
+    int bt = src[i * 3 + B];
+
+    dst[i * 3 + R] = rt - red;
+    dst[i * 3 + G] = gt - green;
+    dst[i * 3 + B] = bt - blue;
+
+    red = rt;
+    green = gt;
+    blue = bt;
+  }
+}
+
+int main()
+{
+  int dst[8*3];
+  int src[8*3] = { 160, 73, 194, 17, 33, 99, 0, 12, 283, 87, 73, 11,
+		   9, 7, 1, 23, 19, 13, 77, 233, 97, 78, 2, 5 };
+  int dst2[8*3] = {-27, 7, 41, -143, -40, -95, -17, -21, 184, 87, 61,
+      -272, -78, -66, -10, 14, 12, 12, 54, 214, 84, 1, -231, -92};
+
+  check_vect ();
+
+  sub_left_prediction_bgr32(dst, src);
+
+#pragma GCC novector
+  for (int i = 0; i < 8*3; ++i)
+    if (dst[i] != dst2[i])
+      __builtin_abort();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c
new file mode 100644
index 0000000..c895e94
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c
@@ -0,0 +1,54 @@
+/* { dg-additional-options "-mavx2" { target avx2_runtime } } */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define B 0
+#define G 1
+#define R 2
+#define A 3
+
+int red = 153;
+int green = 66;
+int blue = 187;
+int alpha = 255;
+
+static void __attribute__((noipa))
+sub_left_prediction_bgr32(uint8_t *restrict dst, uint8_t *restrict src, int w)
+{
+  for (int i = 0; i < 8; i++) {
+    int rt = src[i * 4 + R];
+    int gt = src[i * 4 + G];
+    int bt = src[i * 4 + B];
+    int at = src[i * 4 + A];
+
+    dst[i * 4 + R] = rt - red;
+    dst[i * 4 + G] = gt - green;
+    dst[i * 4 + B] = bt - blue;
+    dst[i * 4 + A] = at - alpha;
+
+    red = rt;
+    green = gt;
+    blue = bt;
+    alpha = at;
+  }
+}
+
+int main()
+{
+  check_vect ();
+
+  uint8_t *dst = calloc(36, sizeof(uint8_t));
+  uint8_t *src = calloc(36, sizeof(uint8_t));
+
+  src[R] = 160;
+  src[G] = 73;
+  src[B] = 194;
+  src[A] = 255;
+
+  sub_left_prediction_bgr32(dst, src, 33);
+  if (dst[R] != 7 || dst[B] != 7 || dst[A] != 0)
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-1.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-1.c
new file mode 100644
index 0000000..258f17e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-1.c
@@ -0,0 +1,60 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_condition } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+/* PR tree-optimization/119920 */
+
+#define N 32
+
+unsigned int ub[N];
+
+/* Test vectorization of reduction of unsigned-int.  */
+
+__attribute__ ((noinline, noipa))
+void init(void)
+{
+  #pragma GCC novector
+  for(int i = 0;i < N; i++)
+    ub[i] = i;
+}
+
+
+__attribute__ ((noinline, noipa))
+void main1 (unsigned int b, unsigned int c)
+{
+  int i;
+  unsigned int usum = 0;
+
+  init();
+
+  /* Summation.  */
+  for (i = 0; i < N; i++) {
+    if ( ub[i] < N/2 )
+    {
+      usum += b;
+    }
+    else
+    {
+      usum += c;
+    }
+  }
+
+  /* check results:  */
+ /* __builtin_printf("%d : %d\n", usum, (N/2*b + N/2*c)); */
+  if (usum != N/2*b + N/2*c)
+    abort ();
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  main1 (0, 0);
+  main1 (1, 1);
+  main1 (10, 1);
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_int_add } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-2.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-2.c
new file mode 100644
index 0000000..126a50f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-2.c
@@ -0,0 +1,62 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_condition } */
+/* { dg-additional-options "-fdump-tree-ifcvt-details" } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+/* PR tree-optimization/119920 */
+
+#define N 32
+
+unsigned int ub[N];
+unsigned int ua[N];
+
+/* Test vectorization of reduction of unsigned-int.  */
+
+__attribute__ ((noinline, noipa))
+void init(void)
+{
+  #pragma GCC novector
+  for(int i = 0;i < N; i++) {
+    ub[i] = i;
+    ua[i] = 1;
+  }
+}
+
+
+__attribute__ ((noinline, noipa))
+void main1 (unsigned int b, unsigned int c)
+{
+  int i;
+  unsigned int usum = 0;
+
+  init();
+
+  /* Summation.  */
+  for (i = 0; i < N; i++) {
+    unsigned t = ua[i];
+    if ( ub[i] < N/2 )
+      usum += b * t;
+    else
+      usum += c * t;
+  }
+
+  /* check results:  */
+  /* __builtin_printf("%d : %d\n", usum, (N/2*b*1 + N/2*c*1)); */
+  if (usum != N/2*b + N/2*c)
+    abort ();
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  main1 (0, 0);
+  main1 (1, 1);
+  main1 (10, 1);
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_int_add } } } } */
+/* { dg-final { scan-tree-dump-times "changed to factor operation out from COND_EXPR" 2 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-3.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-3.c
new file mode 100644
index 0000000..e425869
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-3.c
@@ -0,0 +1,56 @@
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+/* PR tree-optimization/112324 */
+/* PR tree-optimization/110015 */
+
+#define N 32
+
+int ub[N];
+
+/* Test vectorization of reduction of int max with some extra code involed.  */
+
+__attribute__ ((noinline, noipa))
+void init(void)
+{
+  #pragma GCC novector
+  for(int i = 0;i < N; i++)
+    ub[i] = (i&4) && (i&1) ? -i : i;
+}
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b)) 
+
+__attribute__ ((noinline, noipa))
+void main1 (void)
+{
+  int i;
+  int max = 0;
+
+  init();
+
+  /* Summation.  */
+  for (i = 0; i < N; i++) {
+    int tmp = ub[i];
+    if (tmp < 0)
+      max = MAX (-tmp, max);
+    else
+      max = MAX (tmp, max);
+  }
+
+  /* check results:  */
+  /* __builtin_printf("%d : %d\n", max, N); */
+  if (max != N - 1)
+    abort ();
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  main1 ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_int_min_max } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-simd-pr121130.c b/gcc/testsuite/gcc.dg/vect/vect-simd-pr121130.c
new file mode 100644
index 0000000..c882ded
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-simd-pr121130.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+
+int n2;
+
+__attribute__((simd)) char
+w7(void)
+{
+  short int xb = n2;
+    xb = w7() < 1;
+  return xb;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-1.c b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-1.c
new file mode 100644
index 0000000..7441dd7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-1.c
@@ -0,0 +1,55 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+/* Ensure that we error out in case no hard regs are available for an operand
+   with constraint y.  The position/order of the y-constrained operand does not
+   matter.  */
+
+void
+test (void)
+{
+  int x, a, b, c, d, e, f, g, h;
+
+  __asm__ __volatile__ ("" :
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h));
+
+  __asm__ __volatile__ ("" : /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+      "=y" (x),
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h));
+
+  __asm__ __volatile__ ("" : /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "=y" (x),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h));
+
+  __asm__ __volatile__ ("" : /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h),
+      "=y" (x));
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-2.c b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-2.c
new file mode 100644
index 0000000..7434063
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-2.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a+sve" } */
+
+/* Test register pairs.  */
+
+#include <arm_sve.h>
+
+void
+test (void)
+{
+  svuint32x2_t x, y;
+  svuint32x4_t z;
+
+  __asm__ __volatile__ ("" : "={z4}" (x), "={z6}" (y));
+  __asm__ __volatile__ ("" : "={z5}" (x), "={z6}" (y));  /* { dg-error "multiple outputs to hard register: v6" } */
+  __asm__ __volatile__ ("" : "={z4}" (z), "={z6}" (y));  /* { dg-error "multiple outputs to hard register: v6" } */
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-be.c b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-be.c
new file mode 100644
index 0000000..2e8946b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-be.c
@@ -0,0 +1,23 @@
+/* { dg-do run } */
+/* { dg-require-effective-target aarch64_big_endian } */
+/* { dg-options "-O2 -favoid-store-forwarding" } */
+
+typedef union {
+    char arr[2];
+    short value;
+} DataUnion;
+
+short __attribute__ ((noinline))
+ssll (DataUnion *data, char x, char y)
+{
+  data->arr[0] = x;
+  data->arr[1] = y;
+  return data->value;
+}
+
+int main () {
+  DataUnion data = {};
+  short value = ssll (&data, 0, 1);
+  if (value != 1)
+    __builtin_abort ();
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/cmpbr.c b/gcc/testsuite/gcc.target/aarch64/cmpbr.c
index a86af9d..34630f9 100644
--- a/gcc/testsuite/gcc.target/aarch64/cmpbr.c
+++ b/gcc/testsuite/gcc.target/aarch64/cmpbr.c
@@ -121,7 +121,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_eq_x1:
-**	cbbeq	w1, w0, .L([0-9]+)
+**	cbbeq	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -129,7 +129,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ne_x1:
-**	cbbne	w1, w0, .L([0-9]+)
+**	cbbne	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -137,7 +137,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ult_x1:
-**	cbbhi	w1, w0, .L([0-9]+)
+**	(?:cbbhi	w1, w0|cbblo	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -145,7 +145,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ule_x1:
-**	cbbhs	w1, w0, .L([0-9]+)
+**	(?:cbbhs	w1, w0|cbbls	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -153,7 +153,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ugt_x1:
-**	cbblo	w1, w0, .L([0-9]+)
+**	(?:cbblo	w1, w0|cbbhi	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -161,7 +161,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_uge_x1:
-**	cbbls	w1, w0, .L([0-9]+)
+**	(?:cbbls	w1, w0|cbbhs	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -169,7 +169,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_slt_x1:
-**	cbbgt	w1, w0, .L([0-9]+)
+**	(?:cbbgt	w1, w0|cbblt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -177,7 +177,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_sle_x1:
-**	cbbge	w1, w0, .L([0-9]+)
+**	(?:cbbge	w1, w0|cbble	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -185,7 +185,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_sgt_x1:
-**	cbblt	w1, w0, .L([0-9]+)
+**	(?:cbblt	w1, w0|cbbgt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -193,7 +193,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_sge_x1:
-**	cbble	w1, w0, .L([0-9]+)
+**	(?:cbble	w1, w0|cbbge	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -201,7 +201,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_eq_x1:
-**	cbheq	w1, w0, .L([0-9]+)
+**	cbheq	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -209,7 +209,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ne_x1:
-**	cbhne	w0|w1, w1|w0, .L([0-9]+)
+**	cbhne	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -217,7 +217,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ult_x1:
-**	cbhhi	w1, w0, .L([0-9]+)
+**	(?:cbhhi	w1, w0|cbhlo	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -225,7 +225,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ule_x1:
-**	cbhhs	w1, w0, .L([0-9]+)
+**	(?:cbhhs	w1, w0|cbhls	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -233,7 +233,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ugt_x1:
-**	cbhlo	w1, w0, .L([0-9]+)
+**	(?:cbhlo	w1, w0|cbhhi	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -241,7 +241,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_uge_x1:
-**	cbhls	w1, w0, .L([0-9]+)
+**	(?:cbhls	w1, w0|cbhhs	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -249,7 +249,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_slt_x1:
-**	cbhgt	w1, w0, .L([0-9]+)
+**	(?:cbhgt	w1, w0|cbhlt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -257,7 +257,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_sle_x1:
-**	cbhge	w1, w0, .L([0-9]+)
+**	(?:cbhge	w1, w0|cbhle	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -265,7 +265,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_sgt_x1:
-**	cbhlt	w1, w0, .L([0-9]+)
+**	(?:cbhlt	w1, w0|cbhgt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -273,7 +273,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_sge_x1:
-**	cbhle	w1, w0, .L([0-9]+)
+**	(?:cbhle	w1, w0|cbhge	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-0.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-0.c
new file mode 100644
index 0000000..e544b04f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-0.c
@@ -0,0 +1,12 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-1.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-1.c
new file mode 100644
index 0000000..be70687
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-1.c
@@ -0,0 +1,13 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-2.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-2.c
new file mode 100644
index 0000000..bf594d0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-2.c
@@ -0,0 +1,14 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-3.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-3.c
new file mode 100644
index 0000000..f16d01b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-3.c
@@ -0,0 +1,15 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+  uint64_t hwcap3;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-4.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-4.c
new file mode 100644
index 0000000..1b4ccbd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-4.c
@@ -0,0 +1,16 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+  uint64_t hwcap3;
+  uint64_t hwcap4;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver.in b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver.in
new file mode 100644
index 0000000..ada0b33
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver.in
@@ -0,0 +1,48 @@
+#include <unistd.h>
+#include <string.h>
+#include <sys/mman.h>
+
+/* Allocate memory buffer of size LEN with a protected page
+   following right after the buffer end so that any memory
+   accesses past the end of the buffer would trigger SEGFAUL.  */
+void *allocate_mem (size_t len)
+{
+  size_t pagesize = sysconf (_SC_PAGESIZE);
+  char *m = mmap (NULL, pagesize * 2,
+		  PROT_READ | PROT_WRITE,
+		  MAP_PRIVATE | MAP_ANONYMOUS,
+		  -1, 0);
+  mprotect (m + pagesize, pagesize, PROT_NONE);
+  m = m + pagesize - len;
+  memset(m, 0, len);
+  return m;
+}
+
+int impl ()
+{
+  return 0;
+}
+
+#ifndef _IFUNC_ARG_HWCAP
+#define _IFUNC_ARG_HWCAP (1ULL << 62)
+#endif
+
+void
+__init_cpu_features_resolver (unsigned long hwcap, const void *arg);
+
+static void *
+fun_resolver (uint64_t a0, const uint64_t *a1)
+{
+  ifunc_arg_t *arg = allocate_mem (sizeof (ifunc_arg_t));
+  arg->size = sizeof (ifunc_arg_t);
+  /* Call this function with synthetic ifunc_arg_t arg.  */
+  __init_cpu_features_resolver (_IFUNC_ARG_HWCAP, arg);
+  return (void *)(uintptr_t)impl;
+}
+
+int fun (void) __attribute__ ((ifunc ("fun_resolver")));
+
+int main (int argc, char *argv[])
+{
+  return fun ();
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/inszero_split_1.c b/gcc/testsuite/gcc.target/aarch64/inszero_split_1.c
new file mode 100644
index 0000000..5c739bd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/inszero_split_1.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Avoid INS from WZR register when optimizing for speed.  */
+
+#include <arm_neon.h>
+
+/*
+** foo:
+**	movi?	[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0
+**	ins	v0.h\[2\], v(\1).h\[0\]
+**	ret
+*/
+uint16x8_t foo(uint16x8_t a) {
+  a[2] = 0;
+  return a;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c b/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c
index f57c09d..e8a545a 100644
--- a/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c
+++ b/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c
@@ -33,7 +33,7 @@ TEST(s8_s64, s8, long long)
 /*
 **test_s16_s64:
 **...
-**	ldapursh	x0, \[x[0-9]+\]
+**	ldapursh	x0, \[x[0-9]+, [0-9]+\]
 **	ret
 */
 
@@ -42,7 +42,7 @@ TEST(s16_s64, s16, long long)
 /*
 **test_s32_s64:
 **...
-**	ldapursw	x0, \[x[0-9]+\]
+**	ldapursw	x0, \[x[0-9]+, [0-9]+\]
 **	ret
 */
 
@@ -60,7 +60,7 @@ TEST(s8_s32, s8, int)
 /*
 **test_s16_s32:
 **...
-**	ldapursh	w0, \[x[0-9]+\]
+**	ldapursh	w0, \[x[0-9]+, [0-9]+\]
 **	ret
 */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/ldapur.c b/gcc/testsuite/gcc.target/aarch64/ldapur.c
new file mode 100644
index 0000000..5c68bdd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ldapur.c
@@ -0,0 +1,77 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -std=c99" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+#pragma GCC target "arch=armv8.8-a"
+
+atomic_ullong u64;
+atomic_uint u32;
+atomic_ushort u16;
+atomic_uchar u8[2]; /* Force an offset for u8 */
+
+#define TEST(name, ldsize, rettype)				\
+rettype								\
+test_##name (void)						\
+{								\
+  return atomic_load_explicit (&ldsize, memory_order_acquire);	\
+}								\
+
+
+/*
+** test_u8_u64:
+**	...
+**	ldapurb	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u8_u64, u8[1], uint64_t)
+
+/*
+** test_u16_u64:
+**	...
+**	ldapurh	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u16_u64, u16, uint64_t)
+
+/*
+**test_u32_u64:
+**	...
+**	ldapur	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u32_u64, u32, uint64_t)
+
+/*
+**test_u64_u64:
+**	...
+**	ldapur	x[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u64_u64, u64, uint64_t)
+
+/*
+**test_u8_u32:
+**	...
+**	ldapurb	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u8_u32, u8[1], uint32_t)
+
+/*
+**test_u16_u32:
+**	...
+**	ldapurh	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u16_u32, u16, uint32_t)
+
+/*
+**test_u32_u32:
+**	...
+**	ldapur	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u32_u32, u32, uint32_t)
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/ldapur_avoid.c b/gcc/testsuite/gcc.target/aarch64/ldapur_avoid.c
new file mode 100644
index 0000000..ad87a30
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ldapur_avoid.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -std=c99 -moverride=tune=avoid_ldapur" } */
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+#pragma GCC target "arch=armv8.8-a"
+/* LDAPUR is only avoided for armv8.4 to armv8.7. This checks for the working
+of avoid_ldapur flag. */
+
+/* { dg-final { scan-assembler-not "ldapur\t" } } */
+
+atomic_ullong u64;
+atomic_uint u32;
+atomic_ushort u16;
+atomic_uchar u8[2]; /* Force an offset for u8 */
+
+#define TEST(name, ldsize, rettype)				\
+rettype								\
+test_##name (void)						\
+{								\
+  return atomic_load_explicit (&ldsize, memory_order_acquire);	\
+}								\
+
+TEST(u8_u64, u8[1], uint64_t)
+TEST(u16_u64, u16, uint64_t)
+TEST(u32_u64, u32, uint64_t)
+TEST(u64_u64, u64, uint64_t)
+TEST(u8_u32, u8[1], uint32_t)
+TEST(u16_u32, u16, uint32_t)
+TEST(u32_u32, u32, uint32_t)
+
+/* { dg-final { scan-assembler-times "ldapr\t" 3 } } */
+/* { dg-final { scan-assembler-times "ldaprh\t" 2 } } */
+/* { dg-final { scan-assembler-times "ldaprb\t" 2 } } */
+
+
diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt13.c b/gcc/testsuite/gcc.target/aarch64/popcnt13.c
new file mode 100644
index 0000000..2a30e98
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/popcnt13.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#pragma GCC target "+nocssc+sve"
+
+/*
+** h128:
+**	ldr	q([0-9]+), \[x0\]
+**	ptrue	p([0-9]+).b, vl16
+**	cnt	z([0-9]+).d, p\2/m, z\1.d
+**	addp	d([0-9]+), v\3.2d
+**	fmov	x0, d\4
+**	ret
+*/
+
+unsigned h128 (const unsigned __int128 *a) {
+	  return __builtin_popcountg (a[0]);
+}
+
+/* There should be only one POPCOUNT. */
+/* { dg-final { scan-tree-dump-times "POPCOUNT " 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-not " __builtin_popcount"  "optimized" } } */
+
diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt9.c b/gcc/testsuite/gcc.target/aarch64/popcnt9.c
index c778fc7..cfed8c5 100644
--- a/gcc/testsuite/gcc.target/aarch64/popcnt9.c
+++ b/gcc/testsuite/gcc.target/aarch64/popcnt9.c
@@ -3,7 +3,7 @@
 /* { dg-final { check-function-bodies "**" "" } } */
 /* PR target/113042 */
 
-#pragma GCC target "+nocssc"
+#pragma GCC target "+nocssc+nosve"
 
 /*
 ** h128:
diff --git a/gcc/testsuite/gcc.target/aarch64/pr118348_1.c b/gcc/testsuite/gcc.target/aarch64/pr118348_1.c
index 75f6dad..2715dcb 100644
--- a/gcc/testsuite/gcc.target/aarch64/pr118348_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/pr118348_1.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target aarch64_sve128_hw } } */
+/* { dg-do run { target { aarch64_sve128_hw && fstack_protector } } } */
 /* { dg-options "-O2 -fopenmp-simd -fno-trapping-math -msve-vector-bits=128 --param aarch64-autovec-preference=sve-only -fstack-protector-strong" } */
 
 #pragma GCC target "+sve"
diff --git a/gcc/testsuite/gcc.target/aarch64/pr118348_2.c b/gcc/testsuite/gcc.target/aarch64/pr118348_2.c
index 2e20004..4ce8d20 100644
--- a/gcc/testsuite/gcc.target/aarch64/pr118348_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/pr118348_2.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target aarch64_sve256_hw } } */
+/* { dg-do run { target { aarch64_sve256_hw && fstack_protector } } } */
 /* { dg-options "-O2 -fopenmp-simd -fno-trapping-math -msve-vector-bits=256 --param aarch64-autovec-preference=sve-only -fstack-protector-strong" } */
 
 #include "pr118348_1.c"
diff --git a/gcc/testsuite/gcc.target/aarch64/pr121300.c b/gcc/testsuite/gcc.target/aarch64/pr121300.c
new file mode 100644
index 0000000..5f2cd9a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr121300.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-S -O3 -march=armv9-a+sme2" } */
+
+#include <arm_sme.h>
+
+svfloat16x2_t test (svfloat16x2_t zd, svfloat16x2_t zm) __arm_streaming
+{
+    return svamin_f16_x2 (zd, zm); // { dg-error "ACLE function .svamin_f16_x2. requires ISA extension .faminmax." }
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
index acd2e11..8fc1569 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
@@ -4,24 +4,24 @@
 
 /*
 ** uadd:
-**	dup	v([0-9]+).8b, w1
-**	dup	v([0-9]+).8b, w0
+**	dup	v([0-9]+).8b, w[01]
+**	dup	v([0-9]+).8b, w[01]
 **	uqadd	b([0-9]+), (?:b\2, b\1|b\1, b\2)
 **	umov	w0, v\3.b\[0\]
 **	ret
 */
 /*
 ** uadd2:
-**	dup	v([0-9]+).8b, w1
-**	dup	v([0-9]+).8b, w0
+**	dup	v([0-9]+).8b, w[01]
+**	dup	v([0-9]+).8b, w[01]
 **	uqadd	b([0-9]+), (?:b\2, b\1|b\1, b\2)
 **	umov	w0, v\3.b\[0\]
 **	ret
 */
 /*
 ** usub: { xfail *-*-* }
-**	dup	v([0-9]+).8b, w1
-**	dup	v([0-9]+).8b, w0
+**	dup	v([0-9]+).8b, w[01]
+**	dup	v([0-9]+).8b, w[01]
 **	uqsub	b([0-9]+), b\1, b\2
 **	umov	w0, v\3.b\[0\]
 **	ret
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
index 86c88f8..dd0fefa 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
@@ -4,16 +4,16 @@
 
 /*
 ** uadd:
-**	dup	v([0-9]+).4h, w1
-**	dup	v([0-9]+).4h, w0
+**	dup	v([0-9]+).4h, w[01]
+**	dup	v([0-9]+).4h, w[01]
 **	uqadd	h([0-9]+), (?:h\2, h\1|h\1, h\2)
 **	umov	w0, v\3.h\[0\]
 **	ret
 */
 /*
 ** uadd2:
-**	dup	v([0-9]+).4h, w1
-**	dup	v([0-9]+).4h, w0
+**	dup	v([0-9]+).4h, w[01]
+**	dup	v([0-9]+).4h, w[01]
 **	uqadd	h([0-9]+), (?:h\2, h\1|h\1, h\2)
 **	umov	w0, v\3.h\[0\]
 **	ret
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/bcax_d.c b/gcc/testsuite/gcc.target/aarch64/simd/bcax_d.c
new file mode 100644
index 0000000..a7640c3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/bcax_d.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_neon.h>
+
+#pragma GCC target "+sha3"
+
+#define BCAX(x,y,z)  ((x) ^ ((y) & ~(z)))
+
+/* When the inputs come from GP regs don't form a BCAX.  */
+uint64_t bcax_d_gp (uint64_t a, uint64_t b, uint64_t c) { return BCAX (a, b, c); }
+
+uint64x1_t bcax_d (uint64x1_t a, uint64x1_t b, uint64x1_t c) { return BCAX (a, b, c); }
+uint32x2_t bcax_s (uint32x2_t a, uint32x2_t b, uint32x2_t c) { return BCAX (a, b, c); }
+uint16x4_t bcax_h (uint16x4_t a, uint16x4_t b, uint16x4_t c) { return BCAX (a, b, c); }
+uint8x8_t bcax_b (uint8x8_t a, uint8x8_t b, uint8x8_t c) { return BCAX (a, b, c); }
+
+/* { dg-final { scan-assembler-times {bcax\tv0.16b, v0.16b, v1.16b, v2.16b} 4 } } */
+
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/eor3_d.c b/gcc/testsuite/gcc.target/aarch64/simd/eor3_d.c
new file mode 100644
index 0000000..7f2b2b4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/eor3_d.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_neon.h>
+
+#pragma GCC target "+sha3"
+
+#define EOR3(x,y,z)  ((x) ^ (y) ^ (z))
+
+uint32x2_t bcax_s (uint32x2_t a, uint32x2_t b, uint32x2_t c) { return EOR3 (a, b, c); }
+uint16x4_t bcax_h (uint16x4_t a, uint16x4_t b, uint16x4_t c) { return EOR3 (a, b, c); }
+uint8x8_t bcax_b (uint8x8_t a, uint8x8_t b, uint8x8_t c) { return EOR3 (a, b, c); }
+
+/* { dg-final { scan-assembler-times {eor3\tv0.16b, v[0-9]+.16b, v[0-9]+.16b, v[0-9]+.16b} 3 } } */
+
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_1.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_1.c
new file mode 100644
index 0000000..f082198
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_1.c
@@ -0,0 +1,716 @@
+/* { dg-do compile } */
+/* { dg-options "-O -march=armv9-a+bf16" } */
+
+#include <arm_neon.h>
+
+/* We should use the highpart instruction where doing so would avoid data
+   movement instructions.  This case, where all the arguments are non-constant
+   vector highparts, can be handled by either gimple_fold_builtin or combine.  */
+
+#ifndef TEST_UN_HIGHPARTS
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (INTYPE a)		     \
+  {                                                  \
+    return FN##_##SUFF (vget_high_##SUFF (a));	     \
+  }
+#endif
+
+#ifndef TEST_BIN_W_HIGHPARTS
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)  \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, INTYPE b)	 \
+  {                                                      \
+    return FN##_##SUFF (a, vget_high_##SUFF (b));	 \
+  }
+#endif
+
+#ifndef TEST_BIN_N_HIGHPARTS
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_##FN##_##SUFF (INTYPE a)			   \
+  {                                                        \
+    return FN##_##SUFF (vget_high_##SUFF (a), a[1]);	   \
+  }
+#endif
+
+#ifndef TEST_TERN_N_HIGHPARTS
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)      \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, INTYPE b)	      \
+  {                                                           \
+    return FN##_##SUFF (a, vget_high_##SUFF (b), b[1]);	      \
+  }
+#endif
+
+#ifndef TEST_BIN_HIGHPARTS
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (INTYPE a, INTYPE b)		\
+  {                                                             \
+    return FN##_##SUFF (vget_high_##SUFF (a),			\
+			vget_high_##SUFF (b));			\
+  }
+#endif
+
+#ifndef TEST_TERN_HIGHPARTS
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF)   \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, INTYPE b, INTYPE c)	   \
+  {                                                                \
+    return FN##_##SUFF(a, vget_high_##SUFF (b),			   \
+		       vget_high_##SUFF (c));			   \
+  }
+#endif
+
+#define TEST_UNOP(FN) \
+  TEST_UN_HIGHPARTS (FN, int16x8_t,  int8x16_t,  s8)  \
+  TEST_UN_HIGHPARTS (FN, uint16x8_t, uint8x16_t, u8)  \
+  TEST_UN_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_UN_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_UN_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_UN_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_BINOP(FN)						   \
+  TEST_BIN_HIGHPARTS (FN, int16x8_t,  int8x16_t,  int8x8_t,   s8)  \
+  TEST_BIN_HIGHPARTS (FN, uint16x8_t, uint8x16_t, uint8x8_t,  u8)  \
+  TEST_BIN_HIGHPARTS (FN, int32x4_t,  int16x8_t,  int16x4_t,  s16) \
+  TEST_BIN_HIGHPARTS (FN, uint32x4_t, uint16x8_t, uint16x4_t, u16) \
+  TEST_BIN_HIGHPARTS (FN, int64x2_t,  int32x4_t,  int32x2_t,  s32) \
+  TEST_BIN_HIGHPARTS (FN, uint64x2_t, uint32x4_t, uint32x2_t, u32)
+
+#define TEST_BINOP_N(FN)				 \
+  TEST_BIN_N_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_BIN_N_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_BIN_N_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_BIN_N_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_BINOP_W(FN)				 \
+  TEST_BIN_W_HIGHPARTS (FN, int16x8_t,  int8x16_t,   s8) \
+  TEST_BIN_W_HIGHPARTS (FN, uint16x8_t, uint8x16_t,  u8) \
+  TEST_BIN_W_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_BIN_W_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_BIN_W_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_BIN_W_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_TERNOP_N(FN)				  \
+  TEST_TERN_N_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_TERN_N_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_TERN_N_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_TERN_N_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_TERNOP(FN)                                           \
+  TEST_TERN_HIGHPARTS (FN, int16x8_t,  int8x16_t,  int8x8_t,   s8)  \
+  TEST_TERN_HIGHPARTS (FN, uint16x8_t, uint8x16_t, uint8x8_t,  u8)  \
+  TEST_TERN_HIGHPARTS (FN, int32x4_t,  int16x8_t,  int16x4_t,  s16) \
+  TEST_TERN_HIGHPARTS (FN, uint32x4_t, uint16x8_t, uint16x4_t, u16) \
+  TEST_TERN_HIGHPARTS (FN, int64x2_t,  int32x4_t,  int32x2_t,  s32) \
+  TEST_TERN_HIGHPARTS (FN, uint64x2_t, uint32x4_t, uint32x2_t, u32)
+
+#define TEST_VQDMULL                                                 \
+  TEST_BIN_HIGHPARTS (vqdmull, int32x4_t, int16x8_t, int16x4_t, s16) \
+  TEST_BIN_HIGHPARTS (vqdmull, int64x2_t, int32x4_t, int32x2_t, s32)
+
+#define TEST_VQDMULL_N                                        \
+  TEST_BIN_N_HIGHPARTS (vqdmull_n, int32x4_t, int16x8_t, s16) \
+  TEST_BIN_N_HIGHPARTS (vqdmull_n, int64x2_t, int32x4_t, s32)
+
+#define TEST_VQMLAL                                                   \
+  TEST_TERN_HIGHPARTS (vqdmlal, int32x4_t, int16x8_t, int16x4_t, s16) \
+  TEST_TERN_HIGHPARTS (vqdmlal, int64x2_t, int32x4_t, int32x2_t, s32)
+
+#define TEST_VQMLAL_N                                          \
+  TEST_TERN_N_HIGHPARTS (vqdmlal_n, int32x4_t, int16x8_t, s16) \
+  TEST_TERN_N_HIGHPARTS (vqdmlal_n, int64x2_t, int32x4_t, s32)
+
+#define TEST_VQMLSL                                                   \
+  TEST_TERN_HIGHPARTS (vqdmlsl, int32x4_t, int16x8_t, int16x4_t, s16) \
+  TEST_TERN_HIGHPARTS (vqdmlsl, int64x2_t, int32x4_t, int32x2_t, s32)
+
+#define TEST_VQMLSL_N                                          \
+  TEST_TERN_N_HIGHPARTS (vqdmlsl_n, int32x4_t, int16x8_t, s16) \
+  TEST_TERN_N_HIGHPARTS (vqdmlsl_n, int64x2_t, int32x4_t, s32)
+
+#define TEST_VMOVL \
+  TEST_UNOP (vmovl)
+
+#define TEST_VMULL \
+  TEST_BINOP (vmull) \
+  TEST_BIN_HIGHPARTS (vmull, poly16x8_t, poly8x16_t, poly8x8_t, p8)
+
+#define TEST_VMULL_N \
+  TEST_BINOP_N (vmull_n)
+
+#define TEST_VADDL \
+  TEST_BINOP (vaddl)
+
+#define TEST_VSUBL \
+  TEST_BINOP (vsubl)
+
+#define TEST_VMLAL \
+  TEST_TERNOP (vmlal)
+
+#define TEST_VMLAL_N \
+  TEST_TERNOP_N (vmlal_n)
+
+#define TEST_VMLSL \
+  TEST_TERNOP (vmlsl)
+
+#define TEST_VMLSL_N \
+  TEST_TERNOP_N (vmlsl_n)
+
+#define TEST_VABDL \
+  TEST_BINOP (vabdl)
+
+#define TEST_VABAL \
+  TEST_TERNOP (vabal)
+
+#define TEST_VSUBW \
+  TEST_BINOP_W (vsubw)
+
+#define TEST_VADDW \
+  TEST_BINOP_W (vaddw)
+
+/*
+** test_vmovl_s8:
+**	sxtl2	v0\.8h, v0\.16b
+**	ret
+*/
+
+/*
+** test_vmovl_u8:
+**	uxtl2	v0\.8h, v0\.16b
+**	ret
+*/
+
+/*
+** test_vmovl_s16:
+**	sxtl2	v0\.4s, v0\.8h
+**	ret
+*/
+
+/*
+** test_vmovl_u16:
+**	uxtl2	v0\.4s, v0\.8h
+**	ret
+*/
+
+/*
+** test_vmovl_s32:
+**	sxtl2	v0\.2d, v0\.4s
+**	ret
+*/
+
+/*
+** test_vmovl_u32:
+**	uxtl2	v0\.2d, v0\.4s
+**	ret
+*/
+
+TEST_VMOVL
+
+/*
+** test_vmull_s8:
+**	smull2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vmull_u8:
+**	umull2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vmull_s16:
+**	smull2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vmull_u16:
+**	umull2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vmull_s32:
+**	smull2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vmull_u32:
+**	umull2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vmull_p8:
+**	pmull2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+TEST_VMULL
+
+/*
+** test_vmull_n_s16:
+**	smull2	v0\.4s, v0\.8h, v0\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmull_n_u16:
+**	umull2	v0\.4s, v0\.8h, v0\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmull_n_s32:
+**	smull2	v0\.2d, v0\.4s, v0\.s\[[0-3]\]
+**	ret
+*/
+
+/*
+** test_vmull_n_u32:
+**	umull2	v0\.2d, v0\.4s, v0\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VMULL_N
+
+/*
+** test_vaddl_s8:
+**	saddl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vaddl_u8:
+**	uaddl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vaddl_s16:
+**	saddl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vaddl_u16:
+**	uaddl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vaddl_s32:
+**	saddl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vaddl_u32:
+**	uaddl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+TEST_VADDL
+
+/*
+** test_vsubl_s8:
+**	ssubl2	v0\.8h, v0\.16b, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubl_u8:
+**	usubl2	v0\.8h, v0\.16b, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubl_s16:
+**	ssubl2	v0\.4s, v0\.8h, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubl_u16:
+**	usubl2	v0\.4s, v0\.8h, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubl_s32:
+**	ssubl2	v0\.2d, v0\.4s, v1\.4s
+**	ret
+*/
+
+/*
+** test_vsubl_u32:
+**	usubl2	v0\.2d, v0\.4s, v1\.4s
+**	ret
+*/
+
+TEST_VSUBL
+
+/*
+** test_vabal_s8:
+**	sabal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vabal_u8:
+**	uabal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vabal_s16:
+**	sabal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vabal_u16:
+**	uabal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vabal_s32:
+**	sabal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+/*
+** test_vabal_u32:
+**	uabal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+TEST_VABAL
+
+/*
+** test_vsubw_s8:
+**	ssubw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubw_u8:
+**	usubw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubw_s16:
+**	ssubw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubw_u16:
+**	usubw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubw_s32:
+**	ssubw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+/*
+** test_vsubw_u32:
+**	usubw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+TEST_VSUBW
+
+/*
+** test_vaddw_s8:
+**	saddw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vaddw_u8:
+**	uaddw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vaddw_s16:
+**	saddw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vaddw_u16:
+**	uaddw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vaddw_s32:
+**	saddw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+/*
+** test_vaddw_u32:
+**	uaddw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+TEST_VADDW
+
+/*
+** test_vabdl_s8:
+**	sabdl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vabdl_u8:
+**	uabdl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vabdl_s16:
+**	sabdl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vabdl_u16:
+**	uabdl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vabdl_s32:
+**	sabdl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vabdl_u32:
+**	uabdl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+TEST_VABDL
+
+/*
+** test_vmlal_s8:
+**	smlal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vmlal_u8:
+**	umlal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vmlal_s16:
+**	smlal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vmlal_u16:
+**	umlal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vmlal_s32:
+**	smlal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+/*
+** test_vmlal_u32:
+**	umlal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+TEST_VMLAL
+
+/*
+** test_vmlal_n_s16:
+**	smlal2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlal_n_u16:
+**	umlal2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlal_n_s32:
+**	smlal2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+/*
+** test_vmlal_n_u32:
+**	umlal2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VMLAL_N
+
+/*
+** test_vmlsl_s8:
+**	smlsl2	v0\.8h, v1\.16b, v2\.16b
+**	ret
+*/
+
+/*
+** test_vmlsl_u8:
+**	umlsl2	v0\.8h, v1\.16b, v2\.16b
+**	ret
+*/
+
+/*
+** test_vmlsl_s16:
+**	smlsl2	v0\.4s, v1\.8h, v2\.8h
+**	ret
+*/
+
+/*
+** test_vmlsl_u16:
+**	umlsl2	v0\.4s, v1\.8h, v2\.8h
+**	ret
+*/
+
+/*
+** test_vmlsl_s32:
+**	smlsl2	v0\.2d, v1\.4s, v2\.4s
+**	ret
+*/
+
+/*
+** test_vmlsl_u32:
+**	umlsl2	v0\.2d, v1\.4s, v2\.4s
+**	ret
+*/
+
+TEST_VMLSL
+
+/*
+** test_vmlsl_n_s16:
+**	smlsl2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlsl_n_u16:
+**	umlsl2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlsl_n_s32:
+**	smlsl2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+/*
+** test_vmlsl_n_u32:
+**	umlsl2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VMLSL_N
+
+/*
+** test_vqdmull_s16:
+**	sqdmull2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vqdmull_s32:
+**	sqdmull2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+TEST_VQDMULL
+
+/*
+** test_vqdmull_n_s16:
+**	sqdmull2	v0\.4s, v0\.8h, v0\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vqdmull_n_s32:
+**	sqdmull2	v0\.2d, v0\.4s, v0\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VQDMULL_N
+
+/*
+** test_vqdmlal_s16:
+**	sqdmlal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vqdmlal_s32:
+**	sqdmlal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+TEST_VQMLAL
+
+/*
+** test_vqdmlal_n_s16:
+**	sqdmlal2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vqdmlal_n_s32:
+**	sqdmlal2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VQMLAL_N
+
+/*
+** test_vqdmlsl_s16:
+**	sqdmlsl2	v0\.4s, v1\.8h, v2\.8h
+**	ret
+*/
+
+/*
+** test_vqdmlsl_s32:
+**	sqdmlsl2	v0\.2d, v1\.4s, v2\.4s
+**	ret
+*/
+
+TEST_VQMLSL
+
+/*
+** test_vqdmlsl_n_s16:
+**	sqdmlsl2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vqdmlsl_n_s32:
+**	sqdmlsl2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VQMLSL_N
+
+/* { dg-final { check-function-bodies "**" ""} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_2.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_2.c
new file mode 100644
index 0000000..5885b28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_2.c
@@ -0,0 +1,88 @@
+/* { dg-do compile } */
+/* { dg-options "-O -march=armv9-a+bf16" } */
+
+/* We should not use the highpart instruction unless doing so would avoid
+   data movement instructions.  That is, unless at least one argument is a
+   reference to the highpart of a non-constant vector.  */
+
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_## SUFF ()		     \
+  {                                                  \
+    INTYPE a = vdupq_n_##SUFF (0x1A);		     \
+    return FN##_##SUFF (vget_high_##SUFF (a));	     \
+  }
+
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a)		\
+  {                                                     \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			\
+    return FN##_##SUFF (a, vget_high_##SUFF (b));	\
+  }
+
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)     \
+  RETTYPE test_##FN##_##SUFF (INTYPE c)			    \
+  {							    \
+    INTYPE a = vdupq_n_##SUFF (0x1A);			    \
+    return FN##_##SUFF (vget_high_##SUFF (a), c[1]);	    \
+  }
+
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)      \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a)		      \
+  {                                                           \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			      \
+    return FN##_##SUFF (a, vget_high_##SUFF (b), b[1]);      \
+  }
+
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  RETTYPE test_##FN##_## SUFF (H_INTYPE b)			\
+  {                                                             \
+    INTYPE a = vdupq_n_##SUFF (0x1A);				\
+    return FN##_##SUFF (vget_high_##SUFF (a), b);		\
+  }
+
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, H_INTYPE b)		 \
+  {                                                              \
+    INTYPE c = vdupq_n_##SUFF (0x1A);				 \
+    return FN##_##SUFF (a, vget_high_##SUFF (c), b);		 \
+  }
+
+#include "fold_to_highpart_1.c"
+
+
+/* { dg-final { scan-assembler-not {uxtl2\t} } } */
+/* { dg-final { scan-assembler-not {sxtl2\t} } } */
+
+/* { dg-final { scan-assembler-not {umull2\t} } } */
+/* { dg-final { scan-assembler-not {smull2\t} } } */
+/* { dg-final { scan-assembler-not {pmull2\t} } } */
+
+/* { dg-final { scan-assembler-not {uaddl2\t} } } */
+/* { dg-final { scan-assembler-not {saddl2\t} } } */
+
+/* { dg-final { scan-assembler-not {usubl2\t} } } */
+/* { dg-final { scan-assembler-not {ssubl2\t} } } */
+
+/* { dg-final { scan-assembler-not {uabal2\t} } } */
+/* { dg-final { scan-assembler-not {sabal2\t} } } */
+
+/* { dg-final { scan-assembler-not {uabdl2\t} } } */
+/* { dg-final { scan-assembler-not {sabdl2\t} } } */
+
+/* { dg-final { scan-assembler-not {usubw2\t} } } */
+/* { dg-final { scan-assembler-not {ssubw2\t} } } */
+
+/* { dg-final { scan-assembler-not {uaddw2\t} } } */
+/* { dg-final { scan-assembler-not {saddw2\t} } } */
+
+/* { dg-final { scan-assembler-not {umlal2\t} } } */
+/* { dg-final { scan-assembler-not {smlal2\t} } } */
+
+/* { dg-final { scan-assembler-not {umlsl2\t} } } */
+/* { dg-final { scan-assembler-not {smlsl2\t} } } */
+
+/* { dg-final { scan-assembler-not {sqdmull2\t} } } */
+
+/* { dg-final { scan-assembler-not {sqdmlal2\t} } } */
+
+/* { dg-final { scan-assembler-not {sqdmlsl2\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_3.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_3.c
new file mode 100644
index 0000000..3baf826
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_3.c
@@ -0,0 +1,83 @@
+/* { dg-do compile } */
+/* { dg-options "-O" } */
+
+/* PR117850 */
+
+/* We should use the highpart instruction where doing so would avoid data
+   movement instructions.  We avoid a DUP here after extending the
+   VECTOR_CSTs to 128-bits.  */
+
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+
+#define TEST_BIN_HIGHPART_A1(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_a1_##FN##_##SUFF (INTYPE a)		\
+  {                                                     \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			\
+    return FN##_##SUFF (vget_high_##SUFF (a),		\
+			vget_high_##SUFF (b));		\
+  }
+
+#define TEST_BIN_HIGHPART_A2(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_a2_##FN##_##SUFF (INTYPE a)		\
+  {                                                     \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			\
+    return FN##_##SUFF (vget_high_##SUFF (b),		\
+			vget_high_##SUFF (a));		\
+  }
+
+#define TEST_TERN_HIGHPART_A1(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_a1_##FN##_##SUFF (RETTYPE a, INTYPE b)	    \
+  {                                                         \
+    INTYPE c = vdupq_n_##SUFF (0x1A);			    \
+    return FN##_##SUFF (a, vget_high_##SUFF (b),	    \
+			vget_high_##SUFF (c));		    \
+  }
+
+#define TEST_TERN_HIGHPART_A2(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_a2_##FN##_##SUFF (RETTYPE a, INTYPE b)	    \
+  {                                                         \
+    INTYPE c = vdupq_n_##SUFF (0x1A);			    \
+    return FN##_##SUFF (a, vget_high_##SUFF (c),	    \
+			vget_high_##SUFF (b));		    \
+  }
+
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  TEST_BIN_HIGHPART_A1 (FN, RETTYPE, INTYPE, SUFF)              \
+  TEST_BIN_HIGHPART_A2 (FN, RETTYPE, INTYPE, SUFF)
+
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  TEST_TERN_HIGHPART_A1 (FN, RETTYPE, INTYPE, SUFF)              \
+  TEST_TERN_HIGHPART_A2 (FN, RETTYPE, INTYPE, SUFF)
+
+
+#include "fold_to_highpart_1.c"
+
+/* { dg-final { scan-assembler-not {dup\t} } } */
+
+/* { dg-final { scan-assembler-times {smull2\t} 6} } */
+/* { dg-final { scan-assembler-times {umull2\t} 6} } */
+/* { dg-final { scan-assembler-times {pmull2\t} 2} } */
+
+/* { dg-final { scan-assembler-times {saddl2\t} 6} } */
+/* { dg-final { scan-assembler-times {uaddl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {ssubl2\t} 6} } */
+/* { dg-final { scan-assembler-times {usubl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {sabdl2\t} 6} } */
+/* { dg-final { scan-assembler-times {uabdl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {smlal2\t} 6} } */
+/* { dg-final { scan-assembler-times {umlal2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {smlsl2\t} 6} } */
+/* { dg-final { scan-assembler-times {umlsl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {sqdmull2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlal2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlsl2\t} 4} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_4.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_4.c
new file mode 100644
index 0000000..046c7a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_4.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_little_endian } */
+/* { dg-options "-O -fdump-tree-optimized" } */
+
+#include "arm_neon.h"
+
+#define VEC_CST_u8  0x0102030405060708
+#define VEC_CST_u16 0x0001000200030004
+#define VEC_CST_u32 0x0000000100000002
+
+/* Extend the 64b VECTOR_CST to the type required by the hi builtin.  */
+
+uint16x8_t
+test_u8 (uint8x16_t a)
+{
+  const uint8x8_t b = vcreate_u8 (VEC_CST_u8);
+  return vmull_u8 (vget_high_u8 (a), b);
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 8, 7, 6, 5, 4, 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, 1 \}" 1 "optimized" } } */
+
+uint32x4_t
+test_u16 (uint16x8_t a)
+{
+  const uint16x4_t b = vcreate_u16 (VEC_CST_u16);
+  return vmull_u16 (vget_high_u16 (a), b);
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 4, 3, 2, 1, 4, 3, 2, 1 \}" 1 "optimized" } } */
+
+uint64x2_t
+test_u32 (uint32x4_t a)
+{
+  const uint32x2_t b = vcreate_u32 (VEC_CST_u32);
+  return vmull_u32 (vget_high_u32 (a), b);
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 2, 1, 2, 1 \}" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_5.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_5.c
new file mode 100644
index 0000000..4f39b67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_5.c
@@ -0,0 +1,93 @@
+/* { dg-do compile } */
+/* { dg-options "-O -march=armv9-a+bf16" } */
+
+/* Test that we can still fold when the base type of the vector who's
+   highpart we are referring to is incompatible with that of the hi
+   builtin.
+
+   Use float64x2_t as it is never INTYPE.  */
+
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (float64x2_t a)	     \
+  {                                                  \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (a);	     \
+    return FN##_##SUFF(vget_high_##SUFF (x));	     \
+  }
+
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)       \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, float64x2_t b)	      \
+  {                                                           \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (b);		      \
+    return FN##_##SUFF (a, vget_high_##SUFF (x));	      \
+  }
+
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_##FN##_##SUFF (float64x2_t a)		   \
+  {                                                        \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (a);		   \
+    return FN##_##SUFF (vget_high_##SUFF (x), x[1]);	   \
+  }
+
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)         \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, float64x2_t b)		 \
+  {                                                              \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (b);			 \
+    return FN##_##SUFF (a, vget_high_##SUFF (x), x[1]);	 \
+  }
+
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF)   \
+  RETTYPE test_##FN##_##SUFF (float64x2_t a, float64x2_t b)	  \
+  {                                                               \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (a);			  \
+    INTYPE y = vreinterpretq_##SUFF##_f64 (b);			  \
+    return FN##_##SUFF (vget_high_##SUFF (x),			  \
+			vget_high_##SUFF (y));			  \
+  }
+
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF)	\
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, float64x2_t b, float64x2_t c)	\
+  {									\
+    INTYPE x = vreinterpretq_##SUFF##_f64 (b);				\
+    INTYPE y = vreinterpretq_##SUFF##_f64 (c);				\
+    return FN##_##SUFF (a, vget_high_## SUFF (x),                       \
+			vget_high_## SUFF (y));				\
+  }
+
+#include "fold_to_highpart_1.c"
+
+/* { dg-final { scan-assembler-times {sxtl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uxtl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {smull2\t} 5} } */
+/* { dg-final { scan-assembler-times {umull2\t} 5} } */
+/* { dg-final { scan-assembler-times {pmull2\t} 1} } */
+
+/* { dg-final { scan-assembler-times {saddl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uaddl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {ssubl2\t} 3} } */
+/* { dg-final { scan-assembler-times {usubl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {sabdl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uabdl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {saddw2\t} 3} } */
+/* { dg-final { scan-assembler-times {uaddw2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {ssubw2\t} 3} } */
+/* { dg-final { scan-assembler-times {usubw2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {sabdl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uabdl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {smlal2\t} 5} } */
+/* { dg-final { scan-assembler-times {umlal2\t} 5} } */
+
+/* { dg-final { scan-assembler-times {smlsl2\t} 5} } */
+/* { dg-final { scan-assembler-times {umlsl2\t} 5} } */
+
+/* { dg-final { scan-assembler-times {sqdmull2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlal2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlsl2\t} 4} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_6.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_6.c
new file mode 100644
index 0000000..3570d4d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_6.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_little_endian } */
+/* { dg-options "-O2 -march=armv8-a+sve" } */
+
+#include <arm_neon_sve_bridge.h>
+
+typedef int16_t int16x16_t __attribute__ ((vector_size (32)));
+
+/* Edge cases where we don't/can't fold, reject these gracefully.  */
+
+int8x16_t z;
+
+int16x8_t
+test_addressable ()
+{
+  return vmovl_s8 (vget_high_s8 (z));
+}
+
+int16x8_t
+test_scalable_type (svint8_t scalable)
+{
+  return vmovl_s8 (vget_high_s8 (svget_neonq_s8 (scalable)));
+}
+
+int16x8_t
+test_scalar_type (__int128_t foo)
+{
+  return vmovl_s8 (vget_high_s8 (vreinterpretq_s8_p128 (foo)));
+}
+
+int32x4_t
+test_256b_type (int16x16_t foo)
+{
+  return vmovl_s16 ((int16x4_t) { foo[4], foo[5], foo[6], foo[7] });
+}
+
+/* { dg-final { scan-assembler-not {sxtl2\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c b/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c
index a3fd9b8..79d1ccf 100644
--- a/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c
@@ -1016,7 +1016,12 @@ mfloat8x8_t test_set_lane3(mfloat8x8_t a, const mfloat8_t *ptr)
 
 /*
 ** test_set_lane4:
+**	(
 **	ins	v0.b\[6\], wzr
+**	|
+**	movi?	[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0
+**	ins	v0.b\[6\], v(\1).b\[0\]
+**	)
 **	ret
 */
 mfloat8x8_t test_set_lane4(mfloat8x8_t a)
@@ -1056,7 +1061,12 @@ mfloat8x16_t test_setq_lane3(mfloat8x16_t a, const mfloat8_t *ptr)
 
 /*
 ** test_setq_lane4:
+**	(
 **	ins	v0.b\[14\], wzr
+**	|
+**	movi?	[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0
+**	ins	v0.b\[14\], v(\1).b\[0\]
+**	)
 **	ret
 */
 mfloat8x16_t test_setq_lane4(mfloat8x16_t a)
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/vabal_combine.c b/gcc/testsuite/gcc.target/aarch64/simd/vabal_combine.c
deleted file mode 100644
index c51878a..0000000
--- a/gcc/testsuite/gcc.target/aarch64/simd/vabal_combine.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* { dg-do compile } */
-/* { dg-options "-O" } */
-/* { dg-final { check-function-bodies "**" "" "" } } */
-
-#include <arm_neon.h>
-
-/*
-** test_vabal_s8:
-**      sabal2	v0.8h, v2.16b, v1.16b
-**      ret
-*/
-int16x8_t
-test_vabal_s8 (int16x8_t sadv, int8x16_t pv, int8x16_t sv)
-{
-  return vabal_s8 (sadv, vget_high_s8 (pv), vget_high_s8 (sv));
-}
-
-/*
-** test_vabal_u8:
-**      uabal2	v0.8h, v2.16b, v1.16b
-**      ret
-*/
-uint16x8_t
-test_vabal_u8 (uint16x8_t sadv, uint8x16_t pv, uint8x16_t sv)
-{
-  return vabal_u8 (sadv, vget_high_u8 (pv), vget_high_u8 (sv));
-}
-
-/*
-** test_vabal_s16:
-**      sabal2	v0.4s, v2.8h, v1.8h
-**      ret
-*/
-int32x4_t
-test_vabal_s16 (int32x4_t sadv, int16x8_t pv, int16x8_t sv)
-{
-  return vabal_s16 (sadv, vget_high_s16 (pv), vget_high_s16 (sv));
-}
-
-/*
-** test_vabal_u16:
-**      uabal2	v0.4s, v2.8h, v1.8h
-**      ret
-*/
-uint32x4_t
-test_vabal_u16 (uint32x4_t sadv, uint16x8_t pv, uint16x8_t sv)
-{
-  return vabal_u16 (sadv, vget_high_u16 (pv), vget_high_u16 (sv));
-}
-
-/*
-** test_vabal_s32:
-**      sabal2	v0.2d, v2.4s, v1.4s
-**      ret
-*/
-int64x2_t
-test_vabal_s32 (int64x2_t sadv, int32x4_t pv, int32x4_t sv)
-{
-  return vabal_s32 (sadv, vget_high_s32 (pv), vget_high_s32 (sv));
-}
-
-/*
-** test_vabal_u32:
-**      uabal2	v0.2d, v2.4s, v1.4s
-**      ret
-*/
-uint64x2_t
-test_vabal_u32 (uint64x2_t sadv, uint32x4_t pv, uint32x4_t sv)
-{
-  return vabal_u32 (sadv, vget_high_u32 (pv), vget_high_u32 (sv));
-}
-
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c
index 98922aa..3a63da7 100644
--- a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c
@@ -1,5 +1,5 @@
 // { dg-options "-O -fomit-frame-pointer -fno-optimize-sibling-calls -funwind-tables" }
-// { dg-final { check-function-bodies "**" "" } }
+// { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {\t\.inst} } }
 
 void ns_callee ();
  void s_callee () [[arm::streaming]];
@@ -218,7 +218,7 @@ sc_caller_x1 (int *ptr, int a) [[arm::streaming_compatible]]
 **	bl	ns_callee_stack
 **	ldr	x16, \[x29, #?16\]
 **	tbz	x16, 0, .*
-**	smstart	sm
+**	.inst 0xd503437f // smstart sm
 **	...
 */
 void
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c
index ee6f987..c72d03f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c
+++ b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c
@@ -1,5 +1,6 @@
 // { dg-options "-O -fomit-frame-pointer -fno-optimize-sibling-calls -funwind-tables -mtrack-speculation" }
-// { dg-final { check-function-bodies "**" "" } }
+// { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {\t\.inst} } }
+
 
 void ns_callee ();
  void s_callee () [[arm::streaming]];
@@ -196,7 +197,7 @@ sc_caller_x1 (int *ptr, int a) [[arm::streaming_compatible]]
 **	tst	x16, #?1
 **	beq	[^\n]*
 **	csel	x15, x15, xzr, ne
-**	smstart	sm
+**	.inst 0xd503437f // smstart sm
 **	...
 */
 void
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/pr121028.c b/gcc/testsuite/gcc.target/aarch64/sme/pr121028.c
new file mode 100644
index 0000000..a6aa119
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/pr121028.c
@@ -0,0 +1,46 @@
+// PR121028
+// { dg-do assemble { target aarch64_asm_sme_ok } }
+// { dg-options "-O --save-temps" }
+// { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {\t\.inst} } }
+
+void ns_callee ();
+
+/*
+** sc_caller_sme:
+**	...
+**	mrs	x16, svcr
+**	str	x16, \[x29, #?16\]
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	smstop	sm
+**	bl	ns_callee
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	smstart	sm
+**	...
+*/
+void sc_caller_sme() __arm_streaming_compatible
+{
+    ns_callee ();
+}
+
+#pragma GCC target "+nosme"
+
+/*
+** sc_caller_nosme:
+**	...
+**	bl	__arm_sme_state
+**	str	x0, \[x29, #?16\]
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	.inst 0xd503427f // smstop sm
+**	bl	ns_callee
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	.inst 0xd503437f // smstart sm
+**	...
+*/
+void sc_caller_nosme() __arm_streaming_compatible
+{
+    ns_callee ();
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c
new file mode 100644
index 0000000..b9fd96a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c
@@ -0,0 +1,99 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.h - z29\.h}
+** |
+**	famax	[^\n]+, {z28\.h - z29\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	famax	{z18\.h - z19\.h}, {z18\.h - z19\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat16x2_t, z18,
+	 svamax_f16_x2 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z18\.h - z19\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z18, svfloat16x2_t, z23,
+	 svamax_f16_x2 (z23, z18),
+	 svamax (z23, z18))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.h - z29\.h}, {z28\.h - z29\.h}, {z0\.h - z1\.h}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat16x2_t, z28,
+	 svamax_f16_x2 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+**	famax	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z18\.h - z19\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+** |
+**	famax	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat16x2_t, z4,
+	 svamax_f16_x2 (z4, z23),
+	 svamax (z4, z23))
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c
new file mode 100644
index 0000000..70e2697
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.h - z31\.h}
+** |
+**	famax	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z4\.h - z7\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat16x4_t, z18,
+	 svamax_f16_x4 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z28, svfloat16x4_t, z23,
+	 svamax_f16_x4 (z23, z28),
+	 svamax (z23, z28))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.h - z31\.h}, {z28\.h - z31\.h}, {z0\.h - z3\.h}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat16x4_t, z28,
+	 svamax_f16_x4 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+** |
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+** |
+**	famax	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat16x4_t, z4,
+	 svamax_f16_x4 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c
new file mode 100644
index 0000000..cf57d1b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.s - z29\.s}
+** |
+**	famax	[^\n]+, {z28\.s - z29\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	famax	{z18\.s - z19\.s}, {z18\.s - z19\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat32x2_t, z18,
+	 svamax_f32_x2 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z18\.s - z19\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z18, svfloat32x2_t, z23,
+	 svamax_f32_x2 (z23, z18),
+	 svamax (z23, z18))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.s - z29\.s}, {z28\.s - z29\.s}, {z0\.s - z1\.s}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat32x2_t, z28,
+	 svamax_f32_x2 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+**	famax	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z18\.s - z19\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+** |
+**	famax	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat32x2_t, z4,
+	 svamax_f32_x2 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c
new file mode 100644
index 0000000..10d9175
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c
@@ -0,0 +1,131 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.s - z31\.s}
+** |
+**	famax	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z4\.s - z7\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat32x4_t, z18,
+	 svamax_f32_x4 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z28, svfloat32x4_t, z23,
+	 svamax_f32_x4 (z23, z28),
+	 svamax (z23, z28))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.s - z31\.s}, {z28\.s - z31\.s}, {z0\.s - z3\.s}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat32x4_t, z28,
+	 svamax_f32_x4 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+** |
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+** |
+**	famax	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat32x4_t, z4,
+	 svamax_f32_x4 (z4, z23),
+	 svamax (z4, z23))
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c
new file mode 100644
index 0000000..b7918ab
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.d - z29\.d}
+** |
+**	famax	[^\n]+, {z28\.d - z29\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	famax	{z18\.d - z19\.d}, {z18\.d - z19\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat64x2_t, z18,
+	 svamax_f64_x2 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z18\.d - z19\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z18, svfloat64x2_t, z23,
+	 svamax_f64_x2 (z23, z18),
+	 svamax (z23, z18))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.d - z29\.d}, {z28\.d - z29\.d}, {z0\.d - z1\.d}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat64x2_t, z28,
+	 svamax_f64_x2 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+**	famax	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z18\.d - z19\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+** |
+**	famax	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat64x2_t, z4,
+	 svamax_f64_x2 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c
new file mode 100644
index 0000000..153a37a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.d - z31\.d}
+** |
+**	famax	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z4\.d - z7\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat64x4_t, z18,
+	 svamax_f64_x4 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z28, svfloat64x4_t, z23,
+	 svamax_f64_x4 (z23, z28),
+	 svamax (z23, z28))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.d - z31\.d}, {z28\.d - z31\.d}, {z0\.d - z3\.d}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat64x4_t, z28,
+	 svamax_f64_x4 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+** |
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+** |
+**	famax	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat64x4_t, z4,
+	 svamax_f64_x4 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c
new file mode 100644
index 0000000..bd6e13b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.h - z29\.h}
+** |
+**	famin	[^\n]+, {z28\.h - z29\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	famin	{z18\.h - z19\.h}, {z18\.h - z19\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat16x2_t, z18,
+	 svamin_f16_x2 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z18\.h - z19\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z18, svfloat16x2_t, z23,
+	 svamin_f16_x2 (z23, z18),
+	 svamin (z23, z18))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.h - z29\.h}, {z28\.h - z29\.h}, {z0\.h - z1\.h}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat16x2_t, z28,
+	 svamin_f16_x2 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+**	famin	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z18\.h - z19\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+** |
+**	famin	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat16x2_t, z4,
+	 svamin_f16_x2 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c
new file mode 100644
index 0000000..9f71b1f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.h - z31\.h}
+** |
+**	famin	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z4\.h - z7\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat16x4_t, z18,
+	 svamin_f16_x4 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z28, svfloat16x4_t, z23,
+	 svamin_f16_x4 (z23, z28),
+	 svamin (z23, z28))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.h - z31\.h}, {z28\.h - z31\.h}, {z0\.h - z3\.h}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat16x4_t, z28,
+	 svamin_f16_x4 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+** |
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+** |
+**	famin	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat16x4_t, z4,
+	 svamin_f16_x4 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c
new file mode 100644
index 0000000..aaa6a2e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat32x2_t, z0,
+ svamin_f32_x2 (z0, z4),
+ svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat32x2_t, z0,
+ svamin_f32_x2 (z4, z0),
+ svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.s - z29\.s}
+** |
+**	famin	[^\n]+, {z28\.s - z29\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat32x2_t, z0,
+ svamin_f32_x2 (z4, z28),
+ svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	famin	{z18\.s - z19\.s}, {z18\.s - z19\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat32x2_t, z18,
+ svamin_f32_x2 (z18, z4),
+ svamin (z18, z4))
+
+/*
+** amin_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z18\.s - z19\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z18, svfloat32x2_t, z23,
+ svamin_f32_x2 (z23, z18),
+ svamin (z23, z18))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.s - z29\.s}, {z28\.s - z29\.s}, {z0\.s - z1\.s}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat32x2_t, z28,
+ svamin_f32_x2 (z28, z0),
+ svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+**	famin	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z18\.s - z19\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat32x2_t, z0,
+ svamin_f32_x2 (z0, z18),
+ svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+** |
+**	famin	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat32x2_t, z4,
+ svamin_f32_x2 (z4, z23),
+ svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c
new file mode 100644
index 0000000..34c1098
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.s - z31\.s}
+** |
+**	famin	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z4\.s - z7\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat32x4_t, z18,
+	 svamin_f32_x4 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z28, svfloat32x4_t, z23,
+	 svamin_f32_x4 (z23, z28),
+	 svamin (z23, z28))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.s - z31\.s}, {z28\.s - z31\.s}, {z0\.s - z3\.s}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat32x4_t, z28,
+	 svamin_f32_x4 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+** |
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+** |
+**	famin	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat32x4_t, z4,
+	 svamin_f32_x4 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c
new file mode 100644
index 0000000..e4138e0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.d - z29\.d}
+** |
+**	famin	[^\n]+, {z28\.d - z29\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	famin	{z18\.d - z19\.d}, {z18\.d - z19\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat64x2_t, z18,
+	 svamin_f64_x2 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z18\.d - z19\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z18, svfloat64x2_t, z23,
+	 svamin_f64_x2 (z23, z18),
+	 svamin (z23, z18))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.d - z29\.d}, {z28\.d - z29\.d}, {z0\.d - z1\.d}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat64x2_t, z28,
+	 svamin_f64_x2 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+**	famin	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z18\.d - z19\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+** |
+**	famin	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat64x2_t, z4,
+	 svamin_f64_x2 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c
new file mode 100644
index 0000000..8fbabe7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.d - z31\.d}
+** |
+**	famin	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z4\.d - z7\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat64x4_t, z18,
+	 svamin_f64_x4 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z28, svfloat64x4_t, z23,
+	 svamin_f64_x4 (z23, z28),
+	 svamin (z23, z28))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.d - z31\.d}, {z28\.d - z31\.d}, {z0\.d - z3\.d}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat64x4_t, z28,
+	 svamin_f64_x4 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+** |
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+** |
+**	famin	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat64x4_t, z4,
+	 svamin_f64_x4 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acge_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acge_1.c
new file mode 100644
index 0000000..37428a8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acge_1.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	facge	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacge (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	facge	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacge (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	facge	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacge (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	facge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacge (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	facge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svacge (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	facge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacge (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	facge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svacge (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acgt_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acgt_1.c
new file mode 100644
index 0000000..5829369
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acgt_1.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	facgt	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacgt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	facgt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacgt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	facgt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacgt (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	facgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacgt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	facgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svacgt (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	facgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacgt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	facgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svacgt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acle_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acle_1.c
new file mode 100644
index 0000000..bd5200e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/acle_1.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	facle	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacle (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	facle	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacle (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	facle	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacle (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	facle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacle (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	facle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svacle (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	facle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svacle (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	facle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svacle (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/aclt_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/aclt_1.c
new file mode 100644
index 0000000..876aba9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/aclt_1.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	faclt	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svaclt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	faclt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svaclt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	faclt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svaclt (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	faclt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svaclt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	faclt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svaclt (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	faclt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svaclt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	faclt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svaclt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_1.c
index dd8f6c4..d6aabc8 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_1.c
@@ -18,5 +18,57 @@ test2 (svbool_t pg, svint8_t x, svint64_t y, int *any)
   return svptest_any (pg, res);
 }
 
-/* { dg-final { scan-assembler-times {\tcmpeq\t} 2 } } */
+void
+test3 (svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpeq_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svint8_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpeq_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpeq_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpeq_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint32_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpeq_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint32_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpeq_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpeq\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
 /* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_2.c
index 028d375..df98d27 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_2.c
@@ -33,6 +33,108 @@ test4 (svbool_t pg, svint8_t x, int *any)
   return svptest_any (pg, res);
 }
 
-/* { dg-final { scan-assembler-times {\tcmpeq\t} 4 } } */
-/* { dg-final { scan-assembler-times {\tcmpeq\t[^\n]*, #10} 2 } } */
+void
+test5 (svint16_t x, svint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpeq (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpeq (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpeq (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpeq (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svint32_t x, svint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpeq (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpeq (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpeq (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpeq (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svint64_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpeq (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpeq (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpeq (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpeq (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpeq\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmpeq\t[^\n]*, #10} 8 } } */
 /* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_4.c
new file mode 100644
index 0000000..8e4b931
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_4.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpeq	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpeq (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmpeq	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpeq (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmpeq	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpeq (pg, x, y), pg);
+}
+
+/*
+** test4:
+**	cmpeq	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpeq (pg, x, 10), p0);
+}
+
+/*
+** test5:
+**	cmpeq	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpeq (pg, x, 10), p0);
+}
+
+/*
+** test6:
+**	cmpeq	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpeq (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmpeq	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpeq (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmpeq	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpeq (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmpeq	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpeq (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmpeq	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpeq (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmpeq	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpeq (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmpeq	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpeq (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_5.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_5.c
new file mode 100644
index 0000000..2958bc2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_5.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpeq	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmpeq	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmpeq	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_6.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_6.c
new file mode 100644
index 0000000..9233de9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpeq_6.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	fcmeq	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	fcmeq	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	fcmeq	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	fcmeq	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	fcmeq	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpeq (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	fcmeq	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpeq (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	fcmeq	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpeq (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_1.c
new file mode 100644
index 0000000..f6bb3c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_1.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint8_t y, int *any)
+{
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svint8_t x, int *any)
+{
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svint32_t x, svint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svint64_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpge\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmpge\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_2.c
new file mode 100644
index 0000000..fc92291
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_2.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint8_t y, int *any)
+{
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svuint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svuint8_t x, int *any)
+{
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svuint32_t x, svuint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svuint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svuint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svuint64_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svuint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svuint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpge (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmphs\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmphs\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_3.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_3.c
new file mode 100644
index 0000000..6d50df5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_3.c
@@ -0,0 +1,169 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpge	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmpge	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmpge	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, y), pg);
+}
+
+/*
+** test4:
+** (
+**	cmpge	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmpgt	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test5:
+** (
+**	cmpge	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmpgt	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test6:
+** (
+**	cmpge	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmpgt	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmpge	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmpge	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmpge	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmpge	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmpge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmpge	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_4.c
new file mode 100644
index 0000000..2430e80
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_4.c
@@ -0,0 +1,169 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmphs	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmphs	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmphs	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, y), pg);
+}
+
+/*
+** test4:
+** (
+**	cmphs	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmphi	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test5:
+** (
+**	cmphs	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmphi	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test6:
+** (
+**	cmphs	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmphi	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpge (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmphs	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmphs	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmphs	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmphs	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svuint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmphs	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpge (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmphs	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svuint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpge (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_5.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_5.c
new file mode 100644
index 0000000..f4fa758
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_5.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint64_t y, int *any)
+{
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svint8_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint32_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint32_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpge\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_6.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_6.c
new file mode 100644
index 0000000..979db4c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_6.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint32_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint32_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpge_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmphs\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_7.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_7.c
new file mode 100644
index 0000000..d6abab0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_7.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpge	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmpge	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmpge	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_8.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_8.c
new file mode 100644
index 0000000..70be917
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_8.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmphs	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svuint16_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmphs	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmphs	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_9.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_9.c
new file mode 100644
index 0000000..0d4140e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpge_9.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	fcmge	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	fcmge	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	fcmge	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	fcmge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	fcmge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpge (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	fcmge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpge (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	fcmge	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpge (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_1.c
new file mode 100644
index 0000000..6c28d6f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_1.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint8_t y, int *any)
+{
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svint8_t x, int *any)
+{
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svint32_t x, svint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svint64_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpgt\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmpgt\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_2.c
new file mode 100644
index 0000000..2160484
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_2.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint8_t y, int *any)
+{
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svuint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svuint8_t x, int *any)
+{
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svuint32_t x, svuint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svuint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svuint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svuint64_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svuint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svuint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpgt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmphi\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmphi\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_3.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_3.c
new file mode 100644
index 0000000..cc48b7e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_3.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpgt	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmpgt	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmpgt	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, y), pg);
+}
+
+/*
+** test4:
+**	cmpgt	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test5:
+**	cmpgt	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test6:
+**	cmpgt	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmpgt	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmpgt	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmpgt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmpgt	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmpgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmpgt	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_4.c
new file mode 100644
index 0000000..bd49fe8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_4.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmphi	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmphi	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmphi	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, y), pg);
+}
+
+/*
+** test4:
+**	cmphi	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test5:
+**	cmphi	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test6:
+**	cmphi	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpgt (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmphi	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmphi	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmphi	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmphi	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svuint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmphi	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpgt (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmphi	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svuint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpgt (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_5.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_5.c
new file mode 100644
index 0000000..f9f4c7d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_5.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint64_t y, int *any)
+{
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svint8_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint32_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint32_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpgt\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_6.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_6.c
new file mode 100644
index 0000000..6df15b9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_6.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint32_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint32_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpgt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmphi\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_7.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_7.c
new file mode 100644
index 0000000..0656b29
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_7.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpgt	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmpgt	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmpgt	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_8.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_8.c
new file mode 100644
index 0000000..b0a9ac8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_8.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmphi	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svuint16_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmphi	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmphi	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_9.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_9.c
new file mode 100644
index 0000000..dcd84f7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpgt_9.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	fcmgt	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	fcmgt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	fcmgt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	fcmgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	fcmgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpgt (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	fcmgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpgt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	fcmgt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpgt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_1.c
new file mode 100644
index 0000000..f2d7d2b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_1.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint8_t y, int *any)
+{
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svint8_t x, int *any)
+{
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svint32_t x, svint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svint64_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmple\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmple\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_2.c
new file mode 100644
index 0000000..9d13d7a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_2.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint8_t y, int *any)
+{
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svuint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svuint8_t x, int *any)
+{
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svuint32_t x, svuint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svuint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svuint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svuint64_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svuint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svuint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmple (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpls\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmpls\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_3.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_3.c
new file mode 100644
index 0000000..7a9326c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_3.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmple	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmple	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmple	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, y), pg);
+}
+
+/*
+** test4:
+**	cmple	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test5:
+**	cmple	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test6:
+**	cmple	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmple	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmple	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmple	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmple	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmple	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmple	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_4.c
new file mode 100644
index 0000000..aca4385
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_4.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpls	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmpls	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmpls	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, y), pg);
+}
+
+/*
+** test4:
+**	cmpls	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test5:
+**	cmpls	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test6:
+**	cmpls	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmple (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmpls	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmpls	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmpls	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmpls	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svuint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmpls	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmple (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmpls	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svuint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmple (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_5.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_5.c
new file mode 100644
index 0000000..1caf496
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_5.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint64_t y, int *any)
+{
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svint8_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint32_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint32_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmple\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_6.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_6.c
new file mode 100644
index 0000000..ae85e89
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_6.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint32_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint32_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmple_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpls\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_7.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_7.c
new file mode 100644
index 0000000..3f3ea53
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_7.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmple	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmple	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmple	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_8.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_8.c
new file mode 100644
index 0000000..01281ca
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_8.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpls	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svuint16_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmpls	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmpls	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_9.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_9.c
new file mode 100644
index 0000000..8d008b4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmple_9.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	fcmle	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	fcmle	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	fcmle	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	fcmle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	fcmle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmple (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	fcmle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmple (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	fcmle	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmple (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_1.c
new file mode 100644
index 0000000..a15bb4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_1.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint8_t y, int *any)
+{
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svint8_t x, int *any)
+{
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svint32_t x, svint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svint64_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmplt\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmplt\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_2.c
new file mode 100644
index 0000000..43c53a1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_2.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint8_t y, int *any)
+{
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svuint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svuint8_t x, int *any)
+{
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svuint32_t x, svuint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svuint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svuint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svuint64_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svuint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svuint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmplt (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmplo\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmplo\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_3.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_3.c
new file mode 100644
index 0000000..bddbbeb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_3.c
@@ -0,0 +1,169 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmplt	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmplt	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmplt	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, y), pg);
+}
+
+/*
+** test4:
+** (
+**	cmplt	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmple	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test5:
+** (
+**	cmplt	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmple	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test6:
+** (
+**	cmplt	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmple	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmplt	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmplt	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmplt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmplt	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmplt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmplt	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_4.c
new file mode 100644
index 0000000..b71c8e9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_4.c
@@ -0,0 +1,169 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmplo	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmplo	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmplo	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svuint8_t x, svuint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, y), pg);
+}
+
+/*
+** test4:
+** (
+**	cmplo	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmpls	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test5:
+** (
+**	cmplo	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmpls	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test6:
+** (
+**	cmplo	p0\.b, p0/z, z0\.b, #10
+** |
+**	cmpls	p0\.b, p0/z, z0\.b, #9
+** )
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svuint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmplt (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmplo	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svuint16_t x, svuint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmplo	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svuint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmplo	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svuint32_t x, svuint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmplo	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svuint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmplo	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svuint64_t x, svuint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmplt (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmplo	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svuint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmplt (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_5.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_5.c
new file mode 100644
index 0000000..6885e4d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_5.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint64_t y, int *any)
+{
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svint8_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint32_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint32_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmplt\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_6.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_6.c
new file mode 100644
index 0000000..e9be9e8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_6.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svuint8_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svuint8_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svuint16_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svuint16_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svuint32_t x, svuint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svuint32_t x, svuint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmplt_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmplo\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_7.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_7.c
new file mode 100644
index 0000000..a4de6ab
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_7.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmplt	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmplt	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmplt	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_8.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_8.c
new file mode 100644
index 0000000..0a095eb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_8.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmplo	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svuint16_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmplo	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmplo	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svuint32_t x, svuint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_9.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_9.c
new file mode 100644
index 0000000..4f4b7b4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmplt_9.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	fcmlt	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	fcmlt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	fcmlt	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	fcmlt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	fcmlt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmplt (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	fcmlt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmplt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	fcmlt	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmplt (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_1.c
new file mode 100644
index 0000000..61f7718
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_1.c
@@ -0,0 +1,140 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint8_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpne (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint8_t y, int *any)
+{
+  svbool_t res = svcmpne (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svbool_t pg, svint8_t x, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpne (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svbool_t pg, svint8_t x, int *any)
+{
+  svbool_t res = svcmpne (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint16_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpne (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpne (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint16_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpne (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpne (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test9 (svint32_t x, svint32_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpne (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test10 (svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpne (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test11 (svint32_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpne (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test12 (svint32_t x)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpne (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+void
+test13 (svint64_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpne (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test14 (svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpne (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test15 (svint64_t x, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpne (pg, x, 10);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test16 (svint64_t x)
+{
+  svbool_t pg = svptrue_b64 ();
+  svbool_t res = svcmpne (pg, x, 10);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpne\t} 16 } } */
+/* { dg-final { scan-assembler-times {\tcmpne\t[^\n]*, #10} 8 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_2.c
new file mode 100644
index 0000000..53cedb3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_2.c
@@ -0,0 +1,157 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpne	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test1 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpne (pg, x, y), p0);
+}
+
+/*
+** test2:
+**	cmpne	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test2 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpne (pg, x, y), p0);
+}
+
+/*
+** test3:
+**	cmpne	p0\.b, p0/z, z0\.b, z1\.b
+**	ret
+*/
+svbool_t
+test3 (svbool_t p0, svint8_t x, svint8_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpne (pg, x, y), pg);
+}
+
+/*
+** test4:
+**	cmpne	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test4 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpne (pg, x, 10), p0);
+}
+
+/*
+** test5:
+**	cmpne	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test5 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpne (pg, x, 10), p0);
+}
+
+/*
+** test6:
+**	cmpne	p0\.b, p0/z, z0\.b, #10
+**	ret
+*/
+svbool_t
+test6 (svbool_t p0, svint8_t x)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (p0, svcmpne (pg, x, 10), pg);
+}
+
+/*
+** test7:
+**	cmpne	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test7 (svbool_t p0, svint16_t x, svint16_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpne (pg, x, y), p0);
+}
+
+/*
+** test8:
+**	cmpne	p0\.h, p0/z, z0\.h, #10
+**	ret
+*/
+svbool_t
+test8 (svbool_t p0, svint16_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpne (pg, x, 10), p0);
+}
+
+/*
+** test9:
+**	cmpne	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test9 (svbool_t p0, svint32_t x, svint32_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpne (pg, x, y), p0);
+}
+
+/*
+** test10:
+**	cmpne	p0\.s, p0/z, z0\.s, #10
+**	ret
+*/
+svbool_t
+test10 (svbool_t p0, svint32_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpne (pg, x, 10), p0);
+}
+
+/*
+** test11:
+**	cmpne	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test11 (svbool_t p0, svint64_t x, svint64_t y)
+{
+  svbool_t pg = svptrue_b8 ();
+  return svand_z (pg, svcmpne (pg, x, y), p0);
+}
+
+/*
+** test12:
+**	cmpne	p0\.d, p0/z, z0\.d, #10
+**	ret
+*/
+svbool_t
+test12 (svbool_t p0, svint64_t x)
+{
+  svbool_t pg = svptrue_b16 ();
+  return svand_z (pg, svcmpne (pg, x, 10), p0);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_3.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_3.c
new file mode 100644
index 0000000..c5c3936
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_3.c
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+void
+test1 (svbool_t pg, svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t res = svcmpne_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test2 (svbool_t pg, svint8_t x, svint64_t y, int *any)
+{
+  svbool_t res = svcmpne_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test3 (svint8_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpne_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test4 (svint8_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b8 ();
+  svbool_t res = svcmpne_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test5 (svint16_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpne_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test6 (svint16_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b16 ();
+  svbool_t res = svcmpne_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+void
+test7 (svint32_t x, svint64_t y, int *any, svbool_t *ptr)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpne_wide (pg, x, y);
+  *any = svptest_any (pg, res);
+  *ptr = res;
+}
+
+int
+test8 (svint32_t x, svint64_t y, int *any)
+{
+  svbool_t pg = svptrue_b32 ();
+  svbool_t res = svcmpne_wide (pg, x, y);
+  return svptest_any (pg, res);
+}
+
+/* { dg-final { scan-assembler-times {\tcmpne\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tptrue\t} 6 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_4.c
new file mode 100644
index 0000000..595e024
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_4.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	cmpne	p0\.h, p0/z, z0\.h, z1\.d
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	cmpne	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne_wide (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	cmpne	p0\.s, p0/z, z0\.s, z1\.d
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svint32_t x, svint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne_wide (pg, x, y),
+		  svptrue_b32 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_5.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_5.c
new file mode 100644
index 0000000..94fecd8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpne_5.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	fcmne	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	fcmne	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	fcmne	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	fcmne	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	fcmne	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpne (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	fcmne	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpne (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	fcmne	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpne (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpuo_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpuo_1.c
new file mode 100644
index 0000000..4b124b3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/cmpuo_1.c
@@ -0,0 +1,104 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	fcmuo	p0\.h, p0/z, z0\.h, z1\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svfloat16_t x, svfloat16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpuo (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	fcmuo	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpuo (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	fcmuo	p0\.s, p0/z, z0\.s, z1\.s
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svfloat32_t x, svfloat32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpuo (pg, x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test4:
+**	fcmuo	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpuo (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test5:
+**	fcmuo	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpuo (pg, x, y),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	fcmuo	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svcmpuo (pg, x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test7:
+**	fcmuo	p0\.d, p0/z, z0\.d, z1\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svfloat64_t x, svfloat64_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svcmpuo (pg, x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dup_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dup_1.c
new file mode 100644
index 0000000..c3c4e2d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dup_1.c
@@ -0,0 +1,47 @@
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+svbool_t
+test1 (int x)
+{
+  return svand_z (svptrue_b16 (), svdup_b16 (x), svptrue_b16 ());
+}
+
+svbool_t
+test2 (int x)
+{
+  return svand_z (svptrue_b8 (), svdup_b32 (x), svptrue_b16 ());
+}
+
+svbool_t
+test3 (int x)
+{
+  return svand_z (svptrue_b32 (), svdup_b32 (x), svptrue_b16 ());
+}
+
+svbool_t
+test4 (int x)
+{
+  return svand_z (svptrue_b32 (), svdup_b32 (x), svptrue_b32 ());
+}
+
+svbool_t
+test5 (int x)
+{
+  return svand_z (svptrue_b8 (), svdup_b64 (x), svptrue_b32 ());
+}
+
+svbool_t
+test6 (int x)
+{
+  return svand_z (svptrue_b16 (), svdup_b64 (x), svptrue_b8 ());
+}
+
+svbool_t
+test7 (int x)
+{
+  return svand_z (svptrue_b16 (), svdup_b64 (x), svptrue_b64 ());
+}
+
+/* { dg-final { scan-assembler-not {\tand\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_13.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_13.c
new file mode 100644
index 0000000..6d702b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_13.c
@@ -0,0 +1,45 @@
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+svbool_t
+test1 (int x0, int x1)
+{
+  return svand_z (svptrue_b8 (), svdupq_b64 (x0, x1), svptrue_b16 ());
+}
+
+svbool_t
+test2 (int x0, int x1, int x2, int x3)
+{
+  return svand_z (svptrue_b8 (), svdupq_b32 (x0, x1, x2, x3), svptrue_b16 ());
+}
+
+svbool_t
+test3 (int x0, int x1, int x2, int x3)
+{
+  return svand_z (svptrue_b32 (), svdupq_b32 (x0, x1, x2, x3), svptrue_b16 ());
+}
+
+svbool_t
+test4 (int x0, int x1, int x2, int x3)
+{
+  return svand_z (svptrue_b32 (), svdupq_b32 (x0, x1, x2, x3), svptrue_b32 ());
+}
+
+svbool_t
+test5 (int x0, int x1, int x2, int x3)
+{
+  return svand_z (svptrue_b8 (),
+		  svdupq_b16 (x0, x1, x2, x3, x2, x0, x1, x3),
+		  svptrue_b32 ());
+}
+
+svbool_t
+test6 (int x0, int x1, int x2, int x3)
+{
+  return svand_z (svptrue_b64 (),
+		  svdupq_b16 (x0, x1, x2, x3, x2, x0, x1, x3),
+		  svptrue_b16 ());
+}
+
+/* { dg-final { scan-assembler-not {\tand\tp} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c
index 218a660..13ebb9f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c
@@ -10,6 +10,6 @@ dupq (int x)
   return svdupq_s32 (x, 1, 2, 3);
 }
 
-/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #3, #-1} } } */
+/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #0, #1\n} } } */
 /* { dg-final { scan-assembler {\tins\tv[0-9]+\.s\[0\], w0\n} } } */
 /* { dg-final { scan-assembler {\tdup\tz[0-9]+\.q, z[0-9]+\.q\[0\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c
index cbee6f2..13d27e2 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c
@@ -10,6 +10,6 @@ dupq (int x)
   return svdupq_s32 (0, 1, x, 3);
 }
 
-/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #3, #-1} } } */
+/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #0, #1\n} } } */
 /* { dg-final { scan-assembler {\tins\tv[0-9]+\.s\[2\], w0\n} } } */
 /* { dg-final { scan-assembler {\tdup\tz[0-9]+\.q, z[0-9]+\.q\[0\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_lane_9.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_lane_9.c
new file mode 100644
index 0000000..e3f352b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_lane_9.c
@@ -0,0 +1,8 @@
+/* { dg-options "-O2 -mbig-endian" } */
+
+#pragma GCC aarch64 "arm_sve.h"
+
+svint32_t f(svint32_t x) { return svdupq_lane (x, 17); }
+void g(svint32_t *a, svint32_t *b) { *a = svdupq_lane (*b, 17); }
+
+/* { dg-final { scan-assembler-not {\trevw\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_1.c
new file mode 100644
index 0000000..6b920b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_1.c
@@ -0,0 +1,14 @@
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include <arm_sve.h>
+typedef svbfloat16_t vls_bfloat16_t __attribute__((arm_sve_vector_bits(32 * 8)));
+svbfloat16_t foo(vls_bfloat16_t a, vls_bfloat16_t b)
+{
+  svbfloat16_t zero = svreinterpret_bf16_f32 (svdup_n_f32 (0.0f));
+  return svzip2_bf16(zero, svuzp1_bf16(a,b));
+}
+
+
+/* { dg-final { scan-assembler-times {\tuzp1\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
+/* { dg-final { scan-assembler-not {\ttbl\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_2.c
new file mode 100644
index 0000000..9b3daaa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_2.c
@@ -0,0 +1,96 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	trn1	p0\.h, p[0-3]\.h, \1\.h
+**	ret
+*/
+svbool_t
+test1 ()
+{
+  return svtrn1_b16 (svptrue_b8 (), svptrue_b16 ());
+}
+
+/*
+** test2:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	trn1	p0\.h, \1\.h, p[0-3]\.h
+**	ret
+*/
+svbool_t
+test2 ()
+{
+  return svtrn1_b16 (svptrue_b16 (), svptrue_b8 ());
+}
+
+/*
+** test3:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	trn1	p0\.s, p[0-3]\.s, \1\.s
+**	ret
+*/
+svbool_t
+test3 ()
+{
+  return svtrn1_b32 (svptrue_b8 (), svptrue_b32 ());
+}
+
+/*
+** test4:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	trn1	p0\.s, \1\.s, p[0-3]\.s
+**	ret
+*/
+svbool_t
+test4 ()
+{
+  return svtrn1_b32 (svptrue_b32 (), svptrue_b8 ());
+}
+
+/*
+** test5:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	trn1	p0\.d, p[0-3]\.d, \1\.d
+**	ret
+*/
+svbool_t
+test5 ()
+{
+  return svtrn1_b64 (svptrue_b8 (), svptrue_b64 ());
+}
+
+/*
+** test6:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	trn1	p0\.d, \1\.d, p[0-3]\.d
+**	ret
+*/
+svbool_t
+test6 ()
+{
+  return svtrn1_b64 (svptrue_b64 (), svptrue_b8 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_3.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_3.c
new file mode 100644
index 0000000..678c541
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_3.c
@@ -0,0 +1,96 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	trn2	p0\.h, p[0-3]\.h, \1\.h
+**	ret
+*/
+svbool_t
+test1 ()
+{
+  return svtrn2_b16 (svptrue_b8 (), svptrue_b16 ());
+}
+
+/*
+** test2:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	trn2	p0\.h, \1\.h, p[0-3]\.h
+**	ret
+*/
+svbool_t
+test2 ()
+{
+  return svtrn2_b16 (svptrue_b16 (), svptrue_b8 ());
+}
+
+/*
+** test3:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	trn2	p0\.s, p[0-3]\.s, \1\.s
+**	ret
+*/
+svbool_t
+test3 ()
+{
+  return svtrn2_b32 (svptrue_b8 (), svptrue_b32 ());
+}
+
+/*
+** test4:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	trn2	p0\.s, \1\.s, p[0-3]\.s
+**	ret
+*/
+svbool_t
+test4 ()
+{
+  return svtrn2_b32 (svptrue_b32 (), svptrue_b8 ());
+}
+
+/*
+** test5:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	trn2	p0\.d, p[0-3]\.d, \1\.d
+**	ret
+*/
+svbool_t
+test5 ()
+{
+  return svtrn2_b64 (svptrue_b8 (), svptrue_b64 ());
+}
+
+/*
+** test6:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	trn2	p0\.d, \1\.d, p[0-3]\.d
+**	ret
+*/
+svbool_t
+test6 ()
+{
+  return svtrn2_b64 (svptrue_b64 (), svptrue_b8 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_4.c
new file mode 100644
index 0000000..28c6018
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_4.c
@@ -0,0 +1,96 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	zip1	p0\.h, p[0-3]\.h, \1\.h
+**	ret
+*/
+svbool_t
+test1 ()
+{
+  return svzip1_b16 (svptrue_b8 (), svptrue_b16 ());
+}
+
+/*
+** test2:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	zip1	p0\.h, \1\.h, p[0-3]\.h
+**	ret
+*/
+svbool_t
+test2 ()
+{
+  return svzip1_b16 (svptrue_b16 (), svptrue_b8 ());
+}
+
+/*
+** test3:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	zip1	p0\.s, p[0-3]\.s, \1\.s
+**	ret
+*/
+svbool_t
+test3 ()
+{
+  return svzip1_b32 (svptrue_b8 (), svptrue_b32 ());
+}
+
+/*
+** test4:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	zip1	p0\.s, \1\.s, p[0-3]\.s
+**	ret
+*/
+svbool_t
+test4 ()
+{
+  return svzip1_b32 (svptrue_b32 (), svptrue_b8 ());
+}
+
+/*
+** test5:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	zip1	p0\.d, p[0-3]\.d, \1\.d
+**	ret
+*/
+svbool_t
+test5 ()
+{
+  return svzip1_b64 (svptrue_b8 (), svptrue_b64 ());
+}
+
+/*
+** test6:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	zip1	p0\.d, \1\.d, p[0-3]\.d
+**	ret
+*/
+svbool_t
+test6 ()
+{
+  return svzip1_b64 (svptrue_b64 (), svptrue_b8 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_5.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_5.c
new file mode 100644
index 0000000..a8aec2b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_5.c
@@ -0,0 +1,96 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	zip2	p0\.h, p[0-3]\.h, \1\.h
+**	ret
+*/
+svbool_t
+test1 ()
+{
+  return svzip2_b16 (svptrue_b8 (), svptrue_b16 ());
+}
+
+/*
+** test2:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	zip2	p0\.h, \1\.h, p[0-3]\.h
+**	ret
+*/
+svbool_t
+test2 ()
+{
+  return svzip2_b16 (svptrue_b16 (), svptrue_b8 ());
+}
+
+/*
+** test3:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	zip2	p0\.s, p[0-3]\.s, \1\.s
+**	ret
+*/
+svbool_t
+test3 ()
+{
+  return svzip2_b32 (svptrue_b8 (), svptrue_b32 ());
+}
+
+/*
+** test4:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	zip2	p0\.s, \1\.s, p[0-3]\.s
+**	ret
+*/
+svbool_t
+test4 ()
+{
+  return svzip2_b32 (svptrue_b32 (), svptrue_b8 ());
+}
+
+/*
+** test5:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	zip2	p0\.d, p[0-3]\.d, \1\.d
+**	ret
+*/
+svbool_t
+test5 ()
+{
+  return svzip2_b64 (svptrue_b8 (), svptrue_b64 ());
+}
+
+/*
+** test6:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	zip2	p0\.d, \1\.d, p[0-3]\.d
+**	ret
+*/
+svbool_t
+test6 ()
+{
+  return svzip2_b64 (svptrue_b64 (), svptrue_b8 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_6.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_6.c
new file mode 100644
index 0000000..3405004
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_6.c
@@ -0,0 +1,96 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	uzp1	p0\.h, p[0-3]\.h, \1\.h
+**	ret
+*/
+svbool_t
+test1 ()
+{
+  return svuzp1_b16 (svptrue_b8 (), svptrue_b16 ());
+}
+
+/*
+** test2:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	uzp1	p0\.h, \1\.h, p[0-3]\.h
+**	ret
+*/
+svbool_t
+test2 ()
+{
+  return svuzp1_b16 (svptrue_b16 (), svptrue_b8 ());
+}
+
+/*
+** test3:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	uzp1	p0\.s, p[0-3]\.s, \1\.s
+**	ret
+*/
+svbool_t
+test3 ()
+{
+  return svuzp1_b32 (svptrue_b8 (), svptrue_b32 ());
+}
+
+/*
+** test4:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	uzp1	p0\.s, \1\.s, p[0-3]\.s
+**	ret
+*/
+svbool_t
+test4 ()
+{
+  return svuzp1_b32 (svptrue_b32 (), svptrue_b8 ());
+}
+
+/*
+** test5:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	uzp1	p0\.d, p[0-3]\.d, \1\.d
+**	ret
+*/
+svbool_t
+test5 ()
+{
+  return svuzp1_b64 (svptrue_b8 (), svptrue_b64 ());
+}
+
+/*
+** test6:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	uzp1	p0\.d, \1\.d, p[0-3]\.d
+**	ret
+*/
+svbool_t
+test6 ()
+{
+  return svuzp1_b64 (svptrue_b64 (), svptrue_b8 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_7.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_7.c
new file mode 100644
index 0000000..1758d00
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_7.c
@@ -0,0 +1,96 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	uzp2	p0\.h, p[0-3]\.h, \1\.h
+**	ret
+*/
+svbool_t
+test1 ()
+{
+  return svuzp2_b16 (svptrue_b8 (), svptrue_b16 ());
+}
+
+/*
+** test2:
+**	...
+**	ptrue	(p[0-3])\.h, all
+**	...
+**	uzp2	p0\.h, \1\.h, p[0-3]\.h
+**	ret
+*/
+svbool_t
+test2 ()
+{
+  return svuzp2_b16 (svptrue_b16 (), svptrue_b8 ());
+}
+
+/*
+** test3:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	uzp2	p0\.s, p[0-3]\.s, \1\.s
+**	ret
+*/
+svbool_t
+test3 ()
+{
+  return svuzp2_b32 (svptrue_b8 (), svptrue_b32 ());
+}
+
+/*
+** test4:
+**	...
+**	ptrue	(p[0-3])\.s, all
+**	...
+**	uzp2	p0\.s, \1\.s, p[0-3]\.s
+**	ret
+*/
+svbool_t
+test4 ()
+{
+  return svuzp2_b32 (svptrue_b32 (), svptrue_b8 ());
+}
+
+/*
+** test5:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	uzp2	p0\.d, p[0-3]\.d, \1\.d
+**	ret
+*/
+svbool_t
+test5 ()
+{
+  return svuzp2_b64 (svptrue_b8 (), svptrue_b64 ());
+}
+
+/*
+** test6:
+**	...
+**	ptrue	(p[0-3])\.d, all
+**	...
+**	uzp2	p0\.d, \1\.d, p[0-3]\.d
+**	ret
+*/
+svbool_t
+test6 ()
+{
+  return svuzp2_b64 (svptrue_b64 (), svptrue_b8 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/pnext_3.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/pnext_3.c
new file mode 100644
index 0000000..d9c0090
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/pnext_3.c
@@ -0,0 +1,130 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	pnext	p0\.h, p1, p0\.h
+**	ret
+*/
+svbool_t
+test1 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b8 (),
+		  svpnext_b16 (prev, pg),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	pnext	p0\.h, p1, p0\.h
+**	ret
+*/
+svbool_t
+test2 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b16 (),
+		  svpnext_b16 (prev, pg),
+		  svptrue_b8 ());
+}
+
+/*
+** test3:
+**	pnext	p0\.h, p1, p0\.h
+**	ret
+*/
+svbool_t
+test3 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b16 (),
+		  svpnext_b16 (prev, pg),
+		  svptrue_b16 ());
+}
+
+/*
+** test4:
+**	pnext	p0\.s, p1, p0\.s
+**	ret
+*/
+svbool_t
+test4 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b32 (),
+		  svpnext_b32 (prev, pg),
+		  svptrue_b8 ());
+}
+
+/*
+** test5:
+**	pnext	p0\.s, p1, p0\.s
+**	ret
+*/
+svbool_t
+test5 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b16 (),
+		  svpnext_b32 (prev, pg),
+		  svptrue_b8 ());
+}
+
+/*
+** test6:
+**	pnext	p0\.s, p1, p0\.s
+**	ret
+*/
+svbool_t
+test6 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b8 (),
+		  svpnext_b32 (prev, pg),
+		  svptrue_b32 ());
+}
+
+/*
+** test7:
+**	pnext	p0\.d, p1, p0\.d
+**	ret
+*/
+svbool_t
+test7 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b16 (),
+		  svpnext_b64 (prev, pg),
+		  svptrue_b8 ());
+}
+
+/*
+** test8:
+**	pnext	p0\.d, p1, p0\.d
+**	ret
+*/
+svbool_t
+test8 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b32 (),
+		  svpnext_b64 (prev, pg),
+		  svptrue_b8 ());
+}
+
+/*
+** test9:
+**	pnext	p0\.d, p1, p0\.d
+**	ret
+*/
+svbool_t
+test9 (svbool_t pg, svbool_t prev)
+{
+  return svand_z (svptrue_b8 (),
+		  svpnext_b64 (prev, pg),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/pr121118_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/pr121118_1.c
new file mode 100644
index 0000000..b59a972
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/pr121118_1.c
@@ -0,0 +1,16 @@
+/* { dg-options "-O2 -msve-vector-bits=512" } */
+
+typedef __SVBool_t fixed_bool __attribute__((arm_sve_vector_bits(512)));
+
+#define TEST_CONST(NAME, CONST)						\
+  fixed_bool								\
+  NAME ()								\
+  {									\
+    union { unsigned long long i; fixed_bool pg; } u = { CONST };	\
+    return u.pg;							\
+  }
+
+TEST_CONST (test1, 0x02aaaaaaaa)
+TEST_CONST (test2, 0x0155555557)
+TEST_CONST (test3, 0x0013333333333333ULL)
+TEST_CONST (test4, 0x0011111111111113ULL)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/rev_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/rev_2.c
new file mode 100644
index 0000000..3dc4eb9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/rev_2.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+svbool_t test1()
+{
+  return svrev_b16 (svptrue_b16 ());
+}
+
+svbool_t test2()
+{
+  return svrev_b32 (svptrue_b32 ());
+}
+
+svbool_t test3()
+{
+  return svrev_b64 (svptrue_b64 ());
+}
+
+/* { dg-final { scan-assembler {\tptrue\tp[0-7]\.h} } } */
+/* { dg-final { scan-assembler {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.b} } } */
+/* { dg-final { scan-assembler {\trev\tp0\.h} } } */
+/* { dg-final { scan-assembler {\trev\tp0\.s} } } */
+/* { dg-final { scan-assembler {\trev\tp0\.d} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/unpkhi_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/unpkhi_1.c
new file mode 100644
index 0000000..9c7b4bc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/unpkhi_1.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+svbool_t
+test1 (svbool_t p)
+{
+  return svand_z (svptrue_b8 (), svunpkhi (p), svptrue_b16 ());
+}
+
+svbool_t
+test2 (svbool_t p)
+{
+  return svand_z (svptrue_b16 (), svunpkhi (p), svptrue_b8 ());
+}
+
+svbool_t
+test3 (svbool_t p)
+{
+  return svand_z (svptrue_b16 (), svunpkhi (p), svptrue_b16 ());
+}
+
+/* { dg-final { scan-assembler-not {\tand\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/unpklo_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/unpklo_1.c
new file mode 100644
index 0000000..f072a2f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/unpklo_1.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+svbool_t
+test1 (svbool_t p)
+{
+  return svand_z (svptrue_b8 (), svunpklo (p), svptrue_b16 ());
+}
+
+svbool_t
+test2 (svbool_t p)
+{
+  return svand_z (svptrue_b16 (), svunpklo (p), svptrue_b8 ());
+}
+
+svbool_t
+test3 (svbool_t p)
+{
+  return svand_z (svptrue_b16 (), svunpklo (p), svptrue_b16 ());
+}
+
+/* { dg-final { scan-assembler-not {\tand\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/whilele_13.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/whilele_13.c
new file mode 100644
index 0000000..cf50dc1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/whilele_13.c
@@ -0,0 +1,130 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	whilele	p0\.h, w0, w1
+**	ret
+*/
+svbool_t
+test1 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilele_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	whilele	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test2 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilele_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	whilels	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test3 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilele_b32 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test4:
+**	whilels	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test4 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilele_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test5:
+**	whilele	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test5 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilele_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test6:
+**	whilels	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test6 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svwhilele_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test7:
+**	whilels	p0\.d, w0, w1
+**	ret
+*/
+svbool_t
+test7 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilele_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test8:
+**	whilele	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test8 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilele_b64 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test9:
+**	whilels	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test9 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b64 (),
+		  svwhilele_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/whilelt_6.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/whilelt_6.c
new file mode 100644
index 0000000..27bf0c2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/whilelt_6.c
@@ -0,0 +1,130 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	whilelt	p0\.h, w0, w1
+**	ret
+*/
+svbool_t
+test1 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilelt_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	whilelt	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test2 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilelt_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	whilelo	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test3 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilelt_b32 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test4:
+**	whilelo	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test4 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilelt_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test5:
+**	whilelt	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test5 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilelt_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test6:
+**	whilelo	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test6 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svwhilelt_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test7:
+**	whilelo	p0\.d, w0, w1
+**	ret
+*/
+svbool_t
+test7 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilelt_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test8:
+**	whilelt	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test8 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilelt_b64 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test9:
+**	whilelo	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test9 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b64 (),
+		  svwhilelt_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c b/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c
new file mode 100644
index 0000000..66d9510
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c
@@ -0,0 +1,23 @@
+// { dg-do compile }
+// { dg-options "-march=armv8-a+sve -msve-vector-bits=128 -O3" }
+
+typedef struct Array {
+    int elems[3];
+} Array;
+
+int loop(Array **pp, int len, int idx) {
+    int nRet = 0;
+
+    #pragma GCC unroll 0
+    for (int i = 0; i < len; i++) {
+        Array *p = pp[i];
+        if (p) {
+            nRet += p->elems[idx];
+        }
+    }
+
+    return nRet;
+}
+
+// { dg-final { scan-assembler-times {ld1w\tz[0-9]+\.d, p[0-7]/z} 1 } }
+// { dg-final { scan-assembler-times {add\tz[0-9]+\.s, p[0-7]/m}  1 } }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c
index a8fd4c8..4708d57 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c
index 08cd6a0..4530b18 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c
index f5c9cbf..3097459 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c
index 91ae3c8..5e9d21c 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c
index 12368ce..768a740 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c
index dd52a58..ce14abb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c
index e55ddfb..ceeb5ae 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c
index 6796229..f8b6b82 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c
index 7f2ec4a..45f74ed 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c
index d18427b..fc601a1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c
index 983ab5c..4959f1d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c
index de36b66..d8a8a81 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c
index 333140d..6cf2683 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
index 93d6693..a32b636 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c
index c88686a..72e743b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c
index 5f4b562fc..1178104 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c
index 0fe8ab3..ebd313a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c
index 758f00f..d531987 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c
index f82471f..55c9cef 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c
index ba500b6..5532232 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve+f64mm" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve+f64mm -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c
index 71894c4..78bdb0b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c
index 1b7cc42..e219007 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c
index 7f4ff2d..98897e9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c
index 0a587fc..c6fe6b9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c
index d795f8e..7e2c1b9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c
index 42b37ae..f7f75f6 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-options "-O2 -fdump-tree-optimized -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c
index bd9a980..54b6197 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c
index 62a0755..e8b8a55 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
index 751e60e..1539f58 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c
index 44792d3..21c8f6b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c
index f3820e0..a908289 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c
index e49266d..12b5e14 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c
index acdd141..89873fc 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c
index 7698045..c6d2cfb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c
index 037376b..8a3b3e0 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c
index 1287a70..04bc049 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve+bf16" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve+bf16 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c
index f519266..f39d2c5 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve+bf16" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve+bf16 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c
index fabde3e..4403e50 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c
index 46c9592..f06b067 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c
index b820bde..a851c4a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c
index 1e99b7f..dde812b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_1.c b/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_1.c
index 38dfdd4..e777f03 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mcpu=neoverse-n2" } */
+/* { dg-options "-O2 -mcpu=neoverse-n2 -fdisable-rtl-combine" } */
 /* { dg-final { check-function-bodies "**" "" } } */
 
 #pragma GCC target "+sve"
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_2.c b/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_2.c
index 45363cc..41182e1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_2.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mcpu=neoverse-v2" } */
+/* { dg-options "-O2 -mcpu=neoverse-v2 -fdisable-rtl-combine" } */
 /* { dg-final { check-function-bodies "**" "" } } */
 
 #pragma GCC target "+sve"
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_3.c b/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_3.c
index c50a581..04a9023 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred_clobber_3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mcpu=neoverse-v1" } */
+/* { dg-options "-O2 -mcpu=neoverse-v1 -fdisable-rtl-combine" } */
 /* { dg-final { check-function-bodies "**" "" } } */
 
 #pragma GCC target "+sve"
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c
new file mode 100644
index 0000000..e6aa047
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fmaxf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c
new file mode 100644
index 0000000..87125a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_builtin_fmax_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c
new file mode 100644
index 0000000..b9fded0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fminf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c
new file mode 100644
index 0000000..5923b67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_builtin_fmin_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c
new file mode 100644
index 0000000..d328b37
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fmaxf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c
new file mode 100644
index 0000000..f84ded5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_builtin_fmax_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c
new file mode 100644
index 0000000..1821f03
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fminf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c
new file mode 100644
index 0000000..bceddf9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_builtin_fmin_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c
new file mode 100644
index 0000000..fa4dd15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT)		    \
+  void                                                      \
+  test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out,  \
+				    TYPE1 *__restrict a,    \
+				    TYPE0 *__restrict b,    \
+				    TYPE2 *__restrict p)    \
+  {							    \
+    for (unsigned int i = 0; i < COUNT; i++)                \
+      out[i] = p[i] ?  (TYPE0)a[i] : b[i];		    \
+  }
+
+#define TEST_CVTF(PFX, T)		   \
+  T (_Float16, PFX##int16_t, uint64_t, 32) \
+  T (_Float16, PFX##int16_t, uint32_t, 64) \
+  T (_Float16, PFX##int32_t, uint64_t, 32) \
+  T (_Float16, PFX##int32_t, uint32_t, 64) \
+  T (_Float16, PFX##int64_t, uint64_t, 32) \
+  T (float, PFX##int32_t, uint64_t, 32)    \
+  T (float, PFX##int64_t, uint64_t, 32)
+
+#define TEST_ALL(T) \
+  TEST_CVTF (, T)   \
+  TEST_CVTF (u, T)
+
+TEST_ALL (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c
new file mode 100644
index 0000000..d959aa9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_fabsf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fabsf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fabsf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c
new file mode 100644
index 0000000..666cf89
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c
@@ -0,0 +1,62 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define ADD(A, B) A + B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, NAME, RHS, MERGE)	 \
+  void								 \
+  f_##TYPE0##_##TYPE1##_##NAME##_##MERGE (TYPE0 *__restrict out, \
+					 TYPE0 *__restrict a,	 \
+					 TYPE0 *__restrict b,	 \
+					 TYPE0 *__restrict c,	 \
+					 TYPE1 *__restrict p)	 \
+  {								 \
+    for (unsigned int i = 0; i < COUNT; i++)			 \
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		 \
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1, a_i)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1, b_i)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5, b_i)
+
+TEST_ALL (ADD, _Float16, uint64_t, 32)
+
+TEST_ALL (ADD, _Float16, uint32_t, 64)
+
+TEST_ALL (ADD, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 19 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 10 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c
new file mode 100644
index 0000000..e59864b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fadd_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 22 } } */
+/* { dg-final { scan-assembler-times {\tand} 33 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 19 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 10 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c
new file mode 100644
index 0000000..3caae19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT)		    \
+  void                                                      \
+  test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out,  \
+				    TYPE1 *__restrict a,    \
+				    TYPE0 *__restrict b,    \
+				    TYPE2 *__restrict p)    \
+  {							    \
+    for (unsigned int i = 0; i < COUNT; i++)                \
+      out[i] = p[i] ?  (TYPE0)a[i] : b[i];		    \
+  }
+
+#define TEST_FCVT(T)		     \
+  T (_Float16, float, uint64_t, 32)  \
+  T (_Float16, float, uint32_t, 64)  \
+  T (_Float16, double, uint64_t, 32) \
+  T (float, double, uint64_t, 32)    \
+  T (float, _Float16, uint64_t, 32)  \
+  T (float, _Float16, uint32_t, 64)  \
+  T (double, _Float16, uint64_t,32)  \
+  T (double, float, uint64_t, 32)
+
+TEST_FCVT (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c
new file mode 100644
index 0000000..426d3af
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c
@@ -0,0 +1,55 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT)		    \
+  void                                                      \
+  test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out,  \
+				    TYPE1 *__restrict a,    \
+				    TYPE0 *__restrict b,    \
+				    TYPE2 *__restrict p)    \
+  {							    \
+    for (unsigned int i = 0; i < COUNT; i++)                \
+      out[i] = p[i] ?  (TYPE0)a[i] : b[i];		    \
+  }
+
+#define TEST_FCVTZ(PFX, T)		   \
+  T (PFX##int16_t, _Float16, uint64_t, 32) \
+  T (PFX##int16_t, _Float16, uint32_t, 64) \
+  T (PFX##int32_t, _Float16, uint64_t, 32) \
+  T (PFX##int32_t, _Float16, uint32_t, 64) \
+  T (PFX##int64_t, _Float16, uint64_t, 32) \
+  T (PFX##int32_t, float, uint64_t, 32)    \
+  T (PFX##int64_t, float, uint64_t, 32)    \
+  T (PFX##int32_t, double, uint64_t, 32)
+
+#define TEST_ALL(T) \
+  TEST_FCVTZ (, T)  \
+  TEST_FCVTZ (u, T)
+
+TEST_ALL (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c
new file mode 100644
index 0000000..ec5653e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c
@@ -0,0 +1,47 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define DIV(A, B) A / B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i)
+
+TEST_ALL (DIV, _Float16, uint64_t, 32)
+
+TEST_ALL (DIV, _Float16, uint32_t, 64)
+
+TEST_ALL (DIV, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c
new file mode 100644
index 0000000..1ca3dbf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fdiv_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 6 } } */
+/* { dg-final { scan-assembler-times {\tand} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c
new file mode 100644
index 0000000..d34872f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define MAX(A, B) (A > B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (MAX, _Float16, uint64_t, 32)
+
+TEST_ALL (MAX, _Float16, uint32_t, 64)
+
+TEST_ALL (MAX, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c
new file mode 100644
index 0000000..282f3ed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only" } */
+
+#include "unpacked_cond_fmaxnm_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c
new file mode 100644
index 0000000..d6c3c38
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define MIN(A, B) (A < B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (MIN, _Float16, uint64_t, 32)
+
+TEST_ALL (MIN, _Float16, uint32_t, 64)
+
+TEST_ALL (MIN, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c
new file mode 100644
index 0000000..8226a6f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only" } */
+
+#include "unpacked_cond_fminnm_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c
new file mode 100644
index 0000000..cae9242
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FMLA (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FMLA (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c
new file mode 100644
index 0000000..72e04a4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmla_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c
new file mode 100644
index 0000000..db0f818
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FMLS (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FMLS (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c
new file mode 100644
index 0000000..3012052
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmls_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c
new file mode 100644
index 0000000..1ae7678
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c
@@ -0,0 +1,50 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define MUL(A, B) A * B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, b_i)
+
+TEST_ALL (MUL, _Float16, uint64_t, 32)
+
+TEST_ALL (MUL, _Float16, uint32_t, 64)
+
+TEST_ALL (MUL, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c
new file mode 100644
index 0000000..21713f5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmul_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tand} 15 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c
new file mode 100644
index 0000000..7280f4e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c
@@ -0,0 +1,39 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define NEG(X) -X
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (NEG, _Float16, uint64_t, 32)
+
+TEST_ALL (NEG, _Float16, uint32_t, 64)
+
+TEST_ALL (NEG, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c
new file mode 100644
index 0000000..07bab63
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FNMLA (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FNMLA (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FNMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c
new file mode 100644
index 0000000..daef4e49
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fnmla_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c
new file mode 100644
index 0000000..5526378
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FNMLS (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FNMLS (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FNMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c
new file mode 100644
index 0000000..8a8f348
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fnmls_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c
new file mode 100644
index 0000000..ed4efb6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_roundf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_roundf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_roundf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c
new file mode 100644
index 0000000..f20e2e6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -mtune=generic -ftree-vectorize" } */
+
+#include "unpacked_cond_frinta_1.c"
+
+/* Test that we don't drop SELs without -fno-trapping-math.  */
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tsel\t} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c
new file mode 100644
index 0000000..d682d15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_nearbyintf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_nearbyintf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_nearbyintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c
new file mode 100644
index 0000000..7d429b3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_floorf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_floorf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_floorf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c
new file mode 100644
index 0000000..c6d0c8c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_ceilf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_ceilf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_ceilf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c
new file mode 100644
index 0000000..b8afef1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_rintf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_rintf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_rintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c
new file mode 100644
index 0000000..d55279b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_truncf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_truncf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_truncf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c
new file mode 100644
index 0000000..eafd169
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c
@@ -0,0 +1,56 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define SUBR(A, B) B - A
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, b_i)
+
+TEST_ALL (SUBR, _Float16, uint64_t, 32)
+
+TEST_ALL (SUBR, _Float16, uint32_t, 64)
+
+TEST_ALL (SUBR, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c
new file mode 100644
index 0000000..cd7a0e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fsubr_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c
new file mode 100644
index 0000000..f09cfe8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_fabsf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_fabsf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_fabsf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c
new file mode 100644
index 0000000..9675f56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define ADD(A, B)  A + B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, NAME, RHS)		\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##NAME (TYPE1 *__restrict out,	\
+				       TYPE0 *__restrict a,     \
+				       TYPE0 *__restrict b,     \
+                                       TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)      \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i]) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1)
+
+TEST_ALL (ADD, _Float16, uint64_t, 32)
+
+TEST_ALL (ADD, _Float16, uint32_t, 64)
+
+TEST_ALL (ADD, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 11 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
new file mode 100644
index 0000000..7a74efd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fadd_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 11 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_1.c
new file mode 100644
index 0000000..bf9c127
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_1.c
@@ -0,0 +1,602 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-schedule-insns -fno-schedule-insns2" } */
+
+#include <stdint.h>
+
+#define UNLT(A, B) (!__builtin_isgreaterequal (A, B))
+#define UNLE(A, B) (!__builtin_isgreater (A, B))
+#define UNGT(A, B) (!__builtin_islessequal (A, B))
+#define UNGE(A, B) (!__builtin_isless (A, B))
+#define UNEQ(A, B) (!__builtin_islessgreater (A, B))
+
+#define EQ(A, B) ((A) == (B))
+#define NE(A, B) ((A) != (B))
+#define LE(A, B) ((A) <= (B))
+#define LT(A, B) ((A) < (B))
+#define GE(A, B) ((A) >= (B))
+#define GT(A, B) ((A) > (B))
+#define ORDERED(A, B) (!__builtin_isunordered (A, B))
+#define UNORDERED(A, B) (__builtin_isunordered (A, B))
+
+#define b_i b[i]
+
+#define TEST_FCM(TYPE0, TYPE1, CMP, RHS, COUNT)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##CMP##_##RHS (TYPE0 *__restrict out,	\
+				       TYPE1 *__restrict a,	\
+				       TYPE1 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = CMP (a[i], RHS) ? 3 : out[i];			\
+  }
+
+#define TEST_CC_REG(CMP)		      \
+  TEST_FCM (uint64_t, float, CMP, b_i, 32)    \
+  TEST_FCM (uint32_t, _Float16, CMP, b_i, 64) \
+  TEST_FCM (uint64_t, _Float16, CMP, b_i, 32)
+
+#define TEST_CC_ALL(CMP)		    \
+  TEST_CC_REG (CMP)			    \
+  TEST_FCM (uint64_t, float, CMP, 0, 32)    \
+  TEST_FCM (uint32_t, _Float16, CMP, 0, 64) \
+  TEST_FCM (uint64_t, _Float16, CMP, 0, 32)
+
+
+/*
+** f_uint64_t_float_UNLT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmge	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNLT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmge	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNLT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmge	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNLT)
+
+/*
+** f_uint64_t_float_UNLE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmgt	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNLE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmgt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNLE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmgt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNLE)
+
+/*
+** f_uint64_t_float_UNGT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmle	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNGT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmle	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNGT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmle	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNGT)
+
+/*
+** f_uint64_t_float_UNGE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmlt	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNGE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmlt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNGE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmlt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNGE)
+
+/*
+** f_uint64_t_float_UNEQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmne	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNEQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmne	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNEQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmne	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNEQ)
+
+/*
+** f_uint64_t_float_EQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_EQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_EQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_EQ_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_EQ_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_EQ_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (EQ)
+
+/*
+** f_uint64_t_float_NE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_NE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_NE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_NE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_NE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_NE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (NE)
+
+/*
+** f_uint64_t_float_LE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_LE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (LE)
+
+/*
+** f_uint64_t_float_LT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_LT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (LT)
+
+/*
+** f_uint64_t_float_GE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_GE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (GE)
+
+/*
+** f_uint64_t_float_GT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_GT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (GT)
+
+/*
+** f_uint64_t_float_ORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_ORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_ORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (ORDERED)
+
+/*
+** f_uint64_t_float_UNORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNORDERED)
+
+
+/* { dg-final { check-function-bodies "**" "" ""} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_2.c
new file mode 100644
index 0000000..ab210da
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_2.c
@@ -0,0 +1,50 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-trapping-math" } */
+
+#include "unpacked_fcm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 32 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmuo\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfcmuo\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_1.c
new file mode 100644
index 0000000..d793a6c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_1.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-trapping-math" } */
+
+#include "unpacked_fcm_1.c"
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 32 } } */
+
+/* Drop a PTRUE predicated AND with the loop mask and comparison result in
+   favour of predicating the comparison with the loop mask.  */
+/* { dg-final { scan-assembler-not {\tand\t} } } */
+
+/* Similarly, for codes that are implemented via an inversion, prefer
+   NOT (predicated with the loop mask) over BIC+PTRUE.  */
+/* { dg-final { scan-assembler-not {\tbic\t} } } */
+/* { dg-final { scan-assembler-times {\tnot\t} 15 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_2.c
new file mode 100644
index 0000000..b85391b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_2.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+/* Ensure that we still emit NOR here, rather than two NOTs.  */
+
+#define TEST_FCM_NOR(TYPE0, TYPE1, CMP, COUNT)		\
+  void							\
+  f_##TYPE0##_##TYPE1##_##CMP (TYPE0 *__restrict out,	\
+			       TYPE1 *__restrict a,	\
+			       TYPE1 *__restrict b,	\
+			       TYPE1 *__restrict c)	\
+  {								    \
+    for (unsigned int i = 0; i < COUNT; i++)			    \
+      out[i] = !(CMP (a[i], c[i]) | CMP (b[i], c[i])) ? 3 : out[i]; \
+  }
+
+#define GT(A, B) ((A) > (B))
+
+TEST_FCM_NOR (uint64_t, float, GT, 32)
+TEST_FCM_NOR (uint64_t, _Float16, GT, 32)
+TEST_FCM_NOR (uint32_t, _Float16, GT, 64)
+
+TEST_FCM_NOR (uint64_t, float, __builtin_isunordered, 32)
+TEST_FCM_NOR (uint64_t, _Float16, __builtin_isunordered, 32)
+TEST_FCM_NOR (uint32_t, _Float16, __builtin_isunordered, 64)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 6 } } */
+
+/* { dg-final { scan-assembler-not {\tbic\t} } } */
+/* { dg-final { scan-assembler-not {\tnot\t} } } */
+/* { dg-final { scan-assembler-times {\tnor\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b\n} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c
new file mode 100644
index 0000000..78d0d9c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define DIV(A, B) A / B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (DIV, _Float16, uint64_t, 32)
+
+TEST_ALL (DIV, _Float16, uint32_t, 64)
+
+TEST_ALL (DIV, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c
new file mode 100644
index 0000000..a8f70e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fdiv_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c
new file mode 100644
index 0000000..ecd088f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -moverride=sve_width=2048 -mlow-precision-div" } */
+
+#include "unpacked_fdiv_1.c"
+
+/* { dg-final { scan-assembler-not {\tfrecpe\tz[0-9]+\.h} } } */
+/* { dg-final { scan-assembler-not {\tfrecps\tz[0-9]+\.h} } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrecpe\tz[0-9]+\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrecps\tz[0-9]+\.s} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c
new file mode 100644
index 0000000..5239e4b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c
@@ -0,0 +1,45 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define MAX(A, B) (A > B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (c[i] = FN (a[i], RHS))				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (MAX, _Float16, uint64_t, 32)
+
+TEST_ALL (MAX, _Float16, uint32_t, 64)
+
+TEST_ALL (MAX, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c
new file mode 100644
index 0000000..11aa7c0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -fno-trapping-math -moverride=sve_width=2048" } */
+
+#include "unpacked_fmaxnm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c
new file mode 100644
index 0000000..02a5f46
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define MIN(A, B) (A < B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (c[i] = FN (a[i], RHS) )				\
+	out[i] = 3;						\
+  }
+
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (MIN, _Float16, uint64_t, 32)
+
+TEST_ALL (MIN, _Float16, uint32_t, 64)
+
+TEST_ALL (MIN, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c
new file mode 100644
index 0000000..81f583b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -fno-trapping-math -moverride=sve_width=2048" } */
+
+#include "unpacked_fminnm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c
new file mode 100644
index 0000000..312bccc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FMLA (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FMLA (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c
new file mode 100644
index 0000000..ca3f94d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmla_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c
new file mode 100644
index 0000000..f7cbfb3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FMLS (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FMLS (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c
new file mode 100644
index 0000000..387dbec
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmls_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c
new file mode 100644
index 0000000..a180a07
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define immp5 0.5
+#define MUL(A, B) A * B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, immp5)
+
+TEST_ALL (MUL, _Float16, uint64_t, 32)
+
+TEST_ALL (MUL, _Float16, uint32_t, 64)
+
+TEST_ALL (MUL, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
new file mode 100644
index 0000000..eb05600
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmul_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c
new file mode 100644
index 0000000..d489ecb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define NEG(X) -X
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (NEG, _Float16, uint64_t, 32)
+
+TEST_FN (NEG, _Float16, uint32_t, 64)
+
+TEST_FN (NEG, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c
new file mode 100644
index 0000000..bf13ff5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FNMLA (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FNMLA (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FNMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c
new file mode 100644
index 0000000..64130ba
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fnmla_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c
new file mode 100644
index 0000000..399920a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FNMLS (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FNMLS (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FNMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c
new file mode 100644
index 0000000..59fb7f9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fnmls_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c
new file mode 100644
index 0000000..3cbdef3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_roundf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_roundf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_roundf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c
new file mode 100644
index 0000000..4564686
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frinta_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c
new file mode 100644
index 0000000..7645fed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_nearbyintf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_nearbyintf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_nearbyintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c
new file mode 100644
index 0000000..eadce07
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frinti_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c
new file mode 100644
index 0000000..98f85fb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_floorf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_floorf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_floorf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c
new file mode 100644
index 0000000..56988be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c
new file mode 100644
index 0000000..f233697
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_ceilf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_ceilf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_ceilf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c
new file mode 100644
index 0000000..c24c632
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintp_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c
new file mode 100644
index 0000000..73403a5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_rintf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_rintf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_rintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c
new file mode 100644
index 0000000..e8b8924
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintx_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c
new file mode 100644
index 0000000..7377843
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_truncf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_truncf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_truncf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c
new file mode 100644
index 0000000..1779122
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintz_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c
new file mode 100644
index 0000000..2cc8ec2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define immp5 0.5
+#define SUBR(A, B) B - A
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, immp5)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (SUBR, _Float16, uint64_t, 32)
+
+TEST_ALL (SUBR, _Float16, uint32_t, 64)
+
+TEST_ALL (SUBR, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfsubr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsubr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
new file mode 100644
index 0000000..de9325c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fsubr_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c
index 25910db..5100a87 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -mlittle-endian" } */
 /* { dg-final { check-function-bodies "**" "" "" } } */
 
 typedef char v16qi __attribute__ ((vector_size (16)));
@@ -8,7 +8,7 @@ typedef short v8hi __attribute__ ((vector_size (16)));
 typedef short v4hi __attribute__ ((vector_size (8)));
 typedef int v4si __attribute__ ((vector_size (16)));
 typedef int v2si __attribute__ ((vector_size (8)));
-typedef long v2di __attribute__ ((vector_size (16)));
+typedef long long v2di __attribute__ ((vector_size (16)));
 
 /*
 ** f_v16qi:
@@ -97,3 +97,113 @@ g_v4si (void)
 {
   return (v4si){ 3, -1, -5, -9 };
 }
+
+/*
+** g_min_1:
+**	index	z0\.s, #-16, #1
+**	ret
+*/
+v4si
+g_min_1 (void)
+{
+  return (v4si){ -16, -15, -14, -13 };
+}
+
+/*
+** g_min_min:
+**	index	z0\.s, #-16, #-16
+**	ret
+*/
+v4si
+g_min_min (void)
+{
+  return (v4si){ -16, -32, -48, -64 };
+}
+
+/*
+** g_min_max:
+**	index	z0\.s, #-16, #15
+**	ret
+*/
+v4si
+g_min_max (void)
+{
+  return (v4si){ -16, -1, 14, 29 };
+}
+
+/*
+** g_max_1:
+**	index	z0\.s, #15, #1
+**	ret
+*/
+v4si
+g_max_1 (void)
+{
+  return (v4si){ 15, 16, 17, 18 };
+}
+
+/*
+** g_max_min:
+**	index	z0\.s, #15, #-16
+**	ret
+*/
+v4si
+g_max_min (void)
+{
+  return (v4si){ 15, -1, -17, -33 };
+}
+
+/*
+** g_max_max:
+**	index	z0\.s, #15, #15
+**	ret
+*/
+v4si
+g_max_max (void)
+{
+  return (v4si){ 15, 30, 45, 60 };
+}
+
+/*
+** g_ob_1:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_1 (void)
+{
+  return (v4si){ -17, -16, -15, -14 };
+}
+
+/*
+** g_ob_2:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_2 (void)
+{
+  return (v4si){ 16, 17, 18, 19 };
+}
+
+/*
+** g_ob_3:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_3 (void)
+{
+  return (v4si){ 0, -17, -34, -51 };
+}
+
+/*
+** g_ob_4:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_4 (void)
+{
+  return (v4si){ 0, 16, 32, 48 };
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vec_init_4.c b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_4.c
new file mode 100644
index 0000000..0681d95
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_4.c
@@ -0,0 +1,209 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mbig-endian" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+typedef char v16qi __attribute__ ((vector_size (16)));
+typedef char v8qi __attribute__ ((vector_size (8)));
+typedef short v8hi __attribute__ ((vector_size (16)));
+typedef short v4hi __attribute__ ((vector_size (8)));
+typedef int v4si __attribute__ ((vector_size (16)));
+typedef int v2si __attribute__ ((vector_size (8)));
+typedef long long v2di __attribute__ ((vector_size (16)));
+
+/*
+** f_v16qi:
+**	index	z0\.b, #15, #-1
+**	ret
+*/
+v16qi
+f_v16qi (void)
+{
+  return (v16qi){ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+}
+
+/*
+** f_v8qi:
+**	index	z0\.b, #7, #-1
+**	ret
+*/
+v8qi
+f_v8qi (void)
+{
+  return (v8qi){ 0, 1, 2, 3, 4, 5, 6, 7 };
+}
+
+/*
+** f_v8hi:
+**	index	z0\.h, #7, #-1
+**	ret
+*/
+v8hi
+f_v8hi (void)
+{
+  return (v8hi){ 0, 1, 2, 3, 4, 5, 6, 7 };
+}
+
+/*
+** f_v4hi:
+**	index	z0\.h, #3, #-1
+**	ret
+*/
+v4hi
+f_v4hi (void)
+{
+  return (v4hi){ 0, 1, 2, 3 };
+}
+
+/*
+** f_v4si:
+**	index	z0\.s, #3, #-1
+**	ret
+*/
+v4si
+f_v4si (void)
+{
+  return (v4si){ 0, 1, 2, 3 };
+}
+
+/*
+** f_v2si:
+**	index	z0\.s, #1, #-1
+**	ret
+*/
+v2si
+f_v2si (void)
+{
+  return (v2si){ 0, 1 };
+}
+
+/*
+** f_v2di:
+**	index	z0\.d, #1, #-1
+**	ret
+*/
+v2di
+f_v2di (void)
+{
+  return (v2di){ 0, 1 };
+}
+
+/*
+** g_v4si:
+**	index	z0\.s, #-9, #4
+**	ret
+*/
+v4si
+g_v4si (void)
+{
+  return (v4si){ 3, -1, -5, -9 };
+}
+
+/*
+** g_min_1:
+**	index	z0\.s, #-16, #1
+**	ret
+*/
+v4si
+g_min_1 (void)
+{
+  return (v4si){ -13, -14, -15, -16 };
+}
+
+/*
+** g_min_min:
+**	index	z0\.s, #-16, #-16
+**	ret
+*/
+v4si
+g_min_min (void)
+{
+  return (v4si){ -64, -48, -32, -16 };
+}
+
+/*
+** g_min_max:
+**	index	z0\.s, #-16, #15
+**	ret
+*/
+v4si
+g_min_max (void)
+{
+  return (v4si){ 29, 14, -1, -16 };
+}
+
+/*
+** g_max_1:
+**	index	z0\.s, #15, #1
+**	ret
+*/
+v4si
+g_max_1 (void)
+{
+  return (v4si){ 18, 17, 16, 15 };
+}
+
+/*
+** g_max_min:
+**	index	z0\.s, #15, #-16
+**	ret
+*/
+v4si
+g_max_min (void)
+{
+  return (v4si){ -33, -17, -1, 15 };
+}
+
+/*
+** g_max_max:
+**	index	z0\.s, #15, #15
+**	ret
+*/
+v4si
+g_max_max (void)
+{
+  return (v4si){ 60, 45, 30, 15 };
+}
+
+/*
+** g_ob_1:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_1 (void)
+{
+  return (v4si){ -14, -15, -16, -17 };
+}
+
+/*
+** g_ob_2:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_2 (void)
+{
+  return (v4si){ 19, 18, 17, 16 };
+}
+
+/*
+** g_ob_3:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_3 (void)
+{
+  return (v4si){ -51, -34, -17, 0 };
+}
+
+/*
+** g_ob_4:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_4 (void)
+{
+  return (v4si){ 48, 32, 16, 0 };
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/match_4.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/match_4.c
new file mode 100644
index 0000000..57f625b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/match_4.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svmatch (pg, x, y),
+		  svptrue_b16 ());
+}
+
+svbool_t
+test2 (svbool_t pg, svint16_t x, svint16_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svmatch (pg, x, y),
+		  svptrue_b8 ());
+}
+
+svbool_t
+test3 (svbool_t pg, svint16_t x, svint16_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svmatch (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/* { dg-final { scan-assembler-not {\tand\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/nmatch_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/nmatch_1.c
new file mode 100644
index 0000000..a3b1e2d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/nmatch_1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_sve.h>
+
+svbool_t
+test1 (svbool_t pg, svint16_t x, svint16_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svnmatch (pg, x, y),
+		  svptrue_b16 ());
+}
+
+svbool_t
+test2 (svbool_t pg, svint16_t x, svint16_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svnmatch (pg, x, y),
+		  svptrue_b8 ());
+}
+
+svbool_t
+test3 (svbool_t pg, svint16_t x, svint16_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svnmatch (pg, x, y),
+		  svptrue_b16 ());
+}
+
+/* { dg-final { scan-assembler-not {\tand\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilege_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilege_1.c
new file mode 100644
index 0000000..07b56a8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilege_1.c
@@ -0,0 +1,130 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	whilege	p0\.h, w0, w1
+**	ret
+*/
+svbool_t
+test1 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilege_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	whilege	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test2 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilege_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	whilehs	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test3 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilege_b32 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test4:
+**	whilehs	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test4 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilege_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test5:
+**	whilege	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test5 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilege_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test6:
+**	whilehs	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test6 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svwhilege_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test7:
+**	whilehs	p0\.d, w0, w1
+**	ret
+*/
+svbool_t
+test7 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilege_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test8:
+**	whilege	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test8 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilege_b64 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test9:
+**	whilehs	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test9 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b64 (),
+		  svwhilege_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilegt_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilegt_1.c
new file mode 100644
index 0000000..df707c3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilegt_1.c
@@ -0,0 +1,130 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	whilegt	p0\.h, w0, w1
+**	ret
+*/
+svbool_t
+test1 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilegt_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	whilegt	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test2 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilegt_b16 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	whilehi	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test3 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilegt_b32 (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test4:
+**	whilehi	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test4 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilegt_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test5:
+**	whilegt	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test5 (int32_t x, int32_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilegt_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test6:
+**	whilehi	p0\.s, w0, w1
+**	ret
+*/
+svbool_t
+test6 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b32 (),
+		  svwhilegt_b32 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test7:
+**	whilehi	p0\.d, w0, w1
+**	ret
+*/
+svbool_t
+test7 (uint32_t x, uint32_t y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilegt_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test8:
+**	whilegt	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test8 (int64_t x, int64_t y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilegt_b64 (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test9:
+**	whilehi	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test9 (uint64_t x, uint64_t y)
+{
+  return svand_z (svptrue_b64 (),
+		  svwhilegt_b64 (x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilerw_5.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilerw_5.c
new file mode 100644
index 0000000..0c24199
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilerw_5.c
@@ -0,0 +1,130 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	whilerw	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test1 (int16_t *x, int16_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilerw (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	whilerw	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test2 (uint16_t *x, uint16_t *y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilerw (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	whilerw	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test3 (int32_t *x, int32_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilerw (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test4:
+**	whilerw	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test4 (uint32_t *x, uint32_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilerw (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test5:
+**	whilerw	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test5 (float32_t *x, float32_t *y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilerw (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test6:
+**	whilerw	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test6 (int32_t *x, int32_t *y)
+{
+  return svand_z (svptrue_b32 (),
+		  svwhilerw (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test7:
+**	whilerw	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test7 (int64_t *x, int64_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilerw (x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test8:
+**	whilerw	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test8 (uint64_t *x, uint64_t *y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilerw (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test9:
+**	whilerw	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test9 (float64_t *x, float64_t *y)
+{
+  return svand_z (svptrue_b64 (),
+		  svwhilerw (x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilewr_5.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilewr_5.c
new file mode 100644
index 0000000..38db9af
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/general/whilewr_5.c
@@ -0,0 +1,130 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** test1:
+**	whilewr	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test1 (int16_t *x, int16_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilewr (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test2:
+**	whilewr	p0\.h, x0, x1
+**	ret
+*/
+svbool_t
+test2 (uint16_t *x, uint16_t *y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilewr (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test3:
+**	whilewr	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test3 (int32_t *x, int32_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilewr (x, y),
+		  svptrue_b16 ());
+}
+
+/*
+** test4:
+**	whilewr	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test4 (uint32_t *x, uint32_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilewr (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test5:
+**	whilewr	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test5 (float32_t *x, float32_t *y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilewr (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test6:
+**	whilewr	p0\.s, x0, x1
+**	ret
+*/
+svbool_t
+test6 (int32_t *x, int32_t *y)
+{
+  return svand_z (svptrue_b32 (),
+		  svwhilewr (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test7:
+**	whilewr	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test7 (int64_t *x, int64_t *y)
+{
+  return svand_z (svptrue_b8 (),
+		  svwhilewr (x, y),
+		  svptrue_b64 ());
+}
+
+/*
+** test8:
+**	whilewr	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test8 (uint64_t *x, uint64_t *y)
+{
+  return svand_z (svptrue_b16 (),
+		  svwhilewr (x, y),
+		  svptrue_b32 ());
+}
+
+/*
+** test9:
+**	whilewr	p0\.d, x0, x1
+**	ret
+*/
+svbool_t
+test9 (float64_t *x, float64_t *y)
+{
+  return svand_z (svptrue_b64 (),
+		  svwhilewr (x, y),
+		  svptrue_b64 ());
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c
index 5472e30..9db60b1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svuint64_t fixed_uint64_t __attribute__((arm_sve_vector_bits(256)));
 **	trn1	z0\.d, z0\.d, z0\.d
 **	ret
 */
-fixed_uint64_t
+[[gnu::noipa]] fixed_uint64_t
 f1 (fixed_uint64_t z0)
 {
   return __builtin_shufflevector (z0, z0, 0, 0, 2, 2);
@@ -26,7 +26,7 @@ f1 (fixed_uint64_t z0)
 **	trn2	z0\.d, z0\.d, z0\.d
 **	ret
 */
-fixed_uint64_t
+[[gnu::noipa]] fixed_uint64_t
 f2 (fixed_uint64_t z0)
 {
   return __builtin_shufflevector (z0, z0, 1, 1, 3, 3);
@@ -37,7 +37,7 @@ f2 (fixed_uint64_t z0)
 **	dupq	z0\.s, z0\.s\[0\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f3 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 0, 0, 0, 0, 4, 4, 4, 4);
@@ -48,7 +48,7 @@ f3 (fixed_int32_t z0)
 **	dupq	z0\.s, z0\.s\[1\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f4 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 1, 1, 1, 1, 5, 5, 5, 5);
@@ -59,7 +59,7 @@ f4 (fixed_int32_t z0)
 **	dupq	z0\.s, z0\.s\[2\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f5 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 2, 2, 2, 2, 6, 6, 6, 6);
@@ -70,7 +70,7 @@ f5 (fixed_int32_t z0)
 **	dupq	z0\.s, z0\.s\[3\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f6 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 3, 3, 3, 3, 7, 7, 7, 7);
@@ -81,7 +81,7 @@ f6 (fixed_int32_t z0)
 **	dupq	z0\.h, z0\.h\[0\]
 **	ret
 */
-fixed_uint16_t
+[[gnu::noipa]] fixed_uint16_t
 f7 (fixed_uint16_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -95,7 +95,7 @@ f7 (fixed_uint16_t z0)
 **	dupq	z0\.h, z0\.h\[5\]
 **	ret
 */
-fixed_uint16_t
+[[gnu::noipa]] fixed_uint16_t
 f8 (fixed_uint16_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -108,7 +108,7 @@ f8 (fixed_uint16_t z0)
 **	dupq	z0\.h, z0\.h\[7\]
 **	ret
 */
-fixed_uint16_t
+[[gnu::noipa]] fixed_uint16_t
 f9 (fixed_uint16_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -121,7 +121,7 @@ f9 (fixed_uint16_t z0)
 **	dupq	z0\.b, z0\.b\[0\]
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f10 (fixed_uint8_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -136,7 +136,7 @@ f10 (fixed_uint8_t z0)
 **	dupq	z0\.b, z0\.b\[13\]
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f11 (fixed_uint8_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -151,7 +151,7 @@ f11 (fixed_uint8_t z0)
 **	dupq	z0\.b, z0\.b\[15\]
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f12 (fixed_uint8_t z0)
 {
   return __builtin_shufflevector (z0, z0,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1_run.c
new file mode 100644
index 0000000..fd25034
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1_run.c
@@ -0,0 +1,87 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "dupq_1.c"
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(B) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_uint64_t a64 = { 0x1122, -1, 0x5566, -2 };
+  fixed_int32_t a32 = { 0x1122, -0x3344, 0x5566, -0x7788,
+			0x99aa, -0xbbcc, 0xddee, -0xff00 };
+  fixed_uint16_t a16 = { 0x9a12, 0xbc34, 0xde56, 0xf078,
+			 0x00ff, 0x11ee, 0x22dd, 0x33cc,
+			 0x44bb, 0x55aa, 0x6699, 0x7788,
+			 0xfe01, 0xdc23, 0xba45, 0x9867 };
+  fixed_uint8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		       0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		       0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		       0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+
+  fixed_uint64_t expected1 = { 0x1122, 0x1122, 0x5566, 0x5566 };
+  TEST (f1 (a64), expected1);
+
+  fixed_uint64_t expected2 = { -1, -1, -2, -2 };
+  TEST (f2 (a64), expected2);
+
+  fixed_int32_t expected3 = { 0x1122, 0x1122, 0x1122, 0x1122,
+                              0x99aa, 0x99aa, 0x99aa, 0x99aa };
+  TEST (f3 (a32), expected3);
+
+  fixed_int32_t expected4 = { -0x3344, -0x3344, -0x3344, -0x3344,
+			      -0xbbcc, -0xbbcc, -0xbbcc, -0xbbcc };
+  TEST (f4 (a32), expected4);
+
+  fixed_int32_t expected5 = { 0x5566, 0x5566, 0x5566, 0x5566,
+                              0xddee, 0xddee, 0xddee, 0xddee };
+  TEST (f5 (a32), expected5);
+
+  fixed_int32_t expected6 = { -0x7788, -0x7788, -0x7788, -0x7788,
+			      -0xff00, -0xff00, -0xff00, -0xff00 };
+  TEST (f6 (a32), expected6);
+
+  fixed_uint16_t expected7 = { 0x9a12, 0x9a12, 0x9a12, 0x9a12,
+			       0x9a12, 0x9a12, 0x9a12, 0x9a12,
+			       0x44bb, 0x44bb, 0x44bb, 0x44bb,
+			       0x44bb, 0x44bb, 0x44bb, 0x44bb };
+  TEST (f7 (a16), expected7);
+
+  fixed_uint16_t expected8 = { 0x11ee, 0x11ee, 0x11ee, 0x11ee,
+			       0x11ee, 0x11ee, 0x11ee, 0x11ee,
+			       0xdc23, 0xdc23, 0xdc23, 0xdc23,
+			       0xdc23, 0xdc23, 0xdc23, 0xdc23 };
+  TEST (f8 (a16), expected8);
+
+  fixed_uint16_t expected9 = { 0x33cc, 0x33cc, 0x33cc, 0x33cc,
+			       0x33cc, 0x33cc, 0x33cc, 0x33cc,
+			       0x9867, 0x9867, 0x9867, 0x9867,
+			       0x9867, 0x9867, 0x9867, 0x9867 };
+  TEST (f9 (a16), expected9);
+
+  fixed_uint8_t expected10 = { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+			       0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+			       0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe,
+			       0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe };
+  TEST (f10 (a8), expected10);
+
+  fixed_uint8_t expected11 = { 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde,
+			       0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde,
+			       0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21,
+			       0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21 };
+  TEST (f11 (a8), expected11);
+
+  fixed_uint8_t expected12 = { 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8,
+			       0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8,
+			       0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+			       0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07 };
+  TEST (f12 (a8), expected12);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/eon_bsl2n.c b/gcc/testsuite/gcc.target/aarch64/sve2/eon_bsl2n.c
new file mode 100644
index 0000000..74b4637
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/eon_bsl2n.c
@@ -0,0 +1,52 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <arm_neon.h>
+#include <arm_sve.h>
+
+#define EON(x, y)   (~((x) ^ (y)))
+
+/*
+** eon_d:
+** 	bsl2n	z0.d, z0.d, z0.d, z1.d
+** 	ret
+*/
+uint32x2_t eon_d(uint32x2_t a, uint32x2_t b) { return EON(a, b); }
+
+/*
+** eon_d_mp:
+**	movprfx	z0, z1
+** 	bsl2n	z0.d, z0.d, z1.d, z2.d
+** 	ret
+*/
+uint32x2_t eon_d_mp(uint32x2_t c, uint32x2_t a, uint32x2_t b) { return EON(a, b); }
+
+/*
+** eon_q:
+** 	bsl2n	z0.d, z0.d, z0.d, z1.d
+** 	ret
+*/
+uint64x2_t eon_q(uint64x2_t a, uint64x2_t b) { return EON(a, b); }
+
+/*
+** eon_q_mp:
+**	movprfx	z0, z1
+** 	bsl2n	z0.d, z0.d, z1.d, z2.d
+** 	ret
+*/
+uint64x2_t eon_q_mp(uint64x2_t c, uint64x2_t a, uint64x2_t b) { return EON(a, b); }
+
+/*
+** eon_z:
+** 	bsl2n	z0.d, z0.d, z0.d, z1.d
+** 	ret
+*/
+svuint64_t eon_z(svuint64_t a, svuint64_t b) { return EON(a, b); }
+
+/*
+** eon_z_mp:
+**	movprfx	z0, z1
+** 	bsl2n	z0.d, z0.d, z1.d, z2.d
+** 	ret
+*/
+svuint64_t eon_z_mp(svuint64_t c, svuint64_t a, svuint64_t b) { return EON(a, b); }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c
index 03c5fb1..be5ae71 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svfloat64_t fixed_float64_t __attribute__((arm_sve_vector_bits(256)));
 **	extq	z0\.b, z0\.b, z1\.b, #8
 **	ret
 */
-fixed_float64_t
+[[gnu::noipa]] fixed_float64_t
 f1 (fixed_float64_t z0, fixed_float64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 4, 3, 6);
@@ -26,7 +26,7 @@ f1 (fixed_float64_t z0, fixed_float64_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #4
 **	ret
 */
-fixed_uint32_t
+[[gnu::noipa]] fixed_uint32_t
 f2 (fixed_uint32_t z0, fixed_uint32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 2, 3, 8, 5, 6, 7, 12);
@@ -37,7 +37,7 @@ f2 (fixed_uint32_t z0, fixed_uint32_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #12
 **	ret
 */
-fixed_uint32_t
+[[gnu::noipa]] fixed_uint32_t
 f3 (fixed_uint32_t z0, fixed_uint32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 3, 8, 9, 10, 7, 12, 13, 14);
@@ -48,7 +48,7 @@ f3 (fixed_uint32_t z0, fixed_uint32_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #2
 **	ret
 */
-fixed_float16_t
+[[gnu::noipa]] fixed_float16_t
 f4 (fixed_float16_t z0, fixed_float16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -61,7 +61,7 @@ f4 (fixed_float16_t z0, fixed_float16_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #10
 **	ret
 */
-fixed_float16_t
+[[gnu::noipa]] fixed_float16_t
 f5 (fixed_float16_t z0, fixed_float16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -74,7 +74,7 @@ f5 (fixed_float16_t z0, fixed_float16_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #14
 **	ret
 */
-fixed_float16_t
+[[gnu::noipa]] fixed_float16_t
 f6 (fixed_float16_t z0, fixed_float16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -87,7 +87,7 @@ f6 (fixed_float16_t z0, fixed_float16_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #1
 **	ret
 */
-fixed_int8_t
+[[gnu::noipa]] fixed_int8_t
 f7 (fixed_int8_t z0, fixed_int8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -102,7 +102,7 @@ f7 (fixed_int8_t z0, fixed_int8_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #11
 **	ret
 */
-fixed_int8_t
+[[gnu::noipa]] fixed_int8_t
 f8 (fixed_int8_t z0, fixed_int8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -117,7 +117,7 @@ f8 (fixed_int8_t z0, fixed_int8_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #15
 **	ret
 */
-fixed_int8_t
+[[gnu::noipa]] fixed_int8_t
 f9 (fixed_int8_t z0, fixed_int8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/extq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1_run.c
new file mode 100644
index 0000000..6b72c98
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1_run.c
@@ -0,0 +1,73 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "extq_1.c"
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(B) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_float64_t a64 = { 1.5, 3.75, -5.25, 9 };
+  fixed_float64_t b64 = { -2, 4.125, -6.375, 11.5 };
+  fixed_float64_t expected1 = { 3.75, -2, 9, -6.375 };
+  TEST (f1 (a64, b64), expected1);
+
+  fixed_uint32_t a32 = { 0x1122, -0x3344, 0x5566, -0x7788,
+			 0x99aa, -0xbbcc, 0xddee, -0xff00 };
+  fixed_uint32_t b32 = { 1 << 20, 1 << 21, 1 << 22, 1 << 23,
+			 5 << 6, 5 << 7, 5 << 8, 5 << 9 };
+  fixed_uint32_t expected2 = { -0x3344, 0x5566, -0x7788, 1 << 20,
+			       -0xbbcc, 0xddee, -0xff00, 5 << 6 };
+  fixed_uint32_t expected3 = { -0x7788, 1 << 20, 1 << 21, 1 << 22,
+			       -0xff00, 5 << 6, 5 << 7, 5 << 8 };
+  TEST (f2 (a32, b32), expected2);
+  TEST (f3 (a32, b32), expected3);
+
+  fixed_float16_t a16 = { 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25,
+			  2.5, 2.75, 3, 3.25, 3.5, 3.75, 4, 4.25 };
+  fixed_float16_t b16 = { -0.5, -0.75, -1, -1.25, -1.5, -1.75, -2, -2.25,
+			  -2.5, -2.75, -3, -3.25, -3.5, -3.75, -4, -4.25 };
+  fixed_float16_t expected4 = { 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25, -0.5,
+				2.75, 3, 3.25, 3.5, 3.75, 4, 4.25, -2.5 };
+  fixed_float16_t expected5 = { 1.75, 2, 2.25, -0.5, -0.75, -1, -1.25, -1.5,
+				3.75, 4, 4.25, -2.5, -2.75, -3, -3.25, -3.5 };
+  fixed_float16_t expected6 = { 2.25, -0.5, -0.75, -1,
+				-1.25, -1.5, -1.75, -2,
+				4.25, -2.5, -2.75, -3,
+				-3.25, -3.5, -3.75, -4 };
+  TEST (f4 (a16, b16), expected4);
+  TEST (f5 (a16, b16), expected5);
+  TEST (f6 (a16, b16), expected6);
+
+  fixed_int8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		      0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		      0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		      0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+  fixed_int8_t b8 = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+		      0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+		      0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, 0x8a,
+		      0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1, 0x02 };
+  fixed_int8_t expected7 = { 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70, 0x89,
+			     0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8, 0x11,
+			     0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f, 0x76,
+			     0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07, 0x13 };
+  fixed_int8_t expected8 = { 0xbc, 0xcd, 0xde, 0xef, 0xf8, 0x11, 0x22, 0x33,
+			     0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb,
+			     0x43, 0x32, 0x21, 0x10, 0x07, 0x13, 0x24, 0x35,
+			     0x46, 0x57, 0x68, 0x79, 0x8a, 0x9b, 0xac, 0xbd };
+  fixed_int8_t expected9 = { 0xf8, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+			     0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+			     0x07, 0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79,
+			     0x8a, 0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1 };
+  TEST (f7 (a8, b8), expected7);
+  TEST (f8 (a8, b8), expected8);
+  TEST (f9 (a8, b8), expected9);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c b/gcc/testsuite/gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c
new file mode 100644
index 0000000..09bfc19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c
@@ -0,0 +1,68 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <arm_neon.h>
+
+#define NAND(x, y)  (~((x) & (y)))
+#define NOR(x, y)   (~((x) | (y)))
+
+/*
+** nand_d:
+** 	nbsl	z0.d, z0.d, z1.d, z1.d
+** 	ret
+*/
+uint32x2_t nand_d(uint32x2_t a, uint32x2_t b) { return NAND(a, b); }
+
+/*
+** nand_d_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z2.d
+** 	ret
+*/
+uint32x2_t nand_d_mp(uint32x2_t c, uint32x2_t a, uint32x2_t b) { return NAND(a, b); }
+
+/*
+** nor_d:
+** 	nbsl	z0.d, z0.d, z1.d, z0.d
+** 	ret
+*/
+uint32x2_t nor_d(uint32x2_t a, uint32x2_t b) { return NOR(a, b); }
+
+/*
+** nor_d_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z1.d
+** 	ret
+*/
+uint32x2_t nor_d_mp(uint32x2_t c, uint32x2_t a, uint32x2_t b) { return NOR(a, b); }
+
+/*
+** nand_q:
+** 	nbsl	z0.d, z0.d, z1.d, z1.d
+** 	ret
+*/
+uint64x2_t nand_q(uint64x2_t a, uint64x2_t b) { return NAND(a, b); }
+
+/*
+** nand_q_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z2.d
+** 	ret
+*/
+uint32x4_t nand_q_mp(uint32x4_t c, uint32x4_t a, uint32x4_t b) { return NAND(a, b); }
+
+/*
+** nor_q:
+** 	nbsl	z0.d, z0.d, z1.d, z0.d
+** 	ret
+*/
+uint64x2_t nor_q(uint64x2_t a, uint64x2_t b) { return NOR(a, b); }
+
+/*
+** nor_q_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z1.d
+** 	ret
+*/
+uint32x4_t nor_q_mp(uint32x4_t c, uint32x4_t a, uint32x4_t b) { return NOR(a, b); }
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c
index 94470a5..977fa39 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c
index b8747b8..b816fa1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c
index 7cb7ee5..0e4427a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c
index 787126f..81d0c82 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c
index 6b2b0a42..3920bdb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve2+faminmax" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve2+faminmax -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c
index a0a7f80..c7d10b3 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c
index c13db48..122fba7 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c
index 145b077..7f35859 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c
index da175db..b079a56 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c
index c0476ce..14e77c00 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c
index f644024..b680548 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c
index a48a8a9..6d1a356 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c
index 1fc08a3..9cb4471 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c
index bd2c937..e57a650 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c
index f4994de..710ca73 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c
index 6bec3b3..dc9cf46 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c
index bcb4a14..2728c9b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c
index ba7e931..9f33295 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv9.2-a+sve+sme" } */
+/* { dg-options "-O2 -march=armv9.2-a+sve+sme -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c
index 7aa59ff..68769fe 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c
index 1a4525c..692891f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c
index b64bfc3..7dffa1c 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pr120999.c b/gcc/testsuite/gcc.target/aarch64/sve2/pr120999.c
new file mode 100644
index 0000000..2dca36a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pr120999.c
@@ -0,0 +1,17 @@
+/* PR target/120999.  */
+/* { dg-do assemble } */
+/* { dg-options "-O2 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <arm_sve.h>
+
+#define NOR(x, y)   (~((x) | (y)))
+
+/*
+** nor_z:
+** 	movprfx	z0, z1
+**	nbsl	z0.d, z0.d, z2.d, z1.d
+** 	ret
+*/
+svuint64_t nor_z(svuint64_t c, svuint64_t a, svuint64_t b) { return NOR(a, b); }
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c
index f923e94..587f670 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svint64_t fixed_int64_t __attribute__((arm_sve_vector_bits(256)));
 **	trn1	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f1 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 4, 2, 6);
@@ -26,7 +26,7 @@ f1 (fixed_int64_t z0, fixed_int64_t z1)
 **	trn2	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f2 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 5, 3, 7);
@@ -37,7 +37,7 @@ f2 (fixed_int64_t z0, fixed_int64_t z1)
 **	uzpq1	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f3 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 2, 8, 10, 4, 6, 12, 14);
@@ -48,7 +48,7 @@ f3 (fixed_float32_t z0, fixed_float32_t z1)
 **	uzpq2	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f4 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 3, 9, 11, 5, 7, 13, 15);
@@ -59,7 +59,7 @@ f4 (fixed_float32_t z0, fixed_float32_t z1)
 **	uzpq1	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -72,7 +72,7 @@ f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	uzpq2	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -85,7 +85,7 @@ f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	uzpq1	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -100,7 +100,7 @@ f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 **	uzpq2	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f8 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1_run.c
new file mode 100644
index 0000000..9044cae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1_run.c
@@ -0,0 +1,78 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "uzpq_1.c"
+
+typedef svuint16_t fixed_uint16_t __attribute__((arm_sve_vector_bits(256)));
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(A) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_int64_t a64 = { 0x1122LL << 31, -1LL << 47, 0x5566 << 15, -2 };
+  fixed_int64_t b64 = { 42, -0x3344LL << 19, 303, -0x7788LL << 27 };
+  fixed_int64_t expected1 = { 0x1122LL << 31, 42,
+			      0x5566 << 15, 303 };
+  fixed_int64_t expected2 = { -1LL << 47, -0x3344LL << 19,
+			      -2, -0x7788LL << 27 };
+  TEST (f1 (a64, b64), expected1);
+  TEST (f2 (a64, b64), expected2);
+
+  fixed_float32_t a32 = { 0.5, 0.75, 1, 1.25, 2.5, 2.75, 3, 3.25 };
+  fixed_float32_t b32 = { -0.5, -0.75, -1, -1.25, -2.5, -2.75, -3, -3.25 };
+  fixed_float32_t expected3 = { 0.5, 1, -0.5, -1,
+				2.5, 3, -2.5, -3 };
+  fixed_float32_t expected4 = { 0.75, 1.25, -0.75, -1.25,
+				2.75, 3.25, -2.75, -3.25 };
+  TEST (f3 (a32, b32), expected3);
+  TEST (f4 (a32, b32), expected4);
+
+  fixed_uint16_t a16_i = { 0x9a12, 0xbc34, 0xde56, 0xf078,
+			   0x00ff, 0x11ee, 0x22dd, 0x33cc,
+			   0x44bb, 0x55aa, 0x6699, 0x7788,
+			   0xfe01, 0xdc23, 0xba45, 0x9867 };
+  fixed_uint16_t b16_i = { 0x1010, 0x2020, 0x3030, 0x4040,
+			   0x5050, 0x6060, 0x7070, 0x8080,
+			   0x9090, 0xa0a0, 0xb0b0, 0xc0c0,
+			   0xd0d0, 0xe0e0, 0xf0f0, 0x0f0f };
+  fixed_uint16_t expected5 = { 0x9a12, 0xde56, 0x00ff, 0x22dd,
+			       0x1010, 0x3030, 0x5050, 0x7070,
+			       0x44bb, 0x6699, 0xfe01, 0xba45,
+			       0x9090, 0xb0b0, 0xd0d0, 0xf0f0 };
+  fixed_uint16_t expected6 = { 0xbc34, 0xf078, 0x11ee, 0x33cc,
+			       0x2020, 0x4040, 0x6060, 0x8080,
+			       0x55aa, 0x7788, 0xdc23, 0x9867,
+			       0xa0a0, 0xc0c0, 0xe0e0, 0x0f0f };
+  fixed_bfloat16_t a16, b16;
+  __builtin_memcpy (&a16, &a16_i, sizeof (a16));
+  __builtin_memcpy (&b16, &b16_i, sizeof (b16));
+  TEST (f5 (a16, b16), expected5);
+  TEST (f6 (a16, b16), expected6);
+
+  fixed_uint8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		       0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		       0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		       0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+  fixed_uint8_t b8 = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+		       0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+		       0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, 0x8a,
+		       0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1, 0x02 };
+  fixed_uint8_t expected7 = { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+			      0x11, 0x33, 0x55, 0x77, 0x99, 0xbb, 0xdd, 0xff,
+			      0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+			      0x13, 0x35, 0x57, 0x79, 0x9b, 0xbd, 0xdf, 0xf1 };
+  fixed_uint8_t expected8 = { 0x12, 0x34, 0x56, 0x70, 0x9a, 0xbc, 0xde, 0xf8,
+			      0x22, 0x44, 0x66, 0x88, 0xaa, 0xcc, 0xee, 0x00,
+			      0xed, 0xcb, 0xa9, 0x8f, 0x65, 0x43, 0x21, 0x07,
+			      0x24, 0x46, 0x68, 0x8a, 0xac, 0xce, 0xe0, 0x02 };
+  TEST (f7 (a8, b8), expected7);
+  TEST (f8 (a8, b8), expected8);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c
index fa420a9..76fb4b4 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svint64_t fixed_int64_t __attribute__((arm_sve_vector_bits(256)));
 **	trn1	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f1 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 4, 2, 6);
@@ -26,7 +26,7 @@ f1 (fixed_int64_t z0, fixed_int64_t z1)
 **	trn2	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f2 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 5, 3, 7);
@@ -37,7 +37,7 @@ f2 (fixed_int64_t z0, fixed_int64_t z1)
 **	zipq1	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f3 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 8, 1, 9, 4, 12, 5, 13);
@@ -48,7 +48,7 @@ f3 (fixed_float32_t z0, fixed_float32_t z1)
 **	zipq2	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f4 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 2, 10, 3, 11, 6, 14, 7, 15);
@@ -59,7 +59,7 @@ f4 (fixed_float32_t z0, fixed_float32_t z1)
 **	zipq1	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -72,7 +72,7 @@ f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	zipq2	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -85,7 +85,7 @@ f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	zipq1	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -100,7 +100,7 @@ f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 **	zipq2	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f8 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1_run.c
new file mode 100644
index 0000000..211f9d9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1_run.c
@@ -0,0 +1,78 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "zipq_1.c"
+
+typedef svuint16_t fixed_uint16_t __attribute__((arm_sve_vector_bits(256)));
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(A) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_int64_t a64 = { 0x1122LL << 31, -1LL << 47, 0x5566 << 15, -2 };
+  fixed_int64_t b64 = { 42, -0x3344LL << 19, 303, -0x7788LL << 27 };
+  fixed_int64_t expected1 = { 0x1122LL << 31, 42,
+			      0x5566 << 15, 303 };
+  fixed_int64_t expected2 = { -1LL << 47, -0x3344LL << 19,
+			      -2, -0x7788LL << 27 };
+  TEST (f1 (a64, b64), expected1);
+  TEST (f2 (a64, b64), expected2);
+
+  fixed_float32_t a32 = { 0.5, 0.75, 1, 1.25, 2.5, 2.75, 3, 3.25 };
+  fixed_float32_t b32 = { -0.5, -0.75, -1, -1.25, -2.5, -2.75, -3, -3.25 };
+  fixed_float32_t expected3 = { 0.5, -0.5, 0.75, -0.75,
+				2.5, -2.5, 2.75, -2.75 };
+  fixed_float32_t expected4 = { 1, -1, 1.25, -1.25,
+				3, -3, 3.25, -3.25 };
+  TEST (f3 (a32, b32), expected3);
+  TEST (f4 (a32, b32), expected4);
+
+  fixed_uint16_t a16_i = { 0x9a12, 0xbc34, 0xde56, 0xf078,
+			   0x00ff, 0x11ee, 0x22dd, 0x33cc,
+			   0x44bb, 0x55aa, 0x6699, 0x7788,
+			   0xfe01, 0xdc23, 0xba45, 0x9867 };
+  fixed_uint16_t b16_i = { 0x1010, 0x2020, 0x3030, 0x4040,
+			   0x5050, 0x6060, 0x7070, 0x8080,
+			   0x9090, 0xa0a0, 0xb0b0, 0xc0c0,
+			   0xd0d0, 0xe0e0, 0xf0f0, 0x0f0f };
+  fixed_uint16_t expected5 = { 0x9a12, 0x1010, 0xbc34, 0x2020,
+			       0xde56, 0x3030, 0xf078, 0x4040,
+			       0x44bb, 0x9090, 0x55aa, 0xa0a0,
+			       0x6699, 0xb0b0, 0x7788, 0xc0c0 };
+  fixed_uint16_t expected6 = { 0x00ff, 0x5050, 0x11ee, 0x6060,
+			       0x22dd, 0x7070, 0x33cc, 0x8080,
+			       0xfe01, 0xd0d0, 0xdc23, 0xe0e0,
+			       0xba45, 0xf0f0, 0x9867, 0x0f0f };
+  fixed_bfloat16_t a16, b16;
+  __builtin_memcpy (&a16, &a16_i, sizeof (a16));
+  __builtin_memcpy (&b16, &b16_i, sizeof (b16));
+  TEST (f5 (a16, b16), expected5);
+  TEST (f6 (a16, b16), expected6);
+
+  fixed_uint8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		       0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		       0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		       0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+  fixed_uint8_t b8 = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+		       0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+		       0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, 0x8a,
+		       0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1, 0x02 };
+  fixed_uint8_t expected7 = { 0x01, 0x11, 0x12, 0x22, 0x23, 0x33, 0x34, 0x44,
+			      0x45, 0x55, 0x56, 0x66, 0x67, 0x77, 0x70, 0x88,
+			      0xfe, 0x13, 0xed, 0x24, 0xdc, 0x35, 0xcb, 0x46,
+			      0xba, 0x57, 0xa9, 0x68, 0x98, 0x79, 0x8f, 0x8a };
+  fixed_uint8_t expected8 = { 0x89, 0x99, 0x9a, 0xaa, 0xab, 0xbb, 0xbc, 0xcc,
+			      0xcd, 0xdd, 0xde, 0xee, 0xef, 0xff, 0xf8, 0x00,
+			      0x76, 0x9b, 0x65, 0xac, 0x54, 0xbd, 0x43, 0xce,
+			      0x32, 0xdf, 0x21, 0xe0, 0x10, 0xf1, 0x07, 0x02 };
+  TEST (f7 (a8, b8), expected7);
+  TEST (f8 (a8, b8), expected8);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c b/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c
index b34b902c..ba4696e 100644
--- a/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c
+++ b/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2" } */
+/* { dg-options "-Os" } */
 
 #include "arm_neon.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/vector-compare-5.c b/gcc/testsuite/gcc.target/aarch64/vector-compare-5.c
new file mode 100644
index 0000000..a1a601d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vector-compare-5.c
@@ -0,0 +1,67 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-additional-options "-fdump-tree-original-all" } */
+
+typedef int v4i __attribute__((vector_size(4*sizeof(int))));
+
+/* Ensure we can simplify `VEC_COND_EXPR(a OP1 b) OP2 VEC_COND_EXPR(a OP3 b)`
+ * into `VEC_COND_EXPR(a OP4 b)`
+ */
+
+void use (v4i const *z);
+
+void
+g (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x > *y | *x == *y; // expect >=
+  *t = *x > *y | *x <= *y; // expect true
+}
+
+void
+h (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x <= *y & *x >= *y; // expect x == y
+  *t = *x <= *y & *x != *y; // expect x<y
+}
+
+void
+i (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x == *y | *x != *y; // expect true
+  *t = *x == *y & *x != *y; // expect false
+}
+
+void
+k (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x < *y | *x == *y;  // x <= y
+  *t = *x < *y & *x > *y;   // expect false
+}
+
+void
+m (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x <= *y ^ *x >= *y; /* expect x != y */
+  *t = *x <= *y ^ *x != *y; /* expect x <= y */
+}
+
+void
+n (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x == *y ^ *x != *y; /* expect true */
+  *t = *x == *y ^ *x == *y; /* expect false */
+}
+
+
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*>=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*==\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*<\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*<=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*!=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*>=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*;" "original" } } */
diff --git a/gcc/testsuite/gcc.target/arm/pr121065.c b/gcc/testsuite/gcc.target/arm/pr121065.c
new file mode 100644
index 0000000..dfc6059
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/pr121065.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=cortex-m55" } */
+
+_Accum sa;
+char c;
+
+void
+div_csa ()
+{
+  c /= sa;
+}
diff --git a/gcc/testsuite/gcc.target/avr/torture/pr118591-1.c b/gcc/testsuite/gcc.target/avr/torture/pr118591-1.c
index 814f041..6f54c63 100644
--- a/gcc/testsuite/gcc.target/avr/torture/pr118591-1.c
+++ b/gcc/testsuite/gcc.target/avr/torture/pr118591-1.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { ! avr_tiny } } } */
-/* { dg-additional-options "-std=c99 -mlra" } */
+/* { dg-additional-options "-std=c99" } */
 
 __attribute__((noipa))
 void func2 (long long a1, long long a2, long b)
diff --git a/gcc/testsuite/gcc.target/avr/torture/pr118591-2.c b/gcc/testsuite/gcc.target/avr/torture/pr118591-2.c
index 83d3606..222af09 100644
--- a/gcc/testsuite/gcc.target/avr/torture/pr118591-2.c
+++ b/gcc/testsuite/gcc.target/avr/torture/pr118591-2.c
@@ -1,6 +1,6 @@
 /* Test case failed on avrtiny.  */
 /* { dg-do run } */
-/* { dg-additional-options "-std=c99 -mlra" } */
+/* { dg-additional-options "-std=c99" } */
 
 __attribute__((noipa))
 void func2 (long a, long b)
diff --git a/gcc/testsuite/gcc.target/i386/20020224-1.c b/gcc/testsuite/gcc.target/i386/20020224-1.c
index 2905719..769332b 100644
--- a/gcc/testsuite/gcc.target/i386/20020224-1.c
+++ b/gcc/testsuite/gcc.target/i386/20020224-1.c
@@ -4,6 +4,7 @@
    while callee was actually not poping it up (as the hidden argument
    was passed in register).  */
 /* { dg-do run } */
+/* { dg-require-effective-target ia32 } */
 /* { dg-options "-O2 -fomit-frame-pointer" } */
 
 extern void abort (void);
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c
index cfd5644..c9a2d19 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_cvtrowd2ps
 void test_amx_avx512_cvtrowd2ps();
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c
index acd5f76..2014ec6 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_cvtrowps2bf16
 void test_amx_avx512_cvtrowps2bf16();
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c
index 1fd28de..ca53ed00 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_cvtrowps2ph
 void test_amx_avx512_cvtrowps2ph();
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c
index ea28d82..b2dee14 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_movrow
 void test_amx_avx512_movrow();
diff --git a/gcc/testsuite/gcc.target/i386/apx-1.c b/gcc/testsuite/gcc.target/i386/apx-1.c
index 4e580ec..b118928 100644
--- a/gcc/testsuite/gcc.target/i386/apx-1.c
+++ b/gcc/testsuite/gcc.target/i386/apx-1.c
@@ -3,6 +3,6 @@
 /* { dg-error "'-mapxf' is not supported for 32-bit code" "" { target ia32 } 0 } */
 
 void
-apx_hanlder ()
+apx_handler ()
 {
 }
diff --git a/gcc/testsuite/gcc.target/i386/asm-hard-reg-1.c b/gcc/testsuite/gcc.target/i386/asm-hard-reg-1.c
new file mode 100644
index 0000000..8080f56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/asm-hard-reg-1.c
@@ -0,0 +1,80 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+void
+test (void)
+{
+  int x, y;
+
+  __asm__ __volatile__ ("" : "=a" (x), "={rbx}" (y));
+  __asm__ __volatile__ ("" : "=a" (x), "={rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=a" (x) : "{rax}" (y));
+  __asm__ __volatile__ ("" : "=&a" (x) : "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "a" (x), "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rbx}" (x), "=a" (y));
+  __asm__ __volatile__ ("" : "={rax}" (x), "=a" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rax}" (x) : "a" (y));
+  __asm__ __volatile__ ("" : "=&{rax}" (x) : "a" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rax}" (x), "a" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=b" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=b" (x), "={rbx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=b" (x) : "{rbx}" (y));
+  __asm__ __volatile__ ("" : "=&b" (x) : "{rbx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "b" (x), "{rbx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=b" (y));
+  __asm__ __volatile__ ("" : "={rbx}" (x), "=b" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rbx}" (x) : "b" (y));
+  __asm__ __volatile__ ("" : "=&{rbx}" (x) : "b" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rbx}" (x), "b" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=c" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=c" (x), "={rcx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=c" (x) : "{rcx}" (y));
+  __asm__ __volatile__ ("" : "=&c" (x) : "{rcx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "c" (x), "{rcx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=c" (y));
+  __asm__ __volatile__ ("" : "={rcx}" (x), "=c" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rcx}" (x) : "c" (y));
+  __asm__ __volatile__ ("" : "=&{rcx}" (x) : "c" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rcx}" (x), "c" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=d" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=d" (x), "={rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=d" (x) : "{rdx}" (y));
+  __asm__ __volatile__ ("" : "=&d" (x) : "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "d" (x), "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=d" (y));
+  __asm__ __volatile__ ("" : "={rdx}" (x), "=d" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rdx}" (x) : "d" (y));
+  __asm__ __volatile__ ("" : "=&{rdx}" (x) : "d" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rdx}" (x), "d" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=S" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=S" (x), "={rsi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=S" (x) : "{rsi}" (y));
+  __asm__ __volatile__ ("" : "=&S" (x) : "{rsi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "S" (x), "{rsi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=S" (y));
+  __asm__ __volatile__ ("" : "={rsi}" (x), "=S" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rsi}" (x) : "S" (y));
+  __asm__ __volatile__ ("" : "=&{rsi}" (x) : "S" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rsi}" (x), "S" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=D" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=D" (x), "={rdi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=D" (x) : "{rdi}" (y));
+  __asm__ __volatile__ ("" : "=&D" (x) : "{rdi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "D" (x), "{rdi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=D" (y));
+  __asm__ __volatile__ ("" : "={rdi}" (x), "=D" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rdi}" (x) : "D" (y));
+  __asm__ __volatile__ ("" : "=&{rdi}" (x) : "D" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rdi}" (x), "D" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/asm-hard-reg-2.c b/gcc/testsuite/gcc.target/i386/asm-hard-reg-2.c
new file mode 100644
index 0000000..b35cf53
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/asm-hard-reg-2.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+void
+test (void)
+{
+  int x, y, yy;
+#ifdef __x86_64__
+  int z __attribute__ ((mode (TI)));
+#else
+  long z;
+#endif
+
+  __asm__ __volatile__ ("" : "=A" (z), "={rbx}" (y));
+  __asm__ __volatile__ ("" : "=A" (z), "={rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=A" (z), "={rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=A" (z) : "{rax}" (y));
+  __asm__ __volatile__ ("" : "=A" (z) : "{rdx}" (y));
+  __asm__ __volatile__ ("" : "=&A" (z) : "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=&A" (z) : "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "A" (z), "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "A" (z), "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rbx}" (y), "=A" (z));
+  __asm__ __volatile__ ("" : "={rax}" (y), "=A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rdx}" (y), "=A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rax}" (y) : "A" (z));
+  __asm__ __volatile__ ("" : "={rdx}" (y) : "A" (z));
+  __asm__ __volatile__ ("" : "=&{rax}" (y) : "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=&{rdx}" (y) : "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rax}" (y), "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rdx}" (y), "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  /* Note, we do not error for */
+  __asm__ __volatile__ ("" : "=A" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=A" (x), "={rdx}" (y));
+  /* This is due to how constraint A is implemented.  RA has the freedom to
+     choose between rax or rdx for operand 0 since x fits into a single
+     register and does not require a register pair.  Of course, we error out if
+     rax and rdx are taken by other operands as in the following:  */
+  __asm__ __volatile__ ("" : "=A" (x), "={rax}" (y), "={rdx}" (yy)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=A" (x), "={rdx}" (y), "={rax}" (yy)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/attributes-error.c b/gcc/testsuite/gcc.target/i386/attributes-error.c
index 405eda5..5d1c77d 100644
--- a/gcc/testsuite/gcc.target/i386/attributes-error.c
+++ b/gcc/testsuite/gcc.target/i386/attributes-error.c
@@ -1,12 +1,40 @@
+/* { dg-options "-msse2" } */
 /* { dg-do compile } */
 /* { dg-require-effective-target ia32 } */
 
-void foo1(int i, int j) __attribute__((fastcall, cdecl)); /* { dg-error "not compatible" } */
-void foo2(int i, int j) __attribute__((fastcall, stdcall)); /* { dg-error "not compatible" } */
+void foo1(int i, int j) __attribute__((cdecl, regparm(2)));
+void foo2(int i, int j) __attribute__((stdcall, regparm(2)));
 void foo3(int i, int j) __attribute__((fastcall, regparm(2))); /* { dg-error "not compatible" } */
-void foo4(int i, int j) __attribute__((stdcall, cdecl)); /* { dg-error "not compatible" } */
-void foo5(int i, int j) __attribute__((stdcall, fastcall)); /* { dg-error "not compatible" } */
-void foo6(int i, int j) __attribute__((cdecl, fastcall)); /* { dg-error "not compatible" } */
-void foo7(int i, int j) __attribute__((cdecl, stdcall)); /* { dg-error "not compatible" } */
-void foo8(int i, int j) __attribute__((regparm(2), fastcall)); /* { dg-error "not compatible" } */
+void foo4(int i, int j) __attribute__((thiscall, regparm(2))); /* { dg-error "not compatible" } */
+void foo5(int i, int j) __attribute__((sseregparm, regparm(2)));
+
+void foo6(int i, int j) __attribute__((stdcall, fastcall)); /* { dg-error "not compatible" } */
+void foo7(int i, int j) __attribute__((regparm(2), fastcall)); /* { dg-error "not compatible" } */
+void foo8(int i, int j) __attribute__((cdecl, fastcall)); /* { dg-error "not compatible" } */
+void foo9(int i, int j) __attribute__((thiscall, fastcall)); /* { dg-error "not compatible" } */
+void foo10(int i, int j) __attribute__((sseregparm, fastcall));
+
+void foo11(int i, int j) __attribute__((cdecl, stdcall)); /* { dg-error "not compatible" } */
+void foo12(int i, int j) __attribute__((fastcall, stdcall)); /* { dg-error "not compatible" } */
+void foo13(int i, int j) __attribute__((thiscall, stdcall)); /* { dg-error "not compatible" } */
+void foo14(int i, int j) __attribute__((regparm(2), stdcall));
+void foo15(int i, int j) __attribute__((sseregparm, stdcall));
+
+void foo16(int i, int j) __attribute__((stdcall, cdecl)); /* { dg-error "not compatible" } */
+void foo17(int i, int j) __attribute__((fastcall, cdecl)); /* { dg-error "not compatible" } */
+void foo18(int i, int j) __attribute__((thiscall, cdecl)); /* { dg-error "not compatible" } */
+void foo19(int i, int j) __attribute__((regparm(2), cdecl));
+void foo20(int i, int j) __attribute__((sseregparm, cdecl));
+
+void foo21(int i, int j) __attribute__((stdcall, thiscall)); /* { dg-error "not compatible" } */
+void foo22(int i, int j) __attribute__((fastcall, thiscall)); /* { dg-error "not compatible" } */
+void foo23(int i, int j) __attribute__((cdecl, thiscall)); /* { dg-error "not compatible" } */
+void foo24(int i, int j) __attribute__((regparm(2), thiscall)); /* { dg-error "not compatible" } */
+void foo25(int i, int j) __attribute__((sseregparm, thiscall));
+
+void foo26(int i, int j) __attribute__((cdecl, sseregparm));
+void foo27(int i, int j) __attribute__((fastcall, sseregparm));
+void foo28(int i, int j) __attribute__((stdcall, sseregparm));
+void foo29(int i, int j) __attribute__((thiscall, sseregparm));
+void foo30(int i, int j) __attribute__((regparm(2), sseregparm));
 
diff --git a/gcc/testsuite/gcc.target/i386/attributes-ignore.c b/gcc/testsuite/gcc.target/i386/attributes-ignore.c
new file mode 100644
index 0000000..93a3770
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/attributes-ignore.c
@@ -0,0 +1,8 @@
+/* { dg-do compile { target { ! ia32 } } } */
+
+void foo1(int i, int j) __attribute__((regparm(0))); /* { dg-warning "ignored" } */
+void foo2(int i, int j) __attribute__((stdcall)); /* { dg-warning "ignored" } */
+void foo3(int i, int j) __attribute__((fastcall)); /* { dg-warning "ignored" } */
+void foo4(int i, int j) __attribute__((cdecl)); /* { dg-warning "ignored" } */
+void foo5(int i, int j) __attribute__((thiscall)); /* { dg-warning "ignored" } */
+void foo6(int i, int j) __attribute__((sseregparm)); /* { dg-warning "ignored" } */
diff --git a/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c b/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c
index 102217c..4f26aa4 100644
--- a/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c
+++ b/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c
@@ -8,17 +8,28 @@
 /*
 **foo:
 **...
+**	leaq	-160\(%rbp\), %rax
+**	movq	%rax, %rcx
 **	pxor	%xmm0, %xmm0
-**...
+**	movl	\$160, %edx
+**	movl	%edx, %edi
+**	andl	\$-64, %edi
+**	movl	\$0, %esi
 **.L[0-9]+:
-**	movl	%esi, %ecx
-**	movaps	%xmm0, \(%rdx,%rcx\)
-**	movaps	%xmm0, 16\(%rdx,%rcx\)
-**	movaps	%xmm0, 32\(%rdx,%rcx\)
-**	movaps	%xmm0, 48\(%rdx,%rcx\)
+**	movl	%esi, %edx
+**	movaps	%xmm0, \(%rax,%rdx\)
+**	movaps	%xmm0, 16\(%rax,%rdx\)
+**	movaps	%xmm0, 32\(%rax,%rdx\)
+**	movaps	%xmm0, 48\(%rax,%rdx\)
 **	addl	\$64, %esi
 **	cmpl	%edi, %esi
 **	jb	.L[0-9]+
+**	movl	%esi, %eax
+**	addq	%rax, %rcx
+**	movaps	%xmm0, \(%rcx\)
+**	movaps	%xmm0, 16\(%rcx\)
+**	movzbl	-116\(%rbp\), %eax
+**	movsbl	%al, %eax
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-1.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-1.c
new file mode 100644
index 0000000..b1f6678
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-1.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mno-sse -mmemcpy-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movq	221\(%rsi\), %rax
+**	xorl	%edx, %edx
+**	movq	%rax, 221\(%rdi\)
+**	movq	229\(%rsi\), %rax
+**	movq	%rax, 229\(%rdi\)
+**	movq	237\(%rsi\), %rax
+**	movq	%rax, 237\(%rdi\)
+**	movq	245\(%rsi\), %rax
+**	movq	%rax, 245\(%rdi\)
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$32, %edx
+**	movq	\(%rsi,%rax\), %r10
+**	movq	8\(%rsi,%rax\), %r9
+**	movq	16\(%rsi,%rax\), %r8
+**	movq	24\(%rsi,%rax\), %rcx
+**	movq	%r10, \(%rdi,%rax\)
+**	movq	%r9, 8\(%rdi,%rax\)
+**	movq	%r8, 16\(%rdi,%rax\)
+**	movq	%rcx, 24\(%rdi,%rax\)
+**	cmpl	\$224, %edx
+**	jb	.L[0-9]+
+**	ret
+**...
+*/
+
+void
+foo (char *dest, char *src)
+{
+  __builtin_memcpy (dest, src, 253);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-2.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-2.c
new file mode 100644
index 0000000..0d0e348
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-2.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$64, %edx
+**	movdqa	src\(%rax\), %xmm3
+**	movdqa	src\+16\(%rax\), %xmm2
+**	movdqa	src\+32\(%rax\), %xmm1
+**	movdqa	src\+48\(%rax\), %xmm0
+**	movaps	%xmm3, dest\(%rax\)
+**	movaps	%xmm2, dest\+16\(%rax\)
+**	movaps	%xmm1, dest\+32\(%rax\)
+**	movaps	%xmm0, dest\+48\(%rax\)
+**	cmpl	\$256, %edx
+**	jb	.L[0-9]+
+**	movdqa	src\(%rdx\), %xmm0
+**	movaps	%xmm0, dest\(%rdx\)
+**	ret
+**...
+*/
+
+#define SIZE (16 + 1) * 16
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-3.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-3.c
new file mode 100644
index 0000000..e5aca32
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-3.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$64, %edx
+**	movdqa	src\(%rax\), %xmm3
+**	movdqa	src\+16\(%rax\), %xmm2
+**	movdqa	src\+32\(%rax\), %xmm1
+**	movdqa	src\+48\(%rax\), %xmm0
+**	movaps	%xmm3, dest\(%rax\)
+**	movaps	%xmm2, dest\+16\(%rax\)
+**	movaps	%xmm1, dest\+32\(%rax\)
+**	movaps	%xmm0, dest\+48\(%rax\)
+**	cmpl	\$256, %edx
+**	jb	.L[0-9]+
+**	movdqa	src\(%rdx\), %xmm0
+**	movaps	%xmm0, dest\(%rdx\)
+**	movdqu	src\+15\(%rdx\), %xmm0
+**	movups	%xmm0, dest\+15\(%rdx\)
+**	ret
+**...
+*/
+
+#define SIZE 16 * 16 + 31
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-4.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-4.c
new file mode 100644
index 0000000..27f7bed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-4.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	subl	\$-128, %edx
+**	vmovdqa	src\(%rax\), %ymm3
+**	vmovdqa	src\+32\(%rax\), %ymm2
+**	vmovdqa	src\+64\(%rax\), %ymm1
+**	vmovdqa	src\+96\(%rax\), %ymm0
+**	vmovdqa	%ymm3, dest\(%rax\)
+**	vmovdqa	%ymm2, dest\+32\(%rax\)
+**	vmovdqa	%ymm1, dest\+64\(%rax\)
+**	vmovdqa	%ymm0, dest\+96\(%rax\)
+**	cmpl	\$512, %edx
+**	jb	.L[0-9]+
+**	vmovdqa	src\(%rdx\), %ymm0
+**	vmovdqa	%ymm0, dest\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE (16 + 1) * 32
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-5.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-5.c
new file mode 100644
index 0000000..34a7408
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-5.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	subl	\$-128, %edx
+**	vmovdqa	src\(%rax\), %ymm3
+**	vmovdqa	src\+32\(%rax\), %ymm2
+**	vmovdqa	src\+64\(%rax\), %ymm1
+**	vmovdqa	src\+96\(%rax\), %ymm0
+**	vmovdqa	%ymm3, dest\(%rax\)
+**	vmovdqa	%ymm2, dest\+32\(%rax\)
+**	vmovdqa	%ymm1, dest\+64\(%rax\)
+**	vmovdqa	%ymm0, dest\+96\(%rax\)
+**	cmpl	\$512, %edx
+**	jb	.L[0-9]+
+**	vmovdqa	src\(%rdx\), %ymm0
+**	vmovdqa	%ymm0, dest\(%rdx\)
+**	vmovdqu	src\+31\(%rdx\), %ymm0
+**	vmovdqu	%ymm0, dest\+31\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE 16 * 32 + 32 + 31
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-6.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-6.c
new file mode 100644
index 0000000..aa5d90d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-6.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$256, %edx
+**	vmovdqa64	src\(%rax\), %zmm3
+**	vmovdqa64	src\+64\(%rax\), %zmm2
+**	vmovdqa64	src\+128\(%rax\), %zmm1
+**	vmovdqa64	src\+192\(%rax\), %zmm0
+**	vmovdqa64	%zmm3, dest\(%rax\)
+**	vmovdqa64	%zmm2, dest\+64\(%rax\)
+**	vmovdqa64	%zmm1, dest\+128\(%rax\)
+**	vmovdqa64	%zmm0, dest\+192\(%rax\)
+**	cmpl	\$1024, %edx
+**	jb	.L[0-9]+
+**	vmovdqa64	src\(%rdx\), %zmm0
+**	vmovdqa64	%zmm0, dest\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE (16 + 1) * 64
+
+char dest[SIZE] __attribute__((aligned(64)));
+char src[SIZE] __attribute__((aligned(64)));
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-7.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-7.c
new file mode 100644
index 0000000..63d8a15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-7.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$256, %edx
+**	vmovdqa64	src\(%rax\), %zmm3
+**	vmovdqa64	src\+64\(%rax\), %zmm2
+**	vmovdqa64	src\+128\(%rax\), %zmm1
+**	vmovdqa64	src\+192\(%rax\), %zmm0
+**	vmovdqa64	%zmm3, dest\(%rax\)
+**	vmovdqa64	%zmm2, dest\+64\(%rax\)
+**	vmovdqa64	%zmm1, dest\+128\(%rax\)
+**	vmovdqa64	%zmm0, dest\+192\(%rax\)
+**	cmpl	\$1024, %edx
+**	jb	.L[0-9]+
+**	vmovdqa	src\(%rdx\), %ymm0
+**	vmovdqa	%ymm0, dest\(%rdx\)
+**	vmovdqu	src\+31\(%rdx\), %ymm0
+**	vmovdqu	%ymm0, dest\+31\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE 16 * 64 + 63
+
+char dest[SIZE] __attribute__((aligned(64)));
+char src[SIZE] __attribute__((aligned(64)));
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c b/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c
index d0316ef..c60cef0 100644
--- a/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c
+++ b/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c
@@ -1,14 +1,21 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mtune=generic -mno-sse" } */
+/* { dg-options "-O2 -mtune=generic -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm" } */
 /* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
 /* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
 
 /*
 **foo:
 **.LFB[0-9]+:
-**...
+**	.cfi_startproc
+**	movq	221\(%rsi\), %rax
 **	xorl	%edx, %edx
-**...
+**	movq	%rax, 221\(%rdi\)
+**	movq	229\(%rsi\), %rax
+**	movq	%rax, 229\(%rdi\)
+**	movq	237\(%rsi\), %rax
+**	movq	%rax, 237\(%rdi\)
+**	movq	245\(%rsi\), %rax
+**	movq	%rax, 245\(%rdi\)
 **.L[0-9]+:
 **	movl	%edx, %eax
 **	addl	\$32, %edx
@@ -22,6 +29,7 @@
 **	movq	%rcx, 24\(%rdi,%rax\)
 **	cmpl	\$224, %edx
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-1.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-1.c
new file mode 100644
index 0000000..06e3892
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-1.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**	movups	%xmm0, 190\(%rdi\)
+**	movups	%xmm0, 206\(%rdi\)
+**	movups	%xmm0, 222\(%rdi\)
+**	movups	%xmm0, 238\(%rdi\)
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movups	%xmm0, \(%rdi,%rdx\)
+**	movups	%xmm0, 16\(%rdi,%rdx\)
+**	movups	%xmm0, 32\(%rdi,%rdx\)
+**	movups	%xmm0, 48\(%rdi,%rdx\)
+**	cmpl	\$192, %eax
+**	jb	.L[0-9]+
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 254);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-10.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-10.c
new file mode 100644
index 0000000..36a924d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-10.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movq	\$0, 48\(%rdi\)
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, 8\(%rdi\)
+**	movq	\$0, 16\(%rdi\)
+**	movq	\$0, 24\(%rdi\)
+**	movq	\$0, 32\(%rdi\)
+**	movq	\$0, 40\(%rdi\)
+**	movq	\$0, 53\(%rdi\)
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 61);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-11.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-11.c
new file mode 100644
index 0000000..4868e56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-11.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	movq	%rax, 48\(%rdi\)
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, 8\(%rdi\)
+**	movq	%rax, 16\(%rdi\)
+**	movq	%rax, 24\(%rdi\)
+**	movq	%rax, 32\(%rdi\)
+**	movq	%rax, 40\(%rdi\)
+**	movq	%rax, 53\(%rdi\)
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 4, 61);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-12.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-12.c
new file mode 100644
index 0000000..9112897
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-12.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	movq	%rsi, 48\(%rdi\)
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, 8\(%rdi\)
+**	movq	%rsi, 16\(%rdi\)
+**	movq	%rsi, 24\(%rdi\)
+**	movq	%rsi, 32\(%rdi\)
+**	movq	%rsi, 40\(%rdi\)
+**	movq	%rsi, 53\(%rdi\)
+**	ret
+**...
+*/
+
+void
+foo (char *dest, int c)
+{
+  __builtin_memset (dest, c, 61);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-13.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-13.c
new file mode 100644
index 0000000..69ec6c6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-13.c
@@ -0,0 +1,36 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$192, %eax
+**	jb	.L[0-9]+
+**	movaps	%xmm0, dest\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	ret
+**...
+*/
+
+char dest[240];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-14.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-14.c
new file mode 100644
index 0000000..209cd67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-14.c
@@ -0,0 +1,91 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	cmpq	\$64, %rsi
+**	jnb	.L2
+**	testb	\$32, %sil
+**	jne	.L19
+**	testb	\$16, %sil
+**	jne	.L20
+**	testb	\$8, %sil
+**	jne	.L21
+**	testb	\$4, %sil
+**	jne	.L22
+**	testq	%rsi, %rsi
+**	jne	.L23
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	movups	%xmm0, -64\(%rdi,%rsi\)
+**	movups	%xmm0, -48\(%rdi,%rsi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$64, %rsi
+**	jb	.L1
+**	andq	\$-64, %rsi
+**	xorl	%eax, %eax
+**.L9:
+**	movups	%xmm0, \(%rdi,%rax\)
+**	movups	%xmm0, 16\(%rdi,%rax\)
+**	movups	%xmm0, 32\(%rdi,%rax\)
+**	movups	%xmm0, 48\(%rdi,%rax\)
+**	addq	\$64, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L9
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	movb	\$0, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L1
+**	xorl	%eax, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L19:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, 16\(%rdi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L20:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	movl	\$0, \(%rdi\)
+**	movl	\$0, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 0, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-15.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-15.c
new file mode 100644
index 0000000..d19188f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-15.c
@@ -0,0 +1,103 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	cmpq	\$128, %rsi
+**	jnb	.L2
+**	testb	\$64, %sil
+**	jne	.L22
+**	testb	\$32, %sil
+**	jne	.L23
+**	testb	\$16, %sil
+**	jne	.L24
+**	testb	\$8, %sil
+**	jne	.L25
+**	testb	\$4, %sil
+**	jne	.L26
+**	testq	%rsi, %rsi
+**	jne	.L27
+**.L20:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu	%ymm0, -128\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -96\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$128, %rsi
+**	jb	.L19
+**	andq	\$-128, %rsi
+**	xorl	%eax, %eax
+**.L10:
+**	vmovdqu	%ymm0, \(%rdi,%rax\)
+**	vmovdqu	%ymm0, 32\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 64\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 96\(%rdi,%rax\)
+**	subq	\$-128, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L10
+**.L19:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movb	\$0, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L20
+**	xorl	%eax, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movl	\$0, \(%rdi\)
+**	movl	\$0, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 0, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-16.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-16.c
new file mode 100644
index 0000000..539714c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-16.c
@@ -0,0 +1,112 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	cmpq	\$256, %rsi
+**	jnb	.L2
+**	testb	\$-128, %sil
+**	jne	.L23
+**	testb	\$64, %sil
+**	jne	.L24
+**	testb	\$32, %sil
+**	jne	.L25
+**	testb	\$16, %sil
+**	jne	.L26
+**	testb	\$8, %sil
+**	jne	.L27
+**	testb	\$4, %sil
+**	jne	.L28
+**	testq	%rsi, %rsi
+**	jne	.L29
+**.L21:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu64	%zmm0, -256\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -192\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$256, %rsi
+**	jb	.L20
+**	xorb	%sil, %sil
+**	xorl	%eax, %eax
+**.L11:
+**	vmovdqu64	%zmm0, \(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 64\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 128\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 192\(%rdi,%rax\)
+**	addq	\$256, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L11
+**.L20:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L29:
+**	movb	\$0, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L21
+**	xorl	%eax, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, 64\(%rdi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L28:
+**	movl	\$0, \(%rdi\)
+**	movl	\$0, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 0, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-17.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-17.c
new file mode 100644
index 0000000..f58cb28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-17.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$128, %eax
+**	jb	.L[0-9]+
+**	movq	\$0, dest\+48\(%rax\)
+**	movaps	%xmm0, dest\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	ret
+**...
+*/
+
+char dest[184];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-18.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-18.c
new file mode 100644
index 0000000..a127028
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-18.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$128, %eax
+**	jb	.L[0-9]+
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	movaps	%xmm0, dest\(%rax\)
+**	movl	\$0, dest\+47\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	ret
+**...
+*/
+
+char dest[179];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-19.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-19.c
new file mode 100644
index 0000000..8dd5ae6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-19.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$128, %eax
+**	jb	.L[0-9]+
+**	movb	\$0, dest\+48\(%rax\)
+**	movaps	%xmm0, dest\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	ret
+**...
+*/
+
+char dest[177];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-2.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-2.c
new file mode 100644
index 0000000..3b84b29
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-2.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	vmovdqu	%ymm0, 192\(%rdi\)
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, 64\(%rdi\)
+**	vmovdqu	%ymm0, 96\(%rdi\)
+**	vmovdqu	%ymm0, 128\(%rdi\)
+**	vmovdqu	%ymm0, 160\(%rdi\)
+**	vmovdqu	%ymm0, 222\(%rdi\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 254);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-20.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-20.c
new file mode 100644
index 0000000..b8b9cb7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-20.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movd	%edi, %xmm0
+**	punpcklbw	%xmm0, %xmm0
+**	punpcklwd	%xmm0, %xmm0
+**	pshufd	\$0, %xmm0, %xmm0
+**	movaps	%xmm0, dest\+160\(%rip\)
+**	movaps	%xmm0, dest\(%rip\)
+**	movaps	%xmm0, dest\+16\(%rip\)
+**	movaps	%xmm0, dest\+32\(%rip\)
+**	movaps	%xmm0, dest\+48\(%rip\)
+**	movaps	%xmm0, dest\+64\(%rip\)
+**	movaps	%xmm0, dest\+80\(%rip\)
+**	movaps	%xmm0, dest\+96\(%rip\)
+**	movaps	%xmm0, dest\+112\(%rip\)
+**	movaps	%xmm0, dest\+128\(%rip\)
+**	movaps	%xmm0, dest\+144\(%rip\)
+**	movd	%xmm0, dest\+175\(%rip\)
+**	ret
+**...
+*/
+
+char dest[179];
+
+void
+foo (int c)
+{
+  __builtin_memset (dest, c, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-21.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-21.c
new file mode 100644
index 0000000..3c7bb7c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-21.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movd	%edi, %xmm0
+**	movb	%dil, dest\+176\(%rip\)
+**	punpcklbw	%xmm0, %xmm0
+**	punpcklwd	%xmm0, %xmm0
+**	pshufd	\$0, %xmm0, %xmm0
+**	movaps	%xmm0, dest\(%rip\)
+**	movaps	%xmm0, dest\+16\(%rip\)
+**	movaps	%xmm0, dest\+32\(%rip\)
+**	movaps	%xmm0, dest\+48\(%rip\)
+**	movaps	%xmm0, dest\+64\(%rip\)
+**	movaps	%xmm0, dest\+80\(%rip\)
+**	movaps	%xmm0, dest\+96\(%rip\)
+**	movaps	%xmm0, dest\+112\(%rip\)
+**	movaps	%xmm0, dest\+128\(%rip\)
+**	movaps	%xmm0, dest\+144\(%rip\)
+**	movaps	%xmm0, dest\+160\(%rip\)
+**	ret
+**...
+*/
+
+char dest[177];
+
+void
+foo (int c)
+{
+  __builtin_memset (dest, c, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-22.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-22.c
new file mode 100644
index 0000000..96a21c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-22.c
@@ -0,0 +1,27 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=rep_8byte:8192:align,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movl	\$25, %ecx
+**	xorl	%eax, %eax
+**	movl	\$dest, %edi
+**	rep stosq
+**	movl	\$0, \(%rdi\)
+**	ret
+**...
+*/
+
+#define SIZE 204
+
+char dest[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-23.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-23.c
new file mode 100644
index 0000000..f3f5d80
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-23.c
@@ -0,0 +1,67 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -minline-all-stringops -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movzbl	%dil, %edi
+**	movl	\$p, %eax
+**	movabsq	\$72340172838076673, %rdx
+**	imulq	%rdx, %rdi
+**	movq	%rdi, %xmm0
+**	punpcklqdq	%xmm0, %xmm0
+**	cmpq	\$64, %rsi
+**	jnb	.L18
+**.L2:
+**	movq	%rsi, %rcx
+**	andl	\$63, %ecx
+**	je	.L1
+**	xorl	%edx, %edx
+**	andl	\$1, %esi
+**	je	.L5
+**	movl	\$1, %edx
+**	movb	%dil, \(%rax\)
+**	cmpq	%rcx, %rdx
+**	jnb	.L19
+**.L5:
+**	movb	%dil, \(%rax,%rdx\)
+**	movb	%dil, 1\(%rax,%rdx\)
+**	addq	\$2, %rdx
+**	cmpq	%rcx, %rdx
+**	jb	.L5
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L18:
+**	movq	%rsi, %rdx
+**	xorl	%eax, %eax
+**	andq	\$-64, %rdx
+**.L3:
+**	movaps	%xmm0, p\(%rax\)
+**	addq	\$64, %rax
+**	movaps	%xmm0, p-48\(%rax\)
+**	movaps	%xmm0, p-32\(%rax\)
+**	movaps	%xmm0, p-16\(%rax\)
+**	cmpq	%rdx, %rax
+**	jb	.L3
+**	addq	\$p, %rax
+**	jmp	.L2
+**.L19:
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+
+#define WRITE_CHUNK 256
+char p[WRITE_CHUNK];
+
+void
+foo (int c, __SIZE_TYPE__ nbyte)
+{
+ __builtin_memset (p, c, nbyte);
+}
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-3.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-3.c
new file mode 100644
index 0000000..faa47ca
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-3.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	vmovdqu8	%zmm0, 128\(%rdi\)
+**	vmovdqu8	%zmm0, \(%rdi\)
+**	vmovdqu8	%zmm0, 64\(%rdi\)
+**	vmovdqu8	%zmm0, 190\(%rdi\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 254);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-4.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-4.c
new file mode 100644
index 0000000..dc3aa57b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-4.c
@@ -0,0 +1,93 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	movq	%rax, %xmm0
+**	punpcklqdq	%xmm0, %xmm0
+**	cmpq	\$64, %rsi
+**	jnb	.L2
+**	testb	\$32, %sil
+**	jne	.L19
+**	testb	\$16, %sil
+**	jne	.L20
+**	testb	\$8, %sil
+**	jne	.L21
+**	testb	\$4, %sil
+**	jne	.L22
+**	testq	%rsi, %rsi
+**	jne	.L23
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	movups	%xmm0, -64\(%rdi,%rsi\)
+**	movups	%xmm0, -48\(%rdi,%rsi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$64, %rsi
+**	jb	.L1
+**	andq	\$-64, %rsi
+**	xorl	%eax, %eax
+**.L9:
+**	movups	%xmm0, \(%rdi,%rax\)
+**	movups	%xmm0, 16\(%rdi,%rax\)
+**	movups	%xmm0, 32\(%rdi,%rax\)
+**	movups	%xmm0, 48\(%rdi,%rax\)
+**	addq	\$64, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L9
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	movb	\$4, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L1
+**	movl	\$1028, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L19:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, 16\(%rdi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L20:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	movl	\$67372036, \(%rdi\)
+**	movl	\$67372036, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 4, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-5.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-5.c
new file mode 100644
index 0000000..a324f8e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-5.c
@@ -0,0 +1,102 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	vmovq	%rax, %xmm1
+**	vpbroadcastq	%xmm1, %ymm0
+**	cmpq	\$128, %rsi
+**	jnb	.L2
+**	testb	\$64, %sil
+**	jne	.L21
+**	testb	\$32, %sil
+**	jne	.L22
+**	testb	\$16, %sil
+**	jne	.L23
+**	testb	\$8, %sil
+**	jne	.L24
+**	testb	\$4, %sil
+**	jne	.L25
+**	testq	%rsi, %rsi
+**	jne	.L26
+**.L19:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu	%ymm0, -128\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -96\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$128, %rsi
+**	jb	.L19
+**	andq	\$-128, %rsi
+**	xorl	%eax, %eax
+**.L10:
+**	vmovdqu	%ymm0, \(%rdi,%rax\)
+**	vmovdqu	%ymm0, 32\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 64\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 96\(%rdi,%rax\)
+**	subq	\$-128, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L10
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movb	\$4, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L19
+**	movl	\$1028, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, -8\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	movl	\$67372036, \(%rdi\)
+**	movl	\$67372036, -4\(%rdi,%rsi\)
+**	jmp	.L19
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 4, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-6.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-6.c
new file mode 100644
index 0000000..64e7589
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-6.c
@@ -0,0 +1,109 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	vpbroadcastq	%rax, %zmm0
+**	cmpq	\$256, %rsi
+**	jnb	.L2
+**	testb	\$-128, %sil
+**	jne	.L22
+**	testb	\$64, %sil
+**	jne	.L23
+**	testb	\$32, %sil
+**	jne	.L24
+**	testb	\$16, %sil
+**	jne	.L25
+**	testb	\$8, %sil
+**	jne	.L26
+**	testb	\$4, %sil
+**	jne	.L27
+**	testq	%rsi, %rsi
+**	jne	.L28
+**.L20:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu64	%zmm0, -256\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -192\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$256, %rsi
+**	jb	.L20
+**	xorb	%sil, %sil
+**	xorl	%eax, %eax
+**.L11:
+**	vmovdqu64	%zmm0, \(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 64\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 128\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 192\(%rdi,%rax\)
+**	addq	\$256, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L11
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L28:
+**	movb	\$4, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L20
+**	movl	\$1028, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, 64\(%rdi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, -8\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movl	\$67372036, \(%rdi\)
+**	movl	\$67372036, -4\(%rdi,%rsi\)
+**	jmp	.L20
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 4, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-7.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-7.c
new file mode 100644
index 0000000..022f6f9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-7.c
@@ -0,0 +1,94 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	movq	%rsi, %xmm0
+**	punpcklqdq	%xmm0, %xmm0
+**	cmpq	\$64, %rdx
+**	jnb	.L2
+**	testb	\$32, %dl
+**	jne	.L19
+**	testb	\$16, %dl
+**	jne	.L20
+**	testb	\$8, %dl
+**	jne	.L21
+**	testb	\$4, %dl
+**	jne	.L22
+**	testq	%rdx, %rdx
+**	jne	.L23
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	movups	%xmm0, -64\(%rdi,%rdx\)
+**	movups	%xmm0, -48\(%rdi,%rdx\)
+**	movups	%xmm0, -32\(%rdi,%rdx\)
+**	movups	%xmm0, -16\(%rdi,%rdx\)
+**	subq	\$1, %rdx
+**	cmpq	\$64, %rdx
+**	jb	.L1
+**	andq	\$-64, %rdx
+**	xorl	%eax, %eax
+**.L9:
+**	movups	%xmm0, \(%rdi,%rax\)
+**	movups	%xmm0, 16\(%rdi,%rax\)
+**	movups	%xmm0, 32\(%rdi,%rax\)
+**	movups	%xmm0, 48\(%rdi,%rax\)
+**	addq	\$64, %rax
+**	cmpq	%rdx, %rax
+**	jb	.L9
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	movb	%sil, \(%rdi\)
+**	testb	\$2, %dl
+**	je	.L1
+**	movw	%si, -2\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L19:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, 16\(%rdi\)
+**	movups	%xmm0, -32\(%rdi,%rdx\)
+**	movups	%xmm0, -16\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L20:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, -16\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, -8\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	movl	%esi, \(%rdi\)
+**	movl	%esi, -4\(%rdi,%rdx\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, int c, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, c, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-8.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-8.c
new file mode 100644
index 0000000..5254e21
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-8.c
@@ -0,0 +1,103 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	vmovq	%rsi, %xmm1
+**	vpbroadcastq	%xmm1, %ymm0
+**	cmpq	\$128, %rdx
+**	jnb	.L2
+**	testb	\$64, %dl
+**	jne	.L21
+**	testb	\$32, %dl
+**	jne	.L22
+**	testb	\$16, %dl
+**	jne	.L23
+**	testb	\$8, %dl
+**	jne	.L24
+**	testb	\$4, %dl
+**	jne	.L25
+**	testq	%rdx, %rdx
+**	jne	.L26
+**.L19:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu	%ymm0, -128\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -96\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	subq	\$1, %rdx
+**	cmpq	\$128, %rdx
+**	jb	.L19
+**	andq	\$-128, %rdx
+**	xorl	%eax, %eax
+**.L10:
+**	vmovdqu	%ymm0, \(%rdi,%rax\)
+**	vmovdqu	%ymm0, 32\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 64\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 96\(%rdi,%rax\)
+**	subq	\$-128, %rax
+**	cmpq	%rdx, %rax
+**	jb	.L10
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movb	%sil, \(%rdi\)
+**	testb	\$2, %dl
+**	je	.L19
+**	movw	%si, -2\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, -8\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	movl	%esi, \(%rdi\)
+**	movl	%esi, -4\(%rdi,%rdx\)
+**	jmp	.L19
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, int c, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, c, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-9.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-9.c
new file mode 100644
index 0000000..1719de6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-9.c
@@ -0,0 +1,110 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	vpbroadcastq	%rsi, %zmm0
+**	cmpq	\$256, %rdx
+**	jnb	.L2
+**	testb	\$-128, %dl
+**	jne	.L22
+**	testb	\$64, %dl
+**	jne	.L23
+**	testb	\$32, %dl
+**	jne	.L24
+**	testb	\$16, %dl
+**	jne	.L25
+**	testb	\$8, %dl
+**	jne	.L26
+**	testb	\$4, %dl
+**	jne	.L27
+**	testq	%rdx, %rdx
+**	jne	.L28
+**.L20:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu64	%zmm0, -256\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -192\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rdx\)
+**	subq	\$1, %rdx
+**	cmpq	\$256, %rdx
+**	jb	.L20
+**	xorb	%dl, %dl
+**	xorl	%eax, %eax
+**.L11:
+**	vmovdqu64	%zmm0, \(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 64\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 128\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 192\(%rdi,%rax\)
+**	addq	\$256, %rax
+**	cmpq	%rdx, %rax
+**	jb	.L11
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L28:
+**	movb	%sil, \(%rdi\)
+**	testb	\$2, %dl
+**	je	.L20
+**	movw	%si, -2\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, 64\(%rdi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, -8\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movl	%esi, \(%rdi\)
+**	movl	%esi, -4\(%rdi,%rdx\)
+**	jmp	.L20
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, int c, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, c, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-25.c b/gcc/testsuite/gcc.target/i386/memset-strategy-25.c
index 1cc3de7..7bd5d43 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-25.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-25.c
@@ -7,7 +7,11 @@
 **foo:
 **.LFB[0-9]+:
 **	.cfi_startproc
+**	movq	\$0, 221\(%rdi\)
 **	xorl	%eax, %eax
+**	movq	\$0, 229\(%rdi\)
+**	movq	\$0, 237\(%rdi\)
+**	movq	\$0, 245\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$32, %eax
@@ -17,6 +21,7 @@
 **	movq	\$0, 24\(%rdi,%rdx\)
 **	cmpl	\$224, %eax
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-29.c b/gcc/testsuite/gcc.target/i386/memset-strategy-29.c
index 61aef92..a33bf92 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-29.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-29.c
@@ -8,7 +8,11 @@
 **...
 **.LFB[0-9]+:
 **	.cfi_startproc
+**	movq	\$0, 49\(%rdi\)
 **	xorl	%eax, %eax
+**	movq	\$0, 57\(%rdi\)
+**	movq	\$0, 65\(%rdi\)
+**	movq	\$0, 73\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$32, %eax
@@ -18,6 +22,7 @@
 **	movq	\$0, 24\(%rdi,%rdx\)
 **	cmpl	\$64, %eax
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-30.c b/gcc/testsuite/gcc.target/i386/memset-strategy-30.c
index 917f151..f3912f8 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-30.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-30.c
@@ -8,7 +8,11 @@
 **...
 **.LFB[0-9]+:
 **	.cfi_startproc
+**	movq	\$0, 63\(%rdi\)
 **	xorl	%eax, %eax
+**	movq	\$0, 71\(%rdi\)
+**	movq	\$0, 79\(%rdi\)
+**	movq	\$0, 87\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$32, %eax
@@ -18,6 +22,7 @@
 **	movq	\$0, 24\(%rdi,%rdx\)
 **	cmpl	\$64, %eax
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-31.c b/gcc/testsuite/gcc.target/i386/memset-strategy-31.c
index 17a4df2..4791c4d 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-31.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-31.c
@@ -9,6 +9,10 @@
 **...
 **	pxor	%xmm0, %xmm0
 **	xorl	%eax, %eax
+**	movups	%xmm0, 190\(%rdi\)
+**	movups	%xmm0, 206\(%rdi\)
+**	movups	%xmm0, 222\(%rdi\)
+**	movups	%xmm0, 238\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$64, %eax
diff --git a/gcc/testsuite/gcc.target/i386/pr103785.c b/gcc/testsuite/gcc.target/i386/pr103785.c
index 5503b96..49d6c56 100644
--- a/gcc/testsuite/gcc.target/i386/pr103785.c
+++ b/gcc/testsuite/gcc.target/i386/pr103785.c
@@ -11,7 +11,10 @@ struct wrapper_t
 
 struct wrapper_t **table;
 
-__attribute__ ((weak, regparm (2)))
+#ifndef __x86_64__
+__attribute__ ((regparm (2)))
+#endif
+__attribute__ ((weak))
 void
 update (long k, long e)
 {
diff --git a/gcc/testsuite/gcc.target/i386/pr104447.c b/gcc/testsuite/gcc.target/i386/pr104447.c
index cb618c7..145ba90 100644
--- a/gcc/testsuite/gcc.target/i386/pr104447.c
+++ b/gcc/testsuite/gcc.target/i386/pr104447.c
@@ -1,6 +1,7 @@
 /* { dg-do compile } */
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
+/* { dg-additional-options "-mfentry -fno-pic" { target *-*-gnu* } } */
 
 int
 bar (int x)
diff --git a/gcc/testsuite/gcc.target/i386/pr113122-3.c b/gcc/testsuite/gcc.target/i386/pr113122-3.c
index 71aa240..87b76de 100644
--- a/gcc/testsuite/gcc.target/i386/pr113122-3.c
+++ b/gcc/testsuite/gcc.target/i386/pr113122-3.c
@@ -2,6 +2,7 @@
 /* { dg-do assemble { target *-*-linux* } } */
 /* { dg-require-effective-target masm_intel } */
 /* { dg-options "-fprofile -O2 -masm=intel" } */
+/* { dg-additional-options "-mfentry -fno-pic" { target *-*-gnu* } } */
 
 void
 func (void)
diff --git a/gcc/testsuite/gcc.target/i386/pr119386-1.c b/gcc/testsuite/gcc.target/i386/pr119386-1.c
index 9a0dc64..7a56eac 100644
--- a/gcc/testsuite/gcc.target/i386/pr119386-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr119386-1.c
@@ -1,7 +1,9 @@
 /* PR target/119386 */
 /* { dg-do compile { target *-*-linux* } } */
 /* { dg-options "-O2 -fpic -pg" } */
-/* { dg-final { scan-assembler "call\[ \t\]+mcount@PLT" } } */
+/* { dg-additional-options "-mfentry" { target { *-*-gnu* && { ! ia32 } } } } */
+/* { dg-final { scan-assembler "call\[ \t\]+mcount@PLT" { target ia32 } } } */
+/* { dg-final { scan-assembler "call\[ \t\]+__fentry__@PLT" { target { *-*-gnu* && { ! ia32 } } } } } */
 
 int
 main ()
diff --git a/gcc/testsuite/gcc.target/i386/pr119386-2.c b/gcc/testsuite/gcc.target/i386/pr119386-2.c
index 3ea978e..cddaaf0 100644
--- a/gcc/testsuite/gcc.target/i386/pr119386-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr119386-2.c
@@ -1,7 +1,8 @@
 /* PR target/119386 */
 /* { dg-do compile { target *-*-linux* } } */
 /* { dg-options "-O2 -fpic -fno-plt -pg" } */
-/* { dg-final { scan-assembler "call\[ \t\]+\\*mcount@GOTPCREL\\(" { target { ! ia32 } } } } */
+/* { dg-additional-options "-mfentry" { target { *-*-gnu* && { ! ia32 } } } } */
+/* { dg-final { scan-assembler "call\[ \t\]+\\*__fentry__@GOTPCREL" { target { *-*-gnu* && { ! ia32 } } } } } */
 /* { dg-final { scan-assembler "call\[ \t\]+\\*mcount@GOT\\(" { target ia32 } } } */
 
 
diff --git a/gcc/testsuite/gcc.target/i386/pr119795.c b/gcc/testsuite/gcc.target/i386/pr119795.c
new file mode 100644
index 0000000..03c91cc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr119795.c
@@ -0,0 +1,26 @@
+/* { dg-do run } */
+/* { dg-options "-O -fschedule-insns -favoid-store-forwarding" } */
+
+unsigned a, b, c;
+
+void
+foo (_BitInt(2) b2, unsigned _BitInt(255) by, unsigned _BitInt(5) b5,
+     unsigned _BitInt(256) *ret)
+{
+  unsigned _BitInt(255) bx = b2;
+  by += 0x80000000000000000000000000000000wb;
+  __builtin_memmove (&b, &c, 3);
+  unsigned d = b;
+  unsigned e = __builtin_stdc_rotate_right (0x1uwb % b5, a);
+  unsigned _BitInt(256) r = by + bx + d + e;
+  *ret = r;
+}
+
+int
+main ()
+{
+  unsigned  _BitInt(256) x;
+  foo (0, -1, 2, &x);
+  if (x != 0x80000000000000000000000000000000wb)
+    __builtin_abort();
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/i386/pr120427-5.c b/gcc/testsuite/gcc.target/i386/pr120427-5.c
new file mode 100644
index 0000000..7199aef
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120427-5.c
@@ -0,0 +1,10 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-Oz" } */
+
+long long
+func1 (void)
+{
+  return -1;
+}
+/* { dg-final { scan-assembler-times "pushq\[ \\t\]+\\\$-1" 1 } } */
+/* { dg-final { scan-assembler-times "popq\[ \\t\]+%rax" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1a.c b/gcc/testsuite/gcc.target/i386/pr120881-1a.c
new file mode 100644
index 0000000..3d9ac0e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1a.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target fpic } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fno-pic" } */
+/* { dg-message "'-pg' without '-mfentry' may be unreliable with shrink wrapping" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1b.c b/gcc/testsuite/gcc.target/i386/pr120881-1b.c
new file mode 100644
index 0000000..0826407
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target { fpic && { ! ia32 } } } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fpic" } */
+/* { dg-message "'-pg' without '-mfentry' may be unreliable with shrink wrapping" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1c.c b/gcc/testsuite/gcc.target/i386/pr120881-1c.c
new file mode 100644
index 0000000..c21979f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1c.c
@@ -0,0 +1,3 @@
+/* { dg-do compile { target { fpic && ia32 } } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fpic" } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1d.c b/gcc/testsuite/gcc.target/i386/pr120881-1d.c
new file mode 100644
index 0000000..f74af23
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1d.c
@@ -0,0 +1,3 @@
+/* { dg-do compile { target { fpic && ia32 } } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fno-shrink-wrap -fno-pic" } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-2a.c b/gcc/testsuite/gcc.target/i386/pr120881-2a.c
new file mode 100644
index 0000000..52e3e52
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-2a.c
@@ -0,0 +1,21 @@
+/* { dg-do compile { target fentry } } */
+/* { dg-options "-O2 -pg" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {^\t?\.} } } */
+
+/*
+**f2:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	call	__fentry__
+**...
+*/
+
+extern void f1 (void);
+
+void
+f2 (int count)
+{
+  for (int i = 0; i < count; ++i)
+    f1 ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-2b.c b/gcc/testsuite/gcc.target/i386/pr120881-2b.c
new file mode 100644
index 0000000..43a12f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-2b.c
@@ -0,0 +1,6 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-rtl-pro_and_epilogue -march=x86-64" } */
+/* { dg-final { scan-rtl-dump "Now spread 1 times" "pro_and_epilogue" } } */
+
+#include "pr120881-2a.c"
+
diff --git a/gcc/testsuite/gcc.target/i386/pr120941-1.c b/gcc/testsuite/gcc.target/i386/pr120941-1.c
new file mode 100644
index 0000000..b4fc6ac
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120941-1.c
@@ -0,0 +1,49 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -march=x86-64-v3" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**bar:
+**.LFB[0-9]+:
+**...
+**	vbroadcastsd	.LC4\(%rip\), %ymm2
+**	leal	2\(%rbx\), %eax
+**	vbroadcastsd	.LC2\(%rip\), %ymm4
+**	negl	%eax
+**...
+*/
+
+extern void foo (int);
+
+enum { N_CELL_ENTRIES1 = 2 }
+typedef LBM_Grid1[64];
+enum { N_CELL_ENTRIES2 = 2 }
+typedef LBM_Grid2[64];
+LBM_Grid1 grid1;
+LBM_Grid2 grid2;
+extern int n;
+
+void
+LBM_handleInOutFlow()
+{
+  int i, j;
+  for (; i; i += 2)
+    {
+      for (j = 0; j < n; j++)
+	{
+	  grid1[i] = 1.0 / 36.0 * i;
+	  grid2[i] = 1.0 / 36.0 * i;
+	}
+    }
+}
+
+int main_t;
+void
+bar (void)
+{
+  for (; main_t; main_t++) {
+    LBM_handleInOutFlow();
+    foo (main_t);
+  }
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121015.c b/gcc/testsuite/gcc.target/i386/pr121015.c
new file mode 100644
index 0000000..57c8bff
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121015.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3" } */
+
+extern union {
+  int i;
+  float f;
+} int_as_float_u;
+
+extern int render_result_from_bake_w;
+extern int render_result_from_bake_h_seed_pass;
+extern float *render_result_from_bake_h_primitive;
+extern float *render_result_from_bake_h_seed;
+
+float
+int_as_float(int i)
+{
+  int_as_float_u.i = i;
+  return int_as_float_u.f;
+}
+
+void
+render_result_from_bake_h(int tx)
+{
+  while (render_result_from_bake_w) {
+    for (; tx < render_result_from_bake_w; tx++)
+      render_result_from_bake_h_primitive[1] =
+          render_result_from_bake_h_primitive[2] = int_as_float(-1);
+    if (render_result_from_bake_h_seed_pass) {
+      *render_result_from_bake_h_seed = 0;
+    }
+  }
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-1.c b/gcc/testsuite/gcc.target/i386/pr121062-1.c
new file mode 100644
index 0000000..799f856
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-1.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3" } */
+
+extern union {
+  int i;
+  float f;
+} int_as_float_u;
+
+extern int render_result_from_bake_w;
+extern int render_result_from_bake_h_seed_pass;
+extern float *render_result_from_bake_h_primitive;
+extern float *render_result_from_bake_h_seed;
+
+float
+int_as_float(int i)
+{
+  int_as_float_u.i = i;
+  return int_as_float_u.f;
+}
+
+void
+render_result_from_bake_h(int tx)
+{
+  while (render_result_from_bake_w) {
+    for (; tx < render_result_from_bake_w; tx++)
+      render_result_from_bake_h_primitive[1] =
+          render_result_from_bake_h_primitive[2] = int_as_float(-1);
+    if (render_result_from_bake_h_seed_pass) {
+      *render_result_from_bake_h_seed = 0;
+    }
+  }
+}
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, %r\[a-z0-9\]+" 2 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-2.c b/gcc/testsuite/gcc.target/i386/pr121062-2.c
new file mode 100644
index 0000000..723d68a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-Og -fno-dce -mtune=generic" } */
+
+typedef int __attribute__((__vector_size__ (4))) S;
+extern void bar (S);
+
+void
+foo ()
+{
+  bar ((S){-1});
+}
+
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$-1, \\(%esp\\)" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$-1, %edi" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-3a.c b/gcc/testsuite/gcc.target/i386/pr121062-3a.c
new file mode 100644
index 0000000..effd4ff
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-3a.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target fpic } } */
+/* { dg-options "-O2 -march=x86-64 -fpic" } */
+
+typedef struct {
+  struct {
+    unsigned short lo4;
+    unsigned short lo3;
+    unsigned short lo2;
+    unsigned short lo1;
+  } i;
+} BID_BINARY80LDOUBLE;
+extern BID_BINARY80LDOUBLE __bid64_to_binary80_x_out;
+void
+__bid64_to_binary80 (void)
+{
+  __bid64_to_binary80_x_out.i.lo4
+    = __bid64_to_binary80_x_out.i.lo3
+    = __bid64_to_binary80_x_out.i.lo2
+    = __bid64_to_binary80_x_out.i.lo1 = 65535;
+}
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+%xmm\[0-9\]+, " 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, \\(%(e|r)\[a-z0-9\]+\\)" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-3b.c b/gcc/testsuite/gcc.target/i386/pr121062-3b.c
new file mode 100644
index 0000000..eb89b5d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-3b.c
@@ -0,0 +1,6 @@
+/* { dg-do compile { target { fpic && lp64 } } } */
+/* { dg-options "-O2 -march=x86-64 -fno-pic -mcmodel=large" } */
+
+#include "pr121062-3a.c"
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, \\(%r\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-3c.c b/gcc/testsuite/gcc.target/i386/pr121062-3c.c
new file mode 100644
index 0000000..4c07029
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-3c.c
@@ -0,0 +1,6 @@
+/* { dg-do compile { target { fpic && lp64 } } } */
+/* { dg-options "-O2 -march=x86-64 -fpic -mcmodel=large" } */
+
+#include "pr121062-3a.c"
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, \\(%r\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-4.c b/gcc/testsuite/gcc.target/i386/pr121062-4.c
new file mode 100644
index 0000000..77a0c2e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-4.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64" } */
+
+typedef long long int __attribute__((__vector_size__ (8))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){0x12345678badbeefULL};
+}
+
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+%xmm\[0-9\]+, " 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movabsq\[ \\t\]+\\\$81985529250168559, %r\[a-z0-9\]+" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-5.c b/gcc/testsuite/gcc.target/i386/pr121062-5.c
new file mode 100644
index 0000000..22c09a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-5.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64" } */
+
+typedef int __attribute__((__vector_size__ (4))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){0x12345678};
+}
+
+
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$305419896, \\(%(e|r)\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-6.c b/gcc/testsuite/gcc.target/i386/pr121062-6.c
new file mode 100644
index 0000000..780b496
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-6.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-Og -fno-dce -mtune=generic" } */
+
+typedef int __attribute__((__vector_size__ (8))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){0x12345678,0xbadbeefULL};
+}
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+%xmm\[0-9\]+, " 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movabsq\[ \\t\]+\\\$841538639400031864, %r\[a-z0-9\]+" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-7.c b/gcc/testsuite/gcc.target/i386/pr121062-7.c
new file mode 100644
index 0000000..f1834f8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-7.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64" } */
+
+typedef __bf16 __attribute__((__vector_size__ (4))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){-0.1, 2.1};
+}
+
+
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$1074183629, \\(%(e|r)\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-1a.c b/gcc/testsuite/gcc.target/i386/pr121208-1a.c
new file mode 100644
index 0000000..cb8bd0b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-1a.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mno-80387 -mtls-dialect=gnu" } */
+
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((no_caller_saved_registers))
+void
+foo (int error)
+{
+  bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+  if (error == 0)
+    func ();
+  bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-1b.c b/gcc/testsuite/gcc.target/i386/pr121208-1b.c
new file mode 100644
index 0000000..037e9a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-1b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mno-80387 -mtls-dialect=gnu2" } */
+
+#include "pr121208-1a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-2a.c b/gcc/testsuite/gcc.target/i386/pr121208-2a.c
new file mode 100644
index 0000000..c1891ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-2a.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu" } */
+
+typedef unsigned int uword_t __attribute__ ((mode (__word__)));
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((target("general-regs-only")))
+__attribute__((interrupt))
+void
+foo (void *frame, uword_t error)
+{
+  bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+  if (error == 0)
+    func ();
+  bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-2b.c b/gcc/testsuite/gcc.target/i386/pr121208-2b.c
new file mode 100644
index 0000000..269b120
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-2b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu2" } */
+
+#include "pr121208-2a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-3a.c b/gcc/testsuite/gcc.target/i386/pr121208-3a.c
new file mode 100644
index 0000000..26fe687
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-3a.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu" } */
+
+typedef unsigned int uword_t __attribute__ ((mode (__word__)));
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((target("general-regs-only")))
+__attribute__((interrupt))
+void
+foo (void *frame)
+{
+  bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+  if (frame == 0)
+    func ();
+  bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-3b.c b/gcc/testsuite/gcc.target/i386/pr121208-3b.c
new file mode 100644
index 0000000..b672d75
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-3b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu2" } */
+
+#include "pr121208-3a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121274.c b/gcc/testsuite/gcc.target/i386/pr121274.c
new file mode 100644
index 0000000..16760cf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121274.c
@@ -0,0 +1,24 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-march=x86-64-v4 -O2" } */
+/* { dg-final { scan-assembler-not "vpextrq" } } */
+/* { dg-final { scan-assembler-not "vpinsrq" } } */
+
+typedef int v16si __attribute__((vector_size(64)));
+typedef int v4si __attribute__((vector_size(16)));
+
+v4si f(v16si x)
+{
+  return __builtin_shufflevector(x, x, 0, 1, 2, 3);
+}
+
+v4si g(v16si x)
+{
+return __builtin_shufflevector(x, x, 4, 5, 6, 7);
+}
+
+v4si f1(__int128 *x)
+{
+  __int128 t = *x;
+  asm("":"+x"(t));
+  return (v4si)t;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121303.c b/gcc/testsuite/gcc.target/i386/pr121303.c
new file mode 100644
index 0000000..7900bce
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121303.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -favoid-store-forwarding" } */
+
+typedef struct {
+  bool is_ssa;
+} nir_src;
+
+nir_src nir_src_init;
+
+typedef struct {
+  nir_src src;
+  char swizzle[6];
+} nir_alu_src;
+
+void nir_src_bit_size(nir_src);
+
+void nir_lower_fb_read_instr() {
+  {
+    nir_alu_src alu_src = {nir_src_init}, src = alu_src;
+    nir_src_bit_size(src.src);
+  }
+  {
+    nir_alu_src alu_src = {nir_src_init}, src = alu_src;
+    nir_src_bit_size(src.src);
+  }
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr15184-2.c b/gcc/testsuite/gcc.target/i386/pr15184-2.c
index cb8201f..dd50c42 100644
--- a/gcc/testsuite/gcc.target/i386/pr15184-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr15184-2.c
@@ -1,4 +1,4 @@
-/* PR 15184 second two tests
+/* PR 15184 second two tests */
 /* { dg-do compile { target ia32 } } */
 /* { dg-options "-O2 -march=pentiumpro" } */
 /* { dg-additional-options "-fno-PIE" { target ia32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr36533.c b/gcc/testsuite/gcc.target/i386/pr36533.c
index 8d71ece..8699d26 100644
--- a/gcc/testsuite/gcc.target/i386/pr36533.c
+++ b/gcc/testsuite/gcc.target/i386/pr36533.c
@@ -55,14 +55,22 @@ typedef struct
   S1 *s18;
 } S7;
 
-__attribute__((regparm (3), noinline)) int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+int
 fn1 (const char *x, void *y, S1 *z)
 {
   asm volatile ("" : : : "memory");
   return *x + (y != 0);
 }
 
-__attribute__((regparm (3), noinline)) int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+int
 fn2 (const char *x, int y, S2 *z)
 {
   asm volatile ("" : : : "memory");
@@ -84,7 +92,11 @@ fn3 (S3 *p)
   return (S3 *) ((char *) p + fn4 (p->s9));
 }
 
-__attribute__((regparm (3), noinline)) int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+int
 fn5 (void)
 {
   asm volatile ("" : : : "memory");
@@ -116,7 +128,11 @@ fn6 (S3 *w, int x, S2 *y, S4 *z)
   return a;
 }
 
-__attribute__((regparm (3), noinline)) unsigned int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+unsigned int
 test (void *u, S6 *v, S1 **w, S7 *x, S2 *y, S1 *z)
 {
   unsigned b = v->s17->s16;
diff --git a/gcc/testsuite/gcc.target/i386/pr59099.c b/gcc/testsuite/gcc.target/i386/pr59099.c
index cf4a8da..21dfbc2 100644
--- a/gcc/testsuite/gcc.target/i386/pr59099.c
+++ b/gcc/testsuite/gcc.target/i386/pr59099.c
@@ -13,10 +13,17 @@ struct s
 };
 
 
-void* f (struct s *, struct s *) __attribute__ ((noinline, regparm(1)));
+void* f (struct s *, struct s *)
+#ifndef __x86_64__
+__attribute__ ((regparm(1)))
+#endif
+__attribute__ ((noinline))
+;
 
 void*
+#ifndef __x86_64__
 __attribute__ ((regparm(1)))
+#endif
 f (struct s *p, struct s *p2)
 {
   void *gp, *gp1;
diff --git a/gcc/testsuite/gcc.target/i386/pr82699-1.c b/gcc/testsuite/gcc.target/i386/pr82699-1.c
index 272d079..96e3ccb 100644
--- a/gcc/testsuite/gcc.target/i386/pr82699-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr82699-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile { target *-*-linux* } } */
-/* { dg-options "-O2 -fno-pic -fcf-protection -pg -fasynchronous-unwind-tables" } */
+/* { dg-options "-O2 -mfentry -fno-pic -fcf-protection -pg -fasynchronous-unwind-tables" } */
 /* { dg-final { scan-assembler-times {\t\.cfi_startproc\n\tendbr} 1 } } */
 
 extern int bar (int);
diff --git a/gcc/testsuite/gcc.target/i386/pr91384-1.c b/gcc/testsuite/gcc.target/i386/pr91384-1.c
new file mode 100644
index 0000000..4f8823d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr91384-1.c
@@ -0,0 +1,20 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2 -mapxf" } */
+
+void foo (void);
+void bar (void);
+
+int
+test (int a)
+{
+  int r;
+
+  if (r = -a)
+    foo ();
+  else
+    bar ();
+
+  return r;
+}
+
+/* { dg-final { scan-assembler-not "testl" } } */
diff --git a/gcc/testsuite/gcc.target/i386/sibcall-8.c b/gcc/testsuite/gcc.target/i386/sibcall-8.c
index 3ab3809..29ebfe5 100644
--- a/gcc/testsuite/gcc.target/i386/sibcall-8.c
+++ b/gcc/testsuite/gcc.target/i386/sibcall-8.c
@@ -1,23 +1,29 @@
 /* { dg-do run } */
 /* { dg-options "-O2" } */
 
+#ifndef __x86_64__
+#define REGPARM __attribute__((regparm(1)))
+#else
+#define REGPARM
+#endif
+
 extern void abort (void);
 
-static int __attribute__((regparm(1)))
+static int REGPARM
 bar(void *arg)
 {
   return arg != bar;
 }
 
-static int __attribute__((noinline,noclone,regparm(1)))
-foo(int (__attribute__((regparm(1))) **bar)(void*))
+static int __attribute__((noinline,noclone)) REGPARM
+foo(int (REGPARM **bar)(void*))
 {
   return (*bar)(*bar);
 }
 
 int main()
 {
-  int (__attribute__((regparm(1))) *p)(void*) = bar;
+  int (REGPARM *p)(void*) = bar;
   if (foo(&p))
     abort();
   return 0;
diff --git a/gcc/testsuite/gcc.target/i386/sw-1.c b/gcc/testsuite/gcc.target/i386/sw-1.c
index 14db3ce..025f0e1 100644
--- a/gcc/testsuite/gcc.target/i386/sw-1.c
+++ b/gcc/testsuite/gcc.target/i386/sw-1.c
@@ -7,7 +7,10 @@
 
 int c;
 int x[2000];
-__attribute__((regparm(1))) void foo (int a, int b)
+#ifndef __x86_64__
+__attribute__((regparm(1)))
+#endif
+void foo (int a, int b)
  {
    int t[200];
    if (a == 0 || c == 0)
diff --git a/gcc/testsuite/gcc.target/i386/uintr-2.c b/gcc/testsuite/gcc.target/i386/uintr-2.c
index 0a83c66..a0d2514 100644
--- a/gcc/testsuite/gcc.target/i386/uintr-2.c
+++ b/gcc/testsuite/gcc.target/i386/uintr-2.c
@@ -15,6 +15,6 @@ foo (void *frame, uword_t uirrv)
 
 void
 __attribute__((interrupt))
-UINTR_hanlder (struct __uintr_frame *frame, uword_t uirrv)
+UINTR_handler (struct __uintr_frame *frame, uword_t uirrv)
 {
 }
diff --git a/gcc/testsuite/gcc.target/i386/uintr-5.c b/gcc/testsuite/gcc.target/i386/uintr-5.c
index 49cb2ec..7c7c12f 100644
--- a/gcc/testsuite/gcc.target/i386/uintr-5.c
+++ b/gcc/testsuite/gcc.target/i386/uintr-5.c
@@ -7,6 +7,6 @@
 typedef unsigned int uword_t __attribute__ ((mode (__word__)));
 
 void
-UINTR_hanlder (struct __uintr_frame *frame, uword_t uirrv)
+UINTR_handler (struct __uintr_frame *frame, uword_t uirrv)
 {
 }
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
index 0ee610f..e88ab30 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O3 -mavx512bw -mtune=znver4 -fdump-tree-vect-optimized" } */
+/* { dg-options "-O3 -mavx512bw -mtune=znver4 --param vect-partial-vector-usage=0 -fdump-tree-vect-optimized" } */
 
 int test (signed char *data, int n)
 {
diff --git a/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-1.c b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-1.c
new file mode 100644
index 0000000..55519aa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=znver5 -fdump-tree-vect-optimized" } */
+
+void bar (double *a, double *b, double c, int n, int m)
+{
+  for (int j = 0; j < m; ++j)
+    for (int i = 0; i < n; ++i)
+      a[j*n + i] = b[j*n + i] + c;
+}
+
+/* { dg-final { scan-tree-dump "epilogue loop vectorized using masked 64 byte vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-2.c b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-2.c
new file mode 100644
index 0000000..3dc28b3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=znver5 -fdump-tree-vect-optimized" } */
+
+void foo (double *a, double b, double c, int n, int m)
+{
+  for (int j = 0; j < m; ++j)
+    for (int i = 0; i < n; ++i)
+      a[j*n + i] = a[j*n + i] * b + c;
+}
+
+/* We do not want to use a masked epilogue for the inner loop as the next
+   outer iteration will possibly immediately read from elements masked of
+   the previous inner loop epilogue and that never forwards.  */
+/* { dg-final { scan-tree-dump "epilogue loop vectorized using 32 byte vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/loongarch/pr121064.c b/gcc/testsuite/gcc.target/loongarch/pr121064.c
new file mode 100644
index 0000000..a466c7a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/loongarch/pr121064.c
@@ -0,0 +1,38 @@
+/* { dg-require-effective-target loongarch_sx_hw } */
+/* { dg-do run } */
+/* { dg-options "-march=loongarch64 -mfpu=64 -mlsx -O3" } */
+
+typedef __INT32_TYPE__ int32_t;
+typedef unsigned __INT32_TYPE__ uint32_t;
+
+__attribute__ ((noipa)) static int32_t
+long_filter_ehigh_3830_1 (int32_t *buffer, int length)
+{
+  int i, j;
+  int32_t dotprod = 0;
+  int32_t delay[4] = { 0 };
+  uint32_t coeffs[4] = { 0 };
+
+  for (i = 0; i < length; i++)
+    {
+      dotprod = 0;
+      for (j = 3; j >= 0; j--)
+        {
+          dotprod += delay[j] * coeffs[j];
+          coeffs[j] += ((delay[j] >> 31) | 1);
+        }
+      for (j = 3; j > 0; j--)
+        delay[j] = delay[j - 1];
+      delay[0] = buffer[i];
+    }
+
+  return dotprod;
+}
+
+int
+main ()
+{
+  int32_t buffer[] = { -1, 1 };
+  if (long_filter_ehigh_3830_1 (buffer, 2) != -1)
+    __builtin_trap ();
+}
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c
new file mode 100644
index 0000000..e759a11
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c
new file mode 100644
index 0000000..153ed1e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c
new file mode 100644
index 0000000..9bb9127
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c
new file mode 100644
index 0000000..06b3ceb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c
new file mode 100644
index 0000000..0cca3f3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c
new file mode 100644
index 0000000..9548be5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c
new file mode 100644
index 0000000..5731249
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c
new file mode 100644
index 0000000..aea501e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c
new file mode 100644
index 0000000..59d8987
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c
new file mode 100644
index 0000000..d28a671
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c
new file mode 100644
index 0000000..613dd65
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c
new file mode 100644
index 0000000..1b23350
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c
new file mode 100644
index 0000000..240332b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c
new file mode 100644
index 0000000..1e7fb70
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c
new file mode 100644
index 0000000..2cbec51
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/powerpc/pr121007.c b/gcc/testsuite/gcc.target/powerpc/pr121007.c
new file mode 100644
index 0000000..9e6b1be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr121007.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mdejagnu-cpu=power9" } */
+
+typedef struct { int a; } A;
+unsigned char *a;
+char b;
+int c;
+void foo (vector char, vector char, vector char);
+
+void
+bar (long stride)
+{
+  vector char v0, v1, v2, v3, v5;
+  vector char r0 = __builtin_vec_vsx_ld (0, a);
+  vector char r2 = __builtin_vec_vsx_ld (2 * stride, a - 3);
+  vector char r3 = __builtin_vec_vsx_ld (3 * stride, a - 3);
+  vector char r4;
+  vector char r6 = __builtin_vec_vsx_ld (6 * stride, a - 3);
+  vector char r7 = __builtin_vec_vsx_ld (7 * stride, a - 3);
+  vector char r14, h, i, j;
+  if (b)
+    return;
+  v1 = __builtin_vec_vsx_ld (9 * stride, a);
+  v2 = __builtin_vec_vsx_ld (10 * stride, a - 3);
+  v3 = __builtin_vec_vsx_ld (11 * stride, a - 3);
+  r3 = __builtin_vec_mergeh (r3, v3);
+  v5 = __builtin_vec_mergel (r2, r6);
+  r14 = __builtin_vec_mergeh (r3, r7);
+  r4 = __builtin_vec_mergeh (v2, r14);
+  v0 = __builtin_vec_mergeh (r0, r4);
+  union { unsigned char a[16]; A b; } temp;
+  vector signed char k;
+  h = __builtin_vec_ld (0, temp.a);
+  i = __builtin_vec_splat (h, 1);
+  temp.b.a = c;
+  k = __builtin_vec_ld (0, (signed char *) temp.a);
+  j = __builtin_vec_and (i, (vector char) k);
+  foo (v1, v0, j);
+  foo (v1, v5, j);
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c b/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
index 5095d50..312043b 100644
--- a/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
+++ b/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
@@ -1,8 +1,16 @@
 /* { dg-do compile { target { powerpc*-*-* } } } */
 /* { dg-skip-if "" { powerpc*-*-darwin* } } */
-/* { dg-options "-O2 -mdejagnu-cpu=power7 -fno-inline-functions" } */
+/* { dg-options "-O2 -mdejagnu-cpu=power7 -fno-inline-functions -fno-ipa-icf" } */
 /* { dg-require-effective-target powerpc_vsx } */
 
+/* PR testsuite/119382
+   Note: Added -fno-ipa-icf to disable Interprocedural Identical Code
+   Folding (ICF). Without this, insert_di_0_v2 is merged with insert_di_0
+   due to improved alias analysis introduced in commit r15-7961-gdc47161c1f32c3.
+   This results in the compiler replacing insert_di_0_v2 with a tail call to
+   insert_di_0, altering expected test behavior. Disabling ICF ensures correct
+   execution of the test.  */
+
 /* Test simple extract/insert/slat operations.  Make sure all types are
    supported with various options.  */
 
diff --git a/gcc/testsuite/gcc.target/pru/pragma-ctable_entry-2.c b/gcc/testsuite/gcc.target/pru/pragma-ctable_entry-2.c
new file mode 100644
index 0000000..a1c707d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/pru/pragma-ctable_entry-2.c
@@ -0,0 +1,22 @@
+/* Test for base addresses with bit 31 set (PR121124).  */
+
+/* { dg-do compile } */
+/* { dg-options "-O1" } */
+
+/* -O1 in the options is significant.  Without it LBCO/SBCO operations may
+   not be optimized to the respective instructions.  */
+
+
+#pragma ctable_entry 12 0x80beef00
+
+unsigned int
+test_ctable (unsigned int val1, unsigned int val2)
+{
+  ((volatile unsigned short int *)0x80beef00)[0] = val2;
+  ((volatile unsigned int *)0x80beef00)[val1] = val2;
+  return ((volatile unsigned int *)0x80beef00)[5];
+}
+
+/* { dg-final { scan-assembler "sbco\\tr15.b\[012\]?, 12, 0, 2" } } */
+/* { dg-final { scan-assembler "sbco\\tr15.b0, 12, r14, 4" } } */
+/* { dg-final { scan-assembler "lbco\\tr14.b0, 12, 20, 4" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/amo/zabha-zacas-atomic-cas.c b/gcc/testsuite/gcc.target/riscv/amo/zabha-zacas-atomic-cas.c
new file mode 100644
index 0000000..d3d84fd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/amo/zabha-zacas-atomic-cas.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* PR target/120995 ICE unrecognized subword atomic cas */
+/* { dg-options "-O" } */
+/* { dg-add-options riscv_zacas } */
+/* { dg-add-options riscv_zabha } */
+
+_Bool b;
+void atomic_bool_cmpxchg()
+{
+  __sync_bool_compare_and_swap(&b, 1, 0);
+}
diff --git a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c
index 4cf617d..0dfe816 100644
--- a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c
+++ b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c
@@ -9,7 +9,7 @@
 
 /*
 ** atomic_add_fetch_int_relaxed:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -23,7 +23,7 @@ void atomic_add_fetch_int_relaxed (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acquire:
-**	1:
+**...
 **	lr.w.aq\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -37,7 +37,7 @@ void atomic_add_fetch_int_acquire (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_release:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -51,7 +51,7 @@ void atomic_add_fetch_int_release (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acq_rel:
-**	1:
+**...
 **	lr.w.aq\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -65,7 +65,7 @@ void atomic_add_fetch_int_acq_rel (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_seq_cst:
-**	1:
+**...
 **	lr.w.aqrl\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
diff --git a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c
index 3fb16c0..658b040 100644
--- a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c
+++ b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c
@@ -9,7 +9,7 @@
 
 /*
 ** atomic_add_fetch_int_relaxed:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -23,7 +23,7 @@ void atomic_add_fetch_int_relaxed (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acquire:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -37,7 +37,7 @@ void atomic_add_fetch_int_acquire (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_release:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -51,7 +51,7 @@ void atomic_add_fetch_int_release (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acq_rel:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -65,7 +65,7 @@ void atomic_add_fetch_int_acq_rel (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_seq_cst:
-**	1:
+**...
 **	lr.w.aqrl\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c b/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
index 81ebf5f..15cc3ee 100644
--- a/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
+++ b/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
@@ -1,7 +1,7 @@
 /* Verify proper errors are generated for conflicted interrupt type.  */
 /* { dg-do compile } */
 /* { dg-options "" } */
-void __attribute__ ((interrupt ("user")))
+void __attribute__ ((interrupt ("supervisor")))
 foo(void);
 
 void __attribute__ ((interrupt ("machine")))
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c b/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c
new file mode 100644
index 0000000..f340108
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c
@@ -0,0 +1,11 @@
+/* Verify the return instruction is mnret.  */
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc_smrnmi" { target { rv32 } } } */
+/* { dg-options "-march=rv64gc_smrnmi" { target { rv64 } } } */
+
+void __attribute__ ((interrupt ("rnmi")))
+foo (void)
+{
+}
+
+/* { dg-final { scan-assembler {\mmnret} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-umode.c b/gcc/testsuite/gcc.target/riscv/interrupt-umode.c
deleted file mode 100644
index 042abf0..0000000
--- a/gcc/testsuite/gcc.target/riscv/interrupt-umode.c
+++ /dev/null
@@ -1,8 +0,0 @@
-/* Verify the return instruction is mret.  */
-/* { dg-do compile } */
-/* { dg-options "" } */
-void __attribute__ ((interrupt ("user")))
-foo (void)
-{
-}
-/* { dg-final { scan-assembler {\muret} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/mipscondmov.c b/gcc/testsuite/gcc.target/riscv/mipscondmov.c
new file mode 100644
index 0000000..5485133
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/mipscondmov.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32imafd_xmipscmov" { target { rv32 } } } */
+/* { dg-options "-march=rv64imafd_xmipscmov -mabi=lp64d" { target { rv64 } } } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-Og" } } */
+
+#define MYTEST(name, mytype) \
+mytype test1_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a == b) ? c : d; } \
+mytype test2_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a != b) ? c : d; } \
+mytype test3_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a > b) ? c : d; } \
+mytype test4_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a >= b) ? c : d; } \
+mytype test5_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a < b) ? c : d; } \
+mytype test6_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a <= b) ? c : d; } \
+mytype test7_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a == 1) ? c : d; } \
+mytype test8_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a != 1) ? c : d; } \
+mytype test9_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a > 1) ? c : d; } \
+mytype test10_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a >= 1) ? c : d; } \
+mytype test11_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a < 1) ? c : d; } \
+mytype test12_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a <= 1) ? c : d; }
+
+MYTEST(1, long)
+MYTEST(2, unsigned long)
+MYTEST(3, int)
+MYTEST(4, unsigned int)
+MYTEST(5, short)
+MYTEST(6, unsigned short)
+MYTEST(7, signed char)
+MYTEST(8, unsigned char)
+
+/* { dg-final { scan-assembler-times "mips.ccmov" 96 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/pr118241-b.cc b/gcc/testsuite/gcc.target/riscv/pr118241-b.cc
new file mode 100644
index 0000000..b2cc73f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/pr118241-b.cc
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=rv64imafdc_zba_zbb_zbs_zicbom_zicbop -mabi=lp64d" } */
+
+/* Reduced from libsanitizer::asan_allocator.  */
+
+enum a { c };
+class d;
+struct e {
+  long count;
+  void *batch[];
+};
+template <typename> class f {
+public:
+  void g() {
+    if (e *b = h->i())
+      for (; b->count;)
+        if (6 < b->count)
+          __builtin_prefetch(b->batch[6]);
+  }
+  d *h;
+};
+class d {
+public:
+  e *i();
+};
+struct j {
+  f<int> k;
+  j(a);
+  void l() { k.g(); }
+} a(c);
+void m() { a.l(); }
+
+/* { dg-final { scan-assembler-times "prefetch.r\t0\\(\[a-x0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h
index 4aeb637..2de7d7c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h
@@ -3,6 +3,11 @@
 
 #include <stdint.h>
 
+#if __riscv_xlen == 64
+typedef unsigned __int128 uint128_t;
+typedef signed __int128 int128_t;
+#endif
+
 #define DEF_AVG_0(NT, WT, NAME)                                 \
 __attribute__((noinline))                                       \
 void                                                            \
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c
index 138124c..31d3b43 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int32_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c
index 30438c9..7f30b9e 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int64_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c
index 2e9cfa5..2e06d0a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int32_t
 #define WT int64_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c
new file mode 100644
index 0000000..ca23066
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d" } */
+
+#include "avg.h"
+
+#define NT int64_t
+#define WT int128_t
+
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
+
+/* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c
index 2ebf294..dda84a6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int16_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c
index 64fec913..dfd2bb3 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int32_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c
index a72642c..d1060cc 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int64_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c
index 1fa080b..3d872a8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c
index deec763..eda9736 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c
index fa72000..21cbb94 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c
new file mode 100644
index 0000000..ee5330c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
+
+#include "avg.h"
+#include "avg_data.h"
+
+#define WT   int128_t
+#define NT   int64_t
+#define NAME avg_ceil
+
+DEF_AVG_1_WRAP(NT, WT, NAME)
+
+#define TEST_DATA                            TEST_AVG_DATA_WRAP(NT, NAME)
+#define TEST_RUN(NT, WT, NAME, a, b, out, n) RUN_AVG_1_WRAP(NT, WT, NAME, a, b, out, n)
+
+#include "avg_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c
index 6865cf2..fd91b6f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c
index 78620f4..38f4920 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c
index b2c763c..f65ee15 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h
index 12b464a..49103f3 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h
@@ -169,8 +169,8 @@ int64_t TEST_AVG_DATA(int64_t, avg_floor)[][3][N] =
     },
     {
        9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,
-       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
                         -2ull,                   -2ull,                   -2ull,                   -2ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
       -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
     },
     {
@@ -345,8 +345,8 @@ int64_t TEST_AVG_DATA(int64_t, avg_ceil)[][3][N] =
     },
     {
        9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,
-       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
                         -2ull,                   -2ull,                   -2ull,                   -2ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
       -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
     },
     {
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c
index 16ba967..fc7943c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int32_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c
index b229b4b..e02e5df 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int64_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c
index 5f946bb..e36e424 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int32_t
 #define WT int64_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c
new file mode 100644
index 0000000..3e2d97d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d" } */
+
+#include "avg.h"
+
+#define NT int64_t
+#define WT int128_t
+
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
+
+/* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c
index 5d9297a..cdbb299 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int16_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c
index 5c5d4ea..53508b0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int32_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c
index f297953..9a6d1a2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int64_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c
index 9d0dd61..92239a2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c
index 2736baa..5716c29 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c
index 2334045..705e091 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c
new file mode 100644
index 0000000..91e9809
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
+
+#include "avg.h"
+#include "avg_data.h"
+
+#define WT   int128_t
+#define NT   int64_t
+#define NAME avg_floor
+
+DEF_AVG_0_WRAP(NT, WT, NAME)
+
+#define TEST_DATA                            TEST_AVG_DATA_WRAP(NT, NAME)
+#define TEST_RUN(NT, WT, NAME, a, b, out, n) RUN_AVG_0_WRAP(NT, WT, NAME, a, b, out, n)
+
+#include "avg_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c
index 8364748..abe5c5b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c
index 157c936..355b90f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c
index 2db0d3c..a9ae96f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h
index f78bdc0..93c29f0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h
@@ -385,6 +385,8 @@ vec_sat_u_sub_##T##_fmt_1 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = (x - y) & (-(T)(x >= y));                             \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_1(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_2(T)                                   \
 void __attribute__((noinline))                                       \
@@ -398,6 +400,8 @@ vec_sat_u_sub_##T##_fmt_2 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = (x - y) & (-(T)(x > y));                              \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_2(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_3(T)                                   \
 void __attribute__((noinline))                                       \
@@ -411,6 +415,8 @@ vec_sat_u_sub_##T##_fmt_3 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x > y ? x - y : 0;                                    \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_3(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_4(T)                                   \
 void __attribute__((noinline))                                       \
@@ -424,6 +430,8 @@ vec_sat_u_sub_##T##_fmt_4 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x >= y ? x - y : 0;                                   \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_4(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_5(T)                                   \
 void __attribute__((noinline))                                       \
@@ -437,6 +445,8 @@ vec_sat_u_sub_##T##_fmt_5 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x < y ? 0 : x - y;                                    \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_5(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_6(T)                                   \
 void __attribute__((noinline))                                       \
@@ -450,6 +460,8 @@ vec_sat_u_sub_##T##_fmt_6 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x <= y ? 0 : x - y;                                   \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_6(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_7(T)                                   \
 void __attribute__((noinline))                                       \
@@ -465,6 +477,8 @@ vec_sat_u_sub_##T##_fmt_7 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = ret & (T)(overflow - 1);                              \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_7(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_8(T)                                   \
 void __attribute__((noinline))                                       \
@@ -480,6 +494,8 @@ vec_sat_u_sub_##T##_fmt_8 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = ret & (T)-(!overflow);                                \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_8(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_9(T)                                   \
 void __attribute__((noinline))                                       \
@@ -495,6 +511,8 @@ vec_sat_u_sub_##T##_fmt_9 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = overflow ? 0 : ret;                                   \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_9(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_10(T)                                   \
 void __attribute__((noinline))                                        \
@@ -510,6 +528,42 @@ vec_sat_u_sub_##T##_fmt_10 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = !overflow ? ret : 0;                                   \
     }                                                                 \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_10(T)
+
+#define DEF_VEC_SAT_U_SUB_FMT_11(T)                                   \
+void __attribute__((noinline))                                       \
+vec_sat_u_sub_##T##_fmt_11 (T *out, T *op_1, T *op_2, unsigned limit) \
+{                                                                    \
+  unsigned i;                                                        \
+  for (i = 0; i < limit; i++)                                        \
+    {                                                                \
+      T x = op_1[i];                                                 \
+      T y = op_2[i];                                                 \
+      T ret;                                                         \
+      T overflow = __builtin_sub_overflow (x, y, &ret);           \
+      out[i] = overflow ? 0 : ret;                                   \
+    }                                                                \
+}
+#define DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_11(T)
+
+#define DEF_VEC_SAT_U_SUB_FMT_12(T)                                   \
+void __attribute__((noinline))                                        \
+vec_sat_u_sub_##T##_fmt_12 (T *out, T *op_1, T *op_2, unsigned limit) \
+{                                                                     \
+  unsigned i;                                                         \
+  for (i = 0; i < limit; i++)                                         \
+    {                                                                 \
+      T x = op_1[i];                                                  \
+      T y = op_2[i];                                                  \
+      T ret;                                                          \
+      T overflow = __builtin_sub_overflow (x, y, &ret);            \
+      out[i] = !overflow ? ret : 0;                                   \
+    }                                                                 \
+}
+#define DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_12(T)
 
 #define DEF_VEC_SAT_U_SUB_ZIP(T1, T2)                             \
 void __attribute__((noinline))                                    \
@@ -669,33 +723,63 @@ vec_sat_s_sub_##T##_fmt_4 (T *out, T *op_1, T *op_2, unsigned limit) \
 
 #define RUN_VEC_SAT_U_SUB_FMT_1(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_1(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_2(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_2(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_3(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_3(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_4(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_4(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_5(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_5(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_6(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_6(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_7(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_7(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_8(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_8(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_9(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_9(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_10(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_10(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10(T, out, op_1, op_2, N)
+
+#define RUN_VEC_SAT_U_SUB_FMT_11(T, out, op_1, op_2, N) \
+  vec_sat_u_sub_##T##_fmt_11(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11(T, out, op_1, op_2, N)
+
+#define RUN_VEC_SAT_U_SUB_FMT_12(T, out, op_1, op_2, N) \
+  vec_sat_u_sub_##T##_fmt_12(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_ZIP(T1, T2, x, b, N) \
   vec_sat_u_sub_##T1##_##T2##_fmt_zip(x, b, N)
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h
index 9f05c0c..7647439 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h
@@ -744,6 +744,258 @@ uint64_t TEST_UNARY_DATA(uint64_t, sat_u_sub_imm)[][2][N] =
   },
 };
 
+uint8_t TEST_UNARY_DATA(uint8_t, ussub)[][3][N] = {
+  {
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* arg_0 */
+    {
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+    }, /* arg_1 */
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* expect */
+  },
+  {
+    {
+	0, 255, 255, 255,
+	0, 255, 255, 255,
+	0, 255, 255, 255,
+	0, 255, 255, 255,
+    },
+    {
+	1, 255, 254, 251,
+	1, 255, 254, 251,
+	1, 255, 254, 251,
+	1, 255, 254, 251,
+    },
+    {
+	0,   0,   1,   4,
+	0,   0,   1,   4,
+	0,   0,   1,   4,
+	0,   0,   1,   4,
+    },
+  },
+  {
+    {
+	0,   0,   1,   0,
+	1,   2,   3,   0,
+	1,   2,   3, 255,
+	5, 254, 255,   9,
+    },
+    {
+	0,   1,   0, 254,
+      254, 254, 254, 255,
+      255, 255,   0, 252,
+      255, 255, 255,   1,
+    },
+    {
+	0,   0,   1,   0,
+	0,   0,   0,   0,
+	0,   0,   3,   3,
+	0,   0,   0,   8,
+    },
+  },
+};
+
+uint16_t TEST_UNARY_DATA(uint16_t, ussub)[][3][N] = {
+  {
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* arg_0 */
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* arg_1 */
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* expect */
+  },
+  {
+    {
+      65535, 65535, 65535, 65535,
+      65535, 65535, 65535, 65535,
+      65535, 65535, 65535, 65535,
+      65535, 65535, 65535, 65535,
+    },
+    {
+      55535, 45535, 35535, 25535,
+      55535, 45535, 35535, 25535,
+      55535, 45535, 35535, 25535,
+      55535, 45535, 35535, 25535,
+    },
+    {
+      10000, 20000, 30000, 40000,
+      10000, 20000, 30000, 40000,
+      10000, 20000, 30000, 40000,
+      10000, 20000, 30000, 40000,
+    },
+  },
+  {
+    {
+	  0,     0,     1,     0,
+	  1,     2,     3,     0,
+	  1, 65535,     3, 65535,
+	  5, 65534, 65535,     9,
+    },
+    {
+	  0,     1,     1, 65534,
+      65534, 65534,     1, 65535,
+	  0, 65535, 65535,     0,
+      65535, 65535,     1,     2,
+    },
+    {
+	  0,     0,     0,     0,
+	  0,     0,     2,     0,
+	  1,     0,     0, 65535,
+	  0,     0, 65534,     7,
+    },
+  },
+};
+
+uint32_t TEST_UNARY_DATA(uint32_t, ussub)[][3][N] = {
+  {
+    {
+      0, 0, 4, 0,
+      0, 0, 4, 0,
+      0, 0, 4, 0,
+      0, 0, 4, 0,
+    }, /* arg_0 */
+    {
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+    }, /* arg_1 */
+    {
+      0, 0, 2, 0,
+      0, 0, 2, 0,
+      0, 0, 2, 0,
+      0, 0, 2, 0,
+    }, /* expect */
+  },
+  {
+    {
+      4294967295, 4294967295, 4294967295, 4294967295,
+      4294967295, 4294967295, 4294967295, 4294967295,
+      4294967295, 4294967295, 4294967295, 4294967295,
+      4294967295, 4294967295, 4294967295, 4294967295,
+    },
+    {
+      1294967295, 2294967295, 3294967295, 4294967295,
+      1294967295, 2294967295, 3294967295, 4294967295,
+      1294967295, 2294967295, 3294967295, 4294967295,
+      1294967295, 2294967295, 3294967295, 4294967295,
+    },
+    {
+      3000000000, 2000000000, 1000000000,          0,
+      3000000000, 2000000000, 1000000000,          0,
+      3000000000, 2000000000, 1000000000,          0,
+      3000000000, 2000000000, 1000000000,          0,
+    },
+  },
+  {
+    {
+	       0,          0,          9,          0,
+	       1, 4294967295,          3,          0,
+	       1,          2,          3,          4,
+	       5, 4294967294, 4294967295, 4294967295,
+    },
+    {
+	       0,          1,          1, 4294967294,
+	       1,          2, 4294967294, 4294967295,
+	       1, 4294967295, 4294967295,          1,
+	       1, 4294967295, 4294967290,          9,
+    },
+    {
+	       0,          0,          8,          0,
+	       0, 4294967293,          0,          0,
+	       0,          0,          0,          3,
+	       4,          0,          5, 4294967286,
+    },
+  },
+};
+
+uint64_t TEST_UNARY_DATA(uint64_t, ussub)[][3][N] = {
+  {
+    {
+      0, 9, 0, 0,
+      0, 9, 0, 0,
+      0, 9, 0, 0,
+      0, 9, 0, 0,
+    }, /* arg_0 */
+    {
+      0, 2, 3, 1,
+      0, 2, 3, 1,
+      0, 2, 3, 1,
+      0, 2, 3, 1,
+    }, /* arg_1 */
+    {
+      0, 7, 0, 0,
+      0, 7, 0, 0,
+      0, 7, 0, 0,
+      0, 7, 0, 0,
+    }, /* expect */
+  },
+  {
+    {
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+    },
+    {
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+    },
+    {
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+    },
+  },
+  {
+    {
+			  0, 18446744073709551615u,                     1,                     0,
+			  1, 18446744073709551615u,                     3,                     0,
+			  1, 18446744073709551614u,                     3,                     4,
+			  5, 18446744073709551614u, 18446744073709551615u,                     9,
+    },
+    {
+			  0,                     1,                     1, 18446744073709551614u,
+      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
+    },
+    {
+			  0, 18446744073709551614u,                     0,                     0,
+			  0,                     1,                     0,                     0,
+			  0,                     0,                     0,                     0,
+			  0,                     0,                     0,                     8,
+    },
+  },
+};
+
 int8_t TEST_UNARY_DATA(int8_t, sat_s_add_imm)[][2][N] =
 {
   { /* For add imm -128 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c
new file mode 100644
index 0000000..57da9e8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c
new file mode 100644
index 0000000..b5264a3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c
new file mode 100644
index 0000000..1a68b5c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c
new file mode 100644
index 0000000..a1c5c19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c
new file mode 100644
index 0000000..fd987e9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c
new file mode 100644
index 0000000..bc380fe
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c
new file mode 100644
index 0000000..c03163f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c
new file mode 100644
index 0000000..91e1909
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c
index 97e5040..5878c5b6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c
index a5428c4..f74979f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c
index bdb65d9..1250e5b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c
index 3fe5fe3..a2a77dd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c
@@ -2,74 +2,15 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c
index 0f4129c..19c8fa0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c
index 8b995eb..ada136f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c
index d12d981..488c158 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c
index 384ef3e..127c27a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c
new file mode 100644
index 0000000..4b49467
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint16_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c
new file mode 100644
index 0000000..80b55ea
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint32_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c
new file mode 100644
index 0000000..6a89d0f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint64_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c
new file mode 100644
index 0000000..974493e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint8_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c
new file mode 100644
index 0000000..28778b9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint16_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c
new file mode 100644
index 0000000..936a39a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint32_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c
new file mode 100644
index 0000000..b8fa65b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint64_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c
new file mode 100644
index 0000000..6bff1e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint8_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c
index 5cf08ac..45bef88 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c
index 85c8454..6d8a653 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c
index 67d5ac5..0132d46 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c
index 809f07f..425f86f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c
index 57839a9..97a8e08 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c
index ffb0dcc..9124899 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c
index 3966677..1e54ede 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c
index e795f62..d8d53b7 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c
index 0eecf82..b293823 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c
index 1d0d16b..f0f1c4f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c
index 98fdfa2..27c28e2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c
index 18a887d..7911825 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c
index ce44c04..6ae7b36 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c
index 36ae7b3..4e6b9e6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c
index 7b40ffd..6b26913 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c
index 3b0807f..2bd28cd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c
index e972078..69b0be9 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c
index 54e2848..2450586 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c
index 33f3be0..0b97910 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c
index 1376038..afb23f6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c
index 83241ef..0466d4c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c
index f20bb21..14b8701 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c
index 4ad0afd..7e0afd8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c
index 3b33b13..40b1a6a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c
index b212550..bd33048 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c
index 1fb707c..36f78f5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c
index da8c09c..3bc5d5d 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c
index 647607f..3964d1b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c
index 9bb0664..4c0809a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c
index f142b8b..3e700bd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c
index 574b91a..81b8dc8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c
index 2c8ee42..8bc52ae 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
index 4dc5703..0fa1ea0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
@@ -72,7 +72,7 @@ f_vnx128qi (int8_t *out)
   *(vnx128qi *) out = v;
 }
 
-/* { dg-final { scan-assembler-times {vmv.v.x\tv[0-9]+,\s*[a-x0-9]+} 6 } } */
+/* { dg-final { scan-assembler-times {vmv.v.x\tv[0-9]+,\s*[a-x0-9]+} 7 } } */
 /* { dg-final { scan-assembler-times {slli\t[a-x0-9]+,\s*[a-x0-9]+,\s*8} 6 } } */
 /* { dg-final { scan-assembler-times {or\t[a-x0-9]+,\s*[a-x0-9]+,\s*[a-x0-9]+} 6 } } */
 /* { dg-final { scan-assembler-times {vslide1down\.vx\tv[0-9]+,\s*v[0-9]+,\s*[a-x0-9]+} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c
index 05cf57c..811f26c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_0 (_Float16, +, +, acc)
 DEF_VF_MULOP_ACC_CASE_0 (_Float16, -, +, sac)
 DEF_VF_MULOP_ACC_CASE_0 (_Float16, +, -, nacc)
 DEF_VF_MULOP_ACC_CASE_0 (_Float16, -, -, nsac)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, -, -, nsac)
 
 /* { dg-final { scan-assembler-times {vfmadd.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfmsub.vf} 1 } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_0 (_Float16, -, -, nsac)
 /* { dg-final { scan-assembler-times {vfmsac.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmacc.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmsac.vf} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c
index 873e315..ca82ead 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_0 (float, +, +, acc)
 DEF_VF_MULOP_ACC_CASE_0 (float, -, +, sac)
 DEF_VF_MULOP_ACC_CASE_0 (float, +, -, nacc)
 DEF_VF_MULOP_ACC_CASE_0 (float, -, -, nsac)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, -, -, nsac)
 
 /* { dg-final { scan-assembler-times {vfmadd.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfmsub.vf} 1 } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_0 (float, -, -, nsac)
 /* { dg-final { scan-assembler-times {vfmsac.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmacc.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmsac.vf} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c
index 78127b6..3a39303 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c
@@ -11,3 +11,9 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler-times {fcvt.s.h} 4 } } */
+/* { dg-final { scan-assembler-times {vfmv.v.f} 12 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c
index 30d57e0..b4618bae 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c
@@ -11,3 +11,9 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler-times {fcvt.d.s} 4 } } */
+/* { dg-final { scan-assembler-times {vfmv.v.f} 12 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c
index 8295ffb..58afaa4 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_1 (_Float16, +, +, acc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (_Float16, -, +, sac, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (_Float16, +, -, nacc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (_Float16, -, -, nsac, VF_MULOP_ACC_BODY_X128)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, -, -, nsac)
 
 /* { dg-final { scan-assembler {vfmadd.vf} } } */
 /* { dg-final { scan-assembler {vfmsub.vf} } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_1 (_Float16, -, -, nsac, VF_MULOP_ACC_BODY_X128)
 /* { dg-final { scan-assembler {vfmsac.vf} } } */
 /* { dg-final { scan-assembler {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwnmsac.vf} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c
index f237f84..0e95774 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_1 (float, +, +, acc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (float, -, +, sac, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (float, +, -, nacc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (float, -, -, nsac, VF_MULOP_ACC_BODY_X128)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, -, -, nsac)
 
 /* { dg-final { scan-assembler {vfmadd.vf} } } */
 /* { dg-final { scan-assembler {vfmsub.vf} } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_1 (float, -, -, nsac, VF_MULOP_ACC_BODY_X128)
 /* { dg-final { scan-assembler {vfmsac.vf} } } */
 /* { dg-final { scan-assembler {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwnmsac.vf} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c
index 7a50f67..559df6c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c
@@ -11,3 +11,8 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler {fcvt.s.h} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c
index fb0493e..03f9c5a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c
@@ -11,3 +11,8 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler {fcvt.d.s} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h
index 1659f78..b1a324f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h
@@ -34,6 +34,21 @@
 #define RUN_VF_MULOP_ACC_CASE_0_WRAP(T, NAME, out, in, x, n)                   \
   RUN_VF_MULOP_ACC_CASE_0 (T, NAME, out, in, x, n)
 
+#define DEF_VF_MULOP_WIDEN_CASE_0(T1, T2, OP, NEG, NAME)                       \
+  void test_vf_mulop_widen_##NAME##_##T1##_case_0 (T2 *restrict out,           \
+						   T1 *restrict in,            \
+						   T1 *restrict f, unsigned n) \
+  {                                                                            \
+    for (unsigned i = 0; i < n; i++)                                           \
+      out[i] = NEG ((T2) * f * (T2) in[i] OP out[i]);                          \
+  }
+#define DEF_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, OP, NEG, NAME)                  \
+  DEF_VF_MULOP_WIDEN_CASE_0 (T1, T2, OP, NEG, NAME)
+#define RUN_VF_MULOP_WIDEN_CASE_0(T1, T2, NAME, out, in, x, n)                 \
+  test_vf_mulop_widen_##NAME##_##T1##_case_0 (out, in, x, n)
+#define RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, x, n)            \
+  RUN_VF_MULOP_WIDEN_CASE_0 (T1, T2, NAME, out, in, x, n)
+
 #define VF_MULOP_BODY(op, neg)                                                 \
   out[k + 0] = neg (tmp * out[k + 0] op in[k + 0]);                            \
   out[k + 1] = neg (tmp * out[k + 1] op in[k + 1]);                            \
@@ -129,4 +144,19 @@
 #define DEF_VF_MULOP_ACC_CASE_1_WRAP(T, OP, NEG, NAME, BODY)                   \
   DEF_VF_MULOP_ACC_CASE_1 (T, OP, NEG, NAME, BODY)
 
+#define DEF_VF_MULOP_WIDEN_CASE_1(TYPE1, TYPE2, OP, NEG, NAME)                 \
+  void test_vf_mulop_widen_##NAME##_##TYPE1##_##TYPE2##_case_1 (               \
+    TYPE2 *__restrict dst, TYPE2 *__restrict dst2, TYPE2 *__restrict dst3,     \
+    TYPE2 *__restrict dst4, TYPE1 *__restrict a, TYPE1 *__restrict b,          \
+    TYPE1 *__restrict a2, TYPE1 *__restrict b2, int n)                         \
+  {                                                                            \
+    for (int i = 0; i < n; i++)                                                \
+      {                                                                        \
+	dst[i] = NEG ((TYPE2) * a * (TYPE2) b[i] OP dst[i]);                   \
+	dst2[i] = NEG ((TYPE2) * a2 * (TYPE2) b[i] OP dst2[i]);                \
+	dst3[i] = NEG ((TYPE2) * a2 * (TYPE2) a[i] OP dst3[i]);                \
+	dst4[i] = NEG ((TYPE2) * a * (TYPE2) b2[i] OP dst4[i]);                \
+      }                                                                        \
+  }
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h
index 3dc3999..3dadfab 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h
@@ -6,6 +6,10 @@
 #define TYPE_FABS(x, T)                                                        \
   (__builtin_types_compatible_p (T, double) ? fabs (x) : fabsf (x))
 
+#define MAX_RELATIVE_DIFF(T)                                                   \
+  (__builtin_types_compatible_p (T, _Float16) ? 0.1f  :                        \
+  (__builtin_types_compatible_p (T, float)    ? 0.01f : 0.01))
+
 int
 main ()
 {
@@ -23,7 +27,8 @@ main ()
       for (k = 0; k < N; k++)
 	{
 	  T diff = expect[k] - TEST_OUT[k];
-	  if (TYPE_FABS (diff, T) > .01 * TYPE_FABS (expect[k], T))
+	  if (TYPE_FABS (diff, T)
+	      > MAX_RELATIVE_DIFF (T) * TYPE_FABS (expect[k], T))
 	    __builtin_abort ();
 	}
     }
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h
new file mode 100644
index 0000000..9f95fbb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h
@@ -0,0 +1,32 @@
+#ifndef HAVE_DEFINED_VF_MULOP_WIDEN_RUN_H
+#define HAVE_DEFINED_VF_MULOP_WIDEN_RUN_H
+
+#include <assert.h>
+
+#define N 512
+
+int main ()
+{
+  T1 f[N];
+  T1 in[N];
+  T2 out[N];
+  T2 out2[N];
+
+  for (int i = 0; i < N; i++) 
+    {
+      f[i] = LIMIT + i % 8723;
+      in[i] = LIMIT + i & 1964;
+      out[i] = LIMIT + i & 628;
+      out2[i] = LIMIT + i & 628;
+      asm volatile ("" ::: "memory");
+    }
+
+  TEST_RUN (T1, T2, NAME, out, in, f, N);
+
+  for (int i = 0; i < N; i++)
+    assert (out[i] == NEG(((T2) *f * (T2) in[i]) OP out2[i]));
+
+  return 0;
+}
+
+#endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
index 5bb926d..fd8aa30 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
index be4dc1d..8fd8552 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
index 812e608..e91fd15 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
index 48c2880..ca7e0db 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
index b960e7a..b38e800 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
index 2be8917..fef5d77 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
index 4f10600..7951d40 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
index 7bed0ed..d0def86 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c
new file mode 100644
index 0000000..d4c527a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME acc
+#define OP +
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c
new file mode 100644
index 0000000..1af5240
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME acc
+#define OP +
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c
new file mode 100644
index 0000000..abce2f2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME sac
+#define OP -
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c
new file mode 100644
index 0000000..13617a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME sac
+#define OP -
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c
new file mode 100644
index 0000000..ddf49d5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME nacc
+#define OP +
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c
new file mode 100644
index 0000000..851c335
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME nacc
+#define OP +
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c
new file mode 100644
index 0000000..a874991
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME nsac
+#define OP -
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c
new file mode 100644
index 0000000..9eacace
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME nsac
+#define OP -
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
index c86d77c..4e1a575 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
index f6524cb..4c4f72d 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
index f1e8627..abf62c2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
@@ -18,3 +18,9 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 { target { no-opts
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
index 9b0cbd2..7744bcb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c
index b064748..cb62e0f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c
index e334bb3..e2a5dbb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
index 3e8ca05..8e7a788 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
@@ -19,3 +19,7 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 2 { target { no-opts {
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
index 1f995cd..d213c18 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
index fb1154c..05801a9 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
index d4baa4b..f05f091 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
index 18c1a78..adf9ccb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
index 5ce3c88..8b3f5bc 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c
index f7fae37..365e650 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c
index b111a4e..c8fd42a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c
index 4640d16..bdb76b4 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c
index 58341ad..fc9c101 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
index 2965924..741f431 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
index e7815e9..1741c22 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
index 063a7a1..d326357 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
index 0efb60c..3137dc0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c
index 406b999..121daeb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c
index 6792b6b..9616e7f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c
index 692a709..cf985f0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c
index 4e30498..3bb382d 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
index 199f8a7..2ae4804 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,11 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=scalable -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
index 392f4fe..88cfc72 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
index d22c387..6b29a72 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,9 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
index 9a832a2..f862eb7 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
index afb5a85..3ecfce6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
index a907e9b..7ce1fe8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,4 +32,6 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vremu.vx} } } */
 /* { dg-final { scan-assembler {vmaxu.vx} } } */
 /* { dg-final { scan-assembler {vminu.vx} } } */
+/* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
index efabf99..c84a30c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} { target { no-opts {
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
index 7b2b088..9f3d7df 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
index f15fec5..df6872c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
index 8d21c47..05ed639 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
index 0660000..6776b1f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,6 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
index ce33461..d3e2785 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
index da1b1be..5497b5a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
index b7ec6c9..3a8e85f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
index dce78b1..060d591 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
index c5c6fb8..86a6c45 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
index 3c9afdd..0bfa2cb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
index b80a6b3..3e3acfc 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
index 15bfe60..531c119 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler-not {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
index 4d529fe..43246bb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
index 5952a7c..f51e7a1 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
index 5bbc585..79b7477 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
index 255ae62..ac5fd69 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler-not {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
index 63cd449..84aa06b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c
new file mode 100644
index 0000000..2b87321
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        8
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint16m1_t
+#define T         int16_t
+#define ELEM_SIZE 16
+#define SUFFIX    i16
+#define FUNC      __riscv_vaadd_vv_i16m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c
new file mode 100644
index 0000000..b95699b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        4
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint32m1_t
+#define T         int32_t
+#define ELEM_SIZE 32
+#define SUFFIX    i32
+#define FUNC      __riscv_vaadd_vv_i32m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c
new file mode 100644
index 0000000..48b6010
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        2
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint64m1_t
+#define T         int64_t
+#define ELEM_SIZE 64
+#define SUFFIX    i64
+#define FUNC      __riscv_vaadd_vv_i64m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c
new file mode 100644
index 0000000..d07a625
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        16
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint8m1_t
+#define T         int8_t
+#define ELEM_SIZE 8
+#define SUFFIX    i8
+#define FUNC      __riscv_vaadd_vv_i8m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c
new file mode 100644
index 0000000..bd36429
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        8
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint16m1_t
+#define T         uint16_t
+#define ELEM_SIZE 16
+#define SUFFIX    u16
+#define FUNC      __riscv_vaaddu_vv_u16m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c
new file mode 100644
index 0000000..f023a76
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        4
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint32m1_t
+#define T         uint32_t
+#define ELEM_SIZE 32
+#define SUFFIX    u32
+#define FUNC      __riscv_vaaddu_vv_u32m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c
new file mode 100644
index 0000000..d9a37ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        2
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint64m1_t
+#define T         uint64_t
+#define ELEM_SIZE 64
+#define SUFFIX    u64
+#define FUNC      __riscv_vaaddu_vv_u64m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c
new file mode 100644
index 0000000..328e5d4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        16
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint8m1_t
+#define T         uint8_t
+#define ELEM_SIZE 8
+#define SUFFIX    u8
+#define FUNC      __riscv_vaaddu_vv_u8m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h
new file mode 100644
index 0000000..438c7ab
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h
@@ -0,0 +1,28 @@
+#ifndef HAVE_DEFINED_VX_FIXED_VXRM_H
+#define HAVE_DEFINED_VX_FIXED_VXRM_H
+
+#include <riscv_vector.h>
+
+int64_t go[VL] = {};
+int64_t ga[VL] = {};
+
+#define DEF_FIXED_BINARY_VX(VT, T, ES, SX, VXRM, FUNC)       \
+void __attribute__((noinline))                               \
+test_fixed_binary_##VT##_##VXRM##_##FUNC##_vx () {           \
+  VT a = __riscv_vle##ES##_v_##SX##m1((T *)ga, VL);          \
+  VT b;                                                      \
+  T *bp = (T *)&b;                                           \
+                                                             \
+  for (int i = 0; i < VL; i++) {                             \
+    bp[i] = 123;                                             \
+  }                                                          \
+                                                             \
+  VT d = FUNC (a, b, VXRM, VL);                              \
+                                                             \
+  __riscv_vse##ES##_v_##SX##m1((T *)&go, d, VL);             \
+}
+
+#define DEF_FIXED_BINARY_VX_WRAP(VT, T, ES, SX, VXRM, FUNC) \
+  DEF_FIXED_BINARY_VX(VT, T, ES, SX, VXRM, FUNC)
+
+#endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
index f12d1d1..4a9daff 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
@@ -3,6 +3,14 @@
 
 #include <stdint.h>
 
+#undef HAS_INT128
+
+#if __riscv_xlen == 64
+#define HAS_INT128
+typedef unsigned __int128 uint128_t;
+typedef signed __int128 int128_t;
+#endif
+
 #define DEF_VX_BINARY_CASE_0(T, OP, NAME)                                \
 void                                                                     \
 test_vx_binary_##NAME##_##T##_case_0 (T * restrict out, T * restrict in, \
@@ -294,41 +302,131 @@ DEF_SAT_U_SUB(uint16_t)
 DEF_SAT_U_SUB(uint32_t)
 DEF_SAT_U_SUB(uint64_t)
 
+#define DEF_SAT_S_ADD(T, UT, MIN, MAX) \
+T                                      \
+test_##T##_sat_add (T x, T y)          \
+{                                      \
+  T sum = (UT)x + (UT)y;               \
+  return (x ^ y) < 0                   \
+    ? sum                              \
+    : (sum ^ x) >= 0                   \
+      ? sum                            \
+      : x < 0 ? MIN : MAX;             \
+}
+
+DEF_SAT_S_ADD(int8_t, uint8_t, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD(int16_t, uint16_t, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD(int32_t, uint32_t, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD(int64_t, uint64_t, INT64_MIN, INT64_MAX)
+
+#define DEF_SAT_S_SUB(T, UT, MIN, MAX) \
+T                                      \
+test_##T##_sat_sub (T x, T y)          \
+{                                      \
+  T minus = (UT)x - (UT)y;             \
+  return (x ^ y) >= 0                  \
+    ? minus                            \
+    : (minus ^ x) >= 0                 \
+      ? minus                          \
+      : x < 0 ? MIN : MAX;             \
+}
+
+DEF_SAT_S_SUB(int8_t, uint8_t, INT8_MIN, INT8_MAX)
+DEF_SAT_S_SUB(int16_t, uint16_t, INT16_MIN, INT16_MAX)
+DEF_SAT_S_SUB(int32_t, uint32_t, INT32_MIN, INT32_MAX)
+DEF_SAT_S_SUB(int64_t, uint64_t, INT64_MIN, INT64_MAX)
+
 #define SAT_U_ADD_FUNC(T) test_##T##_sat_add
 #define SAT_U_ADD_FUNC_WRAP(T) SAT_U_ADD_FUNC(T)
 
 #define SAT_U_SUB_FUNC(T) test_##T##_sat_sub
 #define SAT_U_SUB_FUNC_WRAP(T) SAT_U_SUB_FUNC(T)
 
-#define TEST_BINARY_VX_SIGNED_0(T)                      \
-  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                  \
-  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)         \
-  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                   \
-  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, *, mul)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                  \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)
-
-#define TEST_BINARY_VX_UNSIGNED_0(T)                       \
-  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                     \
-  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)            \
-  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                      \
-  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                     \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_ADD_FUNC(T), sat_add) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_SUB_FUNC(T), sat_sub) \
+#define SAT_S_ADD_FUNC(T) test_##T##_sat_add
+#define SAT_S_ADD_FUNC_WRAP(T) SAT_S_ADD_FUNC(T)
+
+#define SAT_S_SUB_FUNC(T) test_##T##_sat_sub
+#define SAT_S_SUB_FUNC_WRAP(T) SAT_S_SUB_FUNC(T)
+
+#define DEF_AVG_FLOOR(NT, WT)        \
+NT                                   \
+test_##NT##_avg_floor(NT x, NT y)    \
+{                                    \
+  return (NT)(((WT)x + (WT)y) >> 1); \
+}
+
+DEF_AVG_FLOOR(uint8_t, uint16_t)
+DEF_AVG_FLOOR(uint16_t, uint32_t)
+DEF_AVG_FLOOR(uint32_t, uint64_t)
+
+DEF_AVG_FLOOR(int8_t, int16_t)
+DEF_AVG_FLOOR(int16_t, int32_t)
+DEF_AVG_FLOOR(int32_t, int64_t)
+
+#define DEF_AVG_CEIL(NT, WT)             \
+NT                                       \
+test_##NT##_avg_ceil(NT x, NT y)         \
+{                                        \
+  return (NT)(((WT)x + (WT)y + 1) >> 1); \
+}
+
+DEF_AVG_CEIL(uint8_t, uint16_t)
+DEF_AVG_CEIL(uint16_t, uint32_t)
+DEF_AVG_CEIL(uint32_t, uint64_t)
+
+DEF_AVG_CEIL(int8_t, int16_t)
+DEF_AVG_CEIL(int16_t, int32_t)
+DEF_AVG_CEIL(int32_t, int64_t)
+
+#ifdef HAS_INT128
+  DEF_AVG_FLOOR(uint64_t, uint128_t)
+  DEF_AVG_FLOOR(int64_t, int128_t)
+
+  DEF_AVG_CEIL(uint64_t, uint128_t)
+  DEF_AVG_CEIL(int64_t, int128_t)
+#endif
+
+#define AVG_FLOOR_FUNC(T)      test_##T##_avg_floor
+#define AVG_FLOOR_FUNC_WRAP(T) AVG_FLOOR_FUNC(T)
+
+#define AVG_CEIL_FUNC(T)      test_##T##_avg_ceil
+#define AVG_CEIL_FUNC_WRAP(T) AVG_CEIL_FUNC(T)
+
+#define TEST_BINARY_VX_SIGNED_0(T)                                \
+  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                            \
+  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)                   \
+  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                             \
+  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, *, mul)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                            \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_ADD_FUNC(T), sat_add)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_SUB_FUNC(T), sat_sub)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor) \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil)   \
+
+#define TEST_BINARY_VX_UNSIGNED_0(T)                              \
+  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                            \
+  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)                   \
+  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                             \
+  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                            \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_ADD_FUNC(T), sat_add)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_SUB_FUNC(T), sat_sub)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor) \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil)   \
 
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
index f475e36..626347c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
@@ -4514,4 +4514,1180 @@ uint64_t TEST_BINARY_DATA(uint64_t, sat_sub)[][3][N] =
   },
 };
 
+int8_t TEST_BINARY_DATA(int8_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+       127,  127,  127,  127,
+    },
+  },
+  {
+    { -128 },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+      -128, -128, -128, -128,
+         1,    1,    1,    1,
+    },
+    {
+        -1,   -1,   -1,   -1,
+      -128, -128, -128, -128,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+           0,      0,      0,      0,
+       32767,  32767,  32767,  32767,
+    },
+  },
+  {
+    { -32768 },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+      -32768, -32768, -32768, -32768,
+           1,      1,      1,      1,
+    },
+    {
+          -1,     -1,     -1,     -1,
+      -32768, -32768, -32768, -32768,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+                0,           0,           0,           0,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+  },
+  {
+    { -2147483648 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+                1,           1,           1,           1,
+    },
+    {
+               -1,          -1,          -1,          -1,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+  },
+  {
+    { -9223372036854775808ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                           -1,                      -1,                      -1,                      -1,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+                           -1,                      -1,                      -1,                      -1,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+    },
+  },
+};
+
+int8_t TEST_BINARY_DATA(int8_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+         0,    0,    0,    0,
+      -128, -128, -128, -128,
+      -128, -128, -128, -128,
+      -126, -126, -126, -126,
+    },
+  },
+  {
+    { -128 },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+      -128, -128, -128, -128,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       127,  127,  127,  127,
+         0,    0,    0,    0,
+       127,  127,  127,  127,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+           0,      0,      0,      0,
+      -32768, -32768, -32768, -32768,
+      -32768, -32768, -32768, -32768,
+      -32766, -32766, -32766, -32766,
+    },
+  },
+  {
+    { -32768 },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+      -32768, -32768, -32768, -32768,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       32767,  32767,  32767,  32767,
+           0,      0,      0,      0,
+       32767,  32767,  32767,  32767,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+                0,           0,           0,           0,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483646, -2147483646, -2147483646, -2147483646,
+    },
+  },
+  {
+    { -2147483648 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483647,  2147483647,  2147483647,  2147483647,
+                0,           0,           0,           0,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+                            0,                       0,                       0,                       0,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775806ull, -9223372036854775806ull, -9223372036854775806ull, -9223372036854775806ull,
+    },
+  },
+  {
+    { -9223372036854775808ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                           -1,                      -1,                      -1,                      -1,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                            0,                       0,                       0,                       0,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+  },
+};
+
+uint8_t TEST_BINARY_DATA(uint8_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+       128,  128,  128,  128,
+       255,  255,  255,  255,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       127,  127,  127,  127,
+       191,  191,  191,  191,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    { 255 },
+    {
+         0,    0,    0,    0,
+       255,  255,  255,  255,
+       254,  254,  254,  254,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       255,  255,  255,  255,
+       254,  254,  254,  254,
+       128,  128,  128,  128,
+    },
+  },
+};
+
+uint16_t TEST_BINARY_DATA(uint16_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+       32768,  32768,  32768,  32768,
+       65535,  65535,  65535,  65535,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       32767,  32767,  32767,  32767,
+       49151,  49151,  49151,  49151,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    { 65535 },
+    {
+           0,      0,      0,      0,
+       65535,  65535,  65535,  65535,
+       65534,  65534,  65534,  65534,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       65535,  65535,  65535,  65535,
+       65534,  65534,  65534,  65534,
+       32768,  32768,  32768,  32768,
+    },
+  },
+};
+
+uint32_t TEST_BINARY_DATA(uint32_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483648,  2147483648,  2147483648,  2147483648,
+       4294967295,  4294967295,  4294967295,  4294967295,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483647,  2147483647,  2147483647,  2147483647,
+       3221225471,  3221225471,  3221225471,  3221225471,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    { 4294967295 },
+    {
+                0,           0,           0,           0,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967294,  4294967294,  4294967294,  4294967294,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967294,  4294967294,  4294967294,  4294967294,
+       2147483648,  2147483648,  2147483648,  2147483648,
+    },
+  },
+};
+
+uint64_t TEST_BINARY_DATA(uint64_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull,
+       4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    { 18446744073709551615ull },
+    {
+         0,    0,    0,    0,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+       18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+      18446744073709551614ull, 18446744073709551614ull, 18446744073709551614ull, 18446744073709551614ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+    },
+  },
+};
+
+int8_t TEST_BINARY_DATA(int8_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    {-128 },
+    {
+         0,    0,    0,    0,
+      -128, -128, -128, -128,
+       126,  126,  126,  126,
+       127,  127,  127,  127,
+    },
+    {
+       -64,  -64,  -64,  -64,
+      -128, -128, -128, -128,
+        -1,   -1,   -1,   -1,
+        -1,   -1,   -1,   -1,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+           0,      0,      0,      0,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    {-32768 },
+    {
+           0,      0,      0,      0,
+      -32768, -32768, -32768, -32768,
+       32766,  32766,  32766,  32766,
+       32767,  32767,  32767,  32767,
+    },
+    {
+       -16384, -16384, -16384, -16384,
+       -32768, -32768, -32768, -32768,
+           -1,     -1,     -1,     -1,
+           -1,     -1,     -1,     -1,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+                0,           0,           0,           0,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    {-2147483648 },
+    {
+                0,           0,           0,           0,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+       2147483646,  2147483646,  2147483646,  2147483646,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+    {
+      -1073741824, -1073741824, -1073741824, -1073741824,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+               -1,          -1,          -1,          -1,
+               -1,          -1,          -1,          -1,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                           -1,                      -1,                      -1,                      -1,
+                            0,                       0,                       0,                       0,
+      4611686018427387904ull,   4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    {-9223372036854775808ull },
+    {
+                            0,                       0,                       0,                       0,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+    {
+      -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                           -1,                      -1,                      -1,                      -1,
+                           -1,                      -1,                      -1,                      -1,
+    },
+  },
+};
+
+uint8_t TEST_BINARY_DATA(uint8_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+       128,  128,  128,  128,
+       255,  255,  255,  255,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       128,  128,  128,  128,
+       191,  191,  191,  191,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    { 255 },
+    {
+         0,    0,    0,    0,
+       255,  255,  255,  255,
+       254,  254,  254,  254,
+         1,    1,    1,    1,
+    },
+    {
+       128,  128,  128,  128,
+       255,  255,  255,  255,
+       255,  255,  255,  255,
+       128,  128,  128,  128,
+    },
+  },
+};
+
+uint16_t TEST_BINARY_DATA(uint16_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+       32768,  32768,  32768,  32768,
+       65535,  65535,  65535,  65535,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       32768,  32768,  32768,  32768,
+       49151,  49151,  49151,  49151,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    { 65535 },
+    {
+           0,      0,      0,      0,
+       65535,  65535,  65535,  65535,
+       65534,  65534,  65534,  65534,
+           1,      1,      1,      1,
+    },
+    {
+       32768,  32768,  32768,  32768,
+       65535,  65535,  65535,  65535,
+       65535,  65535,  65535,  65535,
+       32768,  32768,  32768,  32768,
+    },
+  },
+};
+
+uint32_t TEST_BINARY_DATA(uint32_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483648,  2147483648,  2147483648,  2147483648,
+       4294967295,  4294967295,  4294967295,  4294967295,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483648,  2147483648,  2147483648,  2147483648,
+       3221225471,  3221225471,  3221225471,  3221225471,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    { 4294967295 },
+    {
+                0,           0,           0,           0,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967294,  4294967294,  4294967294,  4294967294,
+                1,           1,           1,           1,
+    },
+    {
+       2147483648,  2147483648,  2147483648,  2147483648,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       2147483648,  2147483648,  2147483648,  2147483648,
+    },
+  },
+};
+
+uint64_t TEST_BINARY_DATA(uint64_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+      13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull,
+       4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    { 18446744073709551615ull },
+    {
+         0,    0,    0,    0,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+       18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+      18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+      18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+    },
+  },
+};
+
+int8_t TEST_BINARY_DATA(int8_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+         0,    0,    0,    0,
+         0,    0,    0,    0,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    {-128 },
+    {
+         0,    0,    0,    0,
+      -128, -128, -128, -128,
+       126,  126,  126,  126,
+       127,  127,  127,  127,
+    },
+    {
+       -64,  -64,  -64,  -64,
+      -128, -128, -128, -128,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+           0,      0,      0,      0,
+           0,      0,      0,      0,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    {-32768 },
+    {
+           0,      0,      0,      0,
+      -32768, -32768, -32768, -32768,
+       32766,  32766,  32766,  32766,
+       32767,  32767,  32767,  32767,
+    },
+    {
+       -16384, -16384, -16384, -16384,
+       -32768, -32768, -32768, -32768,
+           -1,     -1,     -1,     -1,
+            0,      0,      0,      0,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+                0,           0,           0,           0,
+                0,           0,           0,           0,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    {-2147483648 },
+    {
+                0,           0,           0,           0,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+       2147483646,  2147483646,  2147483646,  2147483646,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+    {
+      -1073741824, -1073741824, -1073741824, -1073741824,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+               -1,          -1,          -1,          -1,
+                0,           0,           0,           0,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                            0,                       0,                       0,                       0,
+                            0,                       0,                       0,                       0,
+       4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    {-9223372036854775808ull },
+    {
+                            0,                       0,                       0,                       0,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+       9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+    {
+      -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                           -1,                      -1,                      -1,                      -1,
+                            0,                       0,                       0,                       0,
+    },
+  },
+};
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c
new file mode 100644
index 0000000..0307b3f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c
new file mode 100644
index 0000000..d73325b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c
new file mode 100644
index 0000000..481774b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c
new file mode 100644
index 0000000..7de89ee
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c
new file mode 100644
index 0000000..73d1a57
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint16_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c
new file mode 100644
index 0000000..60a7aa4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint32_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c
new file mode 100644
index 0000000..803bcba
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint64_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c
new file mode 100644
index 0000000..f28147b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint8_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c
new file mode 100644
index 0000000..8def643
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c
new file mode 100644
index 0000000..d9ca67d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c
new file mode 100644
index 0000000..313109a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c
new file mode 100644
index 0000000..47e4a5d7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c
new file mode 100644
index 0000000..6297672
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint16_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c
new file mode 100644
index 0000000..30db24b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint32_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c
new file mode 100644
index 0000000..db3c911
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint64_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c
new file mode 100644
index 0000000..a7755f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint8_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c
new file mode 100644
index 0000000..1f0fd46
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c
new file mode 100644
index 0000000..4a8df0c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c
new file mode 100644
index 0000000..534cd25
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c
new file mode 100644
index 0000000..de2a9b6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c
new file mode 100644
index 0000000..bd985c2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c
new file mode 100644
index 0000000..c510ea0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c
new file mode 100644
index 0000000..b82278d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c
new file mode 100644
index 0000000..5fae704
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/pr113829.c b/gcc/testsuite/gcc.target/riscv/rvv/base/pr113829.c
new file mode 100644
index 0000000..48c291a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/pr113829.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=rv64gcv -mabi=lp64d" } */
+
+#pragma riscv intrinsic "vector"
+void
+foo (void)
+{
+  __riscv_vfredosum_tu (X); /* { dg-error "undeclared" } */
+  /* { dg-error "too many arguments" "" { target *-*-* } .-1 } */
+}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
index 04dec7b..4f6785a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
@@ -6,9 +6,9 @@
 
 /*
 ** foo:
-** addi\t[a-x0-9]+,\s*[a-x0-9]+,100
+** ...
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv.s.x\tv[0-9]+.*
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
@@ -23,7 +23,7 @@ void foo (void *base, void *out, size_t vl)
 ** foo2:
 ** fld\tfa[0-9]+,\s*100\(a0\)
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
index 0ebb92e..a8c9263c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
@@ -23,7 +23,7 @@ void foo (void *base, void *out, size_t vl)
 ** foo2:
 ** fld\tfa[0-9]+,\s*100\(a0\)
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
@@ -52,7 +52,7 @@ void foo3 (void *base, void *out, size_t vl)
 /*
 ** foo4:
 ** ...
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
 ** ...
 ** ret
 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
index 512fa62..cf53aca 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
@@ -6,9 +6,9 @@
 
 /*
 ** foo:
-** addi\t[a-x0-9]+,\s*[a-x0-9]+,100
+** ...
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv\.v\.x\tv[0-9]+,\s*a[0-9]+
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
@@ -37,7 +37,7 @@ void foo2 (void *base, void *out, size_t vl)
 /*
 ** foo3:
 ** ...
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv\.v\.x\tv[0-9]+,\s*a[0-9]+
 ** ...
 ** ret
 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
index d9d10f3..fd3b7c5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
@@ -175,9 +175,8 @@ void foo12 (void *base, void *out, size_t vl)
 /*
 ** foo13:
 ** ...
-** vmv.v.x\tv[0-9]+,\s*[a-x0-9]+
+** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
 ** ...
-** ret
 */
 void foo13 (void *base, void *out, size_t vl)
 {
@@ -189,7 +188,7 @@ void foo13 (void *base, void *out, size_t vl)
 /*
 ** foo14:
 ** ...
-** vmv.v.x\tv[0-9]+,\s*[a-x0-9]+
+** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
 ** ...
 */
 void foo14 (void *base, void *out, size_t vl)
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
index 80ee1b5..64c22dd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
@@ -23,4 +23,3 @@ vuint64m2_t f3(vuint64m2_t var_17, uint64_t var_60, size_t vl)
 
 /* { dg-final { scan-assembler-times {vsetivli\s+zero,\s*0,\s*e64,\s*m2,\s*t[au],\s*m[au]} 1 } } */
 /* { dg-final { scan-assembler-times {vsetivli\s+zero,\s*1,\s*e64,\s*m2,\s*t[au],\s*m[au]} 1 } } */
-/* { dg-final { scan-assembler-times {sgtu} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/pr120297.c b/gcc/testsuite/gcc.target/riscv/rvv/pr120297.c
new file mode 100644
index 0000000..3d1845d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/pr120297.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -O3 -fwhole-program" } */
+
+unsigned a;
+short c;
+char d;
+unsigned long e;
+_Bool f[10][10];
+unsigned g[10];
+long long ak;
+char i = 7;
+long long t[10];
+short x[10][10][10][10];
+short y[10][10][10][10];
+
+void
+h (char i, long long t[], short x[][10][10][10], short y[][10][10][10],
+   _Bool aa)
+{
+  for (int j = 2; j < 8; j += 2)
+    {
+      for (short k = 0; k < 10; k++)
+	{
+	  for (int l = 3; l < 8; l += 2)
+	    a = x[1][j][k][l];
+	  c = x[c][1][1][c];
+	}
+      for (int k = 0; k < 10; k++)
+	{
+	  f[2][k] |= (_Bool) t[c];
+	  g[c] = t[c + 1];
+	  d += y[j][1][k][k];
+	  e = e > i ? e : i;
+	}
+    }
+}
+
+int
+main ()
+{
+  t[c] = 1;
+  h (i, t, x, y, a);
+  for (int j = 0; j < 10; ++j)
+    for (int k = 0; k < 10; ++k)
+      ak ^= f[j][k] + 238516665 + (ak >> 2);
+  ak ^= g[c] + 238516665 + (ak >> 2);
+  if (ak != 234635118ull)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c b/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c
new file mode 100644
index 0000000..2789d0f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -O3 -mrvv-vector-bits=zvl -fsigned-char -fno-strict-aliasing -fwrapv -Wno-stringop-overflow -Wno-aggressive-loop-optimizations" } */
+
+int a;
+unsigned char p[1][21];
+void init() {
+  for (int s = 0; s < 21; ++s)
+    for (int t = 0; t < 21; ++t)
+      p[s][t] = 39;
+  for (short t = 0; t < 9; t += -5077966496202321318LL + 28071)
+    a = p[3][t] && p[2][t];
+}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120461.c b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120461.c
new file mode 100644
index 0000000..6939157
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120461.c
@@ -0,0 +1,6 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=xt-c920 -mrvv-vector-bits=zvl -fzero-call-used-regs=all" */
+
+void
+foo ()
+{}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120642.c b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120642.c
new file mode 100644
index 0000000..1a72580
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120642.c
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-O -mcpu=xt-c920 -mrvv-vector-bits=zvl" } */
+int __attribute__((__vector_size__(4 * sizeof(int)))) v;
+void foo() { v /= 3; }
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h b/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h
index 6e97cae..e40902a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h
@@ -4,6 +4,10 @@
 #include <stdint-gcc.h>
 #include <stdbool.h>
 
+#if __riscv_xlen == 64
+typedef unsigned __int128 uint128_t;
+#endif
+
 /******************************************************************************/
 /* Saturation Add (unsigned and signed)                                       */
 /******************************************************************************/
@@ -69,6 +73,22 @@ sat_u_add_##WT##_##T##_fmt_7(T x, T y) \
 }
 #define DEF_SAT_U_ADD_FMT_7_WRAP(WT, T) DEF_SAT_U_ADD_FMT_7(WT, T)
 
+#define DEF_SAT_U_ADD_FMT_8(T)                \
+T __attribute__((noinline))                   \
+sat_u_add_##T##_fmt_8(T x, T y)               \
+{                                             \
+  return x <= (T)(x + y) ? (x + y) : -1;      \
+}
+#define DEF_SAT_U_ADD_FMT_8_WRAP(T) DEF_SAT_U_ADD_FMT_8(T)
+
+#define DEF_SAT_U_ADD_FMT_9(T)               \
+T __attribute__((noinline))                  \
+sat_u_add_##T##_fmt_9(T x, T y)              \
+{                                            \
+  return x > (T)(x + y) ? -1 : (x + y);      \
+}
+#define DEF_SAT_U_ADD_FMT_9_WRAP(T) DEF_SAT_U_ADD_FMT_9(T)
+
 #define RUN_SAT_U_ADD_FMT_1(T, x, y) sat_u_add_##T##_fmt_1(x, y)
 #define RUN_SAT_U_ADD_FMT_1_WRAP(T, x, y) RUN_SAT_U_ADD_FMT_1(T, x, y)
 #define RUN_SAT_U_ADD_FMT_2(T, x, y) sat_u_add_##T##_fmt_2(x, y)
@@ -93,6 +113,10 @@ sat_u_add_##WT##_##T##_fmt_7(T x, T y) \
   sat_u_add_uint64_t_##T##_fmt_7(x, y)
 #define RUN_SAT_U_ADD_FMT_7_FROM_U64_WRAP(T, x, y) \
   RUN_SAT_U_ADD_FMT_7_FROM_U64(T, x, y)
+#define RUN_SAT_U_ADD_FMT_8(T, x, y) sat_u_add_##T##_fmt_8(x, y)
+#define RUN_SAT_U_ADD_FMT_8_WRAP(T, x, y) RUN_SAT_U_ADD_FMT_8(T, x, y)
+#define RUN_SAT_U_ADD_FMT_9(T, x, y) sat_u_add_##T##_fmt_9(x, y)
+#define RUN_SAT_U_ADD_FMT_9_WRAP(T, x, y) RUN_SAT_U_ADD_FMT_9(T, x, y)
 
 #define DEF_SAT_U_ADD_IMM_FMT_1(T, IMM)      \
 T __attribute__((noinline))                  \
@@ -227,6 +251,18 @@ sat_s_add_imm_##T##_fmt_1##_##INDEX (T x)             \
 #define RUN_SAT_S_ADD_IMM_FMT_1(INDEX, T, x, expect) \
   if (sat_s_add_imm##_##T##_fmt_1##_##INDEX(x) != expect) __builtin_abort ()
 
+#define DEF_SAT_S_ADD_IMM_FMT_2(INDEX, T, UT, IMM, MIN, MAX) \
+T __attribute__((noinline))                                  \
+sat_s_add_imm_##T##_fmt_2##_##INDEX (T x)                    \
+{                                                            \
+  T sum = (T)((UT)x + (UT)IMM);                                   \
+  return ((x ^ sum) < 0 && (x ^ IMM) >= 0) ?                 \
+    (-(T)(x < 0) ^ MAX) : sum;                         \
+}
+
+#define RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect) \
+  if (sat_s_add_imm##_##T##_fmt_2##_##INDEX(x) != expect) __builtin_abort ()
+
 /******************************************************************************/
 /* Saturation Sub (Unsigned and Signed)                                       */
 /******************************************************************************/
@@ -636,4 +672,25 @@ sat_s_trunc_##WT##_to_##NT##_fmt_8 (WT x)             \
 #define RUN_SAT_S_TRUNC_FMT_8(NT, WT, x) sat_s_trunc_##WT##_to_##NT##_fmt_8 (x)
 #define RUN_SAT_S_TRUNC_FMT_8_WRAP(NT, WT, x) RUN_SAT_S_TRUNC_FMT_8(NT, WT, x)
 
+/******************************************************************************/
+/* Saturation Mult (unsigned and signed)                                  */
+/******************************************************************************/
+
+#define DEF_SAT_U_MUL_FMT_1(NT, WT)             \
+NT __attribute__((noinline))                    \
+sat_u_mul_##NT##_from_##WT##_fmt_1 (NT a, NT b) \
+{                                               \
+  WT x = (WT)a * (WT)b;                         \
+  NT max = -1;                                  \
+  if (x > (WT)(max))                            \
+    return max;                                 \
+  else                                          \
+    return (NT)x;                               \
+}
+
+#define DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT) DEF_SAT_U_MUL_FMT_1(NT, WT)
+#define RUN_SAT_U_MUL_FMT_1(NT, WT, a, b) \
+  sat_u_mul_##NT##_from_##WT##_fmt_1 (a, b)
+#define RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, a, b) RUN_SAT_U_MUL_FMT_1(NT, WT, a, b)
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h b/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h
index f100688..bd33ff1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h
@@ -12,6 +12,7 @@
 
 #define TEST_BINARY_STRUCT_NAME(T, NAME) test_##T##_##NAME##_s
 #define TEST_BINARY_STRUCT_DECL(T, NAME) struct TEST_BINARY_STRUCT_NAME(T, NAME)
+#define TEST_BINARY_STRUCT_DECL_WRAP(T, NAME) TEST_BINARY_STRUCT_DECL(T, NAME)
 #define TEST_BINARY_STRUCT(T, NAME)       \
   struct TEST_BINARY_STRUCT_NAME(T, NAME) \
     {                                     \
@@ -37,6 +38,11 @@ TEST_BINARY_STRUCT (uint16_t, usadd)
 TEST_BINARY_STRUCT (uint32_t, usadd)
 TEST_BINARY_STRUCT (uint64_t, usadd)
 
+TEST_BINARY_STRUCT (uint8_t, usmul)
+TEST_BINARY_STRUCT (uint16_t, usmul)
+TEST_BINARY_STRUCT (uint32_t, usmul)
+TEST_BINARY_STRUCT (uint64_t, usmul)
+
 TEST_BINARY_STRUCT (int8_t,  ssadd)
 TEST_BINARY_STRUCT (int16_t, ssadd)
 TEST_BINARY_STRUCT (int32_t, ssadd)
@@ -433,4 +439,60 @@ TEST_BINARY_STRUCT_DECL(int64_t, sssub) TEST_BINARY_DATA(int64_t, sssub)[] =
   {  9223372036854775806ll,   9223372036854775800ll,                       6},
 };
 
+TEST_BINARY_STRUCT_DECL(uint8_t, usmul) TEST_BINARY_DATA(uint8_t, usmul)[] =
+{
+  {      0,     0,      0, },
+  {      0,     1,      0, },
+  {      1,     1,      1, },
+  {      1,   127,    127, },
+  {      2,   127,    254, },
+  {      3,   127,    255, },
+  {    127,   127,    255, },
+  {      1,   255,    255, },
+  {    127,   255,    255, },
+  {    255,   255,    255, },
+};
+
+TEST_BINARY_STRUCT_DECL(uint16_t, usmul) TEST_BINARY_DATA(uint16_t, usmul)[] =
+{
+  {      0,     0,      0, },
+  {      0,     1,      0, },
+  {      1,     1,      1, },
+  {      1, 32767,  32767, },
+  {      2, 32767,  65534, },
+  {      3, 32767,  65535, },
+  {  32767, 32767,  65535, },
+  {      1, 65535,  65535, },
+  {  32767, 65535,  65535, },
+  {  65535, 65535,  65535, },
+};
+
+TEST_BINARY_STRUCT_DECL(uint32_t, usmul) TEST_BINARY_DATA(uint32_t, usmul)[] =
+{
+  {          0,          0,          0, },
+  {          0,          1,          0, },
+  {          1,          1,          1, },
+  {          1, 2147483647, 2147483647, },
+  {          2, 2147483647, 4294967294, },
+  {          3, 2147483647, 4294967295, },
+  { 2147483647, 2147483647, 4294967295, },
+  {          1, 4294967295, 4294967295, },
+  { 2147483647, 4294967295, 4294967295, },
+  { 4294967295, 4294967295, 4294967295, },
+};
+
+TEST_BINARY_STRUCT_DECL(uint64_t, usmul) TEST_BINARY_DATA(uint64_t, usmul)[] =
+{
+  {                       0,                       0,                       0, },
+  {                       0,                       1,                       0, },
+  {                       1,                       1,                       1, },
+  {                       1,  9223372036854775807ull,  9223372036854775807ull, },
+  {                       2,  9223372036854775807ull, 18446744073709551614ull, },
+  {                       3,  9223372036854775807ull, 18446744073709551615ull, },
+  {  9223372036854775807ull,  9223372036854775807ull, 18446744073709551615ull, },
+  {                       1, 18446744073709551615ull, 18446744073709551615ull, },
+  {  9223372036854775807ull, 18446744073709551615ull, 18446744073709551615ull, },
+  { 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, },
+};
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c
index 55890d8..50f0f1f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c
index 29e843f..dc65817 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c
index 7f29d21..9995bc7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c
index 3ad7bdd..caf745a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c
index 07d3101..f19187d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c
index 81b85b4..88dc37d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c
index 9a3d83e..891d6cf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c
index ecc9a0f..a07172b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c
index 7e93385..5077198 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c
index 09bf497..07af4e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c
index 5652cdb..7c4be5b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c
index 0eb0c84..fc0e1b7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c
index 9dfdb9e..4c0b38a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c
index 74df576..45b4638 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c
index 5937699..294eb52 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c
index af850d0..143fa3c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c
index 34459b8..1023934 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c
index 4d4841f..bccb768 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c
index df81887..34de520 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c
index 9a4ce33..6d136ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c
index cdac5bd..ee8e439 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c
index 4ac952e..8996dd2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c
index 4d25e7f..155c8e9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c
index d57e0a0..4502ed3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c
index 08b961a..21289c9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c
index 3611b6e..3d4a6fa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c
index 3eaa6c2..b55d221 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c
index 6d38e5f..9fef8b0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c
index 2e73450..fd135e5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c
index ec3022d..38ade40 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c
index 911856e..04ba746 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c
index 94d48ef..32aea5c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c
index 2e23af5..414cb61 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c
@@ -1,57 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int16_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*-7
-**	xori\s+[atx][0-9]+,\s*a0,\s*-7
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*32768
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	slliw\s+a0,\s*a0,\s*16
-**	sraiw\s+a0,\s*a0,\s*16
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int16_t, uint16_t, -7, INT16_MIN, INT16_MAX)
-
-/*
-** sat_s_add_imm_int16_t_fmt_1_1:
-** 	addi\s+[atx][0-9]+,\s*a0,\s*-1
-** 	not\s+[atx][0-9]+,\s*a0
-** 	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** 	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** 	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** 	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** 	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** 	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** 	srai\s+a0,\s*a0,\s*63
-** 	li\s+[atx][0-9]+,\s*32768
-** 	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** 	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** 	neg\s+a0,\s*[atx][0-9]+
-** 	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** 	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** 	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** 	or\s+a0,\s*a0,\s*[atx][0-9]+
-** 	slliw\s+a0,\s*a0,\s*16
-** 	sraiw\s+a0,\s*a0,\s*16
-** 	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int16_t, uint16_t, -1, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c
index e63211f..adf5b39 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c
@@ -1,54 +1,11 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int32_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*10
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-**	srli\s+[atx][0-9]+,\s*a0,\s*31
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-2147483648
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,a0,\s*[atx][0-9]+
-**	sext.w\s+a0,\s*a0
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int32_t, uint32_t, 10, INT32_MIN, INT32_MAX)
 
-/*
-** sat_s_add_imm_int32_t_fmt_1_1:
-**	addi\s+[atx][0-9]+,\s*a0,\s*-1
-**	not\s+[atx][0-9]+,\s*a0
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-2147483648
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	sext.w\s+a0,\s*a0
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int32_t, uint32_t, -1, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c
index 3843b71..b88e064 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c
@@ -1,48 +1,11 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int64_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*10
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-**	srli\s+[atx][0-9]+,\s*a0,\s*63
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	srai\s+[atx][0-9]+,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-1
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	neg\s+[atx][0-9]+,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int64_t, uint64_t, 10, INT64_MIN, INT64_MAX)
 
-/*
-** sat_s_add_imm_int64_t_fmt_1_1:
-**	addi\s+[atx][0-9]+,\s*a0,\s*-1
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-**	slti\s+[atx][0-9]+,\s*a0,\s*0
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	srai\s+[atx][0-9]+,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-1
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	neg\s+[atx][0-9]+,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int64_t, uint64_t, -1, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c
index ceae1ea..0e337ef 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c
@@ -1,49 +1,11 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int8_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*9
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-**	srli\s+[atx][0-9]+,\s*a0,\s*7
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	xori\s+[atx][0-9]+,\s*a0,\s*127
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	slliw\s+a0,\s*a0,\s*24
-**	sraiw\s+a0,\s*a0,\s*24
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int8_t, uint8_t, 9, INT8_MIN, INT8_MAX)
 
-/*
-** sat_s_add_imm_int8_t_fmt_1_1:
-** 	addi\s+[atx][0-9]+,\s*a0,\s*-1
-** 	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** 	and\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** 	slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*56
-** 	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** 	srai\s+a0,\s*a0,\s*63
-** 	xori\s+[atx][0-9]+,\s*a0,\s*127
-** 	neg\s+a0,\s*a5
-** 	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** 	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** 	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** 	or\s+a0,\s*a0,\s*[atx][0-9]+
-** 	slliw\s+a0,\s*a0,\s*24
-** 	sraiw\s+a0,\s*a0,\s*24
-** 	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int8_t, uint8_t, -1, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i16.c
new file mode 100644
index 0000000..f217fe1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i16.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int16_t, uint16_t, -7, INT16_MIN, INT16_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int16_t, uint16_t, -1, INT16_MIN, INT16_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i32.c
new file mode 100644
index 0000000..4025b5a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int32_t, uint32_t, 10, INT32_MIN, INT32_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int32_t, uint32_t, -1, INT32_MIN, INT32_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i64.c
new file mode 100644
index 0000000..3fc2514
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int64_t, uint64_t, 10, INT64_MIN, INT64_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int64_t, uint64_t, -1, INT64_MIN, INT64_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i8.c
new file mode 100644
index 0000000..a0e15cf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i8.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int8_t, uint8_t, 9, INT8_MIN, INT8_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int8_t, uint8_t, -1, INT8_MIN, INT8_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c
new file mode 100644
index 0000000..4f24624
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int16_t, uint16_t, -32768, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int16_t, uint16_t, 32767, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int16_t, uint16_t, 100, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int16_t, uint16_t, -100, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int16_t, uint16_t, -1, INT16_MIN, INT16_MAX)
+
+#define T                       int16_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     -32768, },
+  {      2,     -32766, },
+  {      1,      32767, },
+  {    -10,      32757, },
+  {  32669,      32767, },
+  { -32768,     -32668, },
+  { -32768,     -32768, },
+  {      0,       -100, },
+  { -32768,     -32768, },
+  {      0,         -1, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c
new file mode 100644
index 0000000..8d9ddeb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int32_t, uint32_t, -2147483648, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int32_t, uint32_t, 2147483647, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int32_t, uint32_t, 100, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int32_t, uint32_t, -100, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int32_t, uint32_t, -1, INT32_MIN, INT32_MAX)
+
+#define T                       int32_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     -2147483648, },
+  {      2,     -2147483646, },
+  {      1,      2147483647, },
+  {    -10,      2147483637, },
+  {    300,             400, },
+  {   -300,            -200, },
+  {    100,               0, },
+  {      0,            -100, },
+  {    100,              99, },
+  {      0,              -1, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c
new file mode 100644
index 0000000..4523f9a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int64_t, uint64_t, (-9223372036854775807ll - 1), INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int64_t, uint64_t, 9223372036854775807ll, INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int64_t, uint64_t, 100, INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int64_t, uint64_t, -100, INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int64_t, uint64_t, -1, INT64_MIN, INT64_MAX)
+
+#define T                       int64_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     (-9223372036854775807ll - 1), },
+  {      2,           -9223372036854775806ll, },
+  {      1,            9223372036854775807ll, },
+  {     -7,            9223372036854775800ll, },
+  {      0,                              100, },
+  {     -1,                               99, },
+  {      0,                             -100, },
+  {    100,                                0, },
+  {      0,                               -1, },
+  {    100,                               99, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c
new file mode 100644
index 0000000..96445ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int8_t, uint8_t, -128, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int8_t, uint8_t, 127, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int8_t, uint8_t, 6, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int8_t, uint8_t, -6, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int8_t, uint8_t, -1, INT8_MIN, INT8_MAX)
+
+#define T                       int8_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     -128, },
+  {      2,     -126, },
+  {      1,      127, },
+  {    -10,      117, },
+  {    122,      127, },
+  {    -10,       -4, },
+  {   -128,     -128, },
+  {    127,      121, },
+  {   -128,     -128, },
+  {    1,          0, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c
new file mode 100644
index 0000000..a73a77f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int16_t, uint16_t, -32769, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int16_t, uint16_t, 32768, INT16_MIN, INT16_MAX)
+
+/* { dg-final { scan-tree-dump-not ".SAT_ADD " "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c
new file mode 100644
index 0000000..9dae425
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_1(0, int32_t, uint32_t, -2147483649, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_1(1, int32_t, uint32_t, 2147483648, INT32_MIN, INT32_MAX)
+
+/* { dg-final { scan-tree-dump-not ".SAT_ADD " "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c
new file mode 100644
index 0000000..a9cd4b9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int8_t, uint8_t, -129, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int8_t, uint8_t, 128, INT8_MIN, INT8_MAX)
+
+/* { dg-final { scan-tree-dump-not ".SAT_ADD " "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c
index c244eb4..734e8be 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c
index 9d8245d..3aa4c58 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c
index 929de16..4c0caa1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c
index a918d5c..6c1441b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c
index 2da1c0d..57a4327 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c
index 20b28e7..28582fb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c
index a540198..130ca46 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c
index c54057d..cd407b2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c
index 469a113..748d61a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c
index b2c03f6..be7869a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c
index e3fe6c7..d16a7fb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c
index 150cde1..14a2454 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c
index 26d159c..614d1ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c
index d576c38..2f52bd7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c
index f42ffea..cef478b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c
index ee510a6..3ed7790 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c
index e248b73..b2c5735 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c
index bebb4be..6d1518e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c
index f31eb29..adcd1bb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c
index e165e39..31fa0a6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c
index 08a9b5c..0c5ad8c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c
index fc79969..5e89539 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c
index 8d5f745..199e204 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c
index 9f6ef30..4cfe787 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c
index 0523d13..3cf4ecd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c
index e720964..ce2151c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c
index 49ed051..158eeaa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c
index 99b413f..8eb7ab5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c
index c7056ed..339a403 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c
index 7168f94..285733a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c
index 29b2b54..546bac1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c
index 65027b7..dafc86f1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c
index 451a375..6d1fbc4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_1:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c
index 2aafb94..56a6699 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c
index 6e21ee3..10c3320 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_1:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c
index 5e971e4..558d704 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c
index 87e5a52..02bef46 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c
index 22a0dd4..da04904 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_1:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c
index cb307ac..41391e2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_2:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c
index b4bee21..3e5f9e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c
index c467c8d..228eeab 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_2:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c
index 883b77b..78542ca 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c
index bb9ffce..556e8ea 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_2:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c
index a54db48..918a8c3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_2:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c
index 219156c..13c0291 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_3:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c
index 87b8a70..03077b7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c
index 7acd515..e09a88d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_3:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c
index 9141f08..ca071d1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c
index 839a6f7..4acd93c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c
index 5d13f09..362970c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_3:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c
index 34dc804..94d9cc4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_4:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c
index 89c476e..51a6e7b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c
index 03ca7b7..9101b40 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_4:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c
index aafe167..48452e3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c
index 08e5eb3..6757913 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_4:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c
index b0e71fe..9c65582 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_4:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c
index b42c759..f02f866 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_5:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c
index 625372e..6753c03 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_5:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c
index 250e174..3fd17fa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_5:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c
index 4a6ac6d..fba761a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_5:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c
index 02aa6db..8872f7f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_5:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c
index ae1bcb9..13539aa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_5:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c
index 9a740d7..4aa9a8f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_6:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c
index 1e42bfd..a772ee8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_6:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c
index c3bd46d..9c5d88b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_6:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c
index a6575f5..f9f18e9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_6:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c
index fd7b72e..3658fbb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_6:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c
index 242d2d0..f1a7eb8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_6:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c
index 3f258b8..50b06d5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_7:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c
index f37a57e..12be220 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_7:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c
index 4e4a7eb..cb73531 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_7:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c
index 29b64b4..d52394c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_7:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c
index 2bfe898..cf79778 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_7:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c
index 494a314..67485a3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_7:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c
index 678dec6..a34bf4a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_8:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c
index 4acc789..9c25ff0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_8:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c
index 34a992b..9ee75e2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_8:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c
index 1919ba5..8cd361e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_8:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c
index 541e55c..ace064b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_8:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c
index 36a0085..e9a4d3b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_8:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c
index 1f230c5..7ed6809 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c
index 563760b..82e4201 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c
index af50d3e..78be831 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c
index 4ac7025..e8a497f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c
index ca6d31c..1420541 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c
index 697e1bc..31fecc7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c
index 0d9da40..333bb92 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c
index 2e183ef..f494909 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c
index 1950092..d8a619b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c
index b11b097..348832d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c
index 419e909..fc183cf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c
index de3d9f1..dec54d3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c
index 032c83b..2b8700a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c
index 51f4946..cf3f763 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c
index b959bce..20a68bb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c
index ddfb522..5159ab1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c
index 22965e2..edeff90 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c
index 7cba408..7a22637 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c
index 6dfdd4b..65f9aea 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c
index fcf8e47..ab32e5d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c
index 9d911a4..eecfc49 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c
index 3cc2498..410d202 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c
index b9abf50..17518ba 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c
index d90682f..bf0c43e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c
index 1911166..bac1fda 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c
index 28116eb..3a82ea0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c
index 54b1ffb..26a89e7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c
index 633417b..a8bfeef 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c
index c5e4e4a..f79a049 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c
index 9acbee0..eea31af 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c
index db1a698..0ea32f0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c
index e6b52d4..39e44d8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c
index d83836d..cb42b7e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c
index e910edf..f64a46b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c
index 98dd0c2..18e9029 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c
index b843300..d8cda79 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c
index ab51ad5..894d5f5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c
index 9b2c525..1ced757 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c
index ab409f2..ab41a84 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c
index 9013952..c078136 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c
index 67e19e7..af86e69 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c
index a573706..4a2532d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c
index dbd70de..65c82ad 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c
index 25bb42f..a8cb8e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c
index 7c71b3d..5b5f8f4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c
index 61392b5..f489846 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c
index b47e5da..a3f3ae5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c
index 1cd7f80..aafe96b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c
index 3c916bc..8f1b5c0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c
index edded3e..2c66eee 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_1:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c
index 821e4bc..28d7b7c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c
index fd73c3a..ab18336 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c
index a166d28..c03b15d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c
index c06731b..f753c01 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_2:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c
index ae10dff..cad539c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c
index f3977be..b595241 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c
index 5898c3b..08cd820 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c
index a1017c9..e0b73748 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_3:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c
index 83fcb60..7ce0121 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c
index 2c398e0c..48f61c1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c
index c18a5d59..49d5af1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c
index fa2e55d..20ad476 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_4:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c
index 6818c0c..6d2c9a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c
index 1096de8..15e613b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c
index fd4be5c..225ba0c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_5:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c
index 4fbc807..106baf7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_5:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c
index 5bc2948..48e84f6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_5:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c
index 74109c3..9c0d42a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_5:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c
index 3cb9cbe..0b541e0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_6:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c
index fd1cb1a..ee79156 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_6:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c
index c968f33..fd79139 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_6:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c
index 9cd95ad..f826aa4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_6:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c
index 527f8de..446a951 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_uint16_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint32_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c
index e9031de..626effc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_uint16_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint64_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c
index a71bd2f..3014634 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_uint32_t_fmt_7:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint64_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c
index 5892986..541a1d8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_uint8_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint16_t, uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c
index a42a712..26749a8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_uint8_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint32_t, uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c
index f37ef1c..321f662 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_uint8_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint64_t, uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u16.c
new file mode 100644
index 0000000..a7062b5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u32.c
new file mode 100644
index 0000000..2e43c7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u64.c
new file mode 100644
index 0000000..4ad18c1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u8.c
new file mode 100644
index 0000000..608d31b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u16.c
new file mode 100644
index 0000000..b9766d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u32.c
new file mode 100644
index 0000000..2456d39
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u64.c
new file mode 100644
index 0000000..0a0ff24
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u8.c
new file mode 100644
index 0000000..53879dd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c
index fe015cc..548fae3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c
index 8ee6501..e76b636 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c
index d2c6af0..0ea6509 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c
index 154edde..3aa7441 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c
index 1fc08bd..f6f8b9d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c
index a52a230..da8c3eb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c
index d05ed33..03f5960 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c
index fd39335..af898e55 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c
index 7084272..7862a48 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c
index 82de67d..d2fbcf2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c
index d73f305..23b5488 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c
index f572c44..b5931d4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c
index 65c431f..a9937a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c
index 8a73fcd..966831a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c
index 0903e10..08db7a1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c
index ffdd390..f7bbb5a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c
index 72ccd2f..da1782d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c
index 34d1a4e..524106a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c
index d502a58..62fdd25 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c
index f611376..334eb04 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c
index 5ef250d..28a2fb8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c
index ba95dbf..3b19af3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c
index d0e9dfd..f35334a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c
index b3d00df..e04fbf0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c
index 26c2778..3363220 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c
index 1f3e2f3..bc4ca2f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c
index 558f6ce..04abd95 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c
index ec5ac70..c514a86 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c
index aa94eef..b1a644b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c
index 6ac38ba..8664ffa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u16.c
new file mode 100644
index 0000000..aaf13be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint16_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u32.c
new file mode 100644
index 0000000..0ec8d90
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint32_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u64.c
new file mode 100644
index 0000000..f367f67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint64_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u8.c
new file mode 100644
index 0000000..0fd4036
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint8_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u16.c
new file mode 100644
index 0000000..4289e2a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint16_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u32.c
new file mode 100644
index 0000000..d3dd52e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint32_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u64.c
new file mode 100644
index 0000000..a9f0964
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint64_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u8.c
new file mode 100644
index 0000000..91cdb7e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint8_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c
index 3c31ac3..b6388dc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_1:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c
index c6b352c..cae6796 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7_uint32_t_fmt_1:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint32_t, 7)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c
index 1d9df3c..f9d6939 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8_uint64_t_fmt_1:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint64_t, 8)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c
index 101acd8..d90209a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_1:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c
index ac57cc9..a34194d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c
index 6aca60c..9a801d2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7_uint32_t_fmt_2:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint32_t, 7)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c
index d041724..2eb57a3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8_uint64_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint64_t, 8)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c
index 7baeb8d..363b2df8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c
index 6dbabf6..aaf1209 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_3:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c
index 1c52b21..e430b37 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7u_uint32_t_fmt_3:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint32_t, 7u)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c
index ef60ce2..aef5c58 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8ull_uint64_t_fmt_3:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint64_t, 8ull)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c
index 81a4b21..039d982 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_3:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c
index 2f6c0460..baf70c3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c
index 1fc9a50..a4bfe50 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7u_uint32_t_fmt_4:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint32_t, 7u)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c
index 0ca423c..f355de6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8ull_uint64_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint64_t, 8ull)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c
index c8a43fa..54880d7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c
index 090c765..e715bb0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c
index 8dade74..8b8b475 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c
index ace2df8..f6f6408 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c
index 0ce546f..f2154fc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c
index 7b6bd73..8e3aa83 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c
index 8024152..403cf14 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c
index 4a76dbb..17eca5e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c
index 8e8759c..9a277a1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c
index 64924a6..2068037 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c
index 04f3217..5f8f1e6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c
index 8ef6c14..c574521 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c
index 8867361..6b9439a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c
index 0b75206..224c3ae 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c
index e548d0c..5c03e1b7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c
index 4335d82..1ceacd2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c
index 872923e..aef253c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c
new file mode 100644
index 0000000..cd6f2f8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c
new file mode 100644
index 0000000..7409232
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint32_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c
new file mode 100644
index 0000000..43ab563
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc -mabi=ilp32d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c
new file mode 100644
index 0000000..dea9f6d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint32_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c
new file mode 100644
index 0000000..fa3758a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc -mabi=ilp32d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint32_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
+/* { dg-final { scan-assembler-times "mulhu" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c
new file mode 100644
index 0000000..b1bf4fa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint64_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
+/* { dg-final { scan-assembler-times "mulhu" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c
new file mode 100644
index 0000000..dfc9d2e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c
new file mode 100644
index 0000000..ec79e5d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint16_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c
new file mode 100644
index 0000000..eb95184
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint32_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c
new file mode 100644
index 0000000..ee41593
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc -mabi=ilp32d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c
new file mode 100644
index 0000000..b1d33a9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c
new file mode 100644
index 0000000..af5ffecf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint32_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c
new file mode 100644
index 0000000..d65cab0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c
new file mode 100644
index 0000000..79f6297
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint16_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c
new file mode 100644
index 0000000..e212391
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint16_t
+#define WT               uint32_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c
new file mode 100644
index 0000000..79d3fb3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint16_t
+#define WT               uint64_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c
new file mode 100644
index 0000000..e5a9462
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint32_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c
new file mode 100644
index 0000000..ad63db3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint32_t
+#define WT               uint64_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c
new file mode 100644
index 0000000..cbe2a22
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint64_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c
new file mode 100644
index 0000000..1f54c30
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c
new file mode 100644
index 0000000..f5a0ab5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint16_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c
new file mode 100644
index 0000000..32074a4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint32_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c
new file mode 100644
index 0000000..16ca905
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint64_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c
index eb140ae..66a439e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c
index 59ad242..6f40907 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_1:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c
index 47a8382..647fc6d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c
index f01317b..a344c58 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c
index 4b7bd3a..87fb1fc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_10:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c
index a28213f..280236a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_10:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c
index 432da0c..4b7d339 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_10:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c
index 0658d38..191c3a5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_10:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c
index 2e4b875..9dc41e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_11:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c
index 61fb80f..475f944 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_11:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c
index 2a28b1f..61e3584 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_11:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c
index 3033844..7a61055 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_11:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c
index 9cb86df..c4d21cb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_12:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c
index babe768..56beb83 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_12:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c
index 294ef5a..1bef3fe 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_12:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c
index 8b8f924..9004281 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_12:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c
index e724752..7b85582 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c
index 9240406..cfdf66c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c
index 3e1efba..3898817 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c
index 600688a..3318211 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c
index bb2d0b7..61bb5e5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c
index 06635df..73bfa99 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_3:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c
index ac485da..24d1e69 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c
index cdc8776..5523112 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c
index 407ff8f..fb6a604 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c
index cb2cd05..0f7e2d3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_4:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c
index 0ce6269..c762647 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c
index 302206a..3e5d2e6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c
index ce2758f..ab1b375 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_5:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c
index d33cef3..1b8ce84 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_5:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c
index 1bf1e97..3fc4e7a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_5:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c
index b2ed732..5c34ead 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_5:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c
index 20614ec..70dc6ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_6:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c
index 5d7adfd..cc36036 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_6:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c
index b3c6f8d..ea633ff 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_6:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c
index a4f92a8..7c4747a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_6:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c
index ebfe673..cac8471 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_7:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c
index 9884123..18b8e5f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_7:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c
index 67236d5..f5ade61 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_7:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c
index 549d9d2..9b528a4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_7:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c
index aa5aec7..0d093c3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_8:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c
index 89a8cc9..f04ea1d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_8:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c
index a52948d..17dd8f3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_8:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c
index 5606733..b043207 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_8:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c
index 984867a..19b1a5b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_9:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c
index d1109a4..a0026a1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_9:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c
index a9acf15..01c155e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_9:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c
index 47551fa..7b94d40 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_9:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c
index 1534cf9..20e14d6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c
index 5c60d28..1a0c394 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c
index 403764c..ee348b3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c
index 931420a..216af86 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c
index ae87544..109539d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c
index 43414ae..9e35fa2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c
index 3ef70a1..3c7c8db 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c
index 2a157f0..df291e2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c
index 534795c..88dded4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c
index 4d0a34f..239b422 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c
index d74d10d..9a524fd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c
index 949bd0d..b9b84ea 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c
index 80cce95..91bd9de 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c
index 3ecd19c..eaaa256 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c
index 2d7bfc4..04d2a20 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c
index 209965c..caedfe7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c
index 7deaae9a5..06a44f1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c
index d9b1d5c..9d38c9c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c
index 2774c23..5c10409 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c
index 6fa44ca..0ff9827 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c
index ea52ff4..aab99ca 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c
index fdea891..5231d6f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c
index 164ee77..d7462a8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c
index 724adf9..5da7838 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c
index 9b57861..8e69888 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c
index df2eece..9b22dda 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c
index 09e9ac3..abd0a95 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c
index c8ae7a6..d92c0e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c
index 9f575a47..b404bfd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c
index c370455..b746712 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c
index 22d82f9..da90b7a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c
index b282311..38dcabe 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c
index e0dda45..fd55bec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c
index dfd95ef..2e810dd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c
index 7cac446..e86eebc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c
index 0b4cbdb..e749bb5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c
index 10c65fe..eb57d55 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c
index e3b4dde..c1a5bcf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c
index 6e93fcf..27d4b82 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c
index d101d28..feb56e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c
index 4e50e3f..a22f1df 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c
index 3c8f78d..b98931d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c
index 932596a..dff3c0a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c
index 1f74562..d2f3126 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c
index 66a82f2..3740099 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c
index a54b5c3..b6ae459 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c
index 97943b3e..55198d6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c
index ab8b475..ce73d26 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c
index 573ef11..475b31e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32768_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*32768
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 32768)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c
index 0fefbe7..a984f84b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c
index ad6d4f9..b2930d4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65534_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-2
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 65534)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c
index 02dcbc5..362cf48 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c
index 7346fbb..9f17082 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*6
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c
index c7dac8a..801a86e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c
@@ -1,23 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483648_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 2147483648)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c
index 4320db3..e044768 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c
index 765d13c..5518064 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm4294967294_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-2
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 4294967294)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c
index ca11cf1..a4cb49b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c
index 3711930..64808bf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*255
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c
index 2e490f0..493a14d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm18446744073709551614u_uint64_t_fmt_1:
-** li\s+[atx][0-9]+,\s*-2
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint64_t, 18446744073709551614u)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c
index 45baa8f..4faae52 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint64_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint64_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c
index a29a6e9..3f993fd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_1:
-** li\s+[atx][0-9]+,\s*82
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c
index d1c6e94..a0d9235 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm128_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*128
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 128)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c
index 4c8cf90..67dae03 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*253
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c
index b958f5e..0054532 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm254_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*254
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 254)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c
index 1951ec5..c12b560 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c
index 86d0b39..ce9f495 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*11
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c
index 31c1bb8..93d7169 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32768_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*32768
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 32768)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c
index 68807b9..8ac2ce8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c
index 62deec1..740d6ac 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c
@@ -1,18 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint16_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** subw\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c
index f789fee..c82c478 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-6
-** sltiu\s+a0,\s*[atx][0-9]+,\s*6
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c
index 2f4a439..b2f690a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c
@@ -1,23 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483648_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 2147483648)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c
index dcfba62..e62010b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c
index a3f48f7..dd063d8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c
@@ -1,16 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint32_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** subw\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
-
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c
index 0bd8ddc..c0eb8a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*-255
-** sltiu\s+a0,\s*[atx][0-9]+,\s*255
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c
index 7b6d857..ed69313 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c
@@ -1,16 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint64_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** sub\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
-
 DEF_SAT_U_SUB_IMM_FMT_2(uint64_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c
index c334665..fb7db13 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c
@@ -1,18 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-82
-** sltiu\s+a0,\s*[atx][0-9]+,\s*82
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c
index 26e77f0..efe6c00 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm128_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-128
-** sltiu\s+a0,\s*[atx][0-9]+,\s*128
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 128)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c
index c5ac1b0..1262648 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-253
-** sltiu\s+a0,\s*[atx][0-9]+,\s*253
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c
index ee59b5a..108daf2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c
@@ -1,17 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint8_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** subw\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
-
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c
index 69dcc2a..784a97b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-11
-** sltiu\s+a0,\s*[atx][0-9]+,\s*11
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c
index f312362..0f16f9c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32769_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint16_t, 32769)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c
index fa9a9ef..49daab5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c
index b98de41..30fc2bf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*6
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c
index 79457a3..2d3c63d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483649_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint32_t, 2147483649)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c
index 2e8426e..8d96c00 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c
index 845218c..c06c441 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*255
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c
index ee2fbf8..4d2b96d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_3:
-** li\s+[atx][0-9]+,\s*82
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c
index 8cc81e2..8c3eb14 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm134_uint8_t_fmt_3:
-** li\s+[atx][0-9]+,\s*134
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint8_t, 134)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c
index 8d8c70b..b02d832 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_3:
-** li\s+[atx][0-9]+,\s*253
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c
index 348d75b..d8e0a69 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_3:
-** li\s+[atx][0-9]+,\s*11
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c
index 089c168..8f3726f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32768_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*32768
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint16_t, 32768)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c
index b96e3f3..56c377e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c
index 5c209bc..29c6b86 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-6
-** sltiu\s+a0,\s*[atx][0-9]+,\s*6
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c
index 2f4a439..b2f690a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c
@@ -1,23 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483648_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 2147483648)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c
index dcfba62..e62010b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c
index ee1ad9a..6cfb1e4c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_4:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*-255
-** sltiu\s+a0,\s*[atx][0-9]+,\s*255
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c
index c334665..fb7db13 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c
@@ -1,18 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-82
-** sltiu\s+a0,\s*[atx][0-9]+,\s*82
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c
index 3fe4103..49a4150 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm128_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-128
-** sltiu\s+a0,\s*[atx][0-9]+,\s*128
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint8_t, 128)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c
index 18dc505..1022de2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-253
-** sltiu\s+a0,\s*[atx][0-9]+,\s*253
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c
index 5c40f32..48aaeb2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-11
-** sltiu\s+a0,\s*[atx][0-9]+,\s*11
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c
index 2bc3be3..a193d88 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c
index b1d1ee3..e1dd81c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c
index 2539d75..a71526c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c
index 5091872..4fedf96 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c
index 0f4f9e4..f990c43 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c
index ea15d85..44d5e88 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c
index 612da92..91ea986 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c
index fc38095..7da49eb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c
index 150ab2a..8c44ee0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c
index c7d2850..f5c4e5a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c
index 6bf5cd2..393f7f6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c
index dfef1f2..e46463b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c
index 610e021..3062e0f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c
index 1d9e0cb..e621cd2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c
index f864a67..cfc96bf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c
index 603f2ee..771ec4a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c
index b73290a..d368621 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c
index 8af803f..02ca992 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c
index 1c887d4..cc01abd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_1:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c
index 6bcf64b..e28ee5c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_1:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c
index 8a35e72..59302cb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c
index a3b52de..735ea7e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_2:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c
index b9b43f1..8fd3f43 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c
index 7ed3623..bb4ecc5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_1:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c
index 7572c9e..e476897 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_2:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c
index d83b5dd..524d625 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_2:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c
index b7202f9..ba8b238 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c
index e90b853..cba8573 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_2:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c
index e8655b9..5852028 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c
index 41e676a..5d5cf97 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c
index 32eeb88..866e240 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_3:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c
index 5d043ce..f3adfb6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_3:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c
index 7e5906b..4e132a9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c
index e1b0acd..893f43e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_4:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c
index 618d50bd..5c0c7a7e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c
index c9a9a4c..395bb1b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_3:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c
index 418cdc8..8f20c8f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_4:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c
index 4903a04..f7e7ff2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_4:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c
index 6f8191c..2d9b6a6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c
index 24bb846..4fa81fe 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_4:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c
index a5f43e9..72c175c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c
index a76ae08..aef195a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c
index d05ea79..4517418 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c
index adaa421..2e51023 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c
index 38fcba3..8ea83d6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c
index 93705f9..1d0dd5b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c
index c116484..f69968c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c
index 4fbdc91..dcff0b4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c
index 2281610..33f46ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c
index 126c97c..b9c4617 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c
index 61ad79d..21755a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c
index 4142e87..bcf2081 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c
index 8952c06..69f5352 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c
index 8952c06..69f5352 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c
index 20ceda6..f001c39 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c
index 7011e50..1394d9f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c
index e868da1..de5d723 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c
index 7f52283fb..c345bfa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c
index ee13f0a..8ca8cc7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c
index 8471c76..54e00e8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c
index f056bd4..a957cc3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c
index 96c06eb..9691b4d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c
index 1623e52..ff2c2a5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c
index a1b8a5f..918eabb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c b/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
index dc5609c..167fa15 100644
--- a/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
+++ b/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
@@ -20,12 +20,6 @@ void func_machine (void)
 /* { dg-final { scan-assembler-times {\mth\.ipop\M} 2 { target { rv32 } } } } */
 
 
-__attribute__ ((interrupt ("user")))
-void func_usr (void)
-{
-  f ();
-}
-
 __attribute__ ((interrupt ("supervisor")))
 void func_supervisor (void)
 {
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-1.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-1.c
new file mode 100644
index 0000000..671c0ed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-1.c
@@ -0,0 +1,103 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/*
+** test_in_1:
+**     foo	%r2
+**     br	%r14
+*/
+
+int
+test_in_1 (int x)
+{
+  asm ("foo	%0" :: "{r2}" (x));
+  return x;
+}
+
+/*
+** test_in_2:
+**     lgr	(%r[0-9]+),%r2
+**     lr	%r2,%r3
+**     foo	%r2
+**     lgr	%r2,\1
+**     br	%r14
+*/
+
+int
+test_in_2 (int x, int y)
+{
+  asm ("foo	%0" :: "{r2}" (y));
+  return x;
+}
+
+/*
+** test_in_3:
+**     stmg	%r12,%r15,96\(%r15\)
+**     lay	%r15,-160\(%r15\)
+**     lgr	(%r[0-9]+),%r2
+**     ahi	%r2,1
+**     lgfr	%r2,%r2
+**     brasl	%r14,foo@PLT
+**     lr	%r3,%r2
+**     lr	%r2,\1
+**     foo	%r3,%r2
+**     lgr	%r2,\1
+**     lmg	%r12,%r15,256\(%r15\)
+**     br	%r14
+*/
+
+extern int foo (int);
+
+int
+test_in_3 (int x)
+{
+  asm ("foo	%0,%1\n" :: "{r3}" (foo (x + 1)), "{r2}" (x));
+  return x;
+}
+
+/*
+** test_out_1:
+**     foo	%r3
+**     lgfr	%r2,%r3
+**     br	%r14
+*/
+
+int
+test_out_1 (void)
+{
+  int x;
+  asm ("foo	%0" : "={r3}" (x));
+  return x;
+}
+
+/*
+** test_out_2:
+**     lgr	(%r[0-9]+),%r2
+**     foo	%r2
+**     ark	(%r[0-9]+),\1,%r2
+**     lgfr	%r2,\2
+**     br	%r14
+*/
+
+int
+test_out_2 (int x)
+{
+  int y;
+  asm ("foo	%0" : "={r2}" (y));
+  return x + y;
+}
+
+/*
+** test_inout_1:
+**     foo	%r2
+**     lgfr	%r2,%r2
+**     br	%r14
+*/
+
+int
+test_inout_1 (int x)
+{
+  asm ("foo	%0" : "+{r2}" (x));
+  return x;
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-2.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-2.c
new file mode 100644
index 0000000..a892fe8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-2.c
@@ -0,0 +1,43 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler {\.LC0:\n\t\.long\t1078523331\n} } } */
+
+
+/*
+** test_float_into_gpr:
+**     lrl	%r4,.LC0
+**     foo	%r4
+**     br	%r14
+*/
+
+void
+test_float_into_gpr (void)
+{
+  // This is the counterpart to
+  //   register float x asm ("r4") = 3.14f;
+  //   asm ("foo	%0" :: "r" (x));
+  // where the bit-pattern of 3.14f is loaded into GPR.
+  asm ("foo	%0" :: "{r4}" (3.14f));
+}
+
+/*
+** test_float:
+** (
+**     ldr	%f4,%f0
+**     ldr	%f5,%f2
+** |
+**     ldr	%f5,%f2
+**     ldr	%f4,%f0
+** )
+**     aebr	%f5,%f4
+**     ldr	%f0,%f5
+**     br	%r14
+*/
+
+float
+test_float (float x, float y)
+{
+  asm ("aebr	%0,%1" : "+{f5}" (y) : "{f4}" (x));
+  return y;
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-3.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-3.c
new file mode 100644
index 0000000..5df37b5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-3.c
@@ -0,0 +1,42 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler {\.LC0:\n\t\.long\t1074339512\n\t\.long\t1374389535\n} } } */
+
+/*
+** test_double_into_gpr:
+**     lgrl	%r4,.LC0
+**     foo	%r4
+**     br	%r14
+*/
+
+void
+test_double_into_gpr (void)
+{
+  // This is the counterpart to
+  //   register double x asm ("r4") = 3.14;
+  //   asm ("foo	%0" :: "r" (x));
+  // where the bit-pattern of 3.14 is loaded into GPR.
+  asm ("foo	%0" :: "{r4}" (3.14));
+}
+
+/*
+** test_double:
+** (
+**     ldr	%f4,%f0
+**     ldr	%f5,%f2
+** |
+**     ldr	%f5,%f2
+**     ldr	%f4,%f0
+** )
+**     adbr	%f5,%f4
+**     ldr	%f0,%f5
+**     br	%r14
+*/
+
+double
+test_double (double x, double y)
+{
+  asm ("adbr	%0,%1" : "+{f5}" (y) : "{f4}" (x));
+  return y;
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-4.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-4.c
new file mode 100644
index 0000000..29927ce
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-4.c
@@ -0,0 +1,6 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+
+/* Test TARGET_MD_ASM_ADJUST for z13 and long double.  */
+
+#include "asm-hard-reg-longdouble.h"
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-5.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-5.c
new file mode 100644
index 0000000..eaf34d9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-5.c
@@ -0,0 +1,6 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z14 -mzarch" } */
+
+/* Test TARGET_MD_ASM_ADJUST for z14 and long double.  */
+
+#include "asm-hard-reg-longdouble.h"
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-6.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-6.c
new file mode 100644
index 0000000..d012966
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-6.c
@@ -0,0 +1,152 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+void
+test (void)
+{
+  // GPRs
+  {
+    int a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p;
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14"
+        : "=r" (a),
+          "=r" (b),
+          "=r" (c),
+          "=r" (d),
+          "=r" (e),
+          "=r" (f),
+          "=r" (g),
+          "=r" (h),
+          "=r" (i),
+          "=r" (j),
+          "=r" (k),
+          "=r" (l),
+          "=r" (m),
+          "=r" (n),
+          "=r" (o));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14"
+        : "={r0}" (a),
+          "={r1}" (b),
+          "={r2}" (c),
+          "={r3}" (d),
+          "={r4}" (e),
+          "={r5}" (f),
+          "={r6}" (g),
+          "={r7}" (h),
+          "={r8}" (i),
+          "={r9}" (j),
+          "={r10}" (k),
+          "={r11}" (l),
+          "={r12}" (m),
+          "={r13}" (n),
+          "={r14}" (o));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=r" (a),
+          "=r" (b),
+          "=r" (c),
+          "=r" (d),
+          "=r" (e),
+          "=r" (f),
+          "=r" (g),
+          "=r" (h),
+          "=r" (i),
+          "=r" (j),
+          "=r" (k),
+          "=r" (l),
+          "=r" (m),
+          "=r" (n),
+          "=r" (o),
+          "=r" (p));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=r" (a),
+          "=r" (b),
+          "=r" (c),
+          "=r" (d),
+          "=r" (e),
+          "=r" (f),
+          "=r" (g),
+          "=r" (h),
+          "=r" (i),
+          "=r" (j),
+          "=r" (k),
+          "=r" (l),
+          "=r" (m),
+          "=r" (n),
+          "=r" (o),
+          "={r4}" (p));
+  }
+
+  // FPRs
+  {
+    float a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q;
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15"
+        : "=f" (a),
+          "=f" (b),
+          "=f" (c),
+          "=f" (d),
+          "=f" (e),
+          "=f" (f),
+          "=f" (g),
+          "=f" (h),
+          "=f" (i),
+          "=f" (j),
+          "=f" (k),
+          "=f" (l),
+          "=f" (m),
+          "=f" (n),
+          "=f" (o),
+          "=f" (p));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15"
+        : "={f0}" (a),
+          "={f1}" (b),
+          "={f2}" (c),
+          "={f3}" (d),
+          "={f4}" (e),
+          "={f5}" (f),
+          "={f6}" (g),
+          "={f7}" (h),
+          "={f8}" (i),
+          "={f9}" (j),
+          "={f10}" (k),
+          "={f11}" (l),
+          "={f12}" (m),
+          "={f13}" (n),
+          "={f14}" (o),
+          "={f15}" (p));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15 %16" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=f" (a),
+          "=f" (b),
+          "=f" (c),
+          "=f" (d),
+          "=f" (e),
+          "=f" (f),
+          "=f" (g),
+          "=f" (h),
+          "=f" (i),
+          "=f" (j),
+          "=f" (k),
+          "=f" (l),
+          "=f" (m),
+          "=f" (n),
+          "=f" (o),
+          "=f" (p),
+          "=f" (q));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15 %16" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=f" (a),
+          "=f" (b),
+          "=f" (c),
+          "=f" (d),
+          "=f" (e),
+          "=f" (f),
+          "=f" (g),
+          "=f" (h),
+          "=f" (i),
+          "=f" (j),
+          "=f" (k),
+          "=f" (l),
+          "=f" (m),
+          "=f" (n),
+          "=f" (o),
+          "=f" (p),
+          "={f4}" (q));
+  }
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-7.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-7.c
new file mode 100644
index 0000000..923c9d2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-7.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-march=z13" } */
+
+/* Test register pairs.  */
+
+void
+test (void)
+{
+  register double f0 __asm__ ("f0");
+  register double f2 __asm__ ("f2");
+  register long double f0f2 __asm__ ("f0");
+  double x;
+  long double y;
+
+  /* Outputs */
+  __asm__ __volatile__ ("" : "=r" (f0), "=r" (f0f2));
+  __asm__ __volatile__ ("" : "=r" (f0f2), "={f0}" (y));  /* { dg-error "multiple outputs to hard register: %f0" } */
+  __asm__ __volatile__ ("" : "={f0}" (x), "=r" (f0f2));  /* { dg-error "multiple outputs to hard register: %f0" } */
+
+  __asm__ __volatile__ ("" : "=r" (f2), "=r" (f0f2));
+  __asm__ __volatile__ ("" : "={f2}" (x), "={f0}" (y));  /* { dg-error "multiple outputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" : "=r" (f2), "={f0}" (y));  /* { dg-error "multiple outputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" : "={f2}" (x), "=r" (f0f2));  /* { dg-error "multiple outputs to hard register: %f2" } */
+
+  /* Inputs */
+  __asm__ __volatile__ ("" :: "r" (f0), "r" (f0f2));
+  __asm__ __volatile__ ("" :: "r" (f0f2), "{f0}" (y));  /* { dg-error "multiple inputs to hard register: %f0" } */
+  __asm__ __volatile__ ("" :: "{f0}" (x), "r" (f0f2));  /* { dg-error "multiple inputs to hard register: %f0" } */
+
+  __asm__ __volatile__ ("" :: "r" (f2), "r" (f0f2));
+  __asm__ __volatile__ ("" :: "{f2}" (x), "{f0}" (y));  /* { dg-error "multiple inputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" :: "r" (f2), "{f0}" (y));  /* { dg-error "multiple inputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" :: "{f2}" (x), "r" (f0f2));  /* { dg-error "multiple inputs to hard register: %f2" } */
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-longdouble.h b/gcc/testsuite/gcc.target/s390/asm-hard-reg-longdouble.h
new file mode 100644
index 0000000..9f4adad
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-longdouble.h
@@ -0,0 +1,18 @@
+__attribute__ ((noipa))
+long double
+test_longdouble (long double x)
+{
+  long double y;
+  asm ("sqxbr\t%0,%1" : "={f4}" (y) : "{f5}" (x));
+  return y;
+}
+
+int
+main (void)
+{
+  long double x = test_longdouble (42.L);
+  long double y = 6.48074069840786023096596743608799656681773277430814773408787249757445105002862106857719481922686100006103515625L;
+  if (x != y)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/s390/fminmax-1.c b/gcc/testsuite/gcc.target/s390/fminmax-1.c
new file mode 100644
index 0000000..df10905
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/fminmax-1.c
@@ -0,0 +1,77 @@
+/* Check fmin/fmax expanders for scalars on VXE targets.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=z14 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+/*
+** dofmaxl:
+**	vl	(%v.),0\(%r3\),3
+**	vl	(%v.),0\(%r4\),3
+**	wfmaxxb	(%v.),\1,\2,4
+**	vst	\3,0\(%r2\),3
+**	br	%r14
+*/
+long double
+dofmaxl (long double d1, long double d2)
+{
+  return __builtin_fmaxl (d1, d2);
+}
+
+/*
+** dofminl:
+**	vl	(%v.),0\(%r3\),3
+**	vl	(%v.),0\(%r4\),3
+**	wfminxb	(%v.),\1,\2,4
+**	vst	\3,0\(%r2\),3
+**	br	%r14
+*/
+long double
+dofminl (long double d1, long double d2)
+{
+  return __builtin_fminl (d1, d2);
+}
+
+/*
+** dofmax:
+**	wfmaxdb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+double
+dofmax (double d1, double d2)
+{
+  return __builtin_fmax (d1, d2);
+}
+
+/*
+** dofmin:
+**	wfmindb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+double
+dofmin (double d1, double d2)
+{
+  return __builtin_fmin (d1, d2);
+}
+
+/*
+** dofmaxf:
+**	wfmaxsb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+float
+dofmaxf (float f1, float f2)
+{
+  return __builtin_fmaxf (f1, f2);
+}
+
+/*
+** dofminf:
+**	wfminsb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+float
+dofminf (float f1, float f2)
+{
+  return __builtin_fminf (f1, f2);
+}
diff --git a/gcc/testsuite/gcc.target/s390/fminmax-2.c b/gcc/testsuite/gcc.target/s390/fminmax-2.c
new file mode 100644
index 0000000..ea37a0a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/fminmax-2.c
@@ -0,0 +1,29 @@
+/* Check fmin/fmax expanders for scalars on non-VXE targets.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { scan-assembler-times "jg" 4 } } */
+
+double
+dofmax (double d1, double d2)
+{
+  return __builtin_fmax (d1, d2);
+}
+
+double
+dofmin (double d1, double d2)
+{
+  return __builtin_fmin (d1, d2);
+}
+
+float
+dofmaxf (float f1, float f2)
+{
+  return __builtin_fmaxf (f1, f2);
+}
+
+float
+dofminf (float f1, float f2)
+{
+  return __builtin_fminf (f1, f2);
+}
diff --git a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c
index 2ff5a37..e1c7806 100644
--- a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c
+++ b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c
@@ -3,8 +3,10 @@
 
 #include "isfinite-isinf-isnormal-signbit.h"
 
-/* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,1365} 1 } } SIGNBIT long double */
-/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 } } SIGNBIT _Decimal128 */
+/* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,1365} 0 { target lp64 } } } SIGNBIT long double */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 0 { target lp64 } } } SIGNBIT _Decimal128 */
+/* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,1365} 1 { target { ! lp64 } } } } SIGNBIT long double */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 { target { ! lp64 } } } } SIGNBIT _Decimal128 */
 /* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,4032} 1 } } ISFINITE long double */
 /* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,4032} 1 } } ISFINITE _Decimal128 */
 /* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,48} 1 } } ISINF long double */
diff --git a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c
index 8f67553..5c9986d 100644
--- a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c
+++ b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c
@@ -3,8 +3,10 @@
 
 #include "isfinite-isinf-isnormal-signbit.h"
 
-/* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,1365} 1 } } */
-/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 } } */
+/* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,1365} 0 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 0 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,1365} 1 { target { ! lp64 } } } } */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 { target { ! lp64 } } } } */
 /* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,4032} 1 } } */
 /* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,4032} 1 } } */
 /* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,48} 1 } } */
diff --git a/gcc/testsuite/gcc.target/s390/signbit-1.c b/gcc/testsuite/gcc.target/s390/signbit-1.c
new file mode 100644
index 0000000..45f608a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-1.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -march=z900 -save-temps" } */
+/* { dg-final { scan-assembler-times {\ttceb\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttcdb\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttcxb\t} 2 } } */
+
+/* Binary Floating-Point */
+
+__attribute__ ((noipa))
+int signbit_float_reg (float x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_float_mem (float *x) { return __builtin_signbit (*x); }
+__attribute__ ((noipa))
+int signbit_double_reg (double x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_double_mem (double *x) { return __builtin_signbit (*x); }
+
+__attribute__ ((noipa))
+int
+signbit_longdouble_reg (long double x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+__attribute__ ((noipa))
+int signbit_longdouble_mem (long double *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (float, float, __builtin_inff(), __builtin_nanf("42"), 0.f, 42.f)
+TEST (double, double, __builtin_inf(), __builtin_nan("42"), 0., 42.)
+TEST (longdouble, long double, __builtin_infl(), __builtin_nanl("42"), 0.L, 42.L)
+
+int
+main (void)
+{
+  test_float ();
+  test_double ();
+  test_longdouble ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-2.c b/gcc/testsuite/gcc.target/s390/signbit-2.c
new file mode 100644
index 0000000..488c477
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-2.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -march=z9-ec -mzarch -save-temps" } */
+/* { dg-final { scan-assembler-times {\ttdcet\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttdcdt\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttdcxt\t} 2 } } */
+
+/* Decimal Floating-Point */
+
+__attribute__ ((noipa))
+int signbit_dec32_reg (_Decimal32 x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_dec32_mem (_Decimal32 *x) { return __builtin_signbit (*x); }
+__attribute__ ((noipa))
+int signbit_dec64_reg (_Decimal64 x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_dec64_mem (_Decimal64 *x) { return __builtin_signbit (*x); }
+
+__attribute__ ((noipa))
+int
+signbit_dec128_reg (_Decimal128 x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+__attribute__ ((noipa))
+int signbit_dec128_mem (_Decimal128 *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (dec32, _Decimal32, __builtin_infd32(), __builtin_nand32("42"), 0.df, 42.df)
+TEST (dec64, _Decimal64, __builtin_infd64(), __builtin_nand64("42"), 0.dd, 42.dd)
+TEST (dec128, _Decimal128, __builtin_infd128(), __builtin_nand128("42"), 0.dl, 42.dl)
+
+int
+main (void)
+{
+  test_dec32 ();
+  test_dec64 ();
+  test_dec128 ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-3.c b/gcc/testsuite/gcc.target/s390/signbit-3.c
new file mode 100644
index 0000000..2fad58b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-3.c
@@ -0,0 +1,152 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z10 -save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Binary Floating-Point */
+
+/*
+** signbit_float_reg:
+**	lgdr	(%r[0-9]+),%f0
+**	srlg	(%r[0-9]+),\1,63
+**	lgfr	%r2,\2
+**	br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_reg (float x) { return __builtin_signbit (x); }
+
+/*
+** signbit_float_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    srl	\1,31
+**    lgfr	%r2,\1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_mem (float *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_double_reg:
+**    lgdr	(%r[0-9]+),%f0
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_double_reg (double x) { return __builtin_signbit (x); }
+
+/*
+** signbit_double_mem:
+**    lg	(%r[0-9]+),0\(%r2\)
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_double_mem (double *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_longdouble_reg:
+**     ld	%f0,0\(%r2\)
+**     ld	%f2,8\(%r2\)
+**     lgdr	(%r[0-9]+),%f0
+**     srlg	%r2,\1,63
+**     br	%r14
+*/
+__attribute__ ((noipa))
+int
+signbit_longdouble_reg (long double x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+/*
+** signbit_longdouble_mem:
+**      lg	(%r[0-9]+),0\(%r2\)
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_longdouble_mem (long double *x) { return __builtin_signbit (*x); }
+
+/* Decimal Floating-Point */
+
+/*
+** signbit_dec32_reg:
+**	lgdr	(%r[0-9]+),%f0
+**	srlg	(%r[0-9]+),\1,63
+**	lgfr	%r2,\2
+**	br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_reg (_Decimal32 x) { return __builtin_signbit (x); }
+
+/*
+** signbit_dec32_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    srl	\1,31
+**    lgfr	%r2,\1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_mem (_Decimal32 *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_dec64_reg:
+**    lgdr	(%r[0-9]+),%f0
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec64_reg (_Decimal64 x) { return __builtin_signbit (x); }
+
+/*
+** signbit_dec64_mem:
+**    lg	(%r[0-9]+),0\(%r2\)
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec64_mem (_Decimal64 *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_dec128_reg:
+**     ld	%f0,0\(%r2\)
+**     ld	%f2,8\(%r2\)
+**     lgdr	(%r[0-9]+),%f0
+**     srlg	%r2,\1,63
+**     br	%r14
+*/
+__attribute__ ((noipa))
+int
+signbit_dec128_reg (_Decimal128 x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+/*
+** signbit_dec128_mem:
+**      lg	(%r[0-9]+),0\(%r2\)
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec128_mem (_Decimal128 *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (float, float, __builtin_inff(), __builtin_nanf("42"), 0.f, 42.f)
+TEST (double, double, __builtin_inf(), __builtin_nan("42"), 0., 42.)
+TEST (longdouble, long double, __builtin_infl(), __builtin_nanl("42"), 0.L, 42.L)
+TEST (dec32, _Decimal32, __builtin_infd32(), __builtin_nand32("42"), 0.df, 42.df)
+TEST (dec64, _Decimal64, __builtin_infd64(), __builtin_nand64("42"), 0.dd, 42.dd)
+TEST (dec128, _Decimal128, __builtin_infd128(), __builtin_nand128("42"), 0.dl, 42.dl)
+
+int
+main (void)
+{
+  test_float ();
+  test_double ();
+  test_longdouble ();
+  test_dec32 ();
+  test_dec64 ();
+  test_dec128 ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-4.c b/gcc/testsuite/gcc.target/s390/signbit-4.c
new file mode 100644
index 0000000..2cb743e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-4.c
@@ -0,0 +1,55 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-require-effective-target s390_vx } */
+/* { dg-options "-O2 -march=z13 -save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Binary Floating-Point */
+
+/*
+** signbit_float_reg:
+**    vlgvf	(%r[0-9]+),%v0,0
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_reg (float x) { return __builtin_signbit (x); }
+
+/*
+** signbit_float_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_mem (float *x) { return __builtin_signbit (*x); }
+
+/* Decimal Floating-Point */
+
+/*
+** signbit_dec32_reg:
+**    vlgvf	(%r[0-9]+),%v0,0
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_reg (_Decimal32 x) { return __builtin_signbit (x); }
+
+/*
+** signbit_dec32_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_mem (_Decimal32 *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (float, float, __builtin_inff(), __builtin_nanf("42"), 0.f, 42.f)
+TEST (dec32, _Decimal32, __builtin_infd32(), __builtin_nand32("42"), 0.df, 42.df)
+
+int
+main (void)
+{
+  test_float ();
+  test_dec32 ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-5.c b/gcc/testsuite/gcc.target/s390/signbit-5.c
new file mode 100644
index 0000000..6840327
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-5.c
@@ -0,0 +1,35 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z14 -save-temps" } */
+
+/*
+** signbit_longdouble_reg:
+**      ld	%f0,0(%r2);ld	%f2,8+0(%r2)
+**      lgdr	(%r[0-9]+),%f0
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int
+signbit_longdouble_reg (long double x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+/*
+** signbit_longdouble_mem:
+**      lg	(%r[0-9]+),0\(%r2\)
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_longdouble_mem (long double *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (longdouble, long double, __builtin_infl(), __builtin_nanl("42"), 0.L, 42.L)
+
+int
+main (void)
+{
+  test_longdouble ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit.h b/gcc/testsuite/gcc.target/s390/signbit.h
new file mode 100644
index 0000000..730e387
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit.h
@@ -0,0 +1,36 @@
+#define TEST(T, U, I, N, C0, C42)                                              \
+  void test_##T (void)                                                         \
+  {                                                                            \
+    U tmp;                                                                     \
+    int x;                                                                     \
+                                                                               \
+    x = signbit_##T##_reg(C42);                                                \
+    x += signbit_##T##_reg(C0);                                                \
+    x += signbit_##T##_reg(I);                                                 \
+    x += signbit_##T##_reg(N);                                                 \
+    tmp = C42;                                                                 \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = C0;                                                                  \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = I;                                                                   \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = N;                                                                   \
+    x += signbit_##T##_mem(&tmp);                                              \
+    if (x != 0)                                                                \
+      __builtin_abort();                                                       \
+                                                                               \
+    x = signbit_##T##_reg(-C42);                                               \
+    x += signbit_##T##_reg(-C0);                                               \
+    x += signbit_##T##_reg(-I);                                                \
+    x += signbit_##T##_reg(-N);                                                \
+    tmp = -C42;                                                                \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = -C0;                                                                 \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = -I;                                                                  \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = -N;                                                                  \
+    x += signbit_##T##_mem(&tmp);                                              \
+    if (x != 8)                                                                \
+      __builtin_abort();                                                       \
+  }
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c
new file mode 100644
index 0000000..56c3d77
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 2\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else if (x > y)		\
+      return 1;			\
+    else			\
+      return 2;			\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c
new file mode 100644
index 0000000..0c6e6b6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -ffinite-math-only -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 2\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tc[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tk[edx]br\t} } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else if (x > y)		\
+      return 1;			\
+    else			\
+      return 2;			\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c
new file mode 100644
index 0000000..2f567d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 42\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else if (x > y)		\
+      return 1;			\
+    else			\
+      return 42;		\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c
new file mode 100644
index 0000000..4531ecb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c
@@ -0,0 +1,53 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 0\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tloc} } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+/* By time of writing this we emit
+
+	kebr	%f0,%f2
+	jo	.L2
+	je	.L3
+	jnh	.L10
+	jg	f3@PLT
+.L10:
+	jg	f2@PLT
+.L3:
+	jg	f1@PLT
+.L2:
+	jg	f4@PLT
+
+   which is not optimal.  Instead we could fold the conditional branch with the
+   unconditional into something along the lines
+
+	kebr	%f0,%f2
+	jo	f4@PLT
+	je	f1@PLT
+	jnh	f2@PLT
+	jg	f3@PLT
+*/
+
+void f1 (void);
+void f2 (void);
+void f3 (void);
+void f4 (void);
+
+#define TEST(T, U)		\
+  void test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      f1 ();			\
+    else if (x < y)		\
+      f2 ();			\
+    else if (x > y)		\
+      f3 ();			\
+    else			\
+      f4 ();			\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-1.c b/gcc/testsuite/gcc.target/s390/spaceship-int-1.c
new file mode 100644
index 0000000..8ca2677
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 4 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 5 optimized } } */
+/* { dg-final { scan-assembler-times {\tlhi} 9 } } */
+/* { dg-final { scan-assembler-times {\tloc} 18 } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else			\
+      return 1;			\
+  }
+
+TEST(signed char, schar)
+TEST(unsigned char, uchar)
+TEST(char, char)
+
+TEST(short, sshort)
+TEST(unsigned short, ushort)
+
+TEST(int, sint)
+TEST(unsigned int, uint)
+
+TEST(long, slong)
+TEST(unsigned long, ulong)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-2.c b/gcc/testsuite/gcc.target/s390/spaceship-int-2.c
new file mode 100644
index 0000000..5f7975c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 1 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 1 optimized } } */
+/* { dg-final { scan-assembler-times {\tvecg} 1 } } */
+/* { dg-final { scan-assembler-times {\tveclg} 1 } } */
+/* { dg-final { scan-assembler-times {\tvchlgs} 2 } } */
+/* { dg-final { scan-assembler-times {\tvceqgs} 2 } } */
+/* { dg-final { scan-assembler-times {\tlhi} 2 } } */
+/* { dg-final { scan-assembler-times {\tloc} 4 } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else			\
+      return 1;			\
+  }
+
+TEST(__int128, sint128)
+TEST(unsigned __int128, uint128)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-3.c b/gcc/testsuite/gcc.target/s390/spaceship-int-3.c
new file mode 100644
index 0000000..46b0e4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-3.c
@@ -0,0 +1,21 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -march=z17 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 1 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 1 optimized } } */
+/* { dg-final { scan-assembler-times {\tvecq\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tveclq\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tloc} 4 } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else			\
+      return 1;			\
+  }
+
+TEST(__int128, sint128)
+TEST(unsigned __int128, uint128)
diff --git a/gcc/testsuite/gcc.target/s390/stack-protector-guard-tls-1.c b/gcc/testsuite/gcc.target/s390/stack-protector-guard-tls-1.c
new file mode 100644
index 0000000..1efd245
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/stack-protector-guard-tls-1.c
@@ -0,0 +1,39 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-protector-all" } */
+/* { dg-final { scan-assembler-times {\tear\t%r[0-9]+,%a[01]} 8 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tsllg\t%r[0-9]+,%r[0-9]+,32} 4 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tear\t%r[0-9]+,%a[01]} 3 { target { ! lp64 } } } } */
+/* { dg-final { scan-assembler-times {\tmvc\t160\(8,%r15\),40\(%r[0-9]+\)} 2 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tmvc\t100\(4,%r15\),20\(%r[0-9]+\)} 2 { target { ! lp64 } } } } */
+/* { dg-final { scan-assembler-times {\tclc\t160\(8,%r15\),40\(%r[0-9]+\)} 2 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tclc\t100\(4,%r15\),20\(%r[0-9]+\)} 2 { target { ! lp64 } } } } */
+
+/* Computing the address of the thread pointer on s390 involves multiple
+   instructions and therefore bears the risk that the address of the canary or
+   intermediate values of it are spilled and reloaded.  Therefore, as a
+   precaution compute the address always twice, i.e., one time for the prologue
+   and one time for the epilogue.  */
+
+void test_0 (void) { }
+
+void test_1 (void)
+{
+  __asm__ __volatile ("" :::
+      "r0",
+      "r1",
+      "r2",
+      "r3",
+      "r4",
+      "r5",
+      "r6",
+      "r7",
+      "r8",
+      "r9",
+      "r10",
+      "r11",
+#ifndef __PIC__
+      "r12",
+#endif
+      "r13",
+      "r14");
+}
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-avg-1.c b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-1.c
new file mode 100644
index 0000000..285ebc9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-1.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z16 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N)                                                   \
+  void                                                                  \
+  avg##T1 (signed T1 *__restrict res, signed T1 *__restrict a,          \
+           signed T1 *__restrict b)                                     \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((signed T2)a[i] + b[i] + 1) >> 1;                       \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  uavg##T1 (unsigned T1 *__restrict res, unsigned T1 *__restrict a,     \
+            unsigned T1 *__restrict b)                                  \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((unsigned T2)a[i] + b[i] + 1) >> 1;                     \
+  }
+
+TEST(char,short,16)
+TEST(short,int,8)
+TEST(int,long,4)
+
+/* { dg-final { scan-tree-dump-times "\.AVG_CEIL" 6 "optimized" { target lp64 } } } */
+/* { dg-final { scan-tree-dump-times "\.AVG_CEIL" 4 "optimized" { target { ! lp64 } } } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-avg-2.c b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-2.c
new file mode 100644
index 0000000..1cc614e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-2.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O3 -mzarch -march=z16 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N)                                                   \
+  void                                                                  \
+  avg##T1 (signed T1 *__restrict res, signed T1 *__restrict a,          \
+           signed T1 *__restrict b)                                     \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((signed T2)a[i] + b[i] + 1) >> 1;                       \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  uavg##T1 (unsigned T1 *__restrict res, unsigned T1 *__restrict a,     \
+            unsigned T1 *__restrict b)                                  \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((unsigned T2)a[i] + b[i] + 1) >> 1;                     \
+  }
+
+TEST(long,__int128,2)
+
+/* { dg-final { scan-tree-dump-times "\.AVG_CEIL" 2 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-1.c b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-1.c
new file mode 100644
index 0000000..f0b37d6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-1.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=arch15 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N,S)                                                 \
+  void                                                                  \
+  mulh##T1 (signed T1 *__restrict res,                                  \
+            signed T1 *__restrict l,                                    \
+            signed T1 *__restrict r)                                    \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (signed T1) (((signed T2)l[i] * (signed T2)r[i]) >> S);  \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  umulh##T1 (unsigned T1 *__restrict res,                               \
+             unsigned T1 *__restrict l,                                 \
+             unsigned T1 *__restrict r)                                 \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (unsigned T1)                                            \
+        (((unsigned T2)l[i] * (unsigned T2)r[i]) >> S);                 \
+  }
+
+TEST(char,short,16,8)
+TEST(short,int,8,16)
+
+/* { dg-final { scan-tree-dump-times "\.MULH" 4 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-2.c b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-2.c
new file mode 100644
index 0000000..2ff66b7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-2.c
@@ -0,0 +1,27 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O3 -mzarch -march=arch15 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N,S)                                                 \
+  void                                                                  \
+  mulh##T1 (signed T1 *__restrict res,                                  \
+            signed T1 *__restrict l,                                    \
+            signed T1 *__restrict r)                                    \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (signed T1) (((signed T2)l[i] * (signed T2)r[i]) >> S);  \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  umulh##T1 (unsigned T1 *__restrict res,                               \
+             unsigned T1 *__restrict l,                                 \
+             unsigned T1 *__restrict r)                                 \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (unsigned T1)                                            \
+        (((unsigned T2)l[i] * (unsigned T2)r[i]) >> S);                 \
+  }
+
+TEST(int,long,4,32)
+TEST(long,__int128,2,64)
+
+/* { dg-final { scan-tree-dump-times "\.MULH" 4 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/reduc-binops-1.c b/gcc/testsuite/gcc.target/s390/vector/reduc-binops-1.c
new file mode 100644
index 0000000..efd3294
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/reduc-binops-1.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z13 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define T(X,N)                                  \
+  unsigned X                                    \
+  reduce_and_##X (unsigned X *in)               \
+  {                                             \
+  unsigned X acc = (unsigned X)-1;              \
+  for (int i = 0; i < N; i++)                   \
+    acc &= in[i];                               \
+  return acc;                                   \
+  }                                             \
+  unsigned X                                    \
+  reduce_ior_##X (unsigned X *in)               \
+  {                                             \
+  unsigned X acc = 0;                           \
+  for (int i = 0; i < N; i++)                   \
+    acc |= in[i];                               \
+  return acc;                                   \
+  }                                             \
+  unsigned X                                    \
+  redue_xor_##X (unsigned X *in)                \
+  {                                             \
+  unsigned X acc = 0;                           \
+  for (int i = 0; i < N; i++)                   \
+    acc ^= in[i];                               \
+  return acc;                                   \
+  }
+
+T(char,16)
+
+T(short, 8)
+
+T(int,4)
+
+T(long,4)
+
+/* { dg-final { scan-tree-dump-times "\.REDUC_AND" 4 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_IOR" 4 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_XOR" 4 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/reduc-minmax-1.c b/gcc/testsuite/gcc.target/s390/vector/reduc-minmax-1.c
new file mode 100644
index 0000000..5295250
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/reduc-minmax-1.c
@@ -0,0 +1,234 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) > (b) ? (b) : (a))
+
+/* unsigned integers */
+
+unsigned char
+reduce_umax_char (unsigned char *p)
+{
+  unsigned char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned char
+reduce_umin_char (unsigned char *p)
+{
+  unsigned char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+unsigned short
+reduce_umax_short (unsigned short *p)
+{
+  unsigned short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned short
+reduce_umin_short (unsigned short *p)
+{
+  unsigned short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+unsigned int
+reduce_umax_int (unsigned int* p)
+{
+  unsigned int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned int
+reduce_umin_int (unsigned int* p)
+{
+  unsigned int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+unsigned long
+reduce_umax_long (unsigned long* p)
+{
+  unsigned long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned long
+reduce_umin_long (unsigned long* p)
+{
+  unsigned long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+/* signed integers */
+
+signed char
+reduce_smax_char (signed char *p)
+{
+  signed char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed char
+reduce_smin_char (signed char *p)
+{
+  signed char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+signed short
+reduce_smax_short (signed short *p)
+{
+  signed short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed short
+reduce_smin_short (signed short *p)
+{
+  signed short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+signed int
+reduce_smax_int (signed int* p)
+{
+  signed int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed int
+reduce_smin_int (signed int* p)
+{
+  signed int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+signed long
+reduce_smax_long (signed long* p)
+{
+  signed long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed long
+reduce_smin_long (signed long* p)
+{
+  signed long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+float
+__attribute__((optimize("Ofast")))
+reduce_smax_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+float
+__attribute__((optimize("Ofast")))
+reduce_smin_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+double
+__attribute__((optimize("Ofast")))
+reduce_smax_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+double
+__attribute__((optimize("Ofast")))
+reduce_smin_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+float
+reduce_fmax_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fmaxf (res, p[i]);
+  return res;
+}
+
+float
+reduce_fmin_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fminf (res, p[i]);
+  return res;
+}
+
+double
+reduce_fmax_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fmax (res, p[i]);
+  return res;
+}
+
+double
+reduce_fmin_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fmin (res, p[i]);
+  return res;
+}
+
+/* { dg-final { scan-tree-dump-times "\.REDUC_MAX" 10 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_MIN" 10 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_FMAX" 2 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_FMIN" 2 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/reduc-plus-1.c b/gcc/testsuite/gcc.target/s390/vector/reduc-plus-1.c
new file mode 100644
index 0000000..12cdd5f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/reduc-plus-1.c
@@ -0,0 +1,152 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -ftree-vectorize -fdump-tree-optimized" } */
+/* { dg-do run { target { s390_z14_hw } } } */
+
+/* signed integers */
+
+signed char
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_char (signed char* p)
+{
+  signed char sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+short
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_short (short* p)
+{
+  short sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+int
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_int (int* p)
+{
+  int sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+long
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_long (long* p)
+{
+  long sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+/* unsigned integers */
+
+unsigned char
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_uchar (unsigned char* p)
+{
+  unsigned char sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+unsigned short
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_ushort (unsigned short* p)
+{
+  unsigned short sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+unsigned int
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_uint (unsigned int* p)
+{
+  unsigned int sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+unsigned long
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_ulong (unsigned long* p)
+{
+  unsigned long sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+/* floating point */
+
+float
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_float (float* p)
+{
+  float sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+double
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_double (double* p)
+{
+  double sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+int
+main()
+{
+  signed char chararr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+  signed short shortarr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+  signed int intarr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+  signed long longarr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+
+  unsigned char uchararr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  unsigned short ushortarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  unsigned int uintarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  unsigned long ulongarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+
+  float floatarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  double doublearr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+
+  if (reduce_add_char (chararr) != (-136 & 0xff))
+    __builtin_abort();
+  if (reduce_add_short (shortarr) != -136)
+    __builtin_abort();
+  if (reduce_add_int (intarr) != -136)
+    __builtin_abort();
+  if (reduce_add_long (longarr) != -136)
+    __builtin_abort();
+
+  if (reduce_add_uchar (uchararr) != 136)
+    __builtin_abort();
+  if (reduce_add_ushort (ushortarr) != 136)
+    __builtin_abort();
+  if (reduce_add_uint (uintarr) != 136)
+    __builtin_abort();
+  if (reduce_add_ulong (ulongarr) != 136)
+    __builtin_abort();
+
+  if (reduce_add_float (floatarr) != 136)
+    __builtin_abort();
+  if (reduce_add_double (doublearr) != -136)
+    __builtin_abort();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.REDUC_PLUS" 10 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/vec-perm-merge-1.c b/gcc/testsuite/gcc.target/s390/vector/vec-perm-merge-1.c
new file mode 100644
index 0000000..79f8a88
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/vec-perm-merge-1.c
@@ -0,0 +1,242 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -mzvector --save-temps -fno-stack-protector" } */
+/* { dg-do run { target { s390_z14_hw } } } */
+/* { dg-final {check-function-bodies "**" "" } } */
+
+#include "vec-types.h"
+
+/*
+** qi_via_hi_hi:
+**	vmrhh	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_hi_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[0], a[1], b[0], b[1], a[2], a[3], b[2], b[3],
+		 a[4], a[5], b[4], b[5], a[6], a[7], b[6], b[7]};
+}
+
+/*
+** qi_via_hi_lo:
+**	vmrlh	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_hi_lo (v16qi a, v16qi b)
+{
+  return (v16qi){a[8], a[9], b[8], b[9], a[10], a[11], b[10], b[11],
+		 a[12], a[13], b[12], b[13], a[14], a[15], b[14], b[15]};
+}
+
+/*
+** qi_via_si_hi:
+**	vmrhf	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_si_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3],
+		 a[4], a[5], a[6], a[7], b[4], b[5], b[6], b[7]};
+}
+
+/*
+** qi_via_si_lo:
+**	vmrlf	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_si_lo (v16qi a, v16qi b)
+{
+  return (v16qi){a[8], a[9], a[10], a[11], b[8], b[9], b[10], b[11],
+		 a[12], a[13], a[14], a[15], b[12], b[13], b[14], b[15]};
+}
+
+/*
+** qi_via_di_hi:
+**	vmrhg	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_di_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7],
+		 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]};
+}
+
+/*
+** qi_via_di_lo:
+**	vmrlg	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_di_lo (v16qi a, v16qi b)
+{
+  return (v16qi){a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15],
+		 b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]};
+}
+
+/*
+** hi_via_si_hi:
+**	vmrhf	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_si_hi (v8hi a, v8hi b)
+{
+  return (v8hi){a[0], a[1], b[0], b[1], a[2], a[3], b[2], b[3]};
+}
+
+/*
+** hi_via_si_lo:
+**	vmrlf	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_si_lo (v8hi a, v8hi b)
+{
+  return (v8hi){a[4], a[5], b[4], b[5], a[6], a[7], b[6], b[7]};
+}
+
+/*
+** hi_via_di_hi:
+**	vmrhg	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_di_hi (v8hi a, v8hi b)
+{
+  return (v8hi){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]};
+}
+
+/*
+** hi_via_di_lo:
+**	vmrlg	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_di_lo (v8hi a, v8hi b)
+{
+  return (v8hi){a[4], a[5], a[6], a[7], b[4], b[5], b[6], b[7]};
+}
+
+/*
+** si_via_di_hi:
+**	vmrhg	%v24,%v24,%v26
+**	br	%r14
+*/
+v4si __attribute__((noinline,noipa))
+si_via_di_hi (v4si a, v4si b)
+{
+  return (v4si){a[0], a[1], b[0], b[1]};
+}
+
+/*
+** si_via_di_lo:
+**	vmrlg	%v24,%v24,%v26
+**	br	%r14
+*/
+v4si __attribute__((noinline,noipa))
+si_via_di_lo (v4si a, v4si b)
+{
+  return (v4si){a[2], a[3], b[2], b[3]};
+}
+
+int
+main ()
+{
+  static const signed char e_qi_via_hi_hi[16]
+    = {0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23};
+  static const signed char e_qi_via_hi_lo[16]
+    = {8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31};
+  static const signed char e_qi_via_si_hi[16]
+    = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};
+  static const signed char e_qi_via_si_lo[16]
+    = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};
+  static const signed char e_qi_via_di_hi[16]
+    = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
+  static const signed char e_qi_via_di_lo[16]
+    = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
+
+  static const short e_hi_via_si_hi[8] = {0, 1, 8, 9, 2, 3, 10, 11};
+  static const short e_hi_via_si_lo[8] = {4, 5, 12, 13, 6, 7, 14, 15};
+  static const short e_hi_via_di_hi[8] = {0, 1, 2, 3, 8, 9, 10, 11};
+  static const short e_hi_via_di_lo[8] = {4, 5, 6, 7, 12, 13, 14, 15};
+
+  static const int e_si_via_di_hi[4] = {0, 1, 4, 5};
+  static const int e_si_via_di_lo[4] = {2, 3, 6, 7};
+
+  v16qi a_qi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+  v16qi b_qi = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
+  v8hi a_hi = {0, 1, 2, 3, 4, 5, 6, 7};
+  v8hi b_hi = {8, 9, 10, 11, 12, 13, 14, 15};
+  v4si a_si = {0, 1, 2, 3};
+  v4si b_si = {4, 5, 6, 7};
+  v16qi r_qi;
+  v8hi r_hi;
+  v4si r_si;
+  int i;
+
+  r_qi = qi_via_hi_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_hi_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_hi_lo (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_hi_lo[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_si_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_si_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_si_lo (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_si_lo[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_di_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_di_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_di_lo (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_di_lo[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_si_hi (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_si_hi[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_si_lo (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_si_lo[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_di_hi (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_di_hi[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_di_lo (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_di_lo[i])
+      __builtin_abort ();
+
+  r_si = si_via_di_hi (a_si, b_si);
+  for (i = 0; i < 4; ++i)
+    if (r_si[i] != e_si_via_di_hi[i])
+      __builtin_abort ();
+
+  r_si = si_via_di_lo (a_si, b_si);
+  for (i = 0; i < 4; ++i)
+    if (r_si[i] != e_si_via_di_lo[i])
+      __builtin_abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/s390/vector/vec-perm-pack-1.c b/gcc/testsuite/gcc.target/s390/vector/vec-perm-pack-1.c
new file mode 100644
index 0000000..6590c92
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/vec-perm-pack-1.c
@@ -0,0 +1,133 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -mzvector --save-temps -fno-stack-protector" } */
+/* { dg-do run { target { s390_z14_hw } } } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "vec-types.h"
+
+/*
+** qi_via_hi:
+**	vpkh	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[1], a[3], a[5], a[7], a[9], a[11], a[13], a[15],
+		 b[1], b[3], b[5], b[7], b[9], b[11], b[13], b[15]};
+}
+
+/*
+** qi_via_si:
+**	vpkf	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_si (v16qi a, v16qi b)
+{
+  return (v16qi){a[2], a[3], a[6], a[7], a[10], a[11], a[14], a[15],
+		 b[2], b[3], b[6], b[7], b[10], b[11], b[14], b[15]};
+}
+
+/*
+** qi_via_di:
+**	vpkg	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_di (v16qi a, v16qi b)
+{
+  return (v16qi){a[4], a[5], a[6], a[7], a[12], a[13], a[14], a[15],
+		 b[4], b[5], b[6], b[7], b[12], b[13], b[14], b[15]};
+}
+
+/*
+** hi_via_si:
+**	vpkf	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_si (v8hi a, v8hi b)
+{
+  return (v8hi){a[1], a[3], a[5], a[7], b[1], b[3], b[5], b[7]};
+}
+
+/*
+** hi_via_di:
+**	vpkg	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_di (v8hi a, v8hi b)
+{
+  return (v8hi){a[2], a[3], a[6], a[7], b[2], b[3], b[6], b[7]};
+}
+
+/*
+** si_via_di:
+**	vpkg	%v24,%v24,%v26
+**	br	%r14
+*/
+v4si __attribute__((noinline,noipa))
+si_via_di (v4si a, v4si b)
+{
+  return (v4si){a[1], a[3], b[1], b[3]};
+}
+
+int
+main ()
+{
+  static const signed char e_qi_via_hi[16]
+    = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31};
+  static const signed char e_qi_via_si[16]
+    = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31};
+  static const signed char e_qi_via_di[16]
+    = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
+
+  static const short e_hi_via_si[8] = {1, 3, 5, 7, 9, 11, 13, 15};
+  static const short e_hi_via_di[8] = {2, 3, 6, 7, 10, 11, 14, 15};
+
+  static const int e_si_via_di[4] = {1, 3, 5, 7};
+
+  v16qi a_qi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+  v16qi b_qi = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
+  v8hi a_hi = {0, 1, 2, 3, 4, 5, 6, 7};
+  v8hi b_hi = {8, 9, 10, 11, 12, 13, 14, 15};
+  v4si a_si = {0, 1, 2, 3};
+  v4si b_si = {4, 5, 6, 7};
+  v16qi r_qi;
+  v8hi r_hi;
+  v4si r_si;
+  int i;
+
+  r_qi = qi_via_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_si (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_si[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_di (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_di[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_si (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_si[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_di (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_di[i])
+      __builtin_abort ();
+
+  r_si = si_via_di (a_si, b_si);
+  for (i = 0; i < 4; ++i)
+    if (r_si[i] != e_si_via_di[i])
+      __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/s390/vector/vlgv-zero-extend-1.c b/gcc/testsuite/gcc.target/s390/vector/vlgv-zero-extend-1.c
new file mode 100644
index 0000000..11df6c1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/vlgv-zero-extend-1.c
@@ -0,0 +1,71 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target s390_vx } */
+/* { dg-additional-options "-O2" } */
+/* { dg-final { scan-assembler-not {\tllg?[fhc]r\t} } } */
+
+typedef unsigned char __attribute__ ((vector_size (1))) V1QI;
+typedef unsigned char __attribute__ ((vector_size (2))) V2QI;
+typedef unsigned char __attribute__ ((vector_size (4))) V4QI;
+typedef unsigned char __attribute__ ((vector_size (8))) V8QI;
+typedef unsigned char __attribute__ ((vector_size (16))) V16QI;
+
+typedef unsigned short __attribute__ ((vector_size (2))) V1HI;
+typedef unsigned short __attribute__ ((vector_size (4))) V2HI;
+typedef unsigned short __attribute__ ((vector_size (8))) V4HI;
+typedef unsigned short __attribute__ ((vector_size (16))) V8HI;
+
+typedef unsigned int __attribute__ ((vector_size (4))) V1SI;
+typedef unsigned int __attribute__ ((vector_size (8))) V2SI;
+typedef unsigned int __attribute__ ((vector_size (16))) V4SI;
+
+unsigned short ushort;
+unsigned int uint;
+
+#define TEST(T, U, I) \
+  unsigned T test_ ## I ## _ ## U (U x) { return x[I]; } \
+  void       test_ ## I ## _ ## U ## _ushort (U x) { ushort = x[I]; } \
+  void       test_ ## I ## _ ## U ## _uint (U x) { uint = x[I]; }
+
+#define TEST1(T, U) \
+  TEST(T, U, 0)
+
+#define TEST2(T, U) \
+  TEST1 (T, U) \
+  TEST(T, U, 1)
+
+#define TEST4(T, U) \
+  TEST2 (T, U) \
+  TEST(T, U, 2) \
+  TEST(T, U, 3)
+
+#define TEST8(T, U) \
+  TEST4 (T, U) \
+  TEST(T, U, 4) \
+  TEST(T, U, 5) \
+  TEST(T, U, 6) \
+  TEST(T, U, 7)
+
+#define TEST16(T, U) \
+  TEST8 (T, U) \
+  TEST(T, U, 9) \
+  TEST(T, U, 10) \
+  TEST(T, U, 11) \
+  TEST(T, U, 12) \
+  TEST(T, U, 13) \
+  TEST(T, U, 14) \
+  TEST(T, U, 15)
+
+TEST1 (char, V1QI)
+TEST2 (char, V2QI)
+TEST4 (char, V4QI)
+TEST8 (char, V8QI)
+TEST16 (char, V16QI)
+
+TEST1 (short, V1HI)
+TEST2 (short, V2HI)
+TEST4 (short, V4HI)
+TEST8 (short, V8HI)
+
+TEST1 (int, V1SI)
+TEST2 (int, V2SI)
+TEST4 (int, V4SI)
diff --git a/gcc/testsuite/gcc.target/sh/pr54236-2.c b/gcc/testsuite/gcc.target/sh/pr54236-2.c
index 1e2f3bb..78befe4 100644
--- a/gcc/testsuite/gcc.target/sh/pr54236-2.c
+++ b/gcc/testsuite/gcc.target/sh/pr54236-2.c
@@ -4,10 +4,10 @@
 /* { dg-do compile }  */
 /* { dg-options "-O1" } */
 
-/* { dg-final { scan-assembler-times "addc" 36 } } */
+/* { dg-final { scan-assembler-times "addc" 32 } } */
 /* { dg-final { scan-assembler-times "shll" 14 } } */
-/* { dg-final { scan-assembler-times "add\tr" 12 } } */
-/* { dg-final { scan-assembler-not "movt" } } */
+/* { dg-final { scan-assembler-times "add\tr" 16 } } */
+/* { dg-final { scan-assembler-times "movt" 4 } } */
 
 /* { dg-final { scan-assembler-times "add\t#1" 1 } } */
 
@@ -184,28 +184,28 @@ test_022 (int a, int b, int c, int d)
 int
 test_023 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return a + ((b >> 31) & 1);
 }
 
 int
 test_024 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return ((b >> 31) & 1) + a;
 }
 
 int
 test_025 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return ((a >> 31) & 1) + a;
 }
 
 int
 test_026 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return a + ((a >> 31) & 1);
 }
 
diff --git a/gcc/testsuite/gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c b/gcc/testsuite/gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c
new file mode 100644
index 0000000..05873b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+extern void foo(void);
+
+void BGEUI_test(unsigned int a)
+{
+  if (a < 32768U)
+    foo();
+}
+
+void BLTUI_test(unsigned int a)
+{
+  if (a >= 65536U)
+    foo();
+}
+
+/* { dg-final { scan-assembler-times "bgeui" 1 } } */
+/* { dg-final { scan-assembler-times "bltui" 1 } } */
diff --git a/gcc/testsuite/gcc.target/xtensa/pr120888-1.c b/gcc/testsuite/gcc.target/xtensa/pr120888-1.c
new file mode 100644
index 0000000..f438e4c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/xtensa/pr120888-1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-rtl-expand" } */
+
+void u8(unsigned char c);
+void cu8(unsigned char *p)
+{
+  u8(*p);
+}
+
+/* { dg-final { scan-rtl-dump "zero_extend" "expand" } } */
+/* { dg-final { scan-rtl-dump-not "sign_extend" "expand" } } */
diff --git a/gcc/testsuite/gcc.target/xtensa/pr120888-2.c b/gcc/testsuite/gcc.target/xtensa/pr120888-2.c
new file mode 100644
index 0000000..9b5caad
--- /dev/null
+++ b/gcc/testsuite/gcc.target/xtensa/pr120888-2.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-rtl-expand" } */
+
+void s8(signed char c);
+void cs8(signed char *p)
+{
+  s8(*p);
+}
+
+/* { dg-final { scan-rtl-dump "sign_extend" "expand" } } */
+/* { dg-final { scan-rtl-dump-not "zero_extend" "expand" } } */
diff --git a/gcc/testsuite/gfortran.dg/array_constructor_58.f90 b/gcc/testsuite/gfortran.dg/array_constructor_58.f90
new file mode 100644
index 0000000..1473be0
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/array_constructor_58.f90
@@ -0,0 +1,17 @@
+!{ dg-do run }
+
+! Contributed by  Federico Perini <federico.perini@gmail.com>
+! Check that PR fortran/119106 is fixed.
+
+program char_param_array
+implicit none
+character, parameter :: p(5) = ['1','2','3','4','5']
+character, save      :: n(5) = ['1','2','3','4','5']
+integer :: i(10), j
+
+i = 4
+if (any([(n(i(j)),j=1,10)] /= '4')) stop 1 ! OK
+if (any([(p(i(j)),j=1,10)] /= '4')) stop 2 ! used to runtime out-of-bounds error
+
+end program char_param_array
+
diff --git a/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90 b/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90
index 45eafac..a0c5507 100644
--- a/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90
+++ b/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90
@@ -9,6 +9,8 @@ program grow_type_array
     
     type(container), allocatable :: list(:)
 
+    allocate(list(0))
+
     list = [list, new_elem(5)]
 
     deallocate(list)
diff --git a/gcc/testsuite/gfortran.dg/asan/finalize_1.f90 b/gcc/testsuite/gfortran.dg/asan/finalize_1.f90
new file mode 100644
index 0000000..ab53a9e
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/asan/finalize_1.f90
@@ -0,0 +1,67 @@
+!{ dg-do run }
+
+! PR fortran/120637
+
+! Contributed by Antony Lewis  <antony@cosmologist.info>
+! The unused module is needed to trigger the issue of not freeing the
+! memory of second module.
+
+    module MiscUtils
+    implicit none
+
+    contains
+        
+    logical function isFloat0(R)
+    class(*), intent(in) :: R
+
+    select type(R)
+    type is (real)
+        isFloat0 = .true.
+    end select
+    end function isFloat0
+
+    end module MiscUtils
+    
+    module results3
+    implicit none
+    public
+
+    Type ClTransferData2
+      real, dimension(:,:,:), allocatable :: Delta_p_l_k
+    end type ClTransferData2
+
+    type TCLdata2
+       Type(ClTransferData2) :: CTransScal, CTransTens, CTransVec
+    end type TCLdata2 
+
+    type :: CAMBdata2
+        Type(TClData2) :: CLdata2
+    end type  
+
+    end module results3
+
+program driver
+   use results3
+   integer i
+   do i=1, 2
+   call test()   
+   end do
+
+   contains
+
+   subroutine test
+       implicit none
+       class(CAMBdata2), pointer :: Data
+
+       allocate(CAMBdata2::Data)
+
+       allocate(Data%ClData2%CTransScal%Delta_p_l_k(3, 1000, 1000)) 
+       allocate(Data%ClData2%CTransVec%Delta_p_l_k(3, 1000, 1000))
+       deallocate(Data)
+
+   end subroutine test
+
+ end program driver
+
+!{ dg-final { cleanup-modules "miscutils results3" } }
+
diff --git a/gcc/testsuite/gfortran.dg/assign_13.f90 b/gcc/testsuite/gfortran.dg/assign_13.f90
new file mode 100644
index 0000000..262ade0
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/assign_13.f90
@@ -0,0 +1,25 @@
+! { dg-do run }
+!
+! PR fortran/121185
+! The assignment to Y%X in CHECK_T was using a polymorphic array access on the
+! left hand side, using the virtual table of Y.
+
+program p
+  implicit none
+  type t
+     complex, allocatable :: x(:)
+  end type t
+  real :: trace = 2.
+  type(t) :: z
+  z%x = [1,2] * trace
+  call check_t (z)
+contains
+  subroutine check_t (y)
+    class(t) :: y
+    ! print *, y% x
+    if (any(y%x /= [2., 4.])) error stop 11
+    y%x = y%x / trace
+    ! print *, y% x
+    if (any(y%x /= [1., 2.])) error stop 12
+  end subroutine
+end
diff --git a/gcc/testsuite/gfortran.dg/assign_14.f90 b/gcc/testsuite/gfortran.dg/assign_14.f90
new file mode 100644
index 0000000..33b46b9
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/assign_14.f90
@@ -0,0 +1,24 @@
+! { dg-do compile }
+! { dg-additional-options {-fdump-tree-original} }
+!
+! PR fortran/121185
+! Check that an intermediary variable is used to reference component a.
+! { dg-final { scan-tree-dump-not {->b->a} original } }
+
+program p
+  implicit none
+  type t
+     integer, allocatable :: a(:)
+  end type t
+  type u
+     type(t), allocatable :: b
+  end type u
+  type v
+     type(u), allocatable :: c
+  end type v
+  type(v) :: z
+  z%c = u()
+  z%c%b = t()
+  z%c%b%a = [1,2]
+  z%c%b%a = z%c%b%a * 2
+end
diff --git a/gcc/testsuite/gfortran.dg/associate_75.f90 b/gcc/testsuite/gfortran.dg/associate_75.f90
new file mode 100644
index 0000000..c7c461a
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/associate_75.f90
@@ -0,0 +1,50 @@
+! { dg-do run }
+!
+! Test fix for PR121060.
+!
+! Contributed by Damian Rouson  <damian@archaeologic.codes>
+!
+module subdomain_m
+  implicit none
+
+  type subdomain_t 
+    real :: s_ = 99.
+  contains
+    generic :: operator(.laplacian.) => laplacian
+    procedure laplacian
+  end type
+
+contains
+
+  function laplacian(rhs)
+    class(subdomain_t), intent(in) :: rhs
+    type(subdomain_t) laplacian
+    laplacian%s_ = rhs%s_ + 42
+  end function
+
+end module
+
+  use subdomain_m
+  implicit none
+
+  type operands_t
+    real :: s_
+  end type
+
+  type(subdomain_t) phi
+  type(operands_t) operands
+
+  associate(laplacian_phi => .laplacian. phi) ! ICE because specific not found.
+    operands = approximates(laplacian_phi%s_)
+  end associate
+
+  if (int (operands%s_) /= 42) stop 1
+contains
+
+  function approximates(actual)
+    real actual 
+    type(operands_t) approximates
+    approximates%s_ = actual - 99
+  end function
+
+end
diff --git a/gcc/testsuite/gfortran.dg/class_elemental_1.f90 b/gcc/testsuite/gfortran.dg/class_elemental_1.f90
new file mode 100644
index 0000000..547ae98
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/class_elemental_1.f90
@@ -0,0 +1,35 @@
+! { dg-do run }
+!
+! PR fortran/121342
+! The polymorphic function result as actual argument used to force the loop
+! bounds around the elemental call, altering access to the other arrays.
+
+program p
+  implicit none
+  type :: t
+    integer :: i
+  end type
+  type :: u
+    integer :: i, a
+  end type
+  type(u) :: accum(5)
+  integer :: a(3:7), k
+  a = [ (k*k, k=1,5) ]
+  call s(accum, f(), a)
+  ! print *, accum%i
+  ! print *, accum%a
+  if (any(accum%i /= accum%a)) error stop 1
+contains
+  elemental subroutine s(l, c, a)
+    type(u)  , intent(out) :: l
+    class(t) , intent(in)  :: c
+    integer  , intent(in)  :: a
+    l%i = c%i
+    l%a = a
+  end subroutine
+  function f()
+    class(t), allocatable :: f(:)
+    allocate(f(-1:3))
+    f%i = [ (k*k, k=1,5) ]
+  end function
+end program
diff --git a/gcc/testsuite/gfortran.dg/function_charlen_4.f90 b/gcc/testsuite/gfortran.dg/function_charlen_4.f90
new file mode 100644
index 0000000..ed39aca
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/function_charlen_4.f90
@@ -0,0 +1,34 @@
+! { dg-do run }
+! { dg-options "-O2 -std=legacy -fdump-tree-optimized" }
+!
+! PR fortran/121203 - fix passing of character length of function to procedure
+
+program p
+  character(10), external :: f
+  call eval (f,"abc")
+  call eval2(f,"abc")
+contains
+  subroutine eval2(func,c_arg)
+    character(*) c_arg
+    character(*) func
+    external func
+    ! These tests should get optimized:
+    if (len      (c_arg)  /=  3) stop 1
+    if (len (func(c_arg)) /= 10) stop 2
+  end subroutine
+end
+
+character(10) function f(arg)
+  character(*) arg
+  f=arg
+end
+
+subroutine eval(func,c_arg)
+  character(*) c_arg
+  character(*) func
+  external func
+  if (len      (c_arg)  /=  3) error stop 3
+  if (len (func(c_arg)) /= 10) error stop 4
+end subroutine
+
+! { dg-final { scan-tree-dump-not "_gfortran_stop_numeric" "optimized" } }
diff --git a/gcc/testsuite/gfortran.dg/g77/980310-3.f b/gcc/testsuite/gfortran.dg/g77/980310-3.f
index 39bd86c..4bf4d91 100644
--- a/gcc/testsuite/gfortran.dg/g77/980310-3.f
+++ b/gcc/testsuite/gfortran.dg/g77/980310-3.f
@@ -12,7 +12,7 @@ C Date: Wed, 17 Dec 1997 23:20:29 +0000
 C From: Joao Cardoso <jcardoso@inescn.pt>
 C To: egcs-bugs@cygnus.com
 C Subject: egcs-1.0 f77 bug on OSR5
-C When trying to compile the Fortran file that I enclose bellow,
+C When trying to compile the Fortran file that I enclose below,
 C I got an assembler error:
 C 
 C ./g77 -B./ -fpic -O -c scaleg.f
diff --git a/gcc/testsuite/gfortran.dg/goacc/parameter-3.f90 b/gcc/testsuite/gfortran.dg/goacc/parameter-3.f90
new file mode 100644
index 0000000..2c8aa61
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/goacc/parameter-3.f90
@@ -0,0 +1,16 @@
+! { dg-do compile }
+
+subroutine x
+  integer :: var
+  integer, parameter :: ilog = 0
+  integer, parameter :: array(*) = [11,22,33]
+  !$ACC DECLARE COPYIN(ilog, array, var, array) ! { dg-error "Symbol 'array' present on multiple clauses" }
+end subroutine x
+
+integer :: a
+integer, parameter :: b = 4
+integer, parameter :: c(*) = [1,2,3]
+
+!$acc parallel copy(a,c,b,c)  ! { dg-error "Symbol 'c' present on multiple clauses" }
+!$acc end parallel
+end
diff --git a/gcc/testsuite/gfortran.dg/goacc/parameter-4.f90 b/gcc/testsuite/gfortran.dg/goacc/parameter-4.f90
new file mode 100644
index 0000000..aadd7cf
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/goacc/parameter-4.f90
@@ -0,0 +1,26 @@
+! { dg-do compile }
+! { dg-additional-options "-fdump-tree-original" }
+
+subroutine x
+  integer :: var
+  integer, parameter :: ilog = 0
+  integer, parameter :: array(*) = [11,22,33]
+  !$ACC DECLARE COPYIN(ilog, array, var)
+end subroutine x
+
+integer :: a
+integer, parameter :: b = 4
+integer, parameter :: c(*) = [1,2,3]
+
+!$acc parallel copy(a,c,b)
+  a = c(2) + b
+!$acc end parallel
+
+!$acc parallel firstprivate(a,c,b)
+  a = c(2) + b
+!$acc end parallel
+end
+
+! { dg-final { scan-tree-dump-times "#pragma acc data map\\(to:var\\)" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma acc parallel map\\(tofrom:a\\)" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma acc parallel firstprivate\\(a\\)" 1 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/goacc/parameter.f95 b/gcc/testsuite/gfortran.dg/goacc/parameter.f95
index b581338..a9bde4a 100644
--- a/gcc/testsuite/gfortran.dg/goacc/parameter.f95
+++ b/gcc/testsuite/gfortran.dg/goacc/parameter.f95
@@ -1,4 +1,5 @@
 ! { dg-do compile } 
+! { dg-additional-options "-Wsurprising" }
 
 module test
 contains
@@ -6,37 +7,37 @@ contains
     implicit none
     integer :: i
     integer, parameter :: a = 1
-    !$acc declare device_resident (a) ! { dg-error "is not a variable" }
-    !$acc data copy (a) ! { dg-error "not a variable" }
+    !$acc declare device_resident (a) ! (no warning here - for semi-good reasons)
+    !$acc data copy (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
     !$acc end data
-    !$acc data deviceptr (a) ! { dg-error "not a variable" }
+    !$acc data deviceptr (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
     !$acc end data
-    !$acc parallel private (a) ! { dg-error "not a variable" }
+    !$acc parallel private (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     !$acc end parallel
-    !$acc serial private (a) ! { dg-error "not a variable" }
+    !$acc serial private (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     !$acc end serial
-    !$acc host_data use_device (a) ! { dg-error "not a variable" }
+    !$acc host_data use_device (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     !$acc end host_data
-    !$acc parallel loop reduction(+:a) ! { dg-error "not a variable" }
+    !$acc parallel loop reduction(+:a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     do i = 1,5
     enddo
     !$acc end parallel loop
-    !$acc serial loop reduction(+:a) ! { dg-error "not a variable" }
+    !$acc serial loop reduction(+:a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     do i = 1,5
     enddo
     !$acc end serial loop
     !$acc parallel loop
     do i = 1,5
-      !$acc cache (a) ! { dg-error "not a variable" }
+      !$acc cache (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     enddo
     !$acc end parallel loop
     !$acc serial loop
     do i = 1,5
-      !$acc cache (a) ! { dg-error "not a variable" }
+      !$acc cache (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     enddo
     !$acc end serial loop
-    !$acc update device (a) ! { dg-error "not a variable" }
-    !$acc update host (a) ! { dg-error "not a variable" }
-    !$acc update self (a) ! { dg-error "not a variable" }
+    !$acc update device (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
+    !$acc update host (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
+    !$acc update self (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
   end subroutine oacc1
 end module test
diff --git a/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90 b/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90
index 5409f12..279656b 100644
--- a/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90
+++ b/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90
@@ -17,8 +17,8 @@ end
 
 ! { dg-final { scan-tree-dump-not "error_stop" "optimized" } }
 
-! { dg-final { scan-tree-dump-not "omp_get_num_devices;" "optimized" { target { ! offloading_enabled } } } }
+! { dg-final { scan-tree-dump-not "omp_get_num_devices" "optimized" { target { ! offloading_enabled } } } }
 ! { dg-final { scan-tree-dump "return 0;" "optimized" { target { ! offloading_enabled } } } }
 
-! { dg-final { scan-tree-dump-times "omp_get_num_devices;" 1 "optimized" { target offloading_enabled } } }
+! { dg-final { scan-tree-dump-times "omp_get_num_devices" 1 "optimized" { target offloading_enabled } } }
 ! { dg-final { scan-tree-dump "_1 = __builtin_omp_get_num_devices \\(\\);\[\\r\\n\]+\[ \]+return _1;" "optimized" { target offloading_enabled } } }
diff --git a/gcc/testsuite/gfortran.dg/gomp/pr104428.f90 b/gcc/testsuite/gfortran.dg/gomp/pr104428.f90
new file mode 100644
index 0000000..639b331
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/pr104428.f90
@@ -0,0 +1,15 @@
+! { dg-do compile }
+
+program p
+  interface
+    subroutine x
+    end subroutine x
+  end interface
+contains
+  subroutine foo
+    !$omp declare variant(x) match(construct={do})
+  end
+  subroutine bar
+    !$omp declare variant(y) match(construct={do}) ! { dg-error "Cannot find symbol 'y'" }
+  end
+end
diff --git a/gcc/testsuite/gfortran.dg/import12.f90 b/gcc/testsuite/gfortran.dg/import12.f90
new file mode 100644
index 0000000..df1aae6
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/import12.f90
@@ -0,0 +1,302 @@
+! { dg-do compile }
+!
+! Tests the variants of IMPORT introduced in F2018
+!
+! Contributed by Paul Thomas  <pault@gcc.gnu.org>
+!
+MODULE M
+  import, none          ! { dg-error "F2018: C897 IMPORT statement" }
+  IMPLICIT NONE
+  integer :: z
+end module
+
+MODULE N
+  IMPLICIT NONE
+  integer :: z
+end module
+
+! Taken from gfortran.dg/pr103312.f90. These F2008-style invocations should
+! be accepted.
+module example
+  type, abstract :: foo
+    integer :: i
+  contains
+    procedure(foo_size), deferred :: size
+    procedure(foo_func), deferred :: func
+  end type
+  abstract interface
+    pure integer function foo_size (this)
+      import :: foo
+      class(foo), intent(in) :: this
+    end function
+    function foo_func (this) result (string)
+      import :: foo
+      class(foo) :: this
+      character(this%size()) :: string
+    end function
+  end interface
+end module
+
+block data blk
+  import, all          ! { dg-error "F2018: C897 IMPORT statement" }
+  integer a(2)
+  common /my_common/a
+  data a/1,2/
+end
+
+subroutine extern_sub1
+  import               ! { dg-error "F2018: C897 IMPORT statement" }
+end
+
+subroutine extern_sub2 (arg1, arg2, arg3)
+  implicit none
+  integer :: arg1, arg2, arg3
+  arg1 = int_fcn ()
+contains
+  integer function int_fcn () 
+    import, only : arg2, arg3
+    int_fcn = arg2 * arg3
+  end
+end
+
+program p
+  import, all          ! { dg-error "F2018: C897 IMPORT statement" }
+  implicit none
+  integer :: x, y
+  type :: t
+    integer :: i
+  end type
+  type(t) :: progtype
+  type, extends(t) :: s
+    integer :: j
+  end type
+  class(t), allocatable :: progclass
+contains
+
+! OK because arg is just that and x is declared in scope of sub1.
+  subroutine sub1 (arg)
+    import, none
+    implicit none
+    real :: arg, x
+  end
+
+! IMPORT, ALL must be the only IMPORT statement in the scope.
+  subroutine sub2 (arg)
+    import, none
+    import, all         ! { dg-error "F2018: C8100 IMPORT statement" }
+    implicit none
+    real :: arg, x
+  end
+
+! Error message says it all.
+  subroutine sub3 (arg)
+    import, none
+    implicit none
+    integer :: arg
+    print *, arg
+    x = 1              ! { dg-error "F2018: C8102" }
+  end
+
+! Error messages say it all.
+  subroutine sub4 (arg)
+    import, only : y
+    implicit none
+    integer :: arg
+    print *, arg
+    x = 1              ! { dg-error "F2018: C8102" }
+    y = 2
+    print *, x         ! { dg-error "F2018: C8102" }
+  end
+
+! IMPORT eos and IMPORT, ALL must be unique in the scope.
+  subroutine sub5a (arg)
+    import, all
+    import             ! { dg-error "F2018: C8100" }
+    implicit none
+    real :: arg
+    real :: x          ! { dg-error "F2018: C8102" }
+  end
+
+  subroutine sub5b (arg)
+    import, only : x
+    implicit none
+    real :: arg
+    real :: x          ! { dg-error "F2018: C8102" }
+  end
+
+! Error message says it all.
+  integer function func1 ()
+    import, only : x
+    func1 = x * y      ! { dg-error "F2018: C8102" }
+  end
+
+! Error messages say it all.
+  subroutine sub6 (arg)
+    import, only : func1
+    import, only : func2
+    import, only : foobar                ! { dg-error "has no IMPLICIT type" }
+    implicit none
+    integer :: arg
+    arg = func1 () * func2 () * func3 () ! { dg-error "F2018: C8102" }
+  end
+
+! Error message says it all.
+  integer function func2 ()
+    use N
+    import, none
+    implicit none
+    func2 = y          ! { dg-error "F2018: C8102" }
+  end
+
+! OK
+  integer function func3 ()
+    func3 = 42
+  end
+
+  subroutine sub7 (arg)
+    implicit none
+    integer :: arg
+! OK
+    block
+       import, only : arg, func1, func2, func3
+       arg = func1 () * func2 () * func3 ()
+    end block
+    block
+       arg = func1 ()
+       import, only : arg, func1   ! { dg-error "Unexpected IMPORT statement" }
+    end block
+  end
+
+! Error messages say it all.
+  subroutine sub8 (arg)
+    implicit none
+    integer :: arg
+    block
+       import, only : func1
+       import, only : func2
+       import, only : foobar                ! { dg-error "has no IMPLICIT type" }
+       arg = func1 () * func2 () * func3 () ! { dg-error "F2018: C8102" }
+    end block
+  end
+
+! ASSOCIATE does not have a specification part so IMPORT cannot appear.
+  subroutine sub9 (arg)
+    implicit none
+    integer :: arg
+    associate (f3 => func3 ())              ! { dg-error "F2018: C8102" }
+       import, only : arg, func1            ! { dg-error "Unexpected IMPORT statement" }
+       arg = func1 () * func2 () * f3       ! { dg-error "F2018: C8102" }
+    end associate
+  end
+
+! OK
+  subroutine sub10 (arg)
+    import, only : t
+    implicit none
+    type(t) :: arg, mytype
+    mytype%i = 1
+    arg = mytype
+  end
+
+! TYPE t does not appear in the IMPORT list
+  subroutine sub11 (arg)
+    import, only : progtype
+    implicit none
+    type(t) :: arg
+    progtype%i = 1   ! { dg-error "F2018: C8102" }
+    arg = progtype   ! { dg-error "F2018: C8102" }
+  end
+
+! TYPE t is excluded by IMPORT, NONE
+  subroutine sub12 (arg)
+    import, none
+    implicit none
+    type(t) :: arg, mytype
+    mytype%i = 1     ! { dg-error "F2018: C8102" }
+    arg = mytype     ! { dg-error "F2018: C8102" }
+  end
+
+! TYPE t does not appear in the IMPORT list
+  subroutine sub13 (arg)
+    import, only : progclass
+    implicit none
+    class(t) :: arg
+    type(t) :: ca(2) = [t(1), t(2)]  ! { dg-error "F2018: C8102" }
+    progclass%i = t(1) ! { dg-error "F2018: C8102" }
+    arg = progclass    ! { dg-error "F2018: C8102" }
+    ca = [t(1), t(2)]  ! { dg-error "has no IMPLICIT type|F2018: C8102" }
+    arg = ca(2)        ! Note: The preceeding line catches 'ca' having no implicit type.
+  end
+
+! TYPE t is excluded by IMPORT, NONE
+  subroutine sub14 (arg)
+    import, none
+    implicit none
+    class(t) :: arg
+    class(t), allocatable ::  myclass
+    myclass%i =  t(1)  ! { dg-error "F2018: C8102" }
+    arg%i = myclass%i  ! { dg-error "F2018: C8102" }
+    select type (arg)  ! { dg-error "F2018: C8102" }
+      type is (t)
+        arg%i = arg%i + 1
+      type is (s)
+        arg%j = -1
+    end select
+  end
+
+! TYPE s does not appear in the IMPORT, ONLY list
+  subroutine sub15 (arg)
+    import, only : t
+    implicit none
+    class(t) :: arg
+    class(t), allocatable ::  myclass
+    myclass =  t(1)
+    arg%i = myclass%i
+    select type (arg)  ! { dg-error "F2018: C8102" }
+      type is (t)
+        arg%i = arg%i + 1
+      type is (s)
+        arg%j = -1     ! s is caught at the SELECT TYPE statement
+    end select
+  end
+
+! This is OK
+  subroutine sub16 (arg)
+    import, only : t, s
+    implicit none
+    class(t) :: arg
+    class(t), allocatable ::  myclass
+    myclass =  t(1)
+    arg%i = myclass%i
+    select type (arg)
+      type is (t)
+        arg%i = arg%i + 1
+      type is (s)
+        arg%j = -1
+    end select
+  end
+
+  subroutine sub17 (arg)
+    import, only : t
+    implicit none
+    class(t) :: arg
+    call sub16 (arg)  ! { dg-error "F2018: C8102" }
+  end
+
+! Make sure that recursive procedures do not require the procedure itself to be imported.
+  recursive subroutine sub18 (arg)
+    import, none
+    implicit none
+    integer :: arg
+    if (arg <= 0) call sub18 (arg)
+    arg = 1
+  end
+
+  recursive integer function func4 (arg) result (res)
+    import, none
+    implicit none
+    integer :: arg
+    if (arg <= 0) arg = func4 (arg)
+    res = 1
+  end
+end
diff --git a/gcc/testsuite/gfortran.dg/import13.f90 b/gcc/testsuite/gfortran.dg/import13.f90
new file mode 100644
index 0000000..3bcfec3
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/import13.f90
@@ -0,0 +1,21 @@
+! { dg-do compile }
+!
+! Contributed by Steve Kargl  <sgk@troutmask.apl.washington.edu>
+!
+program foo
+   implicit none
+   integer i
+   i = 42
+   if (i /= 42) stop 1
+   call bah
+   contains
+      subroutine bah       ! { dg-error "is already defined at" }
+         i = 43
+         if (i /= 43) stop 2
+      end subroutine bah
+      subroutine bah       ! { dg-error "is already defined at" }
+         ! import statement missing a comma
+         import none       ! { dg-error "Unexpected IMPORT statement" }
+         i = 44            ! { dg-error "Unexpected assignment" }
+      end subroutine bah   ! { dg-error "Expecting END PROGRAM" }
+end program foo
diff --git a/gcc/testsuite/gfortran.dg/import3.f90 b/gcc/testsuite/gfortran.dg/import3.f90
index 74cd527..9288c6b 100644
--- a/gcc/testsuite/gfortran.dg/import3.f90
+++ b/gcc/testsuite/gfortran.dg/import3.f90
@@ -1,6 +1,8 @@
 ! { dg-do compile }
+! { dg-options "-std=f2008" }
 ! { dg-shouldfail "Invalid use of IMPORT" }
 ! Test invalid uses of import
+! Wording of some error messages change for -std>=F2018 but all are caught.
 ! PR fortran/29601
 
 subroutine test()
diff --git a/gcc/testsuite/gfortran.dg/move_alloc_20.f03 b/gcc/testsuite/gfortran.dg/move_alloc_20.f03
new file mode 100644
index 0000000..20403c3
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/move_alloc_20.f03
@@ -0,0 +1,151 @@
+! { dg-do run }
+!
+! Check the presence of the pre and post code of the FROM and TO arguments
+! of the MOVE_ALLOC intrinsic subroutine.
+
+module m
+  implicit none
+  type :: t
+    integer, allocatable :: a(:)
+  end type
+end module 
+
+module pre
+  use m
+  implicit none
+  private
+  public :: check_pre
+
+contains
+
+  subroutine check_pre
+    integer, parameter :: n = 5
+    type(t) :: x(n)
+    integer, allocatable :: tmp(:)
+    integer :: array(4) = [ -1, 0, 1, 2 ]
+    integer :: i
+
+    if (allocated(tmp)) error stop 1
+
+    tmp = [17]
+
+    if (.not. allocated(tmp)) error stop 11
+    if (any(shape(tmp) /= [1])) error stop 12
+    if (any(tmp /= [17])) error stop 13
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 14
+    end do
+
+    ! Check that the index of X is properly computed for the evaluation of TO.
+    call move_alloc(tmp, x(sum(array))%a)
+
+    do i=1,n
+      if (i == 2) cycle
+      if (allocated(x(i)%a)) error stop 21
+    end do
+    if (.not. allocated(x(2)%a)) error stop 22
+    if (any(shape(x(2)%a) /= [1])) error stop 23
+    if (any(x(2)%a /= [17])) error stop 24
+    if (allocated(tmp)) error stop 25
+
+    ! Check that the index of X is properly computed for the evaluation of FROM.
+    call move_alloc(x(sum(array))%a, tmp)
+
+    if (.not. allocated(tmp)) error stop 31
+    if (any(shape(tmp) /= [1])) error stop 32
+    if (any(tmp /= [17])) error stop 33
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 34
+    end do
+  end subroutine
+
+end module
+
+module post
+  use m
+  implicit none
+  private
+  public :: check_post
+  integer, parameter :: n = 5
+  type(t), target :: x(n)
+  type :: u
+    integer :: a
+  contains
+    final :: finalize
+  end type
+  integer :: finalization_count = 0
+
+contains
+
+  function idx(arg)
+    type(u) :: arg
+    integer :: idx
+    idx = mod(arg%a, n)
+  end function
+
+  subroutine check_post
+    type(u) :: y
+    integer, allocatable :: tmp(:)
+    integer, target :: array(4) = [ -1, 0, 1, 2 ]
+    integer :: i
+
+    y%a = 12
+
+    if (allocated(tmp)) error stop 1
+
+    tmp = [37]
+
+    if (.not. allocated(tmp)) error stop 11
+    if (any(shape(tmp) /= [1])) error stop 12
+    if (any(tmp /= [37])) error stop 13
+    if (finalization_count /= 0) error stop 14
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 15
+    end do
+
+    ! Check that the cleanup code for the evaluation of TO is properly
+    ! executed after MOVE_ALLOC: the result of GET_U should be finalized.
+    call move_alloc(tmp, x(idx(get_u(y)))%a)
+
+    do i=1,n
+      if (i == 2) cycle
+      if (allocated(x(i)%a)) error stop 21
+    end do
+    if (.not. allocated(x(2)%a)) error stop 22
+    if (any(shape(x(2)%a) /= [1])) error stop 23
+    if (any(x(2)%a /= [37])) error stop 24
+    if (allocated(tmp)) error stop 25
+    if (finalization_count /= 1) error stop 26
+
+    ! Check that the cleanup code for the evaluation of FROM is properly
+    ! executed after MOVE_ALLOC: the result of GET_U should be finalized.
+    call move_alloc(x(idx(get_u(y)))%a, tmp)
+
+    if (.not. allocated(tmp)) error stop 31
+    if (any(shape(tmp) /= [1])) error stop 32
+    if (any(tmp /= [37])) error stop 33
+    if (finalization_count /= 2) error stop 34
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 35
+    end do
+  end subroutine
+
+  function get_u(arg)
+    type(u) :: arg, get_u
+    get_u = arg
+  end function get_u
+
+  subroutine finalize(obj)
+    type(u) :: obj
+    finalization_count = finalization_count + 1
+  end subroutine
+
+end module
+
+program p
+  use pre
+  use post
+  implicit none
+  call check_pre
+  call check_post
+end program
diff --git a/gcc/testsuite/gfortran.dg/pointer_check_15.f90 b/gcc/testsuite/gfortran.dg/pointer_check_15.f90
new file mode 100644
index 0000000..13c6820
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pointer_check_15.f90
@@ -0,0 +1,46 @@
+! { dg-do run }
+! { dg-additional-options "-O -fcheck=pointer -fdump-tree-original" }
+!
+! PR fortran/121145
+! Erroneous runtime error: Proc-pointer actual argument 'ptr' is not associated
+!
+! Contributed by Federico Perini.
+
+module m
+  implicit none
+
+  abstract interface
+     subroutine fun(x)
+       real, intent(in) :: x
+     end subroutine fun
+  end interface   
+
+contains
+
+  subroutine with_fun(sub)
+    procedure(fun), optional :: sub
+    if (present(sub)) stop 1
+  end subroutine   
+
+  subroutine with_non_optional(sub)
+    procedure(fun) :: sub
+  end subroutine   
+
+end module m
+
+program p
+  use m
+  implicit none
+
+  procedure(fun), pointer :: ptr1 => null()
+  procedure(fun), pointer :: ptr2 => null()
+  
+  call with_fun()
+  call with_fun(sub=ptr1)               ! no runtime check here
+
+  if (associated (ptr2)) then
+     call with_non_optional(sub=ptr2)   ! runtime check here
+  end if
+end  
+
+! { dg-final { scan-tree-dump-times "Proc-pointer actual argument .'ptr2.'" 1 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/split_1.f90 b/gcc/testsuite/gfortran.dg/split_1.f90
new file mode 100644
index 0000000..21659b0
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_1.f90
@@ -0,0 +1,28 @@
+! { dg-do run }
+program b
+  character(len=:), allocatable :: input
+  character(len=2) :: set = ', '
+  integer :: p
+  input = " one,last example,"
+  p = 0
+
+  call split(input, set, p)
+  if (p /= 1) STOP 1
+  call split(input, set, p)
+  if (p /= 5) STOP 2
+  call split(input, set, p)
+  if (p /= 10) STOP 3
+  call split(input, set, p)
+  if (p /= 18) STOP 4
+  call split(input, set, p)
+  if (p /= 19) STOP 5
+
+  call split(input, set, p, .true.)
+  if (p /= 18) STOP 6
+  call split(input, set, p, .true.)
+  if (p /= 10) STOP 7
+  call split(input, set, p, .true.)
+  if (p /= 5) STOP 8
+  call split(input, set, p, .true.)
+  if (p /= 1) STOP 9
+end program b
diff --git a/gcc/testsuite/gfortran.dg/split_2.f90 b/gcc/testsuite/gfortran.dg/split_2.f90
new file mode 100644
index 0000000..9afb30b
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_2.f90
@@ -0,0 +1,22 @@
+! { dg-do run }
+program b
+  integer, parameter :: ucs4 = selected_char_kind('ISO_10646')
+  character(kind=ucs4, len=:), allocatable :: input, set
+  integer :: p = 0
+
+  input = char(int(z'4f60'), ucs4) // char(int(z'597d'), ucs4) // char(int(z'4f60'), ucs4) // char(int(z'4e16'), ucs4)
+  set = char(int(z'597d'), ucs4) // char(int(z'4e16'), ucs4)
+
+  call split(input, set, p)
+  if (p /= 2) stop 1
+  call split(input, set, p)
+  if (p /= 4) stop 2
+  call split(input, set, p)
+  if (p /= 5) stop 3
+  call split(input, set, p, .true.)
+  if (p /= 4) stop 4
+  call split(input, set, p, .true.)
+  if (p /= 2) stop 5
+  call split(input, set, p, .true.)
+  if (p /= 0) stop 6
+end program b
diff --git a/gcc/testsuite/gfortran.dg/split_3.f90 b/gcc/testsuite/gfortran.dg/split_3.f90
new file mode 100644
index 0000000..bec3fdc
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_3.f90
@@ -0,0 +1,11 @@
+! { dg-do run }
+! { dg-shouldfail "Fortran runtime error" }
+
+program b
+  character(len=:), allocatable :: input
+  character(len=2) :: set = ', '
+  integer :: p
+  input = " one,last example,"
+  p = -1
+  call split(input, set, p)
+end program b
diff --git a/gcc/testsuite/gfortran.dg/split_4.f90 b/gcc/testsuite/gfortran.dg/split_4.f90
new file mode 100644
index 0000000..a3c27bb
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_4.f90
@@ -0,0 +1,11 @@
+! { dg-do run }
+! { dg-shouldfail "Fortran runtime error" }
+
+program b
+  character(len=:), allocatable :: input
+  character(len=2) :: set = ', '
+  integer :: p
+  input = " one,last example,"
+  p = 0
+  call split(input, set, p, .true.)
+end program b
diff --git a/gcc/testsuite/gm2/errors/fail/badindrtype.mod b/gcc/testsuite/gm2/errors/fail/badindrtype.mod
new file mode 100644
index 0000000..b393027
--- /dev/null
+++ b/gcc/testsuite/gm2/errors/fail/badindrtype.mod
@@ -0,0 +1,16 @@
+MODULE badindrtype ;
+
+
+PROCEDURE init (VAR ch: CHAR) ;
+VAR
+   c: CARDINAL ;
+BEGIN
+   ch := c
+END init ;
+
+
+VAR
+   ch: CHAR ;
+BEGIN
+   init (ch)
+END badindrtype.
diff --git a/gcc/testsuite/gm2/errors/fail/badindrtype2.mod b/gcc/testsuite/gm2/errors/fail/badindrtype2.mod
new file mode 100644
index 0000000..a31303b
--- /dev/null
+++ b/gcc/testsuite/gm2/errors/fail/badindrtype2.mod
@@ -0,0 +1,16 @@
+MODULE badindrtype2 ;
+
+
+PROCEDURE init (VAR ch: CHAR) ;
+VAR
+   c: CARDINAL ;
+BEGIN
+   c := ch
+END init ;
+
+
+VAR
+   ch: CHAR ;
+BEGIN
+   init (ch)
+END badindrtype2.
diff --git a/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.def b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.def
new file mode 100644
index 0000000..a24f7d3
--- /dev/null
+++ b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.def
@@ -0,0 +1,12 @@
+DEFINITION MODULE arrayofchar ;
+
+FROM FIO IMPORT File ;
+
+(*
+   Description: provides write procedures for ARRAY OF CHAR.
+*)
+
+PROCEDURE Write (f: File; str: ARRAY OF CHAR) ;
+PROCEDURE WriteLn (f: File) ;
+
+END arrayofchar.
diff --git a/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.mod b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.mod
new file mode 100644
index 0000000..4e630a9
--- /dev/null
+++ b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.mod
@@ -0,0 +1,30 @@
+IMPLEMENTATION MODULE arrayofchar ;
+
+FROM FIO IMPORT WriteChar, WriteLine ;
+IMPORT StrLib ;
+
+
+(*
+   Write - writes a string to file f.
+*)
+
+PROCEDURE Write (f: File; a: ARRAY OF CHAR) ;
+VAR
+   len, i: CARDINAL ;
+BEGIN
+   len := StrLib.StrLen (a) ;
+   i := 0 ;
+   WHILE i < len DO
+      WriteChar (f, a[i]) ;
+      INC (i)
+   END
+END Write ;
+
+
+PROCEDURE WriteLn (f: File) ;
+BEGIN
+   WriteLine (f)
+END WriteLn ;
+
+
+END arrayofchar.
diff --git a/gcc/testsuite/gm2/warnings/style/fail/badvarname.mod b/gcc/testsuite/gm2/warnings/style/fail/badvarname.mod
new file mode 100644
index 0000000..e589b0d
--- /dev/null
+++ b/gcc/testsuite/gm2/warnings/style/fail/badvarname.mod
@@ -0,0 +1,14 @@
+MODULE badvarname ;
+
+
+PROCEDURE Foo ;
+VAR
+   end: CARDINAL ;
+BEGIN
+   end := 1
+END Foo ;
+
+
+BEGIN
+   Foo
+END badvarname.
diff --git a/gcc/testsuite/gm2/warnings/style/fail/warnings-style-fail.exp b/gcc/testsuite/gm2/warnings/style/fail/warnings-style-fail.exp
new file mode 100644
index 0000000..f44ed80
--- /dev/null
+++ b/gcc/testsuite/gm2/warnings/style/fail/warnings-style-fail.exp
@@ -0,0 +1,44 @@
+# Expect driver script for GCC Regression Tests
+# Copyright (C) 2025 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+# This file was written by Gaius Mulley (gaius.mulley@southwales.ac.uk)
+# for GNU Modula-2.
+
+if $tracelevel then {
+    strace $tracelevel
+}
+
+# load support procs
+load_lib gm2-torture.exp
+
+gm2_init_pim "${srcdir}/gm2/warnings/style/fail"
+
+global TORTURE_OPTIONS
+
+set old_options $TORTURE_OPTIONS
+set TORTURE_OPTIONS { { -O0 -g -Werror=style } }
+
+foreach testcase [lsort [glob -nocomplain $srcdir/$subdir/*.mod]] {
+    # If we're only testing specific files and this isn't one of them, skip it.
+    if ![runtest_file_p $runtests $testcase] then {
+	continue
+    }
+
+    gm2-torture-fail $testcase
+}
+
+set TORTURE_OPTIONS $old_options
diff --git a/gcc/testsuite/gnat.dg/deref4.adb b/gcc/testsuite/gnat.dg/deref4.adb
new file mode 100644
index 0000000..586a6186
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/deref4.adb
@@ -0,0 +1,9 @@
+-- { dg-do compile }
+-- { dg-options "-gnatX" }
+
+with Deref4_Pkg; use Deref4_Pkg;
+
+procedure Deref4 is
+begin
+  Obj.Proc (null);
+end;
diff --git a/gcc/testsuite/gnat.dg/deref4_pkg.ads b/gcc/testsuite/gnat.dg/deref4_pkg.ads
new file mode 100644
index 0000000..9410d0d
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/deref4_pkg.ads
@@ -0,0 +1,8 @@
+package Deref4_Pkg is
+
+  type A is tagged null record;
+  type A_Ptr is access A;
+  procedure Proc (This : in out A'Class; Some_Parameter : A_Ptr) is null;
+  Obj : A_Ptr;
+
+end Deref4_Pkg;
diff --git a/gcc/testsuite/jit.dg/test-debuginfo.c b/gcc/testsuite/jit.dg/test-debuginfo.c
index 49e8834..e0d6f2d 100644
--- a/gcc/testsuite/jit.dg/test-debuginfo.c
+++ b/gcc/testsuite/jit.dg/test-debuginfo.c
@@ -1,5 +1,5 @@
 /* Essentially this test checks that debug info are generated for globals
-   locals and functions, including type info. The comment bellow is used
+   locals and functions, including type info. The comment below is used
    as fake code (does not affect the test, use for manual debugging). */
 /*
 int a_global_for_test_debuginfo;
diff --git a/gcc/testsuite/lib/gcc-defs.exp b/gcc/testsuite/lib/gcc-defs.exp
index 2f8b7d4..d66c833 100644
--- a/gcc/testsuite/lib/gcc-defs.exp
+++ b/gcc/testsuite/lib/gcc-defs.exp
@@ -599,15 +599,16 @@ proc aarch64-arch-dg-options { args } {
 
     set add_arch 1
     set add_tune 1
+    set add_override 1
     set checks_output [string equal [lindex $do_what 0] "compile"]
     set options [lindex $args 1]
 
     foreach option [split $options] {
 	switch -glob -- $option {
 	    -march=* { set add_arch 0 }
-	    -mcpu=* { set add_arch 0; set add_tune 0 }
-	    -mtune=* { set add_tune 0 }
-	    -moverride=* { set add_tune 0 }
+	    -mcpu=* { set add_arch 0; set add_tune 0; set add_override 0}
+	    -mtune=* { set add_tune 0; set add_override 0 }
+	    -moverride=* { set add_override 0 }
 	    -save-temps { set checks_output 1 }
 	    --save-temps { set checks_output 1 }
 	    -fdump* { set checks_output 1 }
@@ -619,9 +620,14 @@ proc aarch64-arch-dg-options { args } {
 	append options " $aarch64_default_testing_arch"
     }
 
-    if { $add_tune && $checks_output } {
+    if { $checks_output } {
 	# Turn off any default tuning and codegen tweaks.
-	append options " -mtune=generic -moverride=tune=none"
+        if { $add_tune } {
+            append options " -mtune=generic"
+        }
+        if { $add_override } {
+            append options " -moverride=tune=none"
+        }
     }
 
     uplevel 1 aarch64-old-dg-options [lreplace $args 1 1 $options]
diff --git a/gcc/testsuite/lib/gcc-dg.exp b/gcc/testsuite/lib/gcc-dg.exp
index 312c4b8..859a943 100644
--- a/gcc/testsuite/lib/gcc-dg.exp
+++ b/gcc/testsuite/lib/gcc-dg.exp
@@ -1338,8 +1338,8 @@ proc dg-missed { args } {
 }
 
 # Look for messages with 'note: ' prefixes.
-# In addition to standard compiler diagnostics ('DK_NOTE', 'inform' functions,
-# "for additional details on an error message"),
+# In addition to standard compiler diagnostics (diagnostics::kind::note,
+# 'inform' functions, "for additional details on an error message"),
 # this also includes output from '-fopt-info' for 'MSG_NOTE':
 # a general optimization info.
 # By default, any *excess* notes are pruned, meaning their appearance doesn't
diff --git a/gcc/testsuite/lib/profopt.exp b/gcc/testsuite/lib/profopt.exp
index b4d244b..81d86c6 100644
--- a/gcc/testsuite/lib/profopt.exp
+++ b/gcc/testsuite/lib/profopt.exp
@@ -382,6 +382,7 @@ proc profopt-execute { src } {
 	    unsupported "$testcase"
 	    unset testname_with_flags
 	    verbose "$src not supported on this target, skipping it" 3
+	    cleanup-after-saved-dg-test
 	    return
 	}
 
@@ -458,6 +459,7 @@ proc profopt-execute { src } {
 		    unsupported "$testcase -fauto-profile: cannot run create_gcov"
 		    unset testname_with_flags
 		    set status "fail"
+		    cleanup-after-saved-dg-test
 		    return
 		}
 		set status [remote_wait "" 300]
diff --git a/gcc/testsuite/lib/rust.exp b/gcc/testsuite/lib/rust.exp
index 9513e1c..692030c 100644
--- a/gcc/testsuite/lib/rust.exp
+++ b/gcc/testsuite/lib/rust.exp
@@ -168,10 +168,7 @@ proc rust_target_compile { source dest type options } {
     global gluefile wrap_flags
     global ALWAYS_RUSTFLAGS
     global RUST_UNDER_TEST
-    global individual_timeout
-
-    # HACK: guard against infinite loops in the compiler
-    set individual_timeout 10
+    
 
     if { [target_info needs_status_wrapper] != "" && [info exists gluefile] } {
 	lappend options "libs=${gluefile}"
diff --git a/gcc/testsuite/lib/sarif.py b/gcc/testsuite/lib/sarif.py
index 384de2f..d75a87e 100644
--- a/gcc/testsuite/lib/sarif.py
+++ b/gcc/testsuite/lib/sarif.py
@@ -29,10 +29,24 @@ def get_result_by_index(sarif, idx):
     results = run['results']
     return results[idx]
     
-def get_xml_state(events, event_idx):
-    xml_src = events[event_idx]['properties']['gcc/diagnostic_event/xml_state']
+def get_state_graph(events, event_idx):
+    graph = events[event_idx]['properties']['gcc/diagnostics/paths/event/state_graph']
     if 0:
-        print(xml_src)
-    xml = ET.fromstring(xml_src)
-    assert xml.tag == 'state-diagram'
-    return xml    
+        print(graph)
+    assert graph is not None
+    return graph
+
+def get_state_node_attr(obj, attr_name):
+    return obj['properties']['gcc/diagnostic_state_node/%s' % attr_name]
+
+def get_state_node_kind(obj):
+    return get_state_node_attr(obj, 'kind')
+
+def get_state_node_name(obj):
+    return get_state_node_attr(obj, 'name')
+
+def get_state_node_type(obj):
+    return get_state_node_attr(obj, 'type')
+
+def get_state_node_value(obj):
+    return get_state_node_attr(obj, 'value')
diff --git a/gcc/testsuite/lib/scanasm.exp b/gcc/testsuite/lib/scanasm.exp
index 97935cb..51952a6 100644
--- a/gcc/testsuite/lib/scanasm.exp
+++ b/gcc/testsuite/lib/scanasm.exp
@@ -896,6 +896,10 @@ proc configure_check-function-bodies { config } {
 	set up_config(fluff) {^\s*(?://)}
     } elseif { [istarget *-*-darwin*] } {
 	set up_config(fluff) {^\s*(?:\.|//|@)|^L[0-9ABCESV]}
+    } elseif { [istarget s390*-*-*] } {
+	# Additionally to the defaults skip lines beginning with a # resulting
+	# from inline asm.
+	set up_config(fluff) {^\s*(?:\.|//|@|$|#)}
     } else {
 	# Skip lines beginning with labels ('.L[...]:') or other directives
 	# ('.align', '.cfi_startproc', '.quad [...]', '.text', etc.), '//' or
@@ -1109,6 +1113,8 @@ proc check-function-bodies { args } {
 		append function_regexp ".*"
 	    } elseif { [regexp {^\.L} $line] } {
 		append function_regexp $line "\n"
+	    } elseif { [regexp {^[0-9]+:} $line] } {
+		append function_regexp $line "\n"
 	    } else {
 		append function_regexp $config(line_prefix) $line "\n"
 	    }
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 956bc0b..7435519 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -1017,6 +1017,18 @@ proc check_effective_target_label_values {} {
     return 1
 }
 
+
+# Return 1 if builtin_trap expands not into a call but an instruction,
+# 0 otherwise.
+proc check_effective_target_trap { } {
+    return [check_no_messages_and_pattern trap "!\\(call" rtl-expand {
+	void foo ()
+	  {
+	    return __builtin_trap ();
+	  }
+    } "" ]
+}
+
 # Return 1 if builtin_return_address and builtin_frame_address are
 # supported, 0 otherwise.
 
@@ -2428,7 +2440,7 @@ proc check_effective_target_riscv_v_misalign_ok { } {
 		= {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
 	      asm ("vsetivli zero,7,e8,m1,ta,ma");
 	      asm ("addi a7,%0,1" : : "r" (a) : "a7" );
-	      asm ("vle8.v v8,0(a7)" : : : "v8");
+	      asm ("vle16.v v8,0(a7)" : : : "v8");
 	      return 0; } } "-march=${gcc_march}"] } {
 	return 1
     }
@@ -2685,6 +2697,14 @@ proc remove_options_for_riscv_zvbb { flags } {
     return [add_options_for_riscv_z_ext zvbb $flags]
 }
 
+proc add_options_for_riscv_zvfh { flags } {
+    return [add_options_for_riscv_z_ext zvfh $flags]
+}
+
+proc remove_options_for_riscv_zvfh { flags } {
+    return [add_options_for_riscv_z_ext zvfh $flags]
+}
+
 # Return 1 if the target is ia32 or x86_64.
 
 proc check_effective_target_x86 { } {
@@ -5800,6 +5820,13 @@ proc add_options_for_aarch64_sve { flags } {
     return "$flags -march=armv8.2-a+sve"
 }
 
+proc add_options_for_aarch64_sme { flags } {
+    if { ![istarget aarch64*-*-*] || [check_effective_target_aarch64_sme] } {
+	return "$flags"
+    }
+    return "$flags -march=armv9-a+sme"
+}
+
 # Return 1 if this is an ARM target supporting the FP16 alternative
 # format.  Some multilibs may be incompatible with the options needed.  Also
 # set et_arm_fp16_alternative_flags to the best options to add.
@@ -6491,6 +6518,23 @@ proc check_effective_target_aarch64_sve2_hw { } {
     }]
 }
 
+# Return true if this is an AArch64 target that can run SVE2.1 code.
+
+proc check_effective_target_aarch64_sve2p1_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+	return 0
+    }
+    return [check_runtime aarch64_sve2p1_hw_available {
+	#pragma GCC target "+sve2p1"
+	int
+	main (void)
+	{
+	  asm volatile ("dupq z0.b, z0.b[0]");
+	  return 0;
+	}
+    }]
+}
+
 # Return true if this is an AArch64 target that can run SVE code and
 # if its SVE vectors have exactly BITS bits.
 
@@ -6522,6 +6566,22 @@ foreach N { 128 256 512 1024 2048 } {
     }]
 }
 
+# Return true if this is an AArch64 target that can run SME code.
+
+proc check_effective_target_aarch64_sme_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+	return 0
+    }
+    return [check_runtime aarch64_sme_hw_available {
+	int
+	main (void)
+	{
+	  asm volatile ("rdsvl x0, #1");
+	  return 0;
+	}
+    } [add_options_for_aarch64_sme ""]]
+}
+
 proc check_effective_target_arm_neonv2_hw { } {
     return [check_runtime arm_neon_hwv2_available {
 	#include "arm_neon.h"
@@ -9944,7 +10004,8 @@ proc check_effective_target_vect_logical_reduc { } {
 		   || [istarget amdgcn-*-*]
 		   || [check_effective_target_riscv_v]
 		   || [check_effective_target_loongarch_sx]
-		   || [check_effective_target_x86]}]
+		   || [check_effective_target_x86]
+		   || [check_effective_target_s390_vx]}]
 }
 
 # Return 1 if the target supports the fold_extract_last optab.
@@ -12461,10 +12522,16 @@ proc check_effective_target_aarch64_gas_has_build_attributes { } {
 # various architecture extensions via the .arch_extension pseudo-op.
 
 set exts {
-    "bf16" "cmpbr" "crc" "crypto" "dotprod" "f32mm" "f64mm" "fp" "fp8"
-    "fp8dot2" "fp8dot4" "fp8fma" "i8mm" "ls64" "lse" "lut" "sb" "simd"
-    "sme-b16b16" "sme-f16f16" "sme-i16i64" "sme" "sme2" "sme2p1" "ssve-fp8dot2"
-    "ssve-fp8dot4" "ssve-fp8fma" "sve-b16b16" "sve" "sve2"
+    "bf16" "cmpbr" "crc" "crypto" "dotprod" "f32mm" "f64mm" "faminmax"
+    "fp" "fp8" "fp8dot2" "fp8dot4" "fp8fma" "i8mm" "ls64" "lse" "lut"
+    "sb" "simd" "sve-b16b16" "sve" "sve2"
+}
+
+# We don't support SME without SVE2, so we'll use armv9 as the base
+# archiecture for SME and the features that require it.
+set exts_sve2 {
+    "sme-b16b16" "sme-f16f16" "sme-i16i64" "sme" "sme2" "sme2p1"
+    "ssve-fp8dot2" "ssve-fp8dot4" "ssve-fp8fma"
 }
 
 foreach { aarch64_ext } $exts {
@@ -12481,6 +12548,20 @@ foreach { aarch64_ext } $exts {
     }]
 }
 
+foreach { aarch64_ext } $exts_sve2 {
+    eval [string map [list FUNC $aarch64_ext] {
+	proc check_effective_target_aarch64_asm_FUNC_ok { } {
+	  if { [istarget aarch64*-*-*] } {
+		return [check_no_compiler_messages aarch64_FUNC_assembler object {
+			__asm__ (".arch_extension FUNC");
+		} "-march=armv9-a+FUNC"]
+	  } else {
+		return 0
+	  }
+	}
+    }]
+}
+
 proc check_effective_target_aarch64_asm_sve2p1_ok { } {
     if { [istarget aarch64*-*-*] } {
 	return [check_no_compiler_messages aarch64_sve2p1_assembler object {
@@ -14511,3 +14592,51 @@ proc check_effective_target_foldable_pi_based_trigonometry { } {
 	}
     }]
 }
+#
+# Return 1 if the x86-64 target enables -mfentry by default, 0
+# otherwise.  Cache the result.
+
+proc check_effective_target_fentry { } {
+    global tool
+    global GCC_UNDER_TEST
+
+    if { ![check_effective_target_x86] } {
+	return 0
+    }
+
+    # Need auto-host.h to check linker support.
+    if { ![file exists ../../auto-host.h ] } {
+	return 0
+    }
+
+    return [check_cached_effective_target fentry {
+	# Set up and compile to see if ENABLE_X86_64_MFENTRY is
+	# non-zero.  Include the current process ID in the file
+	# names to prevent conflicts with invocations for multiple
+	# testsuites.
+
+	set src pie[pid].c
+	set obj pie[pid].o
+
+	set f [open $src "w"]
+	puts $f "#include \"../../auto-host.h\""
+	puts $f "#if ENABLE_X86_64_MFENTRY == 0 || !defined __x86_64__"
+	puts $f "# error -mfentry is not enabled by default."
+	puts $f "#endif"
+	close $f
+
+	verbose "check_effective_target_fentry compiling testfile $src" 2
+	set lines [${tool}_target_compile $src $obj object ""]
+
+	file delete $src
+	file delete $obj
+
+	if [string match "" $lines] then {
+	    verbose "check_effective_target_fentry testfile compilation passed" 2
+	    return 1
+	} else {
+	    verbose "check_effective_target_fentry testfile compilation failed" 2
+	    return 0
+	}
+    }]
+}
diff --git a/gcc/testsuite/libgdiagnostics.dg/sarif.py b/gcc/testsuite/libgdiagnostics.dg/sarif.py
deleted file mode 100644
index 7daf35b..0000000
--- a/gcc/testsuite/libgdiagnostics.dg/sarif.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import json
-import os
-
-def sarif_from_env():
-    # return parsed JSON content a SARIF_PATH file
-    json_filename = os.environ['SARIF_PATH']
-    json_filename += '.sarif'
-    print('json_filename: %r' % json_filename)
-    with open(json_filename) as f:
-        json_data = f.read()
-    return json.loads(json_data)
-
-def get_location_artifact_uri(location):
-    return location['physicalLocation']['artifactLocation']['uri']
-
-def get_location_physical_region(location):
-    return location['physicalLocation']['region']
-
-def get_location_snippet_text(location):
-    return location['physicalLocation']['contextRegion']['snippet']['text']
-
-def get_location_relationships(location):
-    return location['relationships']
diff --git a/gcc/testsuite/libgdiagnostics.dg/test-message-buffer-c.py b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer-c.py
new file mode 100644
index 0000000..9d14b9a
--- /dev/null
+++ b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer-c.py
@@ -0,0 +1,12 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_message_in_generated_sarif(sarif):
+    result = get_result_by_index(sarif, 0)
+    assert result['level'] == 'error'
+    assert result['message']['text'] == "this is a string; foo; int: 42 str: mostly harmless; [this is a link](https://example.com/) 'this is quoted' highlight A highlight B (1)."
diff --git a/gcc/testsuite/libgdiagnostics.dg/test-message-buffer.c b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer.c
new file mode 100644
index 0000000..a958fc5
--- /dev/null
+++ b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer.c
@@ -0,0 +1,80 @@
+/* Example of using a message buffer to build the text of a diagnostic
+   in pieces before emitting it.  */
+
+#include "libgdiagnostics.h"
+#include "test-helpers.h"
+
+int
+main ()
+{
+  begin_test ("test-message-buffer.c.exe",
+	      "test-message-buffer.c.sarif",
+	      __FILE__, "c");
+
+  diagnostic_event_id event_id = 0;
+
+  /* begin quoted source */
+  diagnostic *d = diagnostic_begin (diag_mgr,
+				    DIAGNOSTIC_LEVEL_ERROR);
+
+  diagnostic_message_buffer *msg_buf = diagnostic_message_buffer_new ();
+
+  /* Add a null-terminated string.  */
+  diagnostic_message_buffer_append_str (msg_buf, "this is a string; ");
+
+  /* Add a length-specified string.  */
+  diagnostic_message_buffer_append_text (msg_buf, "foobar", 3);
+
+  /* "printf"-formatting.  */
+  diagnostic_message_buffer_append_printf (msg_buf,
+					   "; int: %i str: %s; ",
+					   42, "mostly harmless");
+
+  /* Adding a URL.  */
+  diagnostic_message_buffer_begin_url (msg_buf, "https://example.com/");
+  diagnostic_message_buffer_append_str (msg_buf, "this is a link");  
+  diagnostic_message_buffer_end_url (msg_buf);
+
+  diagnostic_message_buffer_append_str (msg_buf, " ");  
+
+  /* Add quoted text.  */
+  diagnostic_message_buffer_begin_quote (msg_buf);
+  diagnostic_message_buffer_append_str (msg_buf, "this is quoted");  
+  diagnostic_message_buffer_end_quote (msg_buf);
+  
+  diagnostic_message_buffer_append_str (msg_buf, " ");  
+
+  /* Add colorized text.  */
+  diagnostic_message_buffer_begin_color (msg_buf, "highlight-a");
+  diagnostic_message_buffer_append_str (msg_buf, "highlight A");  
+  diagnostic_message_buffer_end_color (msg_buf);
+
+  diagnostic_message_buffer_append_str (msg_buf, " ");
+  
+  diagnostic_message_buffer_begin_color (msg_buf, "highlight-b");
+  diagnostic_message_buffer_append_str (msg_buf, "highlight B");  
+  diagnostic_message_buffer_end_color (msg_buf);
+  
+  diagnostic_message_buffer_append_str (msg_buf, " ");
+
+  /* Add an event ID.  This will be printed as "(1)".  */
+  diagnostic_message_buffer_append_event_id (msg_buf, event_id);
+  
+  /* Add an ASCII char.  */
+  diagnostic_message_buffer_append_byte (msg_buf, '.');
+
+  diagnostic_finish_via_msg_buf (d, msg_buf);
+  /* end quoted source */
+
+  return end_test ();
+};
+
+/* Verify the output from the text sink.
+   { dg-regexp "test-message-buffer.c.exe: error: this is a string; foo; int: 42 str: mostly harmless; this is a link 'this is quoted' highlight A highlight B \\(1\\)." } */
+
+/* Verify that some JSON was written to a file with the expected name:
+   { dg-final { verify-sarif-file } } */
+
+/* Use a Python script to verify various properties about the generated
+   .sarif file:
+   { dg-final { run-sarif-pytest test-message-buffer.c "test-message-buffer-c.py" } } */
diff --git a/gcc/testsuite/libgdiagnostics.dg/test-multiple-lines.c b/gcc/testsuite/libgdiagnostics.dg/test-multiple-lines.c
index e761110..39af810 100644
--- a/gcc/testsuite/libgdiagnostics.dg/test-multiple-lines.c
+++ b/gcc/testsuite/libgdiagnostics.dg/test-multiple-lines.c
@@ -66,6 +66,7 @@ main ()
       |                        ~~~~~ 
    23 |                        "bar"
       |                        ~~~~~^
+      |                             ,
    24 |                        "baz"};
       |                        ~~~~~ 
    { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py b/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py
index af1e7b9..61ccb93 100644
--- a/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py
+++ b/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py
@@ -101,7 +101,7 @@ def test_sarif_output_for_warning_with_path(sarif):
         == '    PyList_Append(list, item);\n'
     assert tfl_2_loc['logicalLocations'] == location['logicalLocations']
     assert tfl_2_loc['message']['text'] \
-        == "when calling 'PyList_Append', passing NULL from (1) as argument 1"
+        == "when calling 'PyList_Append', passing NULL from [(1)](sarif:/runs/0/results/0/codeFlows/0/threadFlows/0/locations/0) as argument 1"
     assert tfl_2['nestingLevel'] == 0
     assert tfl_2['executionOrder'] == 3
 
diff --git a/gcc/testsuite/rust/compile/additional-trait-bounds2nr2.rs b/gcc/testsuite/rust/compile/additional-trait-bounds2nr2.rs
index 6764f6e..1c49b75 100644
--- a/gcc/testsuite/rust/compile/additional-trait-bounds2nr2.rs
+++ b/gcc/testsuite/rust/compile/additional-trait-bounds2nr2.rs
@@ -1,5 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
-
 #![feature(optin_builtin_traits)]
 
 pub unsafe auto trait Send {}
diff --git a/gcc/testsuite/rust/compile/all-cast.rs b/gcc/testsuite/rust/compile/all-cast.rs
index fa24373..6d8576c 100644
--- a/gcc/testsuite/rust/compile/all-cast.rs
+++ b/gcc/testsuite/rust/compile/all-cast.rs
@@ -4,7 +4,7 @@ fn main() {
 
     0u32 as char; // { dg-error "cannot cast .u32. as .char., only .u8. can be cast as .char." }
 
-    let x = &[1_usize, 2] as [usize]; // { dg-error "cast to unsized type: .& .usize:CAPACITY.. as ..usize.." }
+    let x = &[1_usize, 2] as [usize]; // { dg-error "cast to unsized type: .& .usize; 2.. as ..usize.." }
 
     let a = &0u8; // Here, `x` is a `&u8`.
     let y: u32 = a as u32; // { dg-error "casting .& u8. as .u32. is invalid" }
diff --git a/gcc/testsuite/rust/compile/arrays2.rs b/gcc/testsuite/rust/compile/arrays2.rs
index 668bcf0..1090059 100644
--- a/gcc/testsuite/rust/compile/arrays2.rs
+++ b/gcc/testsuite/rust/compile/arrays2.rs
@@ -1,5 +1,4 @@
-// { dg-additional-options "-w" }
 fn main() {
     let array: [i32; 5] = [1, 2, 3];
-    // { dg-error "mismatched types, expected an array with a fixed size of 5 elements, found one with 3 elements" "" { target *-*-* } .-1 }
+    // { dg-error "mismatched types, expected ..i32; 5.. but got ...integer.; 3.. .E0308." "" { target *-*-* } .-1 }
 }
diff --git a/gcc/testsuite/rust/compile/auto_traits2.rs b/gcc/testsuite/rust/compile/auto_traits2.rs
index 382d446..7004761 100644
--- a/gcc/testsuite/rust/compile/auto_traits2.rs
+++ b/gcc/testsuite/rust/compile/auto_traits2.rs
@@ -15,7 +15,7 @@ fn foo(a: &(dyn A + Send + Sync)) {
 struct S;
 
 impl A for S {
-    fn a_method(&self) {} // { dg-warning "unused name" }
+    fn a_method(&self) {}
 }
 
 fn main() {
diff --git a/gcc/testsuite/rust/compile/bad-rpit1.rs b/gcc/testsuite/rust/compile/bad-rpit1.rs
new file mode 100644
index 0000000..d8c21b1
--- /dev/null
+++ b/gcc/testsuite/rust/compile/bad-rpit1.rs
@@ -0,0 +1,26 @@
+#[lang = "sized"]
+trait Sized {}
+
+trait Foo {
+    fn id(&self) -> i32;
+}
+
+struct A;
+struct B;
+
+impl Foo for A {
+    fn id(&self) -> i32 {
+        1
+    }
+}
+
+impl Foo for B {
+    fn id(&self) -> i32 {
+        2
+    }
+}
+
+fn make_foo(cond: bool) -> impl Foo {
+    if cond { A } else { B }
+    // { dg-error "mismatched types, expected .A. but got .B. .E0308." "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/bug-with-default-generic.rs b/gcc/testsuite/rust/compile/bug-with-default-generic.rs
new file mode 100644
index 0000000..25f46a0
--- /dev/null
+++ b/gcc/testsuite/rust/compile/bug-with-default-generic.rs
@@ -0,0 +1,15 @@
+#[lang = "sized"]
+pub trait Sized {}
+
+pub trait MyBinaryTrait<Rhs = Self> {
+    fn do_something(&self, rhs: &Rhs);
+}
+
+struct Foo<T> {
+    // { dg-warning "struct is never constructed" "" { target *-*-* } .-1 }
+    value: T,
+}
+
+impl<T> MyBinaryTrait for Foo<T> {
+    fn do_something(&self, _rhs: &Self) {}
+}
diff --git a/gcc/testsuite/rust/compile/const3.rs b/gcc/testsuite/rust/compile/const3.rs
index 22dc3d3..c1d0f29 100644
--- a/gcc/testsuite/rust/compile/const3.rs
+++ b/gcc/testsuite/rust/compile/const3.rs
@@ -3,5 +3,5 @@ fn size() -> usize {
 }
 
 fn main() {
-    let a = [15; size()]; // { dg-error "only functions marked as .const. are allowed to be called from constant contexts" }
+    let a = [15; size()]; // { dg-error "calls in constants are limited to constant functions, tuple structs and tuple variants" }
 }
diff --git a/gcc/testsuite/rust/compile/const_generics_10.rs b/gcc/testsuite/rust/compile/const_generics_10.rs
new file mode 100644
index 0000000..7e3bc86
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_10.rs
@@ -0,0 +1,32 @@
+#[lang = "sized"]
+trait Sized {}
+
+const M: usize = 4;
+
+struct Foo<T, const N: usize = 1> {
+    value: [T; N],
+}
+
+fn main() {
+    let foo = Foo::<i32> { value: [15] };
+    let foo = Foo::<i32, 2> { value: [15, 13] };
+    let foo: Foo<i32, 2> = Foo { value: [15, 13] };
+    let foo: Foo<i32, 2> = Foo::<i32, 2> { value: [15, 13] };
+    let foo: Foo<i32, { 1 + 1 }> = Foo { value: [15, 13] };
+    let foo = Foo::<i32, { 1 + 1 }> { value: [15, 13] };
+    let foo: Foo<i32, { 1 + 1 }> = Foo::<i32, { 1 + 1 }> { value: [15, 13] };
+    let foo: Foo<i32, M> = Foo::<i32, 4> {
+        value: [15, 13, 11, 9],
+    };
+
+    let invalid_foo: Foo<i32, { 1 + 1 }> = Foo::<i32, 3> { value: [15, 13] };
+    // { dg-error {mismatched types, expected ..T=i32; 3.. but got ...integer.; 2.. .E0308.} "" { target *-*-* } .-1 }
+    // { dg-error {mismatched types, expected ..T=i32; 2.. but got ..T=i32; 3.. .E0308.} "" { target *-*-* } .-2 }
+
+    let invalid_foo: Foo<i32, { 1 + 1 }> = Foo::<i32, M> { value: [15, 13] };
+    // { dg-error {mismatched types, expected ..T=i32; 4.. but got ...integer.; 2.. .E0308.} "" { target *-*-* } .-1 }
+    // { dg-error {mismatched types, expected ..T=i32; 2.. but got ..T=i32; 4.. .E0308.} "" { target *-*-* } .-2 }
+
+    let invalid_foo: Foo<i32> = Foo::<i32, 2> { value: [15, 13] };
+    // { dg-error {mismatched types, expected ..T=i32; 1.. but got ..T=i32; 2.. .E0308.} "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/const_generics_11.rs b/gcc/testsuite/rust/compile/const_generics_11.rs
new file mode 100644
index 0000000..de902ee
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_11.rs
@@ -0,0 +1,14 @@
+// { dg-options "-w" }
+
+#[lang = "sized"]
+trait Sized {}
+
+struct Matrix<T, const ROWS: usize, const COLS: usize> {
+    data: [[T; COLS]; ROWS],
+}
+
+fn main() {
+    let _: Matrix<u8, 2, 3> = Matrix {
+        data: [[1, 2, 3], [4, 5, 6]],
+    };
+}
diff --git a/gcc/testsuite/rust/compile/const_generics_12.rs b/gcc/testsuite/rust/compile/const_generics_12.rs
new file mode 100644
index 0000000..a17c525
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_12.rs
@@ -0,0 +1,14 @@
+// { dg-options "-w" }
+
+#[lang = "sized"]
+trait Sized {}
+
+const BASE: usize = 2;
+
+struct Foo<T, const N: usize> {
+    data: [T; N],
+}
+
+fn main() {
+    let _ = Foo::<u8, { BASE + 1 * 2 }> { data: [0; 4] };
+}
diff --git a/gcc/testsuite/rust/compile/const_generics_13.rs b/gcc/testsuite/rust/compile/const_generics_13.rs
new file mode 100644
index 0000000..20dd0b90
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_13.rs
@@ -0,0 +1,11 @@
+#[lang = "sized"]
+trait Sized {}
+
+struct Foo<T, const N: usize> {
+    value: [T; N],
+}
+
+fn main() {
+    let foo: Foo<_, _>;
+    // { dg-error {type provided when a constant was expected .E0747.} "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/const_generics_14.rs b/gcc/testsuite/rust/compile/const_generics_14.rs
new file mode 100644
index 0000000..4d52efb
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_14.rs
@@ -0,0 +1,13 @@
+#[lang = "sized"]
+trait Sized {}
+
+type MyLen = usize;
+struct Foo<T, const N: usize> {
+    data: [T; N],
+}
+
+fn main() {
+    let _ = Foo::<u8, MyLen> { data: [1, 2, 3] };
+    // { dg-error {type provided when a constant was expected .E0747.} "" { target *-*-* } .-1 }
+    // { dg-error {expected an ADT type for constructor} "" { target *-*-* } .-2 }
+}
diff --git a/gcc/testsuite/rust/compile/const_generics_15.rs b/gcc/testsuite/rust/compile/const_generics_15.rs
new file mode 100644
index 0000000..a160abf
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_15.rs
@@ -0,0 +1,16 @@
+#[lang = "sized"]
+trait Sized {}
+
+enum Foo<const N: usize> {
+    A([u8; N]),
+}
+
+union Bar<const N: usize> {
+    a: [i32; N],
+    b: [u8; N],
+}
+
+fn main() {
+    let _ = Foo::<4>::A([1, 2, 3, 4]);
+    let _ = Bar::<4> { a: [0; 4] };
+}
diff --git a/gcc/testsuite/rust/compile/const_generics_16.rs b/gcc/testsuite/rust/compile/const_generics_16.rs
new file mode 100644
index 0000000..060dbda
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_16.rs
@@ -0,0 +1,10 @@
+#[lang = "sized"]
+trait Sized {}
+
+struct Foo<T = u8, const N: usize = 4> {
+    data: [T; N], // { dg-warning "field is never read: .data." }
+}
+
+fn main() {
+    let _x = Foo { data: [1, 2, 3, 4] };
+}
diff --git a/gcc/testsuite/rust/compile/const_generics_3.rs b/gcc/testsuite/rust/compile/const_generics_3.rs
index 524d48d..3415f17 100644
--- a/gcc/testsuite/rust/compile/const_generics_3.rs
+++ b/gcc/testsuite/rust/compile/const_generics_3.rs
@@ -1,28 +1,21 @@
-// { dg-additional-options "-w -frust-name-resolution-2.0" }
-
 #[lang = "sized"]
 trait Sized {}
 
 const M: usize = 4;
 
 struct Foo<T, const N: usize = 1> {
-    value: [T; N],
+    value: [T; N], // { dg-warning "field is never read: .value." }
 }
 
 fn main() {
-    let foo = Foo::<i32> { value: [15] };
-    let foo = Foo::<i32, 2> { value: [15, 13] };
-    let foo: Foo<i32, 2> = Foo { value: [15, 13] };
-    let foo: Foo<i32, 2> = Foo::<i32, 2> { value: [15, 13] };
-    let foo: Foo<i32, { 1 + 1 }> = Foo { value: [15, 13] };
-    let foo = Foo::<i32, { 1 + 1 }> { value: [15, 13] };
-    let foo: Foo<i32, { 1 + 1 }> = Foo::<i32, { 1 + 1 }> { value: [15, 13] };
-    let foo: Foo<i32, M> = Foo::<i32, 4> {
+    let _foo = Foo::<i32> { value: [15] };
+    let _foo = Foo::<i32, 2> { value: [15, 13] };
+    let _foo: Foo<i32, 2> = Foo { value: [15, 13] };
+    let _foo: Foo<i32, 2> = Foo::<i32, 2> { value: [15, 13] };
+    let _foo: Foo<i32, { 1 + 1 }> = Foo { value: [15, 13] };
+    let _foo = Foo::<i32, { 1 + 1 }> { value: [15, 13] };
+    let _foo: Foo<i32, { 1 + 1 }> = Foo::<i32, { 1 + 1 }> { value: [15, 13] };
+    let _foo: Foo<i32, M> = Foo::<i32, 4> {
         value: [15, 13, 11, 9],
     };
-
-    // FIXME: Add proper const typecheck errors here
-    let invalid_foo: Foo<i32, { 1 + 1 }> = Foo::<i32, 3> { value: [15, 13] };
-    let invalid_foo: Foo<i32, { 1 + 1 }> = Foo::<i32, M> { value: [15, 13] };
-    let invalid_foo: Foo<i32> = Foo::<i32, 2> { value: [15, 13] };
 }
diff --git a/gcc/testsuite/rust/compile/const_generics_5.rs b/gcc/testsuite/rust/compile/const_generics_5.rs
index 685229e..4d05569 100644
--- a/gcc/testsuite/rust/compile/const_generics_5.rs
+++ b/gcc/testsuite/rust/compile/const_generics_5.rs
@@ -1,4 +1,3 @@
-// { dg-options "-w" }
 struct Foo<const N: usize = { 14 }>;
 
 const M: usize = 15;
@@ -8,5 +7,6 @@ fn main() {
     let _: Foo<15> = Foo;
     let _: Foo<{ M }> = Foo;
     let _: Foo<M> = Foo;
-    // let _: Foo<N> = Foo; this causes an ICE we need to do const generics
+    let _: Foo<N> = Foo;
+    // { dg-error {type provided when a constant was expected .E0747.} "" { target *-*-* } .-1 }
 }
diff --git a/gcc/testsuite/rust/compile/const_generics_8.rs b/gcc/testsuite/rust/compile/const_generics_8.rs
index bb34652..ce5e1b5 100644
--- a/gcc/testsuite/rust/compile/const_generics_8.rs
+++ b/gcc/testsuite/rust/compile/const_generics_8.rs
@@ -9,12 +9,13 @@ type Bipboupe<const N: i32 = 15> = Bidule;
 trait Fooable<const N: i32 = 15> {}
 
 union Bidoulepe<const N: i32 = 15> {
-    // { dg-error "default values for const generic parameters are not allowed in .union. items"  "" {target *-*-* } .-1 }
     int: i32,
     float: f32,
 }
 
-fn const_default<const N: i32 = 15>() {} // { dg-error "default values for const generic parameters are not allowed in .function. items" }
+fn const_default<const N: i32 = 15>() {} // { dg-error "default values for const generic parameters are not allowed here" }
 
 // Note - missing generic parameter - needs name resolution on const generics
-impl<const N: i32 = 15> Bidule {} // { dg-error "default values for const generic parameters are not allowed in .impl. items" }
+impl<const N: i32 = 15> Bidule {}
+// { dg-error "default values for const generic parameters are not allowed here"   "" {target *-*-* } .-1 }
+// { dg-error "unconstrained type parameter"  "" {target *-*-* } .-2 }
diff --git a/gcc/testsuite/rust/compile/const_generics_9.rs b/gcc/testsuite/rust/compile/const_generics_9.rs
new file mode 100644
index 0000000..98e2d3f
--- /dev/null
+++ b/gcc/testsuite/rust/compile/const_generics_9.rs
@@ -0,0 +1,13 @@
+// { dg-options "-w" }
+
+#[lang = "sized"]
+trait Sized {}
+
+struct ArrayWrapper<T, const N: usize> {
+    data: [T; N],
+}
+
+pub fn test() -> [u8; 4] {
+    let a = ArrayWrapper { data: [1u8; 4] };
+    a.data
+}
diff --git a/gcc/testsuite/rust/compile/deferred_const_inference.rs b/gcc/testsuite/rust/compile/deferred_const_inference.rs
new file mode 100644
index 0000000..25a3b17
--- /dev/null
+++ b/gcc/testsuite/rust/compile/deferred_const_inference.rs
@@ -0,0 +1,7 @@
+// { dg-additional-options "-frust-compile-until=typecheck" }
+
+// #![feature(generic_arg_infer)]
+
+fn main() {
+    let a: [u32; _] = [15u32];
+}
diff --git a/gcc/testsuite/rust/compile/derive-debug1.rs b/gcc/testsuite/rust/compile/derive-debug1.rs
index cf2187d..5927374 100644
--- a/gcc/testsuite/rust/compile/derive-debug1.rs
+++ b/gcc/testsuite/rust/compile/derive-debug1.rs
@@ -23,15 +23,15 @@ mod core {
     }
 }
 
-#[derive(Debug)] // { dg-warning "unused name" }
+#[derive(Debug)]
 // { dg-warning "stub implementation" "" { target *-*-* } .-1 }
 struct Foo { a: i32, b: i64 } // { dg-warning "is never constructed" }
 
-#[derive(Debug)] // { dg-warning "unused name" }
+#[derive(Debug)]
 // { dg-warning "stub implementation" "" { target *-*-* } .-1 }
 struct Bar(i32, i32); // { dg-warning "is never constructed" }
 
-#[derive(Debug)] // { dg-warning "unused name" }
+#[derive(Debug)]
 // { dg-warning "stub implementation" "" { target *-*-* } .-1 }
 enum Baz {
     A,
diff --git a/gcc/testsuite/rust/compile/derive_macro1.rs b/gcc/testsuite/rust/compile/derive_macro1.rs
index bc10d60..8c42aba 100644
--- a/gcc/testsuite/rust/compile/derive_macro1.rs
+++ b/gcc/testsuite/rust/compile/derive_macro1.rs
@@ -7,7 +7,7 @@ pub trait Clone {
 }
 
 // This warning can be removed once we properly handle implems with #[automatically_derived]
-#[derive(Clone)] // { dg-warning "unused name .self." }
+#[derive(Clone)]
 pub struct S;
 
 fn main() {
diff --git a/gcc/testsuite/rust/compile/derive_partial_ord1.rs b/gcc/testsuite/rust/compile/derive_partial_ord1.rs
new file mode 100644
index 0000000..eeca62d
--- /dev/null
+++ b/gcc/testsuite/rust/compile/derive_partial_ord1.rs
@@ -0,0 +1,464 @@
+// { dg-additional-options "-w" }
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+impl Eq for i32 {}
+
+#[derive(PartialEq, PartialOrd)]
+enum Foo {
+    A,
+    B(i32, i32, i32),
+    C { inner: i32, outer: i32 },
+}
+
+#[derive(Ord, PartialOrd, PartialEq, Eq)]
+struct Bar {
+    a: i32,
+}
+
+#[derive(Ord, PartialOrd, PartialEq, Eq)]
+struct BarFull {
+    a: i32,
+    b: i32,
+    c: i32,
+    d: i32,
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() {
+    let a = Foo::A;
+    let b = Foo::B(15, 14, 13);
+
+    match a.partial_cmp(&b) {
+        Option::Some(Ordering::Less) => print("less"),
+        Option::Some(Ordering::Greater) => print("greater"),
+        Option::Some(Ordering::Equal) => print("equal"),
+        _ => print("uuuuh woops lol"),
+    }
+}
diff --git a/gcc/testsuite/rust/compile/enum_variant_name.rs b/gcc/testsuite/rust/compile/enum_variant_name.rs
index 671fced..965acd1 100644
--- a/gcc/testsuite/rust/compile/enum_variant_name.rs
+++ b/gcc/testsuite/rust/compile/enum_variant_name.rs
@@ -1,4 +1,4 @@
-// { dg-additional-options "-w -frust-name-resolution-2.0" }
+// { dg-additional-options "-w" }
 struct E1;
 
 enum Test {
diff --git a/gcc/testsuite/rust/compile/format_args_basic_expansion.rs b/gcc/testsuite/rust/compile/format_args_basic_expansion.rs
index 40bcd3c..cedb62c 100644
--- a/gcc/testsuite/rust/compile/format_args_basic_expansion.rs
+++ b/gcc/testsuite/rust/compile/format_args_basic_expansion.rs
@@ -35,7 +35,6 @@ pub mod core {
 
         impl Display for i32 {
             fn fmt(&self, _: &mut Formatter) -> Result {
-                // { dg-warning "unused name .self." "" { target *-*-* } .-1 }
                 Result
             }
         }
diff --git a/gcc/testsuite/rust/compile/format_args_extra_comma.rs b/gcc/testsuite/rust/compile/format_args_extra_comma.rs
index fcc435c..dc48a3a 100644
--- a/gcc/testsuite/rust/compile/format_args_extra_comma.rs
+++ b/gcc/testsuite/rust/compile/format_args_extra_comma.rs
@@ -35,7 +35,6 @@ pub mod core {
 
         impl Display for i32 {
             fn fmt(&self, _: &mut Formatter) -> Result {
-                // { dg-warning "unused name .self." "" { target *-*-* } .-1 }
                 Result
             }
         }
diff --git a/gcc/testsuite/rust/compile/generics8.rs b/gcc/testsuite/rust/compile/generics8.rs
index 88c4bac..2d30a9e 100644
--- a/gcc/testsuite/rust/compile/generics8.rs
+++ b/gcc/testsuite/rust/compile/generics8.rs
@@ -4,7 +4,7 @@ pub trait Sized {}
 struct Foo<A, B>(A, B);
 
 impl<T> Foo<i32, T> {
-    fn test(a: T) -> T {
+    fn test(a: T) -> T { // { dg-error "duplicate definitions with name .test." }
         a
     }
 }
diff --git a/gcc/testsuite/rust/compile/generics9.rs b/gcc/testsuite/rust/compile/generics9.rs
index 56c6198..949fbb1 100644
--- a/gcc/testsuite/rust/compile/generics9.rs
+++ b/gcc/testsuite/rust/compile/generics9.rs
@@ -1,4 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
 struct Foo<A, B = (A, B)>(A, B);
 // { dg-error "type parameters with a default cannot use forward declared identifiers" "" { target *-*-* } .-1 }
 
diff --git a/gcc/testsuite/rust/compile/glob_import_enum.rs b/gcc/testsuite/rust/compile/glob_import_enum.rs
new file mode 100644
index 0000000..032a1db
--- /dev/null
+++ b/gcc/testsuite/rust/compile/glob_import_enum.rs
@@ -0,0 +1,16 @@
+use self::Ordering::*;
+use Ordering::*;
+
+enum Ordering {
+    A,
+    B,
+}
+
+fn foo(_: Ordering) {}
+
+fn main() {
+    let a = A;
+
+    foo(a);
+    foo(B);
+}
diff --git a/gcc/testsuite/rust/compile/impl_trait_diag.rs b/gcc/testsuite/rust/compile/impl_trait_diag.rs
new file mode 100644
index 0000000..54a0cd2
--- /dev/null
+++ b/gcc/testsuite/rust/compile/impl_trait_diag.rs
@@ -0,0 +1,17 @@
+#[lang = "sized"]
+pub trait Sized {}
+
+trait Foo {
+    fn method(&self);
+}
+
+struct Bar;
+impl Foo for Bar {}
+
+fn main() {
+    let x: impl Foo = Bar; // { dg-error ".impl Trait. not allowed outside of function and inherent method return types .E0562." }
+
+    struct Wrapper {
+        field: impl Foo, // { dg-error ".impl Trait. not allowed outside of function and inherent method return types .E0562." }
+    }
+}
diff --git a/gcc/testsuite/rust/compile/impl_trait_generic_arg.rs b/gcc/testsuite/rust/compile/impl_trait_generic_arg.rs
new file mode 100644
index 0000000..ecdb088
--- /dev/null
+++ b/gcc/testsuite/rust/compile/impl_trait_generic_arg.rs
@@ -0,0 +1,24 @@
+#[lang = "sized"]
+trait Sized {}
+
+trait Foo {
+    fn id(&self) -> u8;
+}
+
+struct Bar;
+
+impl Foo for Bar {
+    fn id(&self) -> u8 {
+        1
+    }
+}
+
+fn takes(val: impl Foo) -> u8 {
+    val.id()
+}
+
+fn main() {
+    let b = Bar;
+    let x = takes::<Bar>(b);
+    // { dg-error "cannot provide explicit generic arguments when .impl Trait. is used in argument position .E0632." "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/invalid_label_name.rs b/gcc/testsuite/rust/compile/invalid_label_name.rs
index 66e40a6..d1c5a33 100644
--- a/gcc/testsuite/rust/compile/invalid_label_name.rs
+++ b/gcc/testsuite/rust/compile/invalid_label_name.rs
@@ -1,4 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
 pub fn function() {
     'continue: loop {
         // { dg-error "invalid label name .'continue." "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/rust/compile/issue-1048.rs b/gcc/testsuite/rust/compile/issue-1048.rs
new file mode 100644
index 0000000..8d4053a
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-1048.rs
@@ -0,0 +1,8 @@
+macro_rules! maybe_return { ($e:expr) => ($e); }
+
+fn frob(x: i32) -> i32{
+    maybe_return! {x}
+    // { dg-error "mismatched types. expected .... but got .i32. .E0308." "" { target *-*-* } .-1 }
+    // should return -1 
+    -1
+}
diff --git a/gcc/testsuite/rust/compile/issue-1485.rs b/gcc/testsuite/rust/compile/issue-1485.rs
new file mode 100644
index 0000000..a0cd5a0
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-1485.rs
@@ -0,0 +1,16 @@
+#[lang = "sized"]
+pub trait Sized {}
+
+#[lang = "fn_once"]
+pub trait FnOnce<Args> {
+    #[lang = "fn_once_output"]
+    type Output;
+
+    extern "rust-call" fn call_once(self, args: Args) -> Self::Output;
+}
+
+struct BinOpInvalid {
+    lhs: i32,
+    rhs: i32,
+    f: impl FnOnce(i32) -> i32, // { dg-error ".impl Trait. not allowed outside of function and inherent method return types .E0562." }
+}
diff --git a/gcc/testsuite/rust/compile/issue-1487.rs b/gcc/testsuite/rust/compile/issue-1487.rs
new file mode 100644
index 0000000..4a4d759
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-1487.rs
@@ -0,0 +1,15 @@
+// { dg-options "-w" }
+#[lang = "sized"]
+trait Sized {}
+
+trait Printable {
+    fn print(&self);
+}
+
+struct Foo;
+
+impl Printable for Foo {
+    fn print(&self) {}
+}
+
+fn take_printable(_: impl Printable) {}
diff --git a/gcc/testsuite/rust/compile/issue-2015.rs b/gcc/testsuite/rust/compile/issue-2015.rs
index 7789ecd..7e03651 100644
--- a/gcc/testsuite/rust/compile/issue-2015.rs
+++ b/gcc/testsuite/rust/compile/issue-2015.rs
@@ -1,4 +1,5 @@
-// { dg-additional-options "-frust-compile-until=lowering" }
+#[lang = "sized"]
+trait Sized {}
 
 macro_rules! impl_foo {
 	() => { impl Foo }
diff --git a/gcc/testsuite/rust/compile/issue-2043.rs b/gcc/testsuite/rust/compile/issue-2043.rs
index efa1ded..92532b7 100644
--- a/gcc/testsuite/rust/compile/issue-2043.rs
+++ b/gcc/testsuite/rust/compile/issue-2043.rs
@@ -6,7 +6,6 @@ struct Foo<'a> {
 impl<'a> Foo<'a> {
     fn bar(self: &mut Foo<'a>) {}
     // { dg-warning "associated function is never used: .bar." "" { target *-*-* } .-1 }
-    // { dg-warning "unused name .self." "" { target *-*-* } .-2 }
 }
 
 fn main() {}
diff --git a/gcc/testsuite/rust/compile/issue-2166.rs b/gcc/testsuite/rust/compile/issue-2166.rs
index 318f0a6..142ed17 100644
--- a/gcc/testsuite/rust/compile/issue-2166.rs
+++ b/gcc/testsuite/rust/compile/issue-2166.rs
@@ -11,7 +11,6 @@ impl Add for u32 {
     type Output = u32;
 
     fn add(self) -> u32 {
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
         0
     }
 }
@@ -20,7 +19,6 @@ impl<'a> Add for &'a u32 {
     type Output = u32;
 
     fn add(self) -> <u32 as Add>::Output {
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
         0
     }
 }
diff --git a/gcc/testsuite/rust/compile/issue-2238.rs b/gcc/testsuite/rust/compile/issue-2238.rs
index 38871b3..6a43a13 100644
--- a/gcc/testsuite/rust/compile/issue-2238.rs
+++ b/gcc/testsuite/rust/compile/issue-2238.rs
@@ -10,7 +10,6 @@ fn main() {
 
     impl Bar for Foo {
         fn foo(&self) {}
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
     }
 
     let s = Foo;
diff --git a/gcc/testsuite/rust/compile/issue-2680.rs b/gcc/testsuite/rust/compile/issue-2680.rs
new file mode 100644
index 0000000..d5ae2ff44
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-2680.rs
@@ -0,0 +1,6 @@
+// { dg-additional-options "-fdump-tree-gimple" }
+pub fn test_cast() {
+    let i = 1;
+    // { dg-final { scan-tree-dump-times {const i32 i;} 1 gimple } }
+    let _j = i as i64;
+}
diff --git a/gcc/testsuite/rust/compile/issue-2907.rs b/gcc/testsuite/rust/compile/issue-2907.rs
index 1af843f..fdf1953 100644
--- a/gcc/testsuite/rust/compile/issue-2907.rs
+++ b/gcc/testsuite/rust/compile/issue-2907.rs
@@ -15,7 +15,6 @@ impl<B: Bar> Foo for B {
     type Ty = u32;
 
     fn foo(self) -> Self::Ty {
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
         14
     }
 }
diff --git a/gcc/testsuite/rust/compile/issue-3144.rs b/gcc/testsuite/rust/compile/issue-3144.rs
new file mode 100644
index 0000000..4e61078
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3144.rs
@@ -0,0 +1,29 @@
+#[lang = "sized"]
+pub trait Sized {}
+
+#[lang = "copy"]
+trait Copy {}
+
+#[lang = "clone"]
+pub trait Clone {
+    fn clone(&self) -> Self;
+}
+
+impl Clone for i32 {
+    fn clone(&self) -> i32 {
+        *self
+    }
+}
+
+struct S {}
+
+#[derive(Clone, Copy)]
+// { dg-error {bounds not satisfied for S .Clone. is not satisfied .E0277.} "" { target *-*-* } .-1 }
+struct S2 {
+    a: i32,
+    s: S,
+}
+
+fn main() -> i32 {
+    0
+}
diff --git a/gcc/testsuite/rust/compile/issue-3304.rs b/gcc/testsuite/rust/compile/issue-3304.rs
index 6ab614f..cc376fa 100644
--- a/gcc/testsuite/rust/compile/issue-3304.rs
+++ b/gcc/testsuite/rust/compile/issue-3304.rs
@@ -1,4 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
 #[lang = "sized"]
 trait Sized {}
 
diff --git a/gcc/testsuite/rust/compile/issue-3454.rs b/gcc/testsuite/rust/compile/issue-3454.rs
new file mode 100644
index 0000000..2a3c0c7
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3454.rs
@@ -0,0 +1,20 @@
+#[lang = "sized"]
+pub trait Sized {}
+
+macro_rules! impl_foo {
+	() => { impl Foo }
+}
+
+pub trait Foo {}
+
+pub trait Bar {
+    type Baz;
+}
+
+pub fn foo(_value: impl Bar<Baz = impl_foo!()>) -> i32 {
+    15
+}
+
+pub fn bar(_value: impl Bar<Baz = impl Foo>) -> i32 {
+    16
+}
diff --git a/gcc/testsuite/rust/compile/issue-3524.rs b/gcc/testsuite/rust/compile/issue-3524.rs
new file mode 100644
index 0000000..62c8c35
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3524.rs
@@ -0,0 +1,9 @@
+struct A {}
+// { dg-warning "struct is never constructed" "" { target *-*-* } .-1 }
+
+impl A {
+    fn main() {}
+    // { dg-warning "associated function is never used: .main." "" { target *-*-* } .-1 }
+}
+
+fn main() {}
diff --git a/gcc/testsuite/rust/compile/issue-3525.rs b/gcc/testsuite/rust/compile/issue-3525.rs
new file mode 100644
index 0000000..84a7ebe
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3525.rs
@@ -0,0 +1,6 @@
+// { dg-options "-w" }
+
+struct Foo(usize);
+
+const B: usize = A.0;
+const A: Foo = Foo(123);
diff --git a/gcc/testsuite/rust/compile/issue-3546.rs b/gcc/testsuite/rust/compile/issue-3546.rs
new file mode 100644
index 0000000..d4ec0bb
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3546.rs
@@ -0,0 +1,16 @@
+const L: usize = 3;
+
+fn main() {
+    let p = Printer {};
+    p.print();
+}
+
+trait Print<const N: usize> {
+    fn print(&self) -> usize {
+        3
+    }
+}
+
+struct Printer {}
+impl Print<L> for Printer {}
+// { dg-error "generic item takes at most 1 type arguments but 1 were supplied" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/rust/compile/issue-3551.rs b/gcc/testsuite/rust/compile/issue-3551.rs
new file mode 100644
index 0000000..6d6a812
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3551.rs
@@ -0,0 +1,15 @@
+#[lang = "sized"]
+pub trait Sized {}
+
+#[lang = "fn_once"]
+pub trait FnOnce<Args> {
+    #[lang = "fn_once_output"]
+    type Output;
+
+    extern "rust-call" fn call_once(self, args: Args) -> Self::Output;
+}
+
+struct Bug {
+    a: [(); (|| 0)()],
+    // { dg-error "calls in constants are limited to constant functions, tuple structs and tuple variants" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/issue-3599.rs b/gcc/testsuite/rust/compile/issue-3599.rs
new file mode 100644
index 0000000..1d29fac
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3599.rs
@@ -0,0 +1,8 @@
+#[lang = "sized"]
+trait Sized {}
+
+trait Bar {}
+
+struct S; // { dg-warning "struct is never constructed" }
+
+pub fn test(foo: impl Bar) {}
diff --git a/gcc/testsuite/rust/compile/issue-3618.rs b/gcc/testsuite/rust/compile/issue-3618.rs
new file mode 100644
index 0000000..3bf2c7e
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3618.rs
@@ -0,0 +1,2 @@
+static _X : ()
+	    = loop{}; // { dg-error "'loop' is not allowed in const context" }
diff --git a/gcc/testsuite/rust/compile/issue-3642.rs b/gcc/testsuite/rust/compile/issue-3642.rs
new file mode 100644
index 0000000..6d9decc
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3642.rs
@@ -0,0 +1,9 @@
+#[lang = "sized"]
+trait Sized {}
+
+pub trait T<X> {
+    const D: i32 = {
+        // { dg-error "mismatched types, expected .i32. but got .()." "" { target *-*-* } .-1 }
+        const C: X;
+    };
+}
diff --git a/gcc/testsuite/rust/compile/issue-3660.rs b/gcc/testsuite/rust/compile/issue-3660.rs
new file mode 100644
index 0000000..1f1c583
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3660.rs
@@ -0,0 +1,3 @@
+pub static A: [u32; 2] = [1, 2];
+
+pub static B: [u8; 2] = [3, 4];
diff --git a/gcc/testsuite/rust/compile/issue-3661.rs b/gcc/testsuite/rust/compile/issue-3661.rs
new file mode 100644
index 0000000..8d03c36
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3661.rs
@@ -0,0 +1,10 @@
+pub macro m($inner_str:expr) {
+    #[m = $inner_str] 
+    // { dg-error "macro not found" "" { target *-*-* } .-1 }
+
+    struct S;
+}
+
+fn main() {
+    m!(stringify!(foo));
+}
diff --git a/gcc/testsuite/rust/compile/issue-3671.rs b/gcc/testsuite/rust/compile/issue-3671.rs
index e800d53..8015653 100644
--- a/gcc/testsuite/rust/compile/issue-3671.rs
+++ b/gcc/testsuite/rust/compile/issue-3671.rs
@@ -1,2 +1,2 @@
-impl Self<0> {}
+impl Foo<0> {}
 // { dg-error "could not resolve type path" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/rust/compile/issue-3836.rs b/gcc/testsuite/rust/compile/issue-3836.rs
new file mode 100644
index 0000000..a228795
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3836.rs
@@ -0,0 +1,67 @@
+// { dg-options "-w" }
+mod core {
+    mod option {
+        pub enum Option<T> {
+            #[lang = "None"]
+            None,
+            #[lang = "Some"]
+            Some(T),
+        }
+    }
+
+    mod marker {
+        #[lang = "sized"]
+        pub trait Sized {}
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        pub enum Ordering {
+            Less = -1,
+            Equal = 0,
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[lang = "partial_ord"]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+        }
+    }
+}
+
+use core::cmp::{Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        false
+    }
+}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Option::Some(Ordering::Equal)
+    }
+}
+
+struct Foo {
+    a: i32,
+}
+
+impl PartialEq for Foo {
+    fn eq(&self, other: &'_ Self) -> bool {
+        ::core::cmp::PartialEq::eq(&self.a, &other.a)
+    }
+}
diff --git a/gcc/testsuite/rust/compile/issue-3874.rs b/gcc/testsuite/rust/compile/issue-3874.rs
new file mode 100644
index 0000000..ebce4b6
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3874.rs
@@ -0,0 +1,4 @@
+fn wow(){
+    &#[serde]
+    // { dg-error "found unexpected token .#. in null denotation" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/issue-3876.rs b/gcc/testsuite/rust/compile/issue-3876.rs
new file mode 100644
index 0000000..17b1590
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3876.rs
@@ -0,0 +1,8 @@
+enum test {
+    A(i32),
+}
+
+fn fun(x: i32) {
+    test::A { x }
+    // { dg-error "unknown field" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/issue-3885.rs b/gcc/testsuite/rust/compile/issue-3885.rs
new file mode 100644
index 0000000..050a59c
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3885.rs
@@ -0,0 +1,7 @@
+pub fn test() {
+    let _u: [_; _] = [15u32];
+    let _v: [u8; _] = [1, 2, 3];
+    let _w: [_; 2] = [1.0, 2.0];
+    let _x = [42; 5];
+    let _y: [_; _] = _x;
+}
diff --git a/gcc/testsuite/rust/compile/issue-3915.rs b/gcc/testsuite/rust/compile/issue-3915.rs
new file mode 100644
index 0000000..7132036
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3915.rs
@@ -0,0 +1,28 @@
+// { dg-options "-w" }
+#[lang = "sized"]
+trait Sized {}
+
+trait Trait {
+    fn do_thing();
+}
+
+struct MyType;
+
+impl Trait for MyType {
+    fn do_thing() {}
+}
+
+struct Wrapper<T: Trait> {
+    value: T,
+}
+
+impl<T: Trait> Wrapper<T> {
+    fn call_it() {
+        T::do_thing();
+    }
+}
+
+fn main() {
+    let _ = Wrapper::<MyType> { value: MyType };
+    Wrapper::<MyType>::call_it();
+}
diff --git a/gcc/testsuite/rust/compile/issue-3916.rs b/gcc/testsuite/rust/compile/issue-3916.rs
new file mode 100644
index 0000000..59b522b
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3916.rs
@@ -0,0 +1,36 @@
+#![feature(rustc_attrs)]
+
+#[lang = "sized"]
+trait Sized {}
+
+#[lang = "add"]
+trait Add<Rhs = Self> {
+    type Output;
+
+    fn add(self, rhs: Rhs) -> Self::Output;
+}
+
+macro_rules! add_impl {
+    ($($t:ty)*) => ($(
+        impl Add for $t {
+            type Output = $t;
+
+            #[inline]
+            #[rustc_inherit_overflow_checks]
+            fn add(self, other: $t) -> $t { self + other }
+        }
+    )*)
+}
+
+add_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 }
+
+pub fn test(len: usize) -> u64 {
+    let mut i = 0;
+    let mut out = 0;
+    if i + 3 < len {
+        out = 123;
+    } else {
+        out = 456;
+    }
+    out
+}
diff --git a/gcc/testsuite/rust/compile/issue-3960.rs b/gcc/testsuite/rust/compile/issue-3960.rs
new file mode 100644
index 0000000..57329f0
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3960.rs
@@ -0,0 +1,7 @@
+fn main() {
+    struct G {
+        g: (),
+    }
+    let g = [0; G { g: () }];
+    // { dg-error "mismatched types, expected .usize. but got .G. .E0308." "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/issue-3978.rs b/gcc/testsuite/rust/compile/issue-3978.rs
new file mode 100644
index 0000000..4f17d3d
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-3978.rs
@@ -0,0 +1,8 @@
+type Dimension = usize;
+
+pub fn main() {}
+
+mod m2 {
+    fn main() {}
+    // { dg-warning "function is never used" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/issue-4006.rs b/gcc/testsuite/rust/compile/issue-4006.rs
new file mode 100644
index 0000000..328c7b6
--- /dev/null
+++ b/gcc/testsuite/rust/compile/issue-4006.rs
@@ -0,0 +1,13 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! asm {
+    () => {};
+}
+
+pub fn main() {
+    asm!(
+        "xor eax, eax"
+        "xor eax, eax");
+    // { dg-error "expected token .;." "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/loop_constant_context.rs b/gcc/testsuite/rust/compile/loop_constant_context.rs
new file mode 100644
index 0000000..ed0782b
--- /dev/null
+++ b/gcc/testsuite/rust/compile/loop_constant_context.rs
@@ -0,0 +1,5 @@
+// { dg-error "'loop' is not allowed in const context" "" { target *-*-* } .+1 }
+const CONST_LOOP : () = loop{};
+
+// { dg-error "'loop' is not allowed in const context" "" { target *-*-* } .+1 }
+static STATIC_LOOP : () = loop{};
\ No newline at end of file
diff --git a/gcc/testsuite/rust/compile/macros/builtin/recurse2.rs b/gcc/testsuite/rust/compile/macros/builtin/recurse2.rs
index 73e6ab4..fbb4b10 100644
--- a/gcc/testsuite/rust/compile/macros/builtin/recurse2.rs
+++ b/gcc/testsuite/rust/compile/macros/builtin/recurse2.rs
@@ -34,7 +34,7 @@ fn print_str(s: &str) {
     }
 }
 
-// { dg-final { scan-assembler {"abheyho"} } }
+// { dg-final { scan-assembler {"abheyho(\\0)?"} } }
 static S: &str = concat!("a", 'b', a!(), a!(b c d e f a!()), '\0');
 
 fn main() {
diff --git a/gcc/testsuite/rust/compile/macros/mbe/macro-issue3708.rs b/gcc/testsuite/rust/compile/macros/mbe/macro-issue3708.rs
index e5b38bb..bcbc8dd 100644
--- a/gcc/testsuite/rust/compile/macros/mbe/macro-issue3708.rs
+++ b/gcc/testsuite/rust/compile/macros/mbe/macro-issue3708.rs
@@ -1,4 +1,4 @@
-// { dg-additional-options "-frust-name-resolution-2.0 -frust-compile-until=lowering" }
+// { dg-additional-options "-frust-compile-until=lowering" }
 
 macro_rules! impl_fn_for_zst {
     ($(
diff --git a/gcc/testsuite/rust/compile/macros/mbe/macro-issue3709-2.rs b/gcc/testsuite/rust/compile/macros/mbe/macro-issue3709-2.rs
index cfc8ab4..47514f1 100644
--- a/gcc/testsuite/rust/compile/macros/mbe/macro-issue3709-2.rs
+++ b/gcc/testsuite/rust/compile/macros/mbe/macro-issue3709-2.rs
@@ -1,4 +1,4 @@
-// { dg-additional-options "-frust-name-resolution-2.0 -frust-compile-until=lowering" }
+// { dg-additional-options "-frust-compile-until=lowering" }
 
 macro_rules! impl_fn_for_zst {
     ($(
diff --git a/gcc/testsuite/rust/compile/macros/mbe/meta-param.rs b/gcc/testsuite/rust/compile/macros/mbe/meta-param.rs
new file mode 100644
index 0000000..ed6e100
--- /dev/null
+++ b/gcc/testsuite/rust/compile/macros/mbe/meta-param.rs
@@ -0,0 +1,7 @@
+macro_rules! foo {
+    ($x:meta) => {0}
+}
+
+pub fn main() -> i32 {
+    foo!(Clone)
+}
diff --git a/gcc/testsuite/rust/compile/match-identifierpattern-enum.rs b/gcc/testsuite/rust/compile/match-identifierpattern-enum.rs
new file mode 100644
index 0000000..c712667
--- /dev/null
+++ b/gcc/testsuite/rust/compile/match-identifierpattern-enum.rs
@@ -0,0 +1,12 @@
+enum Foo {
+    I(i32),
+}
+
+fn main() {
+    let x = Foo::I(1);
+
+    match x {
+        a @ Foo::I(b) => {},
+        _ => {},
+    };
+}
diff --git a/gcc/testsuite/rust/compile/match-identifierpattern.rs b/gcc/testsuite/rust/compile/match-identifierpattern.rs
new file mode 100644
index 0000000..6c558ac
--- /dev/null
+++ b/gcc/testsuite/rust/compile/match-identifierpattern.rs
@@ -0,0 +1,9 @@
+fn main() {
+    let x = 1;
+
+    match x {
+        2 => {},
+        a @ 3 => {},
+        _ => {},
+    }
+}
diff --git a/gcc/testsuite/rust/compile/match-restpattern-tuple-1.rs b/gcc/testsuite/rust/compile/match-restpattern-tuple-1.rs
new file mode 100644
index 0000000..5cce3c4
--- /dev/null
+++ b/gcc/testsuite/rust/compile/match-restpattern-tuple-1.rs
@@ -0,0 +1,8 @@
+fn main() {
+    let x = (1, 2, 3, 4);
+
+    match x {
+        (1, .., 4) => {},
+        _ => {}
+    }
+}
\ No newline at end of file
diff --git a/gcc/testsuite/rust/compile/match-restpattern-tuple-2.rs b/gcc/testsuite/rust/compile/match-restpattern-tuple-2.rs
new file mode 100644
index 0000000..40900a3
--- /dev/null
+++ b/gcc/testsuite/rust/compile/match-restpattern-tuple-2.rs
@@ -0,0 +1,8 @@
+fn main() {
+    let x = (1, 2, 3, 4);
+
+    match x {
+        (1, .., 2, 3, 4, 5) => {}, // { dg-error "expected a tuple with 4 elements, found one with 5 elements" }
+        _ => {}
+    }
+}
\ No newline at end of file
diff --git a/gcc/testsuite/rust/compile/match-slicepattern-array.rs b/gcc/testsuite/rust/compile/match-slicepattern-array.rs
new file mode 100644
index 0000000..e48ca75
--- /dev/null
+++ b/gcc/testsuite/rust/compile/match-slicepattern-array.rs
@@ -0,0 +1,8 @@
+fn main() {
+    let a = [0, 1];
+
+    match a {
+        [0, 1] => {},
+        _ => {}
+    }
+}
diff --git a/gcc/testsuite/rust/compile/match-slicepattern-slice.rs b/gcc/testsuite/rust/compile/match-slicepattern-slice.rs
new file mode 100644
index 0000000..cc33d93
--- /dev/null
+++ b/gcc/testsuite/rust/compile/match-slicepattern-slice.rs
@@ -0,0 +1,10 @@
+fn main() {
+    let arr = [1, 2];
+    let slice: &[i32] = &arr;
+
+    match slice {
+        [1] => {},
+        [_, 2] => {},
+        _ => {}
+    }
+}
\ No newline at end of file
diff --git a/gcc/testsuite/rust/compile/match-tuplestructpattern.rs b/gcc/testsuite/rust/compile/match-tuplestructpattern.rs
new file mode 100644
index 0000000..0dae71e
--- /dev/null
+++ b/gcc/testsuite/rust/compile/match-tuplestructpattern.rs
@@ -0,0 +1,9 @@
+fn main() {
+    struct A (i32, i32);
+    let a = A (0, 1);
+
+    match a {
+        A (0, 1) => {},
+        _ => {}
+    }
+}
diff --git a/gcc/testsuite/rust/compile/min_specialization1.rs b/gcc/testsuite/rust/compile/min_specialization1.rs
index d38167e..ba97f87 100644
--- a/gcc/testsuite/rust/compile/min_specialization1.rs
+++ b/gcc/testsuite/rust/compile/min_specialization1.rs
@@ -9,7 +9,7 @@ pub trait Foo {
 pub struct Bar;
 
 impl Foo for Bar {
-    default fn foo(&self) -> bool { // { dg-warning "unused" }
+    default fn foo(&self) -> bool {
         true
     }
 }
diff --git a/gcc/testsuite/rust/compile/name_resolution10.rs b/gcc/testsuite/rust/compile/name_resolution10.rs
index 33643bd..f156f98 100644
--- a/gcc/testsuite/rust/compile/name_resolution10.rs
+++ b/gcc/testsuite/rust/compile/name_resolution10.rs
@@ -1,4 +1,4 @@
-// { dg-options "-frust-name-resolution-2.0 -frust-compile-until=lowering" }
+// { dg-options "-frust-compile-until=lowering" }
 
 #![feature(decl_macro)]
 
diff --git a/gcc/testsuite/rust/compile/name_resolution11.rs b/gcc/testsuite/rust/compile/name_resolution11.rs
index a464d2a..329567a 100644
--- a/gcc/testsuite/rust/compile/name_resolution11.rs
+++ b/gcc/testsuite/rust/compile/name_resolution11.rs
@@ -1,4 +1,4 @@
-// { dg-additional-options "-frust-name-resolution-2.0 -frust-compile-until=lowering" }
+// { dg-additional-options "-frust-compile-until=lowering" }
 fn foo() {
     let b = 10;
     fn bar() {
diff --git a/gcc/testsuite/rust/compile/name_resolution12.rs b/gcc/testsuite/rust/compile/name_resolution12.rs
index 9cce31c..0f217aa 100644
--- a/gcc/testsuite/rust/compile/name_resolution12.rs
+++ b/gcc/testsuite/rust/compile/name_resolution12.rs
@@ -1,4 +1,4 @@
-// { dg-additional-options "-frust-name-resolution-2.0 -frust-compile-until=lowering" }
+// { dg-additional-options "-frust-compile-until=lowering" }
 
 const TOTO: i32 = 10;
 
diff --git a/gcc/testsuite/rust/compile/name_resolution13.rs b/gcc/testsuite/rust/compile/name_resolution13.rs
index 33edbf9..8356cf6 100644
--- a/gcc/testsuite/rust/compile/name_resolution13.rs
+++ b/gcc/testsuite/rust/compile/name_resolution13.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 pub mod foo {
     pub macro bar() {}
 }
diff --git a/gcc/testsuite/rust/compile/name_resolution14.rs b/gcc/testsuite/rust/compile/name_resolution14.rs
index eaef6a5..44c43aa 100644
--- a/gcc/testsuite/rust/compile/name_resolution14.rs
+++ b/gcc/testsuite/rust/compile/name_resolution14.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 pub mod foo {
     pub macro bar() {}
 }
diff --git a/gcc/testsuite/rust/compile/name_resolution15.rs b/gcc/testsuite/rust/compile/name_resolution15.rs
index 45f38da..e82c90e 100644
--- a/gcc/testsuite/rust/compile/name_resolution15.rs
+++ b/gcc/testsuite/rust/compile/name_resolution15.rs
@@ -1,4 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
 #![feature(decl_macro)]
 
 pub mod foo {
diff --git a/gcc/testsuite/rust/compile/name_resolution16.rs b/gcc/testsuite/rust/compile/name_resolution16.rs
index 230722e..4db7b2e 100644
--- a/gcc/testsuite/rust/compile/name_resolution16.rs
+++ b/gcc/testsuite/rust/compile/name_resolution16.rs
@@ -1,4 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
 #![feature(decl_macro)]
 
 pub mod foo {
diff --git a/gcc/testsuite/rust/compile/name_resolution17.rs b/gcc/testsuite/rust/compile/name_resolution17.rs
index 4859476..84ad380 100644
--- a/gcc/testsuite/rust/compile/name_resolution17.rs
+++ b/gcc/testsuite/rust/compile/name_resolution17.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 struct Foo;
 fn Foo() {} // { dg-error ".Foo. defined multiple times" }
 
diff --git a/gcc/testsuite/rust/compile/name_resolution18.rs b/gcc/testsuite/rust/compile/name_resolution18.rs
index 5940149..17a3352 100644
--- a/gcc/testsuite/rust/compile/name_resolution18.rs
+++ b/gcc/testsuite/rust/compile/name_resolution18.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 struct Marker;
 
 struct Foo {
diff --git a/gcc/testsuite/rust/compile/name_resolution2.rs b/gcc/testsuite/rust/compile/name_resolution2.rs
index 7e4f5a1..564c5d2 100644
--- a/gcc/testsuite/rust/compile/name_resolution2.rs
+++ b/gcc/testsuite/rust/compile/name_resolution2.rs
@@ -4,7 +4,7 @@ pub trait Sized {}
 struct Bar;
 
 trait Foo {
-    fn bar(&self) {} // { dg-warning "unused name" }
+    fn bar(&self) {}
 }
 
 pub fn outer() {
diff --git a/gcc/testsuite/rust/compile/name_resolution20.rs b/gcc/testsuite/rust/compile/name_resolution20.rs
index e6c2dd5..f131bb4 100644
--- a/gcc/testsuite/rust/compile/name_resolution20.rs
+++ b/gcc/testsuite/rust/compile/name_resolution20.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 pub mod foo {
     pub macro bar() {}
 }
diff --git a/gcc/testsuite/rust/compile/name_resolution22.rs b/gcc/testsuite/rust/compile/name_resolution22.rs
index c49331e..bb5edda 100644
--- a/gcc/testsuite/rust/compile/name_resolution22.rs
+++ b/gcc/testsuite/rust/compile/name_resolution22.rs
@@ -1,4 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
 struct Marker;
 struct Foo(Marker);
 
diff --git a/gcc/testsuite/rust/compile/name_resolution23.rs b/gcc/testsuite/rust/compile/name_resolution23.rs
index 50b8e81..843be2a 100644
--- a/gcc/testsuite/rust/compile/name_resolution23.rs
+++ b/gcc/testsuite/rust/compile/name_resolution23.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 mod a {
     pub mod b {
         pub fn foo() {}
diff --git a/gcc/testsuite/rust/compile/name_resolution24.rs b/gcc/testsuite/rust/compile/name_resolution24.rs
index f4eb7b2..4350cd8 100644
--- a/gcc/testsuite/rust/compile/name_resolution24.rs
+++ b/gcc/testsuite/rust/compile/name_resolution24.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 mod a {
     pub mod b {
         pub fn baz() {}
diff --git a/gcc/testsuite/rust/compile/name_resolution25.rs b/gcc/testsuite/rust/compile/name_resolution25.rs
index 3cacac7..0cadd9e 100644
--- a/gcc/testsuite/rust/compile/name_resolution25.rs
+++ b/gcc/testsuite/rust/compile/name_resolution25.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 struct Test; // { dg-warning "struct is never constructed: .Test." }
 
 impl Test {}
diff --git a/gcc/testsuite/rust/compile/name_resolution4.rs b/gcc/testsuite/rust/compile/name_resolution4.rs
index b2eadbe..0fc72f6 100644
--- a/gcc/testsuite/rust/compile/name_resolution4.rs
+++ b/gcc/testsuite/rust/compile/name_resolution4.rs
@@ -2,7 +2,7 @@
 pub trait Sized {}
 
 trait Foo {
-    fn foo(&self) {} // { dg-warning "unused name" }
+    fn foo(&self) {}
 }
 
 struct Bar;
diff --git a/gcc/testsuite/rust/compile/name_resolution6.rs b/gcc/testsuite/rust/compile/name_resolution6.rs
index e4087e6..b2b5f6b 100644
--- a/gcc/testsuite/rust/compile/name_resolution6.rs
+++ b/gcc/testsuite/rust/compile/name_resolution6.rs
@@ -1,4 +1,4 @@
-// { dg-options "-frust-name-resolution-2.0 -frust-compile-until=lowering" }
+// { dg-options "-frust-compile-until=lowering" }
 
 pub mod foo {
     pub mod bar {
diff --git a/gcc/testsuite/rust/compile/name_resolution7.rs b/gcc/testsuite/rust/compile/name_resolution7.rs
index fa84e2f..78cb0b2 100644
--- a/gcc/testsuite/rust/compile/name_resolution7.rs
+++ b/gcc/testsuite/rust/compile/name_resolution7.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 // check that macros by example do not get inserted in ribs like regular items
 pub mod foo {
     pub mod bar {
diff --git a/gcc/testsuite/rust/compile/name_resolution8.rs b/gcc/testsuite/rust/compile/name_resolution8.rs
index 6fb5170..aca1945 100644
--- a/gcc/testsuite/rust/compile/name_resolution8.rs
+++ b/gcc/testsuite/rust/compile/name_resolution8.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 // check that macros by example get exported to the crate's root with #[macro_export]
 pub mod foo {
     pub mod bar {
diff --git a/gcc/testsuite/rust/compile/name_resolution9.rs b/gcc/testsuite/rust/compile/name_resolution9.rs
index 792b3bd..84ba3c5 100644
--- a/gcc/testsuite/rust/compile/name_resolution9.rs
+++ b/gcc/testsuite/rust/compile/name_resolution9.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 pub mod foo {
     pub mod bar {
         fn f() {
diff --git a/gcc/testsuite/rust/compile/nested_macro_definition.rs b/gcc/testsuite/rust/compile/nested_macro_definition.rs
index c0b7250..b71afbd 100644
--- a/gcc/testsuite/rust/compile/nested_macro_definition.rs
+++ b/gcc/testsuite/rust/compile/nested_macro_definition.rs
@@ -1,5 +1,3 @@
-// { dg-options "-frust-name-resolution-2.0" }
-
 macro_rules! toto {
     () => {
         macro_rules! tata {
diff --git a/gcc/testsuite/rust/compile/nr2/compile.exp b/gcc/testsuite/rust/compile/nr2/compile.exp
deleted file mode 100644
index 9e15cdd..0000000
--- a/gcc/testsuite/rust/compile/nr2/compile.exp
+++ /dev/null
@@ -1,149 +0,0 @@
-# Copyright (C) 2021-2024 Free Software Foundation, Inc.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with GCC; see the file COPYING3.  If not see
-# <http://www.gnu.org/licenses/>.
-
-# Run compile tests with name resolution 2.0 enabled
-
-# Load support procs.
-load_lib rust-dg.exp
-
-# These tests don't run runtest_file_p consistently if it
-# doesn't return the same values, so disable parallelization
-# of this *.exp file.  The first parallel runtest to reach
-# this will run all the tests serially.
-if ![gcc_parallel_test_run_p compile] {
-    return
-}
-gcc_parallel_test_enable 0
-
-# Initialize `dg'.
-dg-init
-
-namespace eval rust-nr2-ns {
-    # Exclude tests which aren't passing yet
-    # These should be removed from the exclude file over time
-
-    set exclude_fh [open $srcdir/$subdir/exclude r]
-    set exclude_raw [lsort [split [read $exclude_fh] "\n"]]
-    close $exclude_fh
-    unset exclude_fh
-
-    set exclude ""
-    foreach ent $exclude_raw {
-        if [regexp {^[^#].*} $ent] {
-            lappend exclude $ent
-	}
-    }
-    unset exclude_raw
-
-    # Run tests in directories
-    # Manually specifying these, in case some other test file
-    # does something weird
-    set test_dirs {{} {macros builtin} {macros mbe} {macros proc} {torture}}
-
-    set tests_expect_ok ""
-    set tests_expect_err ""
-
-    set compile_dir [list {*}[file split $srcdir] {*}[file split $subdir]]
-    set compile_dir [lreplace $compile_dir end end]
-
-    foreach test_dir $test_dirs {
-        foreach test [lsort [glob -nocomplain -tails -directory [file join {*}$compile_dir {*}$test_dir] *.rs]] {
-	    # use '/' as the path seperator for entries in the exclude file
-	    set test_lbl [join [list {*}$test_dir $test] "/"]
-	    set idx [lsearch -exact -sorted $exclude $test_lbl]
-	    if {$idx == -1} {
-	        if {[runtest_file_p $runtests [file join {*}$compile_dir {*}$test_dir $test]]} {
-		    lappend tests_expect_ok [list {*}$test_dir $test]
-		}
-	    } else {
-	        if {[runtest_file_p $runtests [file join {*}$compile_dir {*}$test_dir $test]]} {
-		    lappend tests_expect_err [list {*}$test_dir $test]
-		}
-		set exclude [lreplace $exclude $idx $idx]
-	    }
-	}
-    }
-
-    # Generate failures for unmatched tests in the exclude list
-    foreach ent $exclude {
-        fail "$ent: could not exclude test"
-    }
-    unset exclude
-
-    # run a test while catching record_test calls
-    set record_test_out ""
-    proc try_test { test } {
-        variable record_test_out
-	rename ::record_test record_test_old
-
-	proc ::record_test { type msg args } {
-	    namespace eval ::rust-nr2-ns {
-		set type [uplevel 1 {set type}]
-		set msg [uplevel 1 {set msg}]
-		variable record_test_out
-		switch $type {
-		    FAIL {
-			lappend record_test_out [list $type $msg]
-		    }
-		    XPASS {
-			lappend record_test_out [list $type $msg]
-		    }
-		}
-            }
-	}
-
-        namespace eval :: {
-	    set saved-dg-do-what-default ${dg-do-what-default}
-	    set dg-do-what-default "compile"
-            dg-runtest [list [uplevel 1 {set test}]] "-frust-name-resolution-2.0" ""
-	    set dg-do-what-default ${saved-dg-do-what-default}
-	}
-
-        rename ::record_test ""
-	rename record_test_old ::record_test
-
-	set record_test_cache $record_test_out
-        set record_test_out ""
-	return $record_test_cache
-    }
-
-    # check for unexpected failures
-    foreach test $tests_expect_ok {
-        set fails [try_test [file join {*}$compile_dir {*}$test]]
-	if {[llength $fails] != 0} {
-	    foreach ent $fails {
-		record_test [lindex $ent 0] "on nr2: [lindex $ent 1]"
-	    }
-	} else {
-	    record_test PASS "[file join {*}$test] on nr2"
-	}
-    }
-
-    #check for unexpected successes
-    foreach test $tests_expect_err {
-        set fails [try_test [file join {*}$compile_dir {*}$test]]
-	if {[llength $fails] == 0} {
-	    record_test XPASS "[file join {*}$test] on nr2"
-	} else {
-	    record_test XFAIL "[file join {*}$test] on nr2 was rightfully excluded"
-	}
-    }
-}
-
-# All done.
-dg-finish
-
-gcc_parallel_test_enable 1
diff --git a/gcc/testsuite/rust/compile/nr2/exclude b/gcc/testsuite/rust/compile/nr2/exclude
deleted file mode 100644
index c020e36..0000000
--- a/gcc/testsuite/rust/compile/nr2/exclude
+++ /dev/null
@@ -1,17 +0,0 @@
-canonical_paths1.rs
-issue-3315-2.rs
-privacy5.rs
-privacy8.rs
-pub_restricted_1.rs
-pub_restricted_2.rs
-pub_restricted_3.rs
-issue-2905-2.rs
-derive-default1.rs
-derive-eq-invalid.rs
-torture/alt_patterns1.rs
-torture/name_resolve1.rs
-issue-3663.rs
-issue-3671.rs
-issue-3652.rs
-issue-3649.rs
-# please don't delete the trailing newline
diff --git a/gcc/testsuite/rust/compile/offset_of1.rs b/gcc/testsuite/rust/compile/offset_of1.rs
new file mode 100644
index 0000000..5b79699
--- /dev/null
+++ b/gcc/testsuite/rust/compile/offset_of1.rs
@@ -0,0 +1,11 @@
+// { dg-additional-options "-frust-compile-until=lowering -frust-assume-builtin-offset-of" }
+
+pub struct Foo {
+    a: i32,
+}
+
+fn main() {
+    let _ = offset_of!(Foo, a); // valid
+    let _ = offset_of!("bloop", a); // { dg-error "could not parse type" }
+    let _ = offset_of!(Foo, 15); // { dg-error "could not parse field" }
+}
diff --git a/gcc/testsuite/rust/compile/offset_of2.rs b/gcc/testsuite/rust/compile/offset_of2.rs
new file mode 100644
index 0000000..d4ad9c2
--- /dev/null
+++ b/gcc/testsuite/rust/compile/offset_of2.rs
@@ -0,0 +1,9 @@
+// { dg-additional-options "-frust-compile-until=compilation -frust-assume-builtin-offset-of" }
+
+pub struct Foo {
+    a: i32,
+}
+
+fn main() {
+    let _ = offset_of!(Foo, a); // valid
+}
diff --git a/gcc/testsuite/rust/compile/parse_simple_path_fail_1.rs b/gcc/testsuite/rust/compile/parse_simple_path_fail_1.rs
new file mode 100644
index 0000000..c112e40
--- /dev/null
+++ b/gcc/testsuite/rust/compile/parse_simple_path_fail_1.rs
@@ -0,0 +1,3 @@
+pub(in crate::) struct S;
+// { dg-error "expecting ... but .::. found" "" { target *-*-* } .-1 }
+// { dg-error "failed to parse item in crate" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/rust/compile/parse_simple_path_fail_2.rs b/gcc/testsuite/rust/compile/parse_simple_path_fail_2.rs
new file mode 100644
index 0000000..94c49c3
--- /dev/null
+++ b/gcc/testsuite/rust/compile/parse_simple_path_fail_2.rs
@@ -0,0 +1,9 @@
+mod A {
+    struct B;
+}
+
+use A{B};
+// { dg-error "unexpected token" "" { target *-*-* } .-1 }
+// { dg-error "could not parse use tree" "" { target *-*-* } .-2 }
+// { dg-error "failed to parse item in crate" "" { target *-*-* } 10 }
+// ^^^ TODO: should the above error happen at line 10?
diff --git a/gcc/testsuite/rust/compile/pub_restricted_1.rs b/gcc/testsuite/rust/compile/pub_restricted_1.rs
index 9bda968..2afbeb4 100644
--- a/gcc/testsuite/rust/compile/pub_restricted_1.rs
+++ b/gcc/testsuite/rust/compile/pub_restricted_1.rs
@@ -6,8 +6,8 @@ pub mod foo {
     }
 }
 
-pub(in foo::fah::baz) struct A1; // { dg-error "cannot find simple path segment .fah." }
-pub(in fro::bulator::saindoux) struct A2; // { dg-error "cannot find simple path segment .fro." }
-pub(in foo::bar::saindoux) struct A3; // { dg-error "cannot find simple path segment .saindoux." }
+pub(in foo::fah::baz) struct A1; // { dg-error "could not resolve path .foo::fah::baz." }
+pub(in fro::bulator::saindoux) struct A2; // { dg-error "could not resolve path .fro::bulator::saindoux." }
+pub(in foo::bar::saindoux) struct A3; // { dg-error "could not resolve path .foo::bar::saindoux." }
 
 fn main() {}
diff --git a/gcc/testsuite/rust/compile/pub_restricted_2.rs b/gcc/testsuite/rust/compile/pub_restricted_2.rs
index 8588f27..fea9379 100644
--- a/gcc/testsuite/rust/compile/pub_restricted_2.rs
+++ b/gcc/testsuite/rust/compile/pub_restricted_2.rs
@@ -3,16 +3,16 @@
 mod foo {
     mod bar {
         mod baz {
-            pub(in baz) struct A0;
-            pub(in bar::baz) struct A1;
+            pub(in super::baz) struct A0;
+            pub(in super::super::bar::baz) struct A1;
             pub(in foo::bar::baz) struct A2;
 
             mod sain {
                 mod doux {}
             }
 
-            pub(in sain) struct A3; // { dg-error "restricted path is not an ancestor of the current module" }
-            pub(in sain::doux) struct A4; // { dg-error "restricted path is not an ancestor of the current module" }
+            pub(in self::sain) struct A3; // { dg-error "restricted path is not an ancestor of the current module" }
+            pub(in self::sain::doux) struct A4; // { dg-error "restricted path is not an ancestor of the current module" }
         }
     }
 }
diff --git a/gcc/testsuite/rust/execute/same_field_name.rs b/gcc/testsuite/rust/compile/same_field_name.rs
index d57562b..8e5b78c 100644
--- a/gcc/testsuite/rust/execute/same_field_name.rs
+++ b/gcc/testsuite/rust/compile/same_field_name.rs
@@ -1,7 +1,7 @@
 // https://doc.rust-lang.org/error_codes/E0124.html
 fn main() {
     struct Foo {
-        field1: i32, // { dg-error "field .field1. is already declared" }
+        field1: i32,
         field1: i32, // { dg-error "field .field1. is already declared" }
         field1: i32, // { dg-error "field .field1. is already declared" }
     }
diff --git a/gcc/testsuite/rust/compile/self-in-impl.rs b/gcc/testsuite/rust/compile/self-in-impl.rs
new file mode 100644
index 0000000..a567897
--- /dev/null
+++ b/gcc/testsuite/rust/compile/self-in-impl.rs
@@ -0,0 +1,15 @@
+// the error message here is what rustc >=1.66 emits
+// rustc <1.66 emits a "cycle detected" error when
+//   trying to calculate the impl type
+//
+// since we aren't trying to match error messages too closely
+// and the >=1.66 error message is nicer
+// we may as well mimic that
+
+impl ((Self, i32)) {}
+// { dg-error ".Self. is not valid in the self" "" { target *-*-* } .-1 }
+
+trait Foo {}
+
+impl Foo for ((Self, i32)) {}
+// { dg-error ".Self. is not valid in the self" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/rust/compile/self_import_namespace.rs b/gcc/testsuite/rust/compile/self_import_namespace.rs
index 2d9b2ed..a63c1d7 100644
--- a/gcc/testsuite/rust/compile/self_import_namespace.rs
+++ b/gcc/testsuite/rust/compile/self_import_namespace.rs
@@ -1,5 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
-
 mod bar {
     pub mod foo {}
     pub fn foo() {}
diff --git a/gcc/testsuite/rust/compile/silly-order-bug.rs b/gcc/testsuite/rust/compile/silly-order-bug.rs
new file mode 100644
index 0000000..0d9cf1d
--- /dev/null
+++ b/gcc/testsuite/rust/compile/silly-order-bug.rs
@@ -0,0 +1,8 @@
+#[lang = "sized"]
+trait Sized {}
+
+#[lang = "fn_once"]
+pub trait FnOnce<Args> {
+    extern "rust-call" fn call_once(self, args: Args) -> Self::Output;
+    type Output;
+}
diff --git a/gcc/testsuite/rust/compile/slicepattern-size-mismatch.rs b/gcc/testsuite/rust/compile/slicepattern-size-mismatch.rs
new file mode 100644
index 0000000..b54b532
--- /dev/null
+++ b/gcc/testsuite/rust/compile/slicepattern-size-mismatch.rs
@@ -0,0 +1,8 @@
+fn main() {
+    let arr = [0, 1];
+
+    match arr {
+        [0, 1, 2] => {} // { dg-error "pattern requires 3 elements but array has 2 .E0527." }
+        _ => {}
+    }
+}
\ No newline at end of file
diff --git a/gcc/testsuite/rust/compile/torture/extern_mod2.rs b/gcc/testsuite/rust/compile/torture/extern_mod2.rs
index 4984d5d..f3a4f79 100644
--- a/gcc/testsuite/rust/compile/torture/extern_mod2.rs
+++ b/gcc/testsuite/rust/compile/torture/extern_mod2.rs
@@ -12,6 +12,12 @@ mod no_leading_equal;
 #[path       =     "modules/valid_path.rs"]
 mod extra_spaces;
 
+#[path = ""]  // { dg-error "path attributes must contain a filename" }
+mod empty_path; // { dg-error "no candidate found" }
+
+#[path = "          "]  // { dg-error "path attributes must contain a filename" }
+mod path_with_spaces; // { dg-error "no candidate found" }
+
 #[path] // { dg-error "path attributes must contain a filename" }
 mod error; // { dg-error "no candidate found" }
 
diff --git a/gcc/testsuite/rust/compile/torture/generics29.rs b/gcc/testsuite/rust/compile/torture/generics29.rs
index e9c693e..baf53e4 100644
--- a/gcc/testsuite/rust/compile/torture/generics29.rs
+++ b/gcc/testsuite/rust/compile/torture/generics29.rs
@@ -5,7 +5,6 @@ struct Foo<A, B>(A, B);
 
 impl Foo<i32, f32> {
     fn test<X>(self, a: X) -> X {
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
         a
     }
 }
diff --git a/gcc/testsuite/rust/compile/torture/generics30.rs b/gcc/testsuite/rust/compile/torture/generics30.rs
index 24ae58f..a84f140 100644
--- a/gcc/testsuite/rust/compile/torture/generics30.rs
+++ b/gcc/testsuite/rust/compile/torture/generics30.rs
@@ -5,7 +5,6 @@ struct Foo<A, B>(A, B);
 
 impl<T> Foo<T, f32> {
     fn test<X>(self, a: X) -> X {
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
         a
     }
 }
diff --git a/gcc/testsuite/rust/compile/torture/traits3.rs b/gcc/testsuite/rust/compile/torture/traits3.rs
index d805da5..dad6dda 100644
--- a/gcc/testsuite/rust/compile/torture/traits3.rs
+++ b/gcc/testsuite/rust/compile/torture/traits3.rs
@@ -10,7 +10,6 @@ struct Baz;
 
 impl Foo for Baz {
     fn Bar(self) -> i32 {
-        // { dg-warning "unused name .self." "" { target *-*-* } .-1 }
         123
     }
 }
diff --git a/gcc/testsuite/rust/compile/torture/traits7.rs b/gcc/testsuite/rust/compile/torture/traits7.rs
index 8e4472d..545fd39 100644
--- a/gcc/testsuite/rust/compile/torture/traits7.rs
+++ b/gcc/testsuite/rust/compile/torture/traits7.rs
@@ -13,7 +13,6 @@ impl Foo for Bar {
     // { dg-warning "unused name" "" { target *-*-* } .-1 }
 
     fn test(self) {}
-    // { dg-warning "unused name" "" { target *-*-* } .-1 }
 }
 
 fn main() {
diff --git a/gcc/testsuite/rust/compile/torture/unended-raw-byte-string.rs b/gcc/testsuite/rust/compile/torture/unended-raw-byte-string.rs
new file mode 100644
index 0000000..91a3c9a
--- /dev/null
+++ b/gcc/testsuite/rust/compile/torture/unended-raw-byte-string.rs
@@ -0,0 +1,6 @@
+// { dg-excess-errors "...." }
+fn main() {
+    // { dg-error "unended raw byte string literal" "" { target *-*-* } .+1 }
+    let s = br##"123"#
+}
+
diff --git a/gcc/testsuite/rust/compile/traits9.rs b/gcc/testsuite/rust/compile/traits9.rs
index bb3034d..f4308e8 100644
--- a/gcc/testsuite/rust/compile/traits9.rs
+++ b/gcc/testsuite/rust/compile/traits9.rs
@@ -11,6 +11,5 @@ fn main() {
     a = Foo(123);
 
     let b: &dyn Bar = &a;
-    // { dg-error "bounds not satisfied for Foo .Bar. is not satisfied" "" { target *-*-* } .-1 }
-    // { dg-error "expected" "" { target *-*-* } .-2 }
+    // { dg-error "bounds not satisfied for Foo .Bar. is not satisfied .E0277." "" { target *-*-* } .-1 }
 }
diff --git a/gcc/testsuite/rust/compile/try_block1.rs b/gcc/testsuite/rust/compile/try_block1.rs
new file mode 100644
index 0000000..7ae0536
--- /dev/null
+++ b/gcc/testsuite/rust/compile/try_block1.rs
@@ -0,0 +1,89 @@
+// { dg-additional-options "-frust-edition=2018" }
+
+#[lang = "sized"]
+trait Sized {}
+
+enum Result<T, E> {
+    Ok(T),
+    Err(E)
+}
+
+pub trait Try {
+    /// The type of this value when viewed as successful.
+    #[unstable(feature = "try_trait", issue = "42327")]
+    type Ok;
+    /// The type of this value when viewed as failed.
+    #[unstable(feature = "try_trait", issue = "42327")]
+    type Error;
+
+    /// Applies the "?" operator. A return of `Ok(t)` means that the
+    /// execution should continue normally, and the result of `?` is the
+    /// value `t`. A return of `Err(e)` means that execution should branch
+    /// to the innermost enclosing `catch`, or return from the function.
+    ///
+    /// If an `Err(e)` result is returned, the value `e` will be "wrapped"
+    /// in the return type of the enclosing scope (which must itself implement
+    /// `Try`). Specifically, the value `X::from_error(From::from(e))`
+    /// is returned, where `X` is the return type of the enclosing function.
+    #[lang = "into_result"]
+    #[unstable(feature = "try_trait", issue = "42327")]
+    fn into_result(self) -> Result<Self::Ok, Self::Error>;
+
+    /// Wrap an error value to construct the composite result. For example,
+    /// `Result::Err(x)` and `Result::from_error(x)` are equivalent.
+    #[lang = "from_error"]
+    #[unstable(feature = "try_trait", issue = "42327")]
+    fn from_error(v: Self::Error) -> Self;
+
+    /// Wrap an OK value to construct the composite result. For example,
+    /// `Result::Ok(x)` and `Result::from_ok(x)` are equivalent.
+    #[lang = "from_ok"]
+    #[unstable(feature = "try_trait", issue = "42327")]
+    fn from_ok(v: Self::Ok) -> Self;
+}
+
+pub struct NoneError;
+
+
+pub enum Option<T> {
+    /// No value
+    None,
+    /// Some value `T`
+    Some(T),
+}
+
+impl<T> Option<T> {
+    pub fn ok_or<E>(self, err: E) -> Result<T, E> {
+        match self {
+            Some(ok) => Result::Ok(ok),
+            None => Result::Err(err)
+        }
+    }
+}
+
+use Option::*;
+
+#[unstable(feature = "try_trait", issue = "42327")]
+impl<T> Try for Option<T> {
+    type Ok = T;
+    type Error = NoneError;
+
+    #[inline]
+    fn into_result(self) -> Result<T, NoneError> {
+        self.ok_or(NoneError)
+    }
+
+    #[inline]
+    fn from_ok(v: T) -> Self {
+        Some(v)
+    }
+
+    #[inline]
+    fn from_error(_: NoneError) -> Self {
+        None
+    }
+}
+
+fn main() {
+    let _: Option<i32> = try { 15i32 };
+}
diff --git a/gcc/testsuite/rust/compile/tuple_mismatch.rs b/gcc/testsuite/rust/compile/tuple_mismatch.rs
index 828586b..1ff358b 100644
--- a/gcc/testsuite/rust/compile/tuple_mismatch.rs
+++ b/gcc/testsuite/rust/compile/tuple_mismatch.rs
@@ -3,6 +3,7 @@ fn main() {
     let (_,) = (1, 2); // { dg-error "expected a tuple with 2 elements, found one with 1 element" }
     let (_, _) = (1, 2, 3); // { dg-error "expected a tuple with 3 elements, found one with 2 elements" }
     let (_, _) = (1,); // { dg-error "expected a tuple with 1 element, found one with 2 elements" }
+    let (_, .., _) = (1,); // { dg-error "expected a tuple with 1 element, found one with 2 elements" }
 }
 
 // The lhs and rhs sizes don't match, but we still resolve 'a' to be bool, we don't
diff --git a/gcc/testsuite/rust/compile/unify-errors1.rs b/gcc/testsuite/rust/compile/unify-errors1.rs
new file mode 100644
index 0000000..0fe95ef
--- /dev/null
+++ b/gcc/testsuite/rust/compile/unify-errors1.rs
@@ -0,0 +1,49 @@
+#[lang = "sized"]
+trait Sized {}
+
+#[lang = "copy"]
+trait Copy {}
+
+trait MyTrait {}
+
+struct Wrapper<T: MyTrait> {
+    value: T,
+}
+
+struct NotImpl;
+
+trait A {}
+trait B {}
+
+struct Wrapper2<T: A + B> {
+    value: T,
+}
+
+struct NotImpl2;
+
+impl A for NotImpl2 {}
+
+fn takes_tuple(x: (i32, bool)) {}
+
+fn requires_copy<T: Copy>(value: T) {}
+
+pub fn test() {
+    takes_tuple((1, 2));
+    // { dg-error "mismatched types, expected .bool. but got .<integer>. .E0308." "" { target *-*-* } .-1 }
+
+    takes_tuple((1, 2, 3));
+    // { dg-error "mismatched types, expected ..i32, bool.. but got ..<integer>, <integer>, <integer>.. .E0308." "" { target *-*-* } .-1 }
+
+    takes_tuple("hello");
+    // { dg-error "mismatched types, expected ..i32, bool.. but got .& str. .E0308." "" { target *-*-* } .-1 }
+
+    let x = &mut 5;
+    requires_copy(x);
+    // { dg-error "bounds not satisfied for &mut <integer> .Copy. is not satisfied .E0277." "" { target *-*-* } .-1 }
+
+    let _x = Wrapper { value: NotImpl };
+    // { dg-error "bounds not satisfied for NotImpl .MyTrait. is not satisfied .E0277." "" { target *-*-* } .-1 }
+
+    let _x = Wrapper2 { value: NotImpl2 };
+    // { dg-error "bounds not satisfied for NotImpl2 .B. is not satisfied .E0277." "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/rust/compile/use_1.rs b/gcc/testsuite/rust/compile/use_1.rs
index e8e2037..21ee3e1 100644
--- a/gcc/testsuite/rust/compile/use_1.rs
+++ b/gcc/testsuite/rust/compile/use_1.rs
@@ -1,4 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
 mod frob {}
 
 use foo::bar::baz; // { dg-error "unresolved import .foo::bar::baz." }
diff --git a/gcc/testsuite/rust/compile/usize1.rs b/gcc/testsuite/rust/compile/usize1.rs
index 36cb99b..08f6c9c 100644
--- a/gcc/testsuite/rust/compile/usize1.rs
+++ b/gcc/testsuite/rust/compile/usize1.rs
@@ -1,5 +1,5 @@
 fn main() {
     let a = [1, 2, 3];
     let b: u32 = 1;
-    let c = a[b]; // { dg-error "the type ...integer..CAPACITY.. cannot be indexed by .u32." }
+    let c = a[b]; // { dg-error "the type ...integer.; 3.. cannot be indexed by .u32." }
 }
diff --git a/gcc/testsuite/rust/compile/while_let1.rs b/gcc/testsuite/rust/compile/while_let1.rs
new file mode 100644
index 0000000..a3fa305
--- /dev/null
+++ b/gcc/testsuite/rust/compile/while_let1.rs
@@ -0,0 +1,109 @@
+// use self::Ordering::*;
+// use Ordering::*;
+
+// enum Ordering {
+//     A,
+//     B,
+// }
+
+// fn foo(_: Ordering) {}
+
+// fn main() {
+//     let a = A;
+
+//     foo(a);
+//     foo(B);
+// }
+
+#[lang = "sized"]
+trait Sized {}
+
+enum Result<T, E> {
+    Ok(T),
+    Err(E),
+}
+
+pub trait Try {
+    /// The type of this value when viewed as successful.
+    #[unstable(feature = "try_trait", issue = "42327")]
+    type Ok;
+    /// The type of this value when viewed as failed.
+    #[unstable(feature = "try_trait", issue = "42327")]
+    type Error;
+
+    /// Applies the "?" operator. A return of `Ok(t)` means that the
+    /// execution should continue normally, and the result of `?` is the
+    /// value `t`. A return of `Err(e)` means that execution should branch
+    /// to the innermost enclosing `catch`, or return from the function.
+    ///
+    /// If an `Err(e)` result is returned, the value `e` will be "wrapped"
+    /// in the return type of the enclosing scope (which must itself implement
+    /// `Try`). Specifically, the value `X::from_error(From::from(e))`
+    /// is returned, where `X` is the return type of the enclosing function.
+    #[lang = "into_result"]
+    #[unstable(feature = "try_trait", issue = "42327")]
+    fn into_result(self) -> Result<Self::Ok, Self::Error>;
+
+    /// Wrap an error value to construct the composite result. For example,
+    /// `Result::Err(x)` and `Result::from_error(x)` are equivalent.
+    #[lang = "from_error"]
+    #[unstable(feature = "try_trait", issue = "42327")]
+    fn from_error(v: Self::Error) -> Self;
+
+    /// Wrap an OK value to construct the composite result. For example,
+    /// `Result::Ok(x)` and `Result::from_ok(x)` are equivalent.
+    #[lang = "from_ok"]
+    #[unstable(feature = "try_trait", issue = "42327")]
+    fn from_ok(v: Self::Ok) -> Self;
+}
+
+pub struct NoneError;
+
+pub enum Option<T> {
+    /// No value
+    None,
+    /// Some value `T`
+    Some(T),
+}
+
+impl<T> Option<T> {
+    pub fn ok_or<E>(self, err: E) -> Result<T, E> {
+        match self {
+            Some(ok) => Result::Ok(ok),
+            None => Result::Err(err),
+        }
+    }
+}
+
+use Option::*;
+
+#[unstable(feature = "try_trait", issue = "42327")]
+impl<T> Try for Option<T> {
+    type Ok = T;
+    type Error = NoneError;
+
+    #[inline]
+    fn into_result(self) -> Result<T, NoneError> {
+        self.ok_or(NoneError)
+    }
+
+    #[inline]
+    fn from_ok(v: T) -> Self {
+        Some(v)
+    }
+
+    #[inline]
+    fn from_error(_: NoneError) -> Self {
+        None
+    }
+}
+
+fn foo() -> Option<i32> {
+    Option::Some(15)
+}
+
+fn main() {
+    // let _: Option<i32> = try { 15i32 };
+
+    while let Option::Some(15) = foo() {}
+}
diff --git a/gcc/testsuite/rust/compile/while_let_without_label.rs b/gcc/testsuite/rust/compile/while_let_without_label.rs
new file mode 100644
index 0000000..e04e4b5
--- /dev/null
+++ b/gcc/testsuite/rust/compile/while_let_without_label.rs
@@ -0,0 +1,11 @@
+// { dg-additional-options "-frust-compile-until=lowering" }
+
+enum Foo {
+    A(i32),
+}
+
+fn main() {
+    let b = Foo::A(15);
+
+    while let Foo::A(x) = b {}
+}
diff --git a/gcc/testsuite/rust/compile/xfail/name_resolution21.rs b/gcc/testsuite/rust/compile/xfail/name_resolution21.rs
index df48d00..fc8e94b 100644
--- a/gcc/testsuite/rust/compile/xfail/name_resolution21.rs
+++ b/gcc/testsuite/rust/compile/xfail/name_resolution21.rs
@@ -1,5 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
-
 pub mod foo {
     pub macro bar() {}
 }
diff --git a/gcc/testsuite/rust/execute/black_box.rs b/gcc/testsuite/rust/execute/black_box.rs
index 7a9920e..58d10a3 100644
--- a/gcc/testsuite/rust/execute/black_box.rs
+++ b/gcc/testsuite/rust/execute/black_box.rs
@@ -21,10 +21,11 @@ pub fn black_box<T>(mut dummy: T) -> T {
     dummy
 }
 
-fn main() {
+fn main() -> i32 {
     let dummy: i32 = 42;
     let result = black_box(dummy);
     unsafe {
         printf("Value is: %i\n\0" as *const str as *const i8, result);
     }
+    0
 }
diff --git a/gcc/testsuite/rust/execute/execute.exp b/gcc/testsuite/rust/execute/execute.exp
new file mode 100644
index 0000000..3754778
--- /dev/null
+++ b/gcc/testsuite/rust/execute/execute.exp
@@ -0,0 +1,33 @@
+# Copyright (C) 2021-2025 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+# Execute tests.
+
+# Load support procs.
+load_lib rust-dg.exp
+
+# Initialize `dg'.
+dg-init
+
+# Main loop.
+set saved-dg-do-what-default ${dg-do-what-default}
+
+set dg-do-what-default "run"
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/*.rs]] "" ""
+set dg-do-what-default ${saved-dg-do-what-default}
+
+# All done.
+dg-finish
diff --git a/gcc/testsuite/rust/execute/inline_asm_inout_ident.rs b/gcc/testsuite/rust/execute/inline_asm_inout_ident.rs
new file mode 100644
index 0000000..b0a3d25
--- /dev/null
+++ b/gcc/testsuite/rust/execute/inline_asm_inout_ident.rs
@@ -0,0 +1,23 @@
+/* { dg-output "Value is: 5\r*\n" } */
+#![feature(rustc_attrs)]
+
+extern "C" {
+    fn printf(s: *const i8, ...);
+}
+
+#[rustc_builtin_macro]
+macro_rules! asm {
+    () => {};
+}
+
+fn main() -> i32 {
+    let x: i32;
+    // `inout` can also move values to different places
+    unsafe {
+        asm!("inc {}", inout(reg) 4u64=>x);
+    }
+    unsafe {
+        printf("Value is: %i\n\0" as *const str as *const i8, x);
+    }
+    0
+}
diff --git a/gcc/testsuite/rust/execute/inline_asm_inout_var.rs b/gcc/testsuite/rust/execute/inline_asm_inout_var.rs
new file mode 100644
index 0000000..ff101b8
--- /dev/null
+++ b/gcc/testsuite/rust/execute/inline_asm_inout_var.rs
@@ -0,0 +1,24 @@
+/* { dg-output "Value is: 5\r*\n" } */
+#![feature(rustc_attrs)]
+
+extern "C" {
+    fn printf(s: *const i8, ...);
+}
+
+#[rustc_builtin_macro]
+macro_rules! asm {
+    () => {};
+}
+
+fn main() -> i32 {
+    let y: i32 = 4;
+    let x: i32;
+    // `inout` can also move values to different places
+    unsafe {
+        asm!("inc {}", inout(reg) y=>x);
+    }
+    unsafe {
+        printf("Value is: %i\n\0" as *const str as *const i8, x);
+    }
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/basic_partial_ord1.rs b/gcc/testsuite/rust/execute/torture/basic_partial_ord1.rs
new file mode 100644
index 0000000..0431629
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/basic_partial_ord1.rs
@@ -0,0 +1,189 @@
+/* { dg-output "less\r*" }*/
+mod core {
+    mod option {
+        pub enum Option<T> {
+            None,
+            Some(T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {}
+
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {}
+
+        #[lang = "sized"]
+        pub trait Sized {}
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        pub enum Ordering {
+            Less = -1,
+            Equal = 0,
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        pub trait Eq: PartialEq<Self> {
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) | Option::Some(Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) | Option::Some(Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        pub trait Ord: Eq + PartialOrd<Self> {
+            fn cmp(&self, other: &Self) -> Ordering;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::option::Option;
+
+// Needed impls for primitives
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self < *other {
+            Option::Some(Ordering::Less)
+        } else if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Eq for i32 {}
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self < *other {
+            Ordering::Less
+        } else if *self > *other {
+            Ordering::Greater
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+// Manual impl for struct Bar
+struct Bar {
+    a: i32,
+    b: i32,
+}
+
+impl PartialEq for Bar {
+    fn eq(&self, other: &Self) -> bool {
+        self.a.eq(&other.a) && self.b.eq(&other.b)
+    }
+}
+
+impl Eq for Bar {}
+
+impl PartialOrd for Bar {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        match self.a.partial_cmp(&other.a) {
+            Option::Some(Ordering::Equal) => self.b.partial_cmp(&other.b),
+            ord => ord,
+        }
+    }
+}
+
+impl Ord for Bar {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match self.a.cmp(&other.a) {
+            Ordering::Equal => self.b.cmp(&other.b),
+            ord => ord,
+        }
+    }
+}
+
+// External print shim
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let x = Bar { a: 1, b: 2 };
+    let y = Bar { a: 1, b: 3 };
+
+    match x.partial_cmp(&y) {
+        Option::Some(Ordering::Less) => print("less"),
+        Option::Some(Ordering::Greater) => print("greater"),
+        Option::Some(Ordering::Equal) => print("equal"),
+        _ => print("none"),
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/basic_partial_ord2.rs b/gcc/testsuite/rust/execute/torture/basic_partial_ord2.rs
new file mode 100644
index 0000000..b6a9695
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/basic_partial_ord2.rs
@@ -0,0 +1,197 @@
+/* { dg-output "<><=>=\r*" } */
+/* { dg-options "-w" } */
+mod core {
+    mod option {
+        pub enum Option<T> {
+            None,
+            Some(T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {}
+
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {}
+
+        #[lang = "sized"]
+        pub trait Sized {}
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        pub enum Ordering {
+            Less = -1,
+            Equal = 0,
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        pub trait Eq: PartialEq<Self> {
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) | Option::Some(Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) | Option::Some(Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        pub trait Ord: Eq + PartialOrd<Self> {
+            fn cmp(&self, other: &Self) -> Ordering;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::option::Option;
+
+// Needed impls for primitives
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self < *other {
+            Option::Some(Ordering::Less)
+        } else if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Eq for i32 {}
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self < *other {
+            Ordering::Less
+        } else if *self > *other {
+            Ordering::Greater
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+// Manual impl for struct Bar
+struct Bar {
+    a: i32,
+    b: i32,
+}
+
+impl PartialEq for Bar {
+    fn eq(&self, other: &Self) -> bool {
+        self.a.eq(&other.a) && self.b.eq(&other.b)
+    }
+}
+
+impl Eq for Bar {}
+
+impl PartialOrd for Bar {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        match self.a.partial_cmp(&other.a) {
+            Option::Some(Ordering::Equal) => self.b.partial_cmp(&other.b),
+            ord => ord,
+        }
+    }
+}
+
+impl Ord for Bar {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match self.a.cmp(&other.a) {
+            Ordering::Equal => self.b.cmp(&other.b),
+            ord => ord,
+        }
+    }
+}
+
+// External print shim
+extern "C" {
+    fn printf(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        printf(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Bar { a: 1, b: 2 };
+    let b = Bar { a: 1, b: 3 };
+    let c = Bar { a: 1, b: 2 };
+
+    if a < b {
+        print("<");
+    }
+    if b > a {
+        print(">");
+    }
+    if a <= c {
+        print("<=");
+    }
+    if b >= c {
+        print(">=");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/builtin_abort.rs b/gcc/testsuite/rust/execute/torture/builtin_abort.rs
index 9f2d8c2..8c8259a 100644
--- a/gcc/testsuite/rust/execute/torture/builtin_abort.rs
+++ b/gcc/testsuite/rust/execute/torture/builtin_abort.rs
@@ -9,6 +9,6 @@ mod intrinsics {
 }
 
 pub fn main () -> i32 {
-    abort();
+    intrinsics::abort();
     0
 }
diff --git a/gcc/testsuite/rust/execute/torture/const-generics-1.rs b/gcc/testsuite/rust/execute/torture/const-generics-1.rs
new file mode 100644
index 0000000..dbb7afe
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/const-generics-1.rs
@@ -0,0 +1,24 @@
+#[lang = "sized"]
+trait Sized {}
+
+struct Foo<const N: usize>;
+
+impl Foo<1> {
+    fn call(&self) -> i32 {
+        10
+    }
+}
+
+impl Foo<2> {
+    fn call(&self) -> i32 {
+        20
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo::<1> {};
+    let b = Foo::<2> {};
+    let aa = a.call();
+    let bb = b.call();
+    bb - aa - 10
+}
diff --git a/gcc/testsuite/rust/execute/torture/const_block1.rs b/gcc/testsuite/rust/execute/torture/const_block1.rs
new file mode 100644
index 0000000..eaf3432
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/const_block1.rs
@@ -0,0 +1,9 @@
+const X: i32 = const {
+    let a = 15;
+    let b = 14;
+    a + b
+};
+
+fn main() -> i32 {
+    X - 29
+}
diff --git a/gcc/testsuite/rust/execute/torture/derive-partialeq2.rs b/gcc/testsuite/rust/execute/torture/derive-partialeq2.rs
new file mode 100644
index 0000000..e316017
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/derive-partialeq2.rs
@@ -0,0 +1,80 @@
+// { dg-output "true\r*\nfalse\r*\nfalse\r*\nfalse\r*\nfalse\r*\n" }
+
+#![feature(intrinsics)]
+
+pub mod core {
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+#[lang = "sized"]
+trait Sized {}
+
+#[lang = "copy"]
+trait Copy {}
+
+#[lang = "structural_peq"]
+trait StructuralPartialEq {}
+
+#[lang = "eq"]
+pub trait PartialEq<Rhs: ?Sized = Self> {
+    /// This method tests for `self` and `other` values to be equal, and is used
+    /// by `==`.
+    #[must_use]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    fn eq(&self, other: &Rhs) -> bool;
+
+    /// This method tests for `!=`.
+    #[inline]
+    #[must_use]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    fn ne(&self, other: &Rhs) -> bool {
+        !self.eq(other)
+    }
+}
+
+#[derive(PartialEq)]
+enum Foo {
+    A { a: i32, b: i32 },
+    B(i32, i32),
+    C,
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(b: bool) {
+    if b {
+        unsafe { puts("true\0" as *const str as *const i8) }
+    } else {
+        unsafe { puts("false\0" as *const str as *const i8) }
+    }
+}
+
+fn main() -> i32 {
+    let x = Foo::A { a: 15, b: 14 };
+
+    let b1 = x == Foo::A { a: 15, b: 14 };
+    let b12 = x == Foo::A { a: 15, b: 19 };
+    let b13 = x == Foo::A { a: 19, b: 14 };
+    let b2 = x == Foo::B(15, 14);
+    let b3 = x == Foo::C;
+
+    print(b1);
+    print(b12);
+    print(b13);
+    print(b2);
+    print(b3);
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/for-loop1.rs b/gcc/testsuite/rust/execute/torture/for-loop1.rs
index 5a6a70c..3342189 100644
--- a/gcc/testsuite/rust/execute/torture/for-loop1.rs
+++ b/gcc/testsuite/rust/execute/torture/for-loop1.rs
@@ -102,30 +102,30 @@ mod ptr {
     #[lang = "const_ptr"]
     impl<T> *const T {
         pub unsafe fn offset(self, count: isize) -> *const T {
-            intrinsics::offset(self, count)
+            crate::intrinsics::offset(self, count)
         }
     }
 
     #[lang = "mut_ptr"]
     impl<T> *mut T {
         pub unsafe fn offset(self, count: isize) -> *mut T {
-            intrinsics::offset(self, count) as *mut T
+            crate::intrinsics::offset(self, count) as *mut T
         }
     }
 
     pub unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) {
         let x = x as *mut u8;
         let y = y as *mut u8;
-        let len = mem::size_of::<T>() * count;
+        let len = crate::mem::size_of::<T>() * count;
         swap_nonoverlapping_bytes(x, y, len)
     }
 
     pub unsafe fn swap_nonoverlapping_one<T>(x: *mut T, y: *mut T) {
         // For types smaller than the block optimization below,
         // just swap directly to avoid pessimizing codegen.
-        if mem::size_of::<T>() < 32 {
+        if crate::mem::size_of::<T>() < 32 {
             let z = read(x);
-            intrinsics::copy_nonoverlapping(y, x, 1);
+            crate::intrinsics::copy_nonoverlapping(y, x, 1);
             write(y, z);
         } else {
             swap_nonoverlapping(x, y, 1);
@@ -133,12 +133,12 @@ mod ptr {
     }
 
     pub unsafe fn write<T>(dst: *mut T, src: T) {
-        intrinsics::move_val_init(&mut *dst, src)
+        crate::intrinsics::move_val_init(&mut *dst, src)
     }
 
     pub unsafe fn read<T>(src: *const T) -> T {
-        let mut tmp: T = mem::uninitialized();
-        intrinsics::copy_nonoverlapping(src, &mut tmp, 1);
+        let mut tmp: T = crate::mem::uninitialized();
+        crate::intrinsics::copy_nonoverlapping(src, &mut tmp, 1);
         tmp
     }
 
@@ -146,7 +146,7 @@ mod ptr {
         struct Block(u64, u64, u64, u64);
         struct UnalignedBlock(u64, u64, u64, u64);
 
-        let block_size = mem::size_of::<Block>();
+        let block_size = crate::mem::size_of::<Block>();
 
         // Loop through x & y, copying them `Block` at a time
         // The optimizer should unroll the loop fully for most types
@@ -155,31 +155,31 @@ mod ptr {
         while i + block_size <= len {
             // Create some uninitialized memory as scratch space
             // Declaring `t` here avoids aligning the stack when this loop is unused
-            let mut t: Block = mem::uninitialized();
+            let mut t: Block = crate::mem::uninitialized();
             let t = &mut t as *mut _ as *mut u8;
             let x = x.offset(i as isize);
             let y = y.offset(i as isize);
 
             // Swap a block of bytes of x & y, using t as a temporary buffer
             // This should be optimized into efficient SIMD operations where available
-            intrinsics::copy_nonoverlapping(x, t, block_size);
-            intrinsics::copy_nonoverlapping(y, x, block_size);
-            intrinsics::copy_nonoverlapping(t, y, block_size);
+            crate::intrinsics::copy_nonoverlapping(x, t, block_size);
+            crate::intrinsics::copy_nonoverlapping(y, x, block_size);
+            crate::intrinsics::copy_nonoverlapping(t, y, block_size);
             i += block_size;
         }
 
         if i < len {
             // Swap any remaining bytes
-            let mut t: UnalignedBlock = mem::uninitialized();
+            let mut t: UnalignedBlock = crate::mem::uninitialized();
             let rem = len - i;
 
             let t = &mut t as *mut _ as *mut u8;
             let x = x.offset(i as isize);
             let y = y.offset(i as isize);
 
-            intrinsics::copy_nonoverlapping(x, t, rem);
-            intrinsics::copy_nonoverlapping(y, x, rem);
-            intrinsics::copy_nonoverlapping(t, y, rem);
+            crate::intrinsics::copy_nonoverlapping(x, t, rem);
+            crate::intrinsics::copy_nonoverlapping(y, x, rem);
+            crate::intrinsics::copy_nonoverlapping(t, y, rem);
         }
     }
 }
@@ -194,7 +194,7 @@ mod mem {
 
     pub fn swap<T>(x: &mut T, y: &mut T) {
         unsafe {
-            ptr::swap_nonoverlapping_one(x, y);
+            crate::ptr::swap_nonoverlapping_one(x, y);
         }
     }
 
@@ -204,7 +204,7 @@ mod mem {
     }
 
     pub unsafe fn uninitialized<T>() -> T {
-        intrinsics::uninit()
+        crate::intrinsics::uninit()
     }
 }
 
diff --git a/gcc/testsuite/rust/execute/torture/for-loop2.rs b/gcc/testsuite/rust/execute/torture/for-loop2.rs
index 5ba2cd1..4f5dfe1 100644
--- a/gcc/testsuite/rust/execute/torture/for-loop2.rs
+++ b/gcc/testsuite/rust/execute/torture/for-loop2.rs
@@ -101,30 +101,30 @@ mod ptr {
     #[lang = "const_ptr"]
     impl<T> *const T {
         pub unsafe fn offset(self, count: isize) -> *const T {
-            intrinsics::offset(self, count)
+            crate::intrinsics::offset(self, count)
         }
     }
 
     #[lang = "mut_ptr"]
     impl<T> *mut T {
         pub unsafe fn offset(self, count: isize) -> *mut T {
-            intrinsics::offset(self, count) as *mut T
+            crate::intrinsics::offset(self, count) as *mut T
         }
     }
 
     pub unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) {
         let x = x as *mut u8;
         let y = y as *mut u8;
-        let len = mem::size_of::<T>() * count;
+        let len = crate::mem::size_of::<T>() * count;
         swap_nonoverlapping_bytes(x, y, len)
     }
 
     pub unsafe fn swap_nonoverlapping_one<T>(x: *mut T, y: *mut T) {
         // For types smaller than the block optimization below,
         // just swap directly to avoid pessimizing codegen.
-        if mem::size_of::<T>() < 32 {
+        if crate::mem::size_of::<T>() < 32 {
             let z = read(x);
-            intrinsics::copy_nonoverlapping(y, x, 1);
+            crate::intrinsics::copy_nonoverlapping(y, x, 1);
             write(y, z);
         } else {
             swap_nonoverlapping(x, y, 1);
@@ -132,12 +132,12 @@ mod ptr {
     }
 
     pub unsafe fn write<T>(dst: *mut T, src: T) {
-        intrinsics::move_val_init(&mut *dst, src)
+        crate::intrinsics::move_val_init(&mut *dst, src)
     }
 
     pub unsafe fn read<T>(src: *const T) -> T {
-        let mut tmp: T = mem::uninitialized();
-        intrinsics::copy_nonoverlapping(src, &mut tmp, 1);
+        let mut tmp: T = crate::mem::uninitialized();
+        crate::intrinsics::copy_nonoverlapping(src, &mut tmp, 1);
         tmp
     }
 
@@ -145,7 +145,7 @@ mod ptr {
         struct Block(u64, u64, u64, u64);
         struct UnalignedBlock(u64, u64, u64, u64);
 
-        let block_size = mem::size_of::<Block>();
+        let block_size = crate::mem::size_of::<Block>();
 
         // Loop through x & y, copying them `Block` at a time
         // The optimizer should unroll the loop fully for most types
@@ -154,31 +154,31 @@ mod ptr {
         while i + block_size <= len {
             // Create some uninitialized memory as scratch space
             // Declaring `t` here avoids aligning the stack when this loop is unused
-            let mut t: Block = mem::uninitialized();
+            let mut t: Block = crate::mem::uninitialized();
             let t = &mut t as *mut _ as *mut u8;
             let x = x.offset(i as isize);
             let y = y.offset(i as isize);
 
             // Swap a block of bytes of x & y, using t as a temporary buffer
             // This should be optimized into efficient SIMD operations where available
-            intrinsics::copy_nonoverlapping(x, t, block_size);
-            intrinsics::copy_nonoverlapping(y, x, block_size);
-            intrinsics::copy_nonoverlapping(t, y, block_size);
+            crate::intrinsics::copy_nonoverlapping(x, t, block_size);
+            crate::intrinsics::copy_nonoverlapping(y, x, block_size);
+            crate::intrinsics::copy_nonoverlapping(t, y, block_size);
             i += block_size;
         }
 
         if i < len {
             // Swap any remaining bytes
-            let mut t: UnalignedBlock = mem::uninitialized();
+            let mut t: UnalignedBlock = crate::mem::uninitialized();
             let rem = len - i;
 
             let t = &mut t as *mut _ as *mut u8;
             let x = x.offset(i as isize);
             let y = y.offset(i as isize);
 
-            intrinsics::copy_nonoverlapping(x, t, rem);
-            intrinsics::copy_nonoverlapping(y, x, rem);
-            intrinsics::copy_nonoverlapping(t, y, rem);
+            crate::intrinsics::copy_nonoverlapping(x, t, rem);
+            crate::intrinsics::copy_nonoverlapping(y, x, rem);
+            crate::intrinsics::copy_nonoverlapping(t, y, rem);
         }
     }
 }
@@ -193,7 +193,7 @@ mod mem {
 
     pub fn swap<T>(x: &mut T, y: &mut T) {
         unsafe {
-            ptr::swap_nonoverlapping_one(x, y);
+            crate::ptr::swap_nonoverlapping_one(x, y);
         }
     }
 
@@ -203,7 +203,7 @@ mod mem {
     }
 
     pub unsafe fn uninitialized<T>() -> T {
-        intrinsics::uninit()
+        crate::intrinsics::uninit()
     }
 }
 
diff --git a/gcc/testsuite/rust/execute/torture/impl_desugar-2.rs b/gcc/testsuite/rust/execute/torture/impl_desugar-2.rs
new file mode 100644
index 0000000..c73ea34
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_desugar-2.rs
@@ -0,0 +1,32 @@
+#[lang = "sized"]
+trait Sized {}
+
+macro_rules! impl_foo {
+	() => { impl Foo }
+}
+
+pub trait Foo {}
+
+pub trait Bar {
+    type Baz;
+}
+
+struct MyBaz; // { dg-warning "struct is never constructed" }
+impl Foo for MyBaz {}
+
+struct MyBar;
+
+impl Bar for MyBar {
+    type Baz = MyBaz;
+}
+
+pub fn foo(_value: impl Bar<Baz = impl_foo!()>) -> i32 {
+    15
+}
+
+fn main() -> i32 {
+    let bar = MyBar;
+    let result: i32 = foo(bar);
+
+    result - 15
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_desugar.rs b/gcc/testsuite/rust/execute/torture/impl_desugar.rs
new file mode 100644
index 0000000..22d3951
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_desugar.rs
@@ -0,0 +1,32 @@
+#[lang = "sized"]
+trait Sized {}
+
+pub trait Foo {}
+
+pub trait Bar {
+    type Baz;
+}
+
+struct MyBaz; // { dg-warning "struct is never constructed" }
+impl Foo for MyBaz {}
+
+struct MyBar;
+
+impl Bar for MyBar {
+    type Baz = MyBaz;
+}
+
+pub fn foo<T, U>(_value: T) -> i32
+where
+    T: Bar<Baz = U>,
+    U: Foo,
+{
+    15
+}
+
+fn main() -> i32 {
+    let bar = MyBar;
+    let result: i32 = foo::<MyBar, MyBaz>(bar);
+
+    result - 15
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_rpit1.rs b/gcc/testsuite/rust/execute/torture/impl_rpit1.rs
new file mode 100644
index 0000000..8ce5f21
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_rpit1.rs
@@ -0,0 +1,28 @@
+#[lang = "sized"]
+trait Sized {}
+
+trait Foo {
+    fn id(&self) -> i32;
+}
+
+struct Thing(i32);
+
+impl Foo for Thing {
+    fn id(&self) -> i32 {
+        self.0
+    }
+}
+
+fn make_thing(a: i32) -> impl Foo {
+    Thing(a)
+}
+
+fn use_foo(f: impl Foo) -> i32 {
+    f.id()
+}
+
+fn main() -> i32 {
+    let value = make_thing(42);
+    let val = use_foo(value);
+    val - 42
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_rpit2.rs b/gcc/testsuite/rust/execute/torture/impl_rpit2.rs
new file mode 100644
index 0000000..f7cbbb6
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_rpit2.rs
@@ -0,0 +1,36 @@
+#[lang = "sized"]
+trait Sized {}
+
+trait Foo {
+    fn id(&self) -> i32;
+}
+
+struct Thing(i32);
+
+impl Thing {
+    fn double(&self) -> i32 {
+        // { dg-warning "associated function is never used: .double." "" { target *-*-* } .-1 }
+        self.0 * 2
+    }
+}
+
+impl Foo for Thing {
+    fn id(&self) -> i32 {
+        self.0
+    }
+}
+
+fn make_thing(a: i32) -> impl Foo {
+    Thing(a)
+}
+
+fn use_foo(f: impl Foo) -> i32 {
+    f.id()
+}
+
+fn main() -> i32 {
+    let value = make_thing(21);
+    let id = use_foo(value);
+
+    id - 21
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_rpit3.rs b/gcc/testsuite/rust/execute/torture/impl_rpit3.rs
new file mode 100644
index 0000000..dd68eb2
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_rpit3.rs
@@ -0,0 +1,25 @@
+#[lang = "sized"]
+trait Sized {}
+
+trait Foo {
+    fn id(&self) -> i32;
+}
+
+struct Thing(i32);
+
+impl Foo for Thing {
+    fn id(&self) -> i32 {
+        self.0
+    }
+}
+
+fn make_thing() -> impl Foo {
+    Thing(99)
+}
+
+fn main() -> i32 {
+    let v = make_thing();
+    let r = &v;
+    let val = r.id();
+    val - 99
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_trait1.rs b/gcc/testsuite/rust/execute/torture/impl_trait1.rs
new file mode 100644
index 0000000..33a5c8c
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_trait1.rs
@@ -0,0 +1,31 @@
+#[lang = "sized"]
+trait Sized {}
+
+pub trait Value {
+    fn get(&self) -> i32;
+}
+
+struct Foo(i32);
+struct Bar(i32);
+
+impl Value for Foo {
+    fn get(&self) -> i32 {
+        self.0
+    }
+}
+impl Value for Bar {
+    fn get(&self) -> i32 {
+        self.0
+    }
+}
+
+pub fn foo(a: impl Value, b: impl Value) -> i32 {
+    a.get() + b.get()
+}
+
+fn main() -> i32 {
+    let a = Foo(1);
+    let b = Bar(2);
+
+    foo(a, b) - 3
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_trait2.rs b/gcc/testsuite/rust/execute/torture/impl_trait2.rs
new file mode 100644
index 0000000..29f393d
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_trait2.rs
@@ -0,0 +1,31 @@
+#[lang = "sized"]
+trait Sized {}
+
+pub trait Value {
+    fn get(&self) -> i32;
+}
+
+struct Foo(i32);
+struct Bar(i32);
+
+impl Value for Foo {
+    fn get(&self) -> i32 {
+        self.0
+    }
+}
+impl Value for Bar {
+    fn get(&self) -> i32 {
+        self.0
+    }
+}
+
+pub fn foo(a: &impl Value, b: &impl Value) -> i32 {
+    a.get() + b.get()
+}
+
+fn main() -> i32 {
+    let a = Foo(1);
+    let b = Bar(2);
+
+    foo(&a, &b) - 3
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_trait3.rs b/gcc/testsuite/rust/execute/torture/impl_trait3.rs
new file mode 100644
index 0000000..c1cec07
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_trait3.rs
@@ -0,0 +1,45 @@
+/* { dg-output "Hello from Message\r*\n" } */
+#[lang = "sized"]
+pub trait Sized {}
+
+extern "C" {
+    fn printf(s: *const i8, ...);
+}
+
+trait Speak {
+    fn speak(&self) -> &'static str;
+}
+
+trait Printer {
+    fn print(&self, input: impl Speak);
+}
+
+struct Console;
+
+impl Printer for Console {
+    fn print(&self, input: impl Speak) {
+        unsafe {
+            let a = input.speak();
+            let b = a as *const str;
+            let c = b as *const i8;
+
+            printf(c);
+        }
+    }
+}
+
+struct Message(&'static str);
+
+impl Speak for Message {
+    fn speak(&self) -> &'static str {
+        self.0
+    }
+}
+
+fn main() -> i32 {
+    let c = Console;
+    let msg = Message("Hello from Message\n");
+    c.print(msg);
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/impl_trait4.rs b/gcc/testsuite/rust/execute/torture/impl_trait4.rs
new file mode 100644
index 0000000..67d0095
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/impl_trait4.rs
@@ -0,0 +1,31 @@
+#[lang = "sized"]
+trait Sized {}
+
+trait Foo {
+    fn id(&self) -> i32;
+}
+
+struct A(i32);
+struct B(i32);
+
+impl Foo for A {
+    fn id(&self) -> i32 {
+        self.0
+    }
+}
+
+impl Foo for B {
+    fn id(&self) -> i32 {
+        self.0
+    }
+}
+
+fn takes_tuple(pair: (impl Foo, impl Foo)) -> i32 {
+    pair.0.id() + pair.1.id()
+}
+
+fn main() -> i32 {
+    let a = A(1);
+    let b = B(2);
+    takes_tuple((a, b)) - 3
+}
diff --git a/gcc/testsuite/rust/execute/torture/issue-1481.rs b/gcc/testsuite/rust/execute/torture/issue-1481.rs
new file mode 100644
index 0000000..2ff78d9
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/issue-1481.rs
@@ -0,0 +1,35 @@
+/* { dg-output "called Foo::print\\(\\)\r*" } */
+/* { dg-options "-w" } */
+
+#[lang = "sized"]
+trait Sized {}
+
+trait Printable {
+    fn print(&self);
+}
+
+struct Foo;
+
+impl Printable for Foo {
+    fn print(&self) {
+        // Simulate output
+        unsafe {
+            puts("called Foo::print()\0" as *const _ as *const i8);
+        }
+    }
+}
+
+fn get_printable() -> impl Printable {
+    Foo
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn main() -> i32 {
+    let p = get_printable();
+    p.print();
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/issue-1482.rs b/gcc/testsuite/rust/execute/torture/issue-1482.rs
new file mode 100644
index 0000000..ed8dc81
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/issue-1482.rs
@@ -0,0 +1,23 @@
+#[lang = "sized"]
+trait Sized {}
+
+#[lang = "fn_once"]
+pub trait FnOnce<Args> {
+    #[lang = "fn_once_output"]
+    type Output;
+
+    extern "rust-call" fn call_once(self, args: Args) -> Self::Output;
+}
+
+fn takes_fn(a: i32, f: impl FnOnce(i32) -> i32) -> i32 {
+    f(a)
+}
+
+pub fn main() -> i32 {
+    let capture = 2;
+    let a = |i: i32| {
+        let b = i + capture;
+        b
+    };
+    takes_fn(1, a) - 3
+}
diff --git a/gcc/testsuite/rust/execute/torture/issue-2005.rs b/gcc/testsuite/rust/execute/torture/issue-2005.rs
new file mode 100644
index 0000000..87edb95
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/issue-2005.rs
@@ -0,0 +1,465 @@
+// { dg-additional-options "-w" }
+/* { dg-output "WORKS\r?\n" } */
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// --------------
+
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Eq for isize {}
+
+impl Ord for isize {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+// ----------------------------------
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+impl Eq for i32 {}
+
+#[derive(Ord, PartialOrd, PartialEq, Eq)]
+enum Foo {
+    A,
+    B(i32),
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo::A;
+    let b = Foo::B(15);
+
+    if (a != b) {
+        print("WORKS");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/issue-3836.rs b/gcc/testsuite/rust/execute/torture/issue-3836.rs
new file mode 100644
index 0000000..61ad424
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/issue-3836.rs
@@ -0,0 +1,454 @@
+// { dg-options "-w" }
+// { dg-output "less\r*\n" }
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+impl Eq for i32 {}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+// ------------
+
+#[derive(Ord, PartialOrd, PartialEq, Eq)]
+struct Bar {
+    a: i32,
+    b: i32,
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let x = Bar { a: 1, b: 2 };
+    let y = Bar { a: 1, b: 3 };
+
+    match x.partial_cmp(&y) {
+        Option::Some(Ordering::Less) => print("less"),
+        Option::Some(Ordering::Greater) => print("greater"),
+        Option::Some(Ordering::Equal) => print("equal"),
+        _ => print("none"),
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/iter1.rs b/gcc/testsuite/rust/execute/torture/iter1.rs
index c3b6c7b..233eb60 100644
--- a/gcc/testsuite/rust/execute/torture/iter1.rs
+++ b/gcc/testsuite/rust/execute/torture/iter1.rs
@@ -99,30 +99,30 @@ mod ptr {
     #[lang = "const_ptr"]
     impl<T> *const T {
         pub unsafe fn offset(self, count: isize) -> *const T {
-            intrinsics::offset(self, count)
+            crate::intrinsics::offset(self, count)
         }
     }
 
     #[lang = "mut_ptr"]
     impl<T> *mut T {
         pub unsafe fn offset(self, count: isize) -> *mut T {
-            intrinsics::offset(self, count) as *mut T
+            crate::intrinsics::offset(self, count) as *mut T
         }
     }
 
     pub unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) {
         let x = x as *mut u8;
         let y = y as *mut u8;
-        let len = mem::size_of::<T>() * count;
+        let len = crate::mem::size_of::<T>() * count;
         swap_nonoverlapping_bytes(x, y, len)
     }
 
     pub unsafe fn swap_nonoverlapping_one<T>(x: *mut T, y: *mut T) {
         // For types smaller than the block optimization below,
         // just swap directly to avoid pessimizing codegen.
-        if mem::size_of::<T>() < 32 {
+        if crate::mem::size_of::<T>() < 32 {
             let z = read(x);
-            intrinsics::copy_nonoverlapping(y, x, 1);
+            crate::intrinsics::copy_nonoverlapping(y, x, 1);
             write(y, z);
         } else {
             swap_nonoverlapping(x, y, 1);
@@ -130,12 +130,12 @@ mod ptr {
     }
 
     pub unsafe fn write<T>(dst: *mut T, src: T) {
-        intrinsics::move_val_init(&mut *dst, src)
+        crate::intrinsics::move_val_init(&mut *dst, src)
     }
 
     pub unsafe fn read<T>(src: *const T) -> T {
-        let mut tmp: T = mem::uninitialized();
-        intrinsics::copy_nonoverlapping(src, &mut tmp, 1);
+        let mut tmp: T = crate::mem::uninitialized();
+        crate::intrinsics::copy_nonoverlapping(src, &mut tmp, 1);
         tmp
     }
 
@@ -143,7 +143,7 @@ mod ptr {
         struct Block(u64, u64, u64, u64);
         struct UnalignedBlock(u64, u64, u64, u64);
 
-        let block_size = mem::size_of::<Block>();
+        let block_size = crate::mem::size_of::<Block>();
 
         // Loop through x & y, copying them `Block` at a time
         // The optimizer should unroll the loop fully for most types
@@ -152,31 +152,31 @@ mod ptr {
         while i + block_size <= len {
             // Create some uninitialized memory as scratch space
             // Declaring `t` here avoids aligning the stack when this loop is unused
-            let mut t: Block = mem::uninitialized();
+            let mut t: Block = crate::mem::uninitialized();
             let t = &mut t as *mut _ as *mut u8;
             let x = x.offset(i as isize);
             let y = y.offset(i as isize);
 
             // Swap a block of bytes of x & y, using t as a temporary buffer
             // This should be optimized into efficient SIMD operations where available
-            intrinsics::copy_nonoverlapping(x, t, block_size);
-            intrinsics::copy_nonoverlapping(y, x, block_size);
-            intrinsics::copy_nonoverlapping(t, y, block_size);
+            crate::intrinsics::copy_nonoverlapping(x, t, block_size);
+            crate::intrinsics::copy_nonoverlapping(y, x, block_size);
+            crate::intrinsics::copy_nonoverlapping(t, y, block_size);
             i += block_size;
         }
 
         if i < len {
             // Swap any remaining bytes
-            let mut t: UnalignedBlock = mem::uninitialized();
+            let mut t: UnalignedBlock = crate::mem::uninitialized();
             let rem = len - i;
 
             let t = &mut t as *mut _ as *mut u8;
             let x = x.offset(i as isize);
             let y = y.offset(i as isize);
 
-            intrinsics::copy_nonoverlapping(x, t, rem);
-            intrinsics::copy_nonoverlapping(y, x, rem);
-            intrinsics::copy_nonoverlapping(t, y, rem);
+            crate::intrinsics::copy_nonoverlapping(x, t, rem);
+            crate::intrinsics::copy_nonoverlapping(y, x, rem);
+            crate::intrinsics::copy_nonoverlapping(t, y, rem);
         }
     }
 }
@@ -191,7 +191,7 @@ mod mem {
 
     pub fn swap<T>(x: &mut T, y: &mut T) {
         unsafe {
-            ptr::swap_nonoverlapping_one(x, y);
+            crate::ptr::swap_nonoverlapping_one(x, y);
         }
     }
 
@@ -201,7 +201,7 @@ mod mem {
     }
 
     pub unsafe fn uninitialized<T>() -> T {
-        intrinsics::uninit()
+        crate::intrinsics::uninit()
     }
 }
 
diff --git a/gcc/testsuite/rust/execute/torture/match-identifierpattern.rs b/gcc/testsuite/rust/execute/torture/match-identifierpattern.rs
new file mode 100644
index 0000000..b102ad1
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/match-identifierpattern.rs
@@ -0,0 +1,10 @@
+fn main() -> i32 {
+    let mut x = 2;
+
+    match x {
+        a @ 2 => { x = a + 1 },
+        _ => {}
+    }
+
+    x - 3
+}
diff --git a/gcc/testsuite/rust/execute/torture/match-restpattern-tuple.rs b/gcc/testsuite/rust/execute/torture/match-restpattern-tuple.rs
new file mode 100644
index 0000000..2c1418c
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/match-restpattern-tuple.rs
@@ -0,0 +1,27 @@
+// { dg-output "correct\r*" }
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn main() -> i32 {
+    let x = (1, 2, 3, 4);
+    let mut ret = 1;
+
+    match x {
+        (1, .., 2, 4) => {
+            /* should not take this path */
+            unsafe { puts("wrong\0" as *const str as *const i8) }
+        },
+        (2, ..) => {
+            /* should not take this path */
+            unsafe { puts("wrong\0" as *const str as *const i8) }
+        },
+        (b, .., 4) => { 
+            ret -= b;
+            unsafe { puts("correct\0" as *const str as *const i8) }
+        },
+        _ => {}
+    }
+
+    ret
+}
\ No newline at end of file
diff --git a/gcc/testsuite/rust/execute/torture/match-slicepattern-array-1.rs b/gcc/testsuite/rust/execute/torture/match-slicepattern-array-1.rs
new file mode 100644
index 0000000..95c55d8
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/match-slicepattern-array-1.rs
@@ -0,0 +1,23 @@
+// { dg-output "correct\r*" }
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn main() -> i32 {
+    let a = [0, 1];
+    let mut ret = 1;
+
+    match a {
+        [0, 0] => {
+            /* should not take this path */
+            unsafe { puts("wrong\0" as *const str as *const i8) }
+        },
+        [0, b] => {
+            ret -= b;
+            unsafe { puts("correct\0" as *const str as *const i8) }
+        },
+        _ => {}
+    }
+
+    ret
+}
diff --git a/gcc/testsuite/rust/execute/torture/match-slicepattern-slice-1.rs b/gcc/testsuite/rust/execute/torture/match-slicepattern-slice-1.rs
new file mode 100644
index 0000000..3ed0b644
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/match-slicepattern-slice-1.rs
@@ -0,0 +1,24 @@
+// { dg-output "correct\r*" }
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn main() -> i32 {
+    let arr = [0, 1];
+    let a: &[i32] = &arr;
+    let mut ret = 1;
+
+    match a {
+        [0, 0] => {
+            /* should not take this path */
+            unsafe { puts("wrong\0" as *const str as *const i8) }
+        },
+        [0, b] => {
+            ret -= b;
+            unsafe { puts("correct\0" as *const str as *const i8) }
+        },
+        _ => {}
+    }
+
+    ret
+}
diff --git a/gcc/testsuite/rust/execute/torture/match-tuplestructpattern.rs b/gcc/testsuite/rust/execute/torture/match-tuplestructpattern.rs
new file mode 100644
index 0000000..323109c
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/match-tuplestructpattern.rs
@@ -0,0 +1,12 @@
+fn main() -> i32 {
+    struct A (i32, i32);
+    let a = A (0, 1);
+    let mut ret = 1;
+
+    match a {
+        A (0, b) => { ret -= b },
+        _ => {}
+    }
+
+    ret
+}
diff --git a/gcc/testsuite/rust/execute/torture/min_specialization2.rs b/gcc/testsuite/rust/execute/torture/min_specialization2.rs
index d3239ee..74faee4 100644
--- a/gcc/testsuite/rust/execute/torture/min_specialization2.rs
+++ b/gcc/testsuite/rust/execute/torture/min_specialization2.rs
@@ -8,7 +8,7 @@ trait Foo {
 }
 
 impl<T> Foo for T {
-    default fn foo(&self) -> i32 { // { dg-warning "unused" }
+    default fn foo(&self) -> i32 {
         15
     }
 }
diff --git a/gcc/testsuite/rust/execute/torture/name_resolution.rs b/gcc/testsuite/rust/execute/torture/name_resolution.rs
index 7492183..a2eaf48 100644
--- a/gcc/testsuite/rust/execute/torture/name_resolution.rs
+++ b/gcc/testsuite/rust/execute/torture/name_resolution.rs
@@ -1,4 +1,3 @@
-// { dg-additional-options "-frust-name-resolution-2.0" }
 // { dg-output "Value is 10\r*\n" }
 
 const BAZ: i32 = 10;
diff --git a/gcc/testsuite/rust/execute/torture/offset_of1.rs b/gcc/testsuite/rust/execute/torture/offset_of1.rs
new file mode 100644
index 0000000..7d39483
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/offset_of1.rs
@@ -0,0 +1,16 @@
+// { dg-do run { target x86_64*-*-* } }
+// { dg-additional-options "-frust-assume-builtin-offset-of" }
+
+pub struct Foo {
+    pub a: i32,
+    pub b: i32,
+}
+
+fn main() -> i32 {
+    let a = offset_of!(Foo, a); // valid
+    let b = offset_of!(Foo, b); // valid
+
+    let res = a + b - 4;
+
+    res as i32
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-eq-1.rs b/gcc/testsuite/rust/execute/torture/partial-eq-1.rs
new file mode 100644
index 0000000..db123a1
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-eq-1.rs
@@ -0,0 +1,103 @@
+/* { dg-output "a == b\r*\na != c\r*\n" }*/
+/* { dg-options "-w" } */
+
+mod core {
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[lang = "sized"]
+        pub trait Sized {}
+    }
+
+    pub mod cmp {
+        use super::marker::Sized;
+
+        #[lang = "eq"]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        pub trait Eq: PartialEq<Self> {
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+    }
+}
+
+use core::cmp::{Eq, PartialEq};
+
+// PartialEq for i32 and u32 so we can compare across types
+impl PartialEq<u32> for i32 {
+    fn eq(&self, other: &u32) -> bool {
+        *self >= 0 && (*self as u32) == *other
+    }
+}
+impl PartialEq<i32> for u32 {
+    fn eq(&self, other: &i32) -> bool {
+        *other >= 0 && *self == *other as u32
+    }
+}
+
+// Our generic struct
+struct Foo<T> {
+    value: T,
+}
+
+// Manual impl of PartialEq for different generic params
+impl<T, U> PartialEq<Foo<U>> for Foo<T>
+where
+    T: PartialEq<U>,
+{
+    fn eq(&self, other: &Foo<U>) -> bool {
+        self.value.eq(&other.value)
+    }
+}
+
+impl<T: PartialEq> Eq for Foo<T> {}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo { value: 42i32 };
+    let b = Foo { value: 42u32 };
+    let c = Foo { value: 7u32 };
+
+    if a == b {
+        print("a == b");
+    } else {
+        print("a != b");
+    }
+
+    if a == c {
+        print("a == c");
+    } else {
+        print("a != c");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-eq-2.rs b/gcc/testsuite/rust/execute/torture/partial-eq-2.rs
new file mode 100644
index 0000000..debed8c
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-eq-2.rs
@@ -0,0 +1,60 @@
+/* { dg-output "a == b\r*\na != c\r*\n" }*/
+/* { dg-options "-w" } */
+
+#[lang = "sized"]
+pub trait Sized {}
+
+#[lang = "eq"]
+pub trait PartialEq<Rhs: ?Sized = Self> {
+    fn eq(&self, other: &Rhs) -> bool;
+
+    fn ne(&self, other: &Rhs) -> bool {
+        !self.eq(other)
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+struct Foo<T> {
+    value: T,
+}
+
+impl<T: PartialEq> PartialEq for Foo<T> {
+    fn eq(&self, other: &Self) -> bool {
+        self.value.eq(&other.value)
+    }
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo { value: 42i32 };
+    let b = Foo { value: 42i32 };
+    let c = Foo { value: 99i32 };
+
+    if a == b {
+        print("a == b");
+    } else {
+        print("a != b");
+    }
+
+    if a == c {
+        print("a == c");
+    } else {
+        print("a != c");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-eq-3.rs b/gcc/testsuite/rust/execute/torture/partial-eq-3.rs
new file mode 100644
index 0000000..849910a
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-eq-3.rs
@@ -0,0 +1,457 @@
+/* { dg-output "a == b\r*\na != c\r*\n" }*/
+/* { dg-options "-w" } */
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+impl Eq for i32 {}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+#[derive(PartialEq)]
+struct Foo {
+    a: i32,
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo { a: 42i32 };
+    let b = Foo { a: 42i32 };
+    let c = Foo { a: 7i32 };
+
+    if a == b {
+        print("a == b");
+    } else {
+        print("a != b");
+    }
+
+    if a == c {
+        print("a == c");
+    } else {
+        print("a != c");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-eq-4.rs b/gcc/testsuite/rust/execute/torture/partial-eq-4.rs
new file mode 100644
index 0000000..b6997d8
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-eq-4.rs
@@ -0,0 +1,457 @@
+/* { dg-output "a == b\r*\na != c\r*\n" }*/
+/* { dg-options "-w" } */
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+impl Eq for i32 {}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+#[derive(PartialEq, Eq)]
+struct Foo {
+    a: i32,
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo { a: 42i32 };
+    let b = Foo { a: 42i32 };
+    let c = Foo { a: 7i32 };
+
+    if a == b {
+        print("a == b");
+    } else {
+        print("a != b");
+    }
+
+    if a == c {
+        print("a == c");
+    } else {
+        print("a != c");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-ord-1.rs b/gcc/testsuite/rust/execute/torture/partial-ord-1.rs
new file mode 100644
index 0000000..a3558e7
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-ord-1.rs
@@ -0,0 +1,101 @@
+/* { dg-output "x == y\r*\nx > z\r*\n" }*/
+#[lang = "sized"]
+pub trait Sized {}
+
+pub enum Option<T> {
+    #[lang = "None"]
+    None,
+    #[lang = "Some"]
+    Some(T),
+}
+
+use Option::{None, Some};
+
+#[lang = "eq"]
+pub trait PartialEq<Rhs: ?Sized = Self> {
+    fn eq(&self, other: &Rhs) -> bool;
+
+    fn ne(&self, other: &Rhs) -> bool {
+        !self.eq(other)
+    }
+}
+
+pub enum Ordering {
+    Less = -1,
+    Equal = 0,
+    Greater = 1,
+}
+
+#[lang = "partial_ord"]
+pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+    fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+}
+
+// Implement for i32
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// Implement PartialOrd for i32
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self < *other {
+            Some(Ordering::Less)
+        } else if *self > *other {
+            Some(Ordering::Greater)
+        } else {
+            Some(Ordering::Equal)
+        }
+    }
+}
+
+// Struct with manual PartialEq
+struct Foo {
+    a: i32,
+}
+
+impl PartialEq for Foo {
+    fn eq(&self, other: &Self) -> bool {
+        self.a.eq(&other.a)
+    }
+}
+
+impl PartialOrd for Foo {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        self.a.partial_cmp(&other.a)
+    }
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let x = Foo { a: 42 };
+    let y = Foo { a: 42 };
+    let z = Foo { a: 7 };
+
+    match x.partial_cmp(&y) {
+        Some(Ordering::Equal) => print("x == y"),
+        Some(Ordering::Less) => print("x < y"),
+        Some(Ordering::Greater) => print("x > y"),
+        None => print("x ? y"),
+    }
+
+    match x.partial_cmp(&z) {
+        Some(Ordering::Equal) => print("x == z"),
+        Some(Ordering::Less) => print("x < z"),
+        Some(Ordering::Greater) => print("x > z"),
+        None => print("x ? z"),
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-ord-2.rs b/gcc/testsuite/rust/execute/torture/partial-ord-2.rs
new file mode 100644
index 0000000..d3b713f
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-ord-2.rs
@@ -0,0 +1,469 @@
+/* { dg-output "x == y\r*\nx > z\r*\n" }*/
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+impl Eq for i32 {}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+struct Foo {
+    a: i32,
+}
+
+impl PartialEq for Foo {
+    fn eq(&self, other: &'_ Self) -> bool {
+        self.a == other.a
+    }
+}
+
+impl PartialOrd for Foo {
+    fn partial_cmp(&self, other: &'_ Foo) -> Option<::core::cmp::Ordering> {
+        self.a.partial_cmp(&other.a)
+    }
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let x = Foo { a: 42 };
+    let y = Foo { a: 42 };
+    let z = Foo { a: 7 };
+
+    match x.partial_cmp(&y) {
+        Option::Some(Ordering::Equal) => print("x == y"),
+        Option::Some(Ordering::Less) => print("x < y"),
+        Option::Some(Ordering::Greater) => print("x > y"),
+        Option::None => print("x ? y"),
+    }
+
+    match x.partial_cmp(&z) {
+        Option::Some(Ordering::Equal) => print("x == z"),
+        Option::Some(Ordering::Less) => print("x < z"),
+        Option::Some(Ordering::Greater) => print("x > z"),
+        Option::None => print("x ? z"),
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-ord-3.rs b/gcc/testsuite/rust/execute/torture/partial-ord-3.rs
new file mode 100644
index 0000000..7aec07c
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-ord-3.rs
@@ -0,0 +1,489 @@
+/* { dg-output "x == y\r*\nx > z\r*\nx < z\r*\nx >= y\r*\nx <= y\r*\n" } */
+/* { dg-options "-w" } */
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+impl Eq for i32 {}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+struct Foo {
+    a: i32,
+}
+
+impl PartialEq for Foo {
+    fn eq(&self, other: &Self) -> bool {
+        self.a == other.a
+    }
+}
+impl Eq for Foo {}
+
+impl PartialOrd for Foo {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        self.a.partial_cmp(&other.a)
+    }
+}
+
+impl Ord for Foo {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.a.cmp(&other.a)
+    }
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let x = Foo { a: 42 };
+    let y = Foo { a: 42 };
+    let z = Foo { a: 7 };
+
+    // test direct equality
+    if x == y {
+        print("x == y");
+    }
+
+    // test PartialOrd via match
+    match x.partial_cmp(&z) {
+        Option::Some(Ordering::Greater) => print("x > z"),
+        _ => print("x ? z"),
+    }
+
+    // test `<` directly
+    if z < x {
+        print("x < z");
+    }
+
+    // test `>=`
+    if x >= y {
+        print("x >= y");
+    }
+
+    // test `<=`
+    if x <= y {
+        print("x <= y");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-ord-4.rs b/gcc/testsuite/rust/execute/torture/partial-ord-4.rs
new file mode 100644
index 0000000..fd52f32
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-ord-4.rs
@@ -0,0 +1,115 @@
+/* { dg-output "a == b\r*\na != c\r*\n" }*/
+/* { dg-options "-w" } */
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        pub enum Option<T> {
+            #[lang = "None"]
+            None,
+            #[lang = "Some"]
+            Some(T),
+        }
+    }
+
+    mod marker {
+        #[lang = "sized"]
+        pub trait Sized {}
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        pub enum Ordering {
+            Less = -1,
+            Equal = 0,
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[lang = "partial_ord"]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+        }
+    }
+}
+
+use core::cmp::{Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+}
+
+struct Foo {
+    a: i32,
+}
+
+impl PartialEq for Foo {
+    fn eq(&self, other: &'_ Self) -> bool {
+        self.a == other.a
+    }
+}
+
+impl PartialOrd for Foo {
+    fn partial_cmp(&self, other: &'_ Foo) -> Option<::core::cmp::Ordering> {
+        ::core::cmp::PartialOrd::partial_cmp(&self.a, &other.a)
+    }
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo { a: 42i32 };
+    let b = Foo { a: 42i32 };
+    let c = Foo { a: 7i32 };
+
+    if a == b {
+        print("a == b");
+    } else {
+        print("a != b");
+    }
+
+    if a == c {
+        print("a == c");
+    } else {
+        print("a != c");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-ord-5.rs b/gcc/testsuite/rust/execute/torture/partial-ord-5.rs
new file mode 100644
index 0000000..721d2aa
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-ord-5.rs
@@ -0,0 +1,487 @@
+/* { dg-output "a == b\r*\na != c\r*\na >= c\r*\na <= b\r*\na > c\r*\nc < b\r*\n" } */
+/* { dg-options "-w" } */
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+impl Eq for i32 {}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+#[derive(PartialEq, Eq, Ord)]
+struct Foo {
+    a: i32,
+}
+
+impl PartialOrd for Foo {
+    fn partial_cmp(&self, other: &'_ Foo) -> Option<::core::cmp::Ordering> {
+        self.a.partial_cmp(&other.a)
+    }
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    let a = Foo { a: 42 };
+    let b = Foo { a: 42 };
+    let c = Foo { a: 7 };
+
+    if a == b {
+        print("a == b");
+    } else {
+        print("a != b");
+    }
+
+    if a != c {
+        print("a != c");
+    } else {
+        print("a == c");
+    }
+
+    if a < c {
+        print("a < c");
+    } else {
+        print("a >= c");
+    }
+
+    if a <= b {
+        print("a <= b");
+    } else {
+        print("a > b");
+    }
+
+    if a > c {
+        print("a > c");
+    } else {
+        print("a <= c");
+    }
+
+    if c >= b {
+        print("c >= b");
+    } else {
+        print("c < b");
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/partial-ord-6.rs b/gcc/testsuite/rust/execute/torture/partial-ord-6.rs
new file mode 100644
index 0000000..5d64f8c
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/partial-ord-6.rs
@@ -0,0 +1,518 @@
+// { dg-additional-options "-w" }
+/* { dg-output "Foo A < B\r?\nFoo B < C\r?\nFoo C == C\r?\nBar x < y\r?\nBarFull s1 < s2\r?\n" } */
+
+#![feature(intrinsics)]
+
+mod core {
+    mod option {
+        // #[rustc_diagnostic_item = "option_type"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Option<T> {
+            /// No value
+            #[lang = "None"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            None,
+            /// Some value `T`
+            #[lang = "Some"]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Some(#[stable(feature = "rust1", since = "1.0.0")] T),
+        }
+    }
+
+    mod marker {
+        #[lang = "phantom_data"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub struct PhantomData<T: ?Sized>;
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(PartialEq)]`")]
+        #[lang = "structural_peq"]
+        pub trait StructuralPartialEq {
+            // Empty.
+        }
+
+        #[unstable(feature = "structural_match", issue = "31434")]
+        // #[rustc_on_unimplemented(message = "the type `{Self}` does not `#[derive(Eq)]`")]
+        #[lang = "structural_teq"]
+        pub trait StructuralEq {
+            // Empty.
+        }
+
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[lang = "sized"]
+        // #[rustc_on_unimplemented(
+        //     message = "the size for values of type `{Self}` cannot be known at compilation time",
+        //     label = "doesn't have a size known at compile-time"
+        // )]
+        // #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable
+        // #[rustc_specialization_trait]
+        pub trait Sized {
+            // Empty.
+        }
+    }
+
+    mod cmp {
+        use super::marker::Sized;
+        use super::option::Option;
+
+        // #[derive(Clone, Copy, PartialEq, Debug, Hash)]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub enum Ordering {
+            /// An ordering where a compared value is less than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Less = -1,
+            /// An ordering where a compared value is equal to another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Equal = 0,
+            /// An ordering where a compared value is greater than another.
+            #[stable(feature = "rust1", since = "1.0.0")]
+            Greater = 1,
+        }
+
+        #[lang = "eq"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} == {Rhs}`"
+        // )]
+        pub trait PartialEq<Rhs: ?Sized = Self> {
+            /// This method tests for `self` and `other` values to be equal, and is used
+            /// by `==`.
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn eq(&self, other: &Rhs) -> bool;
+
+            fn ne(&self, other: &Rhs) -> bool {
+                !self.eq(other)
+            }
+        }
+
+        #[doc(alias = "==")]
+        #[doc(alias = "!=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Eq: PartialEq<Self> {
+            // this method is used solely by #[deriving] to assert
+            // that every component of a type implements #[deriving]
+            // itself, the current deriving infrastructure means doing this
+            // assertion without using a method on this trait is nearly
+            // impossible.
+            //
+            // This should never be implemented by hand.
+            #[doc(hidden)]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn assert_receiver_is_total_eq(&self) {}
+        }
+
+        #[lang = "partial_ord"]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        // #[rustc_on_unimplemented(
+        //     message = "can't compare `{Self}` with `{Rhs}`",
+        //     label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`"
+        // )]
+        pub trait PartialOrd<Rhs: ?Sized = Self>: PartialEq<Rhs> {
+            /// This method returns an ordering between `self` and `other` values if one exists.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// let result = 1.0.partial_cmp(&2.0);
+            /// assert_eq!(result, Some(Ordering::Less));
+            ///
+            /// let result = 1.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Equal));
+            ///
+            /// let result = 2.0.partial_cmp(&1.0);
+            /// assert_eq!(result, Some(Ordering::Greater));
+            /// ```
+            ///
+            /// When comparison is impossible:
+            ///
+            /// ```
+            /// let result = f64::NAN.partial_cmp(&1.0);
+            /// assert_eq!(result, None);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn partial_cmp(&self, other: &Rhs) -> Option<Ordering>;
+
+            /// This method tests less than (for `self` and `other`) and is used by the `<` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 < 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 < 1.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn lt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests less than or equal to (for `self` and `other`) and is used by the `<=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 <= 2.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 <= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn le(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Less | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than (for `self` and `other`) and is used by the `>` operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 1.0 > 2.0;
+            /// assert_eq!(result, false);
+            ///
+            /// let result = 2.0 > 2.0;
+            /// assert_eq!(result, false);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn gt(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater) => true,
+                    _ => false,
+                }
+            }
+
+            /// This method tests greater than or equal to (for `self` and `other`) and is used by the `>=`
+            /// operator.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// let result = 2.0 >= 1.0;
+            /// assert_eq!(result, true);
+            ///
+            /// let result = 2.0 >= 2.0;
+            /// assert_eq!(result, true);
+            /// ```
+            #[inline]
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn ge(&self, other: &Rhs) -> bool {
+                match self.partial_cmp(other) {
+                    Option::Some(Ordering::Greater | Ordering::Equal) => true,
+                    _ => false,
+                }
+            }
+        }
+
+        #[doc(alias = "<")]
+        #[doc(alias = ">")]
+        #[doc(alias = "<=")]
+        #[doc(alias = ">=")]
+        #[stable(feature = "rust1", since = "1.0.0")]
+        pub trait Ord: Eq + PartialOrd<Self> {
+            /// This method returns an [`Ordering`] between `self` and `other`.
+            ///
+            /// By convention, `self.cmp(&other)` returns the ordering matching the expression
+            /// `self <operator> other` if true.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// use std::cmp::Ordering;
+            ///
+            /// assert_eq!(5.cmp(&10), Ordering::Less);
+            /// assert_eq!(10.cmp(&5), Ordering::Greater);
+            /// assert_eq!(5.cmp(&5), Ordering::Equal);
+            /// ```
+            #[must_use]
+            #[stable(feature = "rust1", since = "1.0.0")]
+            fn cmp(&self, other: &Self) -> Ordering;
+
+            /// Compares and returns the maximum of two values.
+            ///
+            /// Returns the second argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(2, 1.max(2));
+            /// assert_eq!(2, 2.max(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn max(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Compares and returns the minimum of two values.
+            ///
+            /// Returns the first argument if the comparison determines them to be equal.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// assert_eq!(1, 1.min(2));
+            /// assert_eq!(2, 2.min(2));
+            /// ```
+            #[stable(feature = "ord_max_min", since = "1.21.0")]
+            #[must_use]
+            fn min(self, other: Self) -> Self
+            where
+                Self: Sized,
+            {
+                self
+            }
+
+            /// Restrict a value to a certain interval.
+            ///
+            /// Returns `max` if `self` is greater than `max`, and `min` if `self` is
+            /// less than `min`. Otherwise this returns `self`.
+            ///
+            /// # Panics
+            ///
+            /// Panics if `min > max`.
+            ///
+            /// # Examples
+            ///
+            /// ```
+            /// #![feature(clamp)]
+            ///
+            /// assert!((-3).clamp(-2, 1) == -2);
+            /// assert!(0.clamp(-2, 1) == 0);
+            /// assert!(2.clamp(-2, 1) == 1);
+            /// ```
+            #[must_use]
+            #[unstable(feature = "clamp", issue = "44095")]
+            fn clamp(self, min: Self, max: Self) -> Self
+            where
+                Self: Sized,
+            {
+                if self < min {
+                    min
+                } else if self > max {
+                    max
+                } else {
+                    self
+                }
+            }
+        }
+    }
+
+    pub mod intrinsics {
+        #[lang = "discriminant_kind"]
+        pub trait DiscriminantKind {
+            #[lang = "discriminant_type"]
+            type Discriminant;
+        }
+
+        extern "rust-intrinsic" {
+            pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
+        }
+    }
+}
+
+use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
+use core::marker::Sized;
+use core::option::Option;
+
+// for comparing discriminant_value
+impl PartialEq for isize {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+// for comparing discriminant_value
+impl PartialOrd for isize {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl PartialEq for i32 {
+    fn eq(&self, other: &Self) -> bool {
+        *self == *other
+    }
+}
+
+impl PartialOrd for i32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        if *self > *other {
+            Option::Some(Ordering::Greater)
+        } else if *self < *other {
+            Option::Some(Ordering::Less)
+        } else {
+            Option::Some(Ordering::Equal)
+        }
+    }
+
+    fn lt(&self, other: &Self) -> bool {
+        *self < *other
+    }
+    fn le(&self, other: &Self) -> bool {
+        *self <= *other
+    }
+    fn ge(&self, other: &Self) -> bool {
+        *self >= *other
+    }
+    fn gt(&self, other: &Self) -> bool {
+        *self > *other
+    }
+}
+
+impl Ord for i32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        if *self > *other {
+            Ordering::Greater
+        } else if *self < *other {
+            Ordering::Less
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+impl Eq for i32 {}
+
+#[derive(PartialEq, PartialOrd)]
+enum Foo {
+    A,
+    B(i32, i32, i32),
+    C { inner: i32, outer: i32 },
+}
+
+#[derive(Ord, PartialOrd, PartialEq, Eq)]
+struct Bar {
+    a: i32,
+}
+
+#[derive(Ord, PartialOrd, PartialEq, Eq)]
+struct BarFull {
+    a: i32,
+    b: i32,
+    c: i32,
+    d: i32,
+}
+
+extern "C" {
+    fn puts(s: *const i8);
+}
+
+fn print(s: &str) {
+    unsafe {
+        puts(s as *const str as *const i8);
+    }
+}
+
+fn main() -> i32 {
+    // Enum comparison
+    let a = Foo::A;
+    let b = Foo::B(15, 14, 13);
+    let c = Foo::C {
+        inner: 10,
+        outer: 20,
+    };
+
+    match a.partial_cmp(&b) {
+        Option::Some(Ordering::Less) => print("Foo A < B"),
+        Option::Some(Ordering::Greater) => print("Foo A > B"),
+        Option::Some(Ordering::Equal) => print("Foo A == B"),
+        _ => print("Foo A ? B"),
+    }
+
+    match b.partial_cmp(&c) {
+        Option::Some(Ordering::Less) => print("Foo B < C"),
+        Option::Some(Ordering::Greater) => print("Foo B > C"),
+        Option::Some(Ordering::Equal) => print("Foo B == C"),
+        _ => print("Foo B ? C"),
+    }
+
+    match c.partial_cmp(&c) {
+        Option::Some(Ordering::Less) => print("Foo C < C ???"),
+        Option::Some(Ordering::Greater) => print("Foo C > C ???"),
+        Option::Some(Ordering::Equal) => print("Foo C == C"),
+        _ => print("Foo C ? C"),
+    }
+
+    // Struct comparison: Bar
+    let x = Bar { a: 10 };
+    let y = Bar { a: 20 };
+
+    if x < y {
+        print("Bar x < y");
+    } else if x > y {
+        print("Bar x > y");
+    } else {
+        print("Bar x == y");
+    }
+
+    // Struct comparison: BarFull
+    let s1 = BarFull {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+    };
+    let s2 = BarFull {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 5,
+    };
+
+    match s1.cmp(&s2) {
+        Ordering::Less => print("BarFull s1 < s2"),
+        Ordering::Greater => print("BarFull s1 > s2"),
+        Ordering::Equal => print("BarFull s1 == s2"),
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/sip-hasher.rs b/gcc/testsuite/rust/execute/torture/sip-hasher.rs
new file mode 100644
index 0000000..60826a3
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/sip-hasher.rs
@@ -0,0 +1,438 @@
+// { dg-skip-if "" { *-*-* } { "-m32" } { "" } }
+// { dg-options "-w" }
+// { dg-output "Hash: 0x63d53fd2170bbb8c\r*\n" }
+#![feature(intrinsics)]
+#![feature(rustc_attrs)]
+
+#[lang = "sized"]
+trait Sized {}
+
+mod intrinsics {
+    extern "rust-intrinsic" {
+        pub fn wrapping_add<T>(a: T, b: T) -> T;
+        pub fn rotate_left<T>(a: T, b: T) -> T;
+        pub fn offset<T>(ptr: *const T, count: isize) -> *const T;
+    }
+}
+
+#[lang = "add"]
+trait Add<Rhs = Self> {
+    type Output;
+
+    fn add(self, rhs: Rhs) -> Self::Output;
+}
+
+macro_rules! add_impl {
+    ($($t:ty)*) => ($(
+        impl Add for $t {
+            type Output = $t;
+
+            #[inline]
+            #[rustc_inherit_overflow_checks]
+            fn add(self, other: $t) -> $t { self + other }
+        }
+
+
+    )*)
+}
+
+add_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 }
+
+impl<T> *const T {
+    pub unsafe fn add(self, count: usize) -> Self {
+        // SAFETY: the caller must uphold the safety contract for `offset`.
+        unsafe { self.offset(count as isize) }
+    }
+
+    pub unsafe fn offset(self, count: isize) -> *const T {
+        // SAFETY: the caller must uphold the safety contract for `offset`.
+        unsafe { intrinsics::offset(self, count) }
+    }
+}
+
+macro_rules! impl_uint {
+    ($($ty:ident = $lang:literal),*) => {
+        $(
+            #[lang = $lang]
+            impl $ty {
+                pub fn wrapping_add(self, rhs: Self) -> Self {
+                    intrinsics::wrapping_add(self, rhs)
+                }
+
+                pub fn rotate_left(self, n: u32) -> Self {
+                    intrinsics::rotate_left(self, n as Self)
+                }
+
+                pub fn to_le(self) -> Self {
+                    #[cfg(target_endian = "little")]
+                    {
+                        self
+                    }
+                    #[cfg(not(target_endian = "little"))]
+                    {
+                       self.swap_bytes()
+                    }
+                }
+            }
+        )*
+    }
+}
+
+impl_uint!(
+    u8 = "u8",
+    u16 = "u16",
+    u32 = "u32",
+    u64 = "u64",
+    u128 = "u128",
+    usize = "usize"
+);
+
+#[repr(C)]
+pub(crate) struct SliceComponents {
+    pub(crate) data_address: *const (),
+    pub(crate) metadata: usize,
+}
+
+#[repr(C)]
+pub(crate) union SliceRepr<T> {
+    pub(crate) const_ptr: *const [T],
+    pub(crate) mut_ptr: *mut [T],
+    pub(crate) components: SliceComponents,
+}
+
+impl<T> [T] {
+    pub const fn as_ptr(&self) -> *const T {
+        self as *const [T] as *const T
+    }
+
+    pub unsafe fn get_unchecked(&self, index: usize) -> &T {
+        unsafe { &*self.as_ptr().add(index) }
+    }
+
+    pub fn len(&self) -> usize {
+        unsafe {
+            SliceRepr {
+                const_ptr: self as *const _,
+            }
+            .components
+            .metadata
+        }
+    }
+}
+
+trait HasherTrait {
+    fn write(&mut self, msg: &[u8]);
+    fn finish(&self) -> u64;
+}
+
+mod cmp {
+    pub fn min(a: usize, b: usize) -> usize {
+        if a < b {
+            a
+        } else {
+            b
+        }
+    }
+}
+
+struct PhantomData<T>;
+
+mod mem {
+    extern "rust-intrinsic" {
+        fn transmute<T, U>(_: T) -> U;
+        fn size_of<T>() -> usize;
+    }
+}
+
+mod ptr {
+    extern "rust-intrinsic" {
+        fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize);
+    }
+}
+
+#[repr(C)]
+struct State {
+    v0: u64,
+    v2: u64,
+    v1: u64,
+    v3: u64,
+}
+
+struct Hasher<S: Sip> {
+    k0: u64,
+    k1: u64,
+    length: usize, // how many bytes we've processed
+    state: State,  // hash State
+    tail: u64,     // unprocessed bytes le
+    ntail: usize,  // how many bytes in tail are valid
+    _marker: PhantomData<S>,
+}
+
+macro_rules! compress {
+    ($state:expr) => {{
+        compress!($state.v0, $state.v1, $state.v2, $state.v3)
+    }};
+    ($v0:expr, $v1:expr, $v2:expr, $v3:expr) => {{
+        $v0 = $v0.wrapping_add($v1);
+        $v1 = $v1.rotate_left(13);
+        $v1 ^= $v0;
+        $v0 = $v0.rotate_left(32);
+        $v2 = $v2.wrapping_add($v3);
+        $v3 = $v3.rotate_left(16);
+        $v3 ^= $v2;
+        $v0 = $v0.wrapping_add($v3);
+        $v3 = $v3.rotate_left(21);
+        $v3 ^= $v0;
+        $v2 = $v2.wrapping_add($v1);
+        $v1 = $v1.rotate_left(17);
+        $v1 ^= $v2;
+        $v2 = $v2.rotate_left(32);
+    }};
+}
+
+#[doc(hidden)]
+trait Sip {
+    fn c_rounds(_: &mut State);
+    fn d_rounds(_: &mut State);
+}
+
+struct Sip13Rounds;
+
+impl Sip for Sip13Rounds {
+    #[inline]
+    fn c_rounds(state: &mut State) {
+        compress!(state);
+    }
+
+    #[inline]
+    fn d_rounds(state: &mut State) {
+        compress!(state);
+        compress!(state);
+        compress!(state);
+    }
+}
+
+struct Sip24Rounds;
+
+impl Sip for Sip24Rounds {
+    #[inline]
+    fn c_rounds(state: &mut State) {
+        compress!(state);
+        compress!(state);
+    }
+
+    #[inline]
+    fn d_rounds(state: &mut State) {
+        compress!(state);
+        compress!(state);
+        compress!(state);
+        compress!(state);
+    }
+}
+
+pub struct SipHasher13 {
+    hasher: Hasher<Sip13Rounds>,
+}
+
+struct SipHasher24 {
+    hasher: Hasher<Sip24Rounds>,
+}
+
+pub struct SipHasher(SipHasher24);
+
+macro_rules! load_int_le {
+    ($buf:expr, $i:expr, $int_ty:ident) => {{
+        let mut data = 0 as $int_ty;
+        ptr::copy_nonoverlapping(
+            $buf.as_ptr().add($i),
+            &mut data as *mut _ as *mut u8,
+            mem::size_of::<$int_ty>(),
+        );
+        data.to_le()
+    }};
+}
+
+#[inline]
+unsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 {
+    let mut i = 0; // current byte index (from LSB) in the output u64
+    let mut out = 0;
+    if i + 3 < len {
+        // SAFETY: `i` cannot be greater than `len`, and the caller must guarantee
+        // that the index start..start+len is in bounds.
+        out = unsafe { load_int_le!(buf, start + i, u32) } as u64;
+        i += 4;
+    }
+    if i + 1 < len {
+        // SAFETY: same as above.
+        out |= (unsafe { load_int_le!(buf, start + i, u16) } as u64) << ((i * 8) as u64);
+        i += 2
+    }
+    if i < len {
+        // SAFETY: same as above.
+        out |= (unsafe { *buf.get_unchecked(start + i) } as u64) << ((i * 8) as u64);
+        i += 1;
+    }
+    out
+}
+
+impl SipHasher {
+    #[inline]
+    #[must_use]
+    pub fn new() -> SipHasher {
+        SipHasher::new_with_keys(0, 0)
+    }
+
+    #[inline]
+    #[must_use]
+    pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher {
+        SipHasher(SipHasher24 {
+            hasher: Hasher::new_with_keys(key0, key1),
+        })
+    }
+}
+
+impl SipHasher13 {
+    #[inline]
+    pub fn new() -> SipHasher13 {
+        SipHasher13::new_with_keys(0, 0)
+    }
+
+    #[inline]
+    pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher13 {
+        SipHasher13 {
+            hasher: Hasher::new_with_keys(key0, key1),
+        }
+    }
+}
+
+impl<S: Sip> Hasher<S> {
+    #[inline]
+    fn new_with_keys(key0: u64, key1: u64) -> Hasher<S> {
+        let mut state = Hasher {
+            k0: key0,
+            k1: key1,
+            length: 0,
+            state: State {
+                v0: 0,
+                v1: 0,
+                v2: 0,
+                v3: 0,
+            },
+            tail: 0,
+            ntail: 0,
+            _marker: PhantomData,
+        };
+        state.reset();
+        state
+    }
+
+    #[inline]
+    fn reset(&mut self) {
+        self.length = 0;
+        self.state.v0 = self.k0 ^ 0x736f6d6570736575;
+        self.state.v1 = self.k1 ^ 0x646f72616e646f6d;
+        self.state.v2 = self.k0 ^ 0x6c7967656e657261;
+        self.state.v3 = self.k1 ^ 0x7465646279746573;
+        self.ntail = 0;
+    }
+}
+
+impl HasherTrait for SipHasher {
+    #[inline]
+    fn write(&mut self, msg: &[u8]) {
+        self.0.hasher.write(msg)
+    }
+
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.0.hasher.finish()
+    }
+}
+
+impl HasherTrait for SipHasher13 {
+    #[inline]
+    fn write(&mut self, msg: &[u8]) {
+        self.hasher.write(msg)
+    }
+
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.hasher.finish()
+    }
+}
+
+impl<S: Sip> HasherTrait for Hasher<S> {
+    #[inline]
+    fn write(&mut self, msg: &[u8]) {
+        let length = msg.len();
+        self.length += length;
+
+        let mut needed = 0;
+
+        if self.ntail != 0 {
+            needed = 8 - self.ntail;
+            // SAFETY: `cmp::min(length, needed)` is guaranteed to not be over `length`
+            self.tail |=
+                unsafe { u8to64_le(msg, 0, cmp::min(length, needed)) } << ((8 * self.ntail) as u64);
+            if length < needed {
+                self.ntail += length;
+                return;
+            } else {
+                self.state.v3 ^= self.tail;
+                S::c_rounds(&mut self.state);
+                self.state.v0 ^= self.tail;
+                self.ntail = 0;
+            }
+        }
+
+        // Buffered tail is now flushed, process new input.
+        let len = length - needed;
+        let left = len & 0x7; // len % 8
+
+        let mut i = needed;
+        while i < len - left {
+            let mi = unsafe { load_int_le!(msg, i, u64) };
+
+            self.state.v3 ^= mi;
+            S::c_rounds(&mut self.state);
+            self.state.v0 ^= mi;
+
+            i += 8;
+        }
+
+        self.tail = unsafe { u8to64_le(msg, i, left) };
+        self.ntail = left;
+    }
+
+    #[inline]
+    fn finish(&self) -> u64 {
+        let mut state = self.state;
+
+        let b: u64 = ((self.length as u64 & 0xff) << 56) | self.tail;
+
+        state.v3 ^= b;
+        S::c_rounds(&mut state);
+        state.v0 ^= b;
+
+        state.v2 ^= 0xff;
+        S::d_rounds(&mut state);
+
+        state.v0 ^ state.v1 ^ state.v2 ^ state.v3
+    }
+}
+
+extern "C" {
+    fn printf(fmt: *const u8, ...) -> i32;
+}
+
+fn main() -> i32 {
+    let mut hasher = SipHasher::new_with_keys(0x0706050403020100, 0x0f0e0d0c0b0a0908);
+    hasher.write(b"Hello");
+    let result = hasher.finish();
+
+    unsafe {
+        printf("Hash: 0x%016llx\n\0" as *const str as *const u8, result);
+    }
+
+    0
+}
diff --git a/gcc/testsuite/rust/execute/torture/struct-pattern-match.rs b/gcc/testsuite/rust/execute/torture/struct-pattern-match.rs
new file mode 100644
index 0000000..6aec51f
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/struct-pattern-match.rs
@@ -0,0 +1,13 @@
+enum Foo {
+    A { x: i32 },
+    B { y: i32 }
+}
+
+fn main() -> i32 {
+    let x = Foo::A { x: 12 };
+    match x {
+        Foo::A { x: 10 } => 1,
+        Foo::B { y: 11 } => 2,
+        Foo::A { x: abc } => { abc - 12 }
+    }
+}
diff --git a/gcc/testsuite/rust/execute/torture/struct_pattern1.rs b/gcc/testsuite/rust/execute/torture/struct_pattern1.rs
new file mode 100644
index 0000000..7a74092
--- /dev/null
+++ b/gcc/testsuite/rust/execute/torture/struct_pattern1.rs
@@ -0,0 +1,19 @@
+struct A {
+    // the two warnings are invalid but this should be fixed by our lint rework
+    // with this year's GSoC so ok for now
+    a: i32, // { dg-warning "never read" }
+    b: i32, // { dg-warning "never read" }
+}
+
+fn main() -> i32 {
+    let a = A { a: 15, b: 14 };
+
+    let result = match a {
+        A {
+            a: self_a,
+            b: self_b,
+        } => self_a + self_b,
+    };
+
+    result - 29
+}
diff --git a/gcc/testsuite/rust/execute/torture/trait10.rs b/gcc/testsuite/rust/execute/torture/trait10.rs
index a595122..4e576eb 100644
--- a/gcc/testsuite/rust/execute/torture/trait10.rs
+++ b/gcc/testsuite/rust/execute/torture/trait10.rs
@@ -26,7 +26,6 @@ impl Bar for Foo {
 struct S;
 impl S {
     fn dynamic_dispatch(self, t: &dyn Bar) {
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
         t.baz();
     }
 }
diff --git a/gcc/testsuite/rust/execute/torture/trait11.rs b/gcc/testsuite/rust/execute/torture/trait11.rs
index 093343c..cca084a 100644
--- a/gcc/testsuite/rust/execute/torture/trait11.rs
+++ b/gcc/testsuite/rust/execute/torture/trait11.rs
@@ -13,7 +13,6 @@ trait FnLike<A, R> {
 struct S;
 impl<'a, T> FnLike<&'a T, &'a T> for S {
     fn call(&self, arg: &'a T) -> &'a T {
-        // { dg-warning "unused name .self." "" { target *-*-* } .-1 }
         arg
     }
 }
diff --git a/gcc/testsuite/rust/execute/torture/trait12.rs b/gcc/testsuite/rust/execute/torture/trait12.rs
index 83cf107..d174a89 100644
--- a/gcc/testsuite/rust/execute/torture/trait12.rs
+++ b/gcc/testsuite/rust/execute/torture/trait12.rs
@@ -16,7 +16,6 @@ struct Identity;
 
 impl<'a, T> FnLike<&'a T, &'a T> for Identity {
     fn call(&self, arg: &'a T) -> &'a T {
-        // { dg-warning "unused name .self." "" { target *-*-* } .-1 }
         arg
     }
 }
diff --git a/gcc/testsuite/rust/execute/torture/trait13.rs b/gcc/testsuite/rust/execute/torture/trait13.rs
index 928a37c..0d8f894 100644
--- a/gcc/testsuite/rust/execute/torture/trait13.rs
+++ b/gcc/testsuite/rust/execute/torture/trait13.rs
@@ -11,7 +11,6 @@ trait Bar {
     fn baz(&self);
 
     fn qux(&self) {
-        // { dg-warning "unused name" "" { target *-*-* } .-1 }
         unsafe {
             let a = "%i\n\0";
             let b = a as *const str;
diff --git a/gcc/testsuite/rust/execute/torture/trait9.rs b/gcc/testsuite/rust/execute/torture/trait9.rs
index a1642f6..f60554a 100644
--- a/gcc/testsuite/rust/execute/torture/trait9.rs
+++ b/gcc/testsuite/rust/execute/torture/trait9.rs
@@ -13,7 +13,6 @@ trait FnLike<A, R> {
 struct S;
 impl<T> FnLike<&T, &T> for S {
     fn call(&self, arg: &T) -> &T {
-        // { dg-warning "unused name .self." "" { target *-*-* } .-1 }
         arg
     }
 }
diff --git a/gcc/testsuite/rust/execute/xfail/match-identifierpattern-enum.rs b/gcc/testsuite/rust/execute/xfail/match-identifierpattern-enum.rs
new file mode 100644
index 0000000..c3a0f65
--- /dev/null
+++ b/gcc/testsuite/rust/execute/xfail/match-identifierpattern-enum.rs
@@ -0,0 +1,15 @@
+enum Foo {
+    I(i32),
+}
+
+fn main() -> i32 {
+    let x = Foo::I(0);
+    let ret = 1;
+
+    match x {
+        _ @ Foo::I(b) => { ret = b },
+        _ => {},
+    };
+
+    ret
+}
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif
new file mode 100644
index 0000000..a4eb9d4
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif
@@ -0,0 +1,23 @@
+{"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [{"tool": { "driver": { "name": "example" } },
+           "results": [],
+           "graphs": [{"nodes": [{"id": "a", /* { dg-message "'a' already used as node id within graph here" } */
+                                  "children": [{"id": "a"}]}], /* { dg-error "duplicate node id 'a' within graph \\\[SARIF v2.1.0 §3.40.2\\\]" } */
+                       "edges": []}]}]}
+
+/* { dg-begin-multiline-output "" }
+In JSON property '/runs/0/graphs/0/nodes/0/children/0/id':
+   { dg-end-multiline-output "" } */
+/* { dg-begin-multiline-output "" }
+    6 |                                   "children": [{"id": "a"}]}],
+      |                                                       ^~~
+   { dg-end-multiline-output "" }  */
+
+/* { dg-begin-multiline-output "" }
+In JSON property '/runs/0/graphs/0/nodes/0/id':
+   { dg-end-multiline-output "" } */
+/* { dg-begin-multiline-output "" }
+    5 |            "graphs": [{"nodes": [{"id": "a",
+      |                                         ^~~
+   { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif
new file mode 100644
index 0000000..b483346
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif
@@ -0,0 +1,16 @@
+{"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [{"tool": { "driver": { "name": "example" } },
+           "results": [],
+           "graphs": [{"nodes": [],
+                       "edges": [{"id": "edge0",
+                                  "sourceNodeId": "this-does-not-exist", /* { dg-error "no node with id 'this-does-not-exist' in graph \\\[SARIF v2.1.0 §3.41.4\\\]" } */
+                                  "targetNodeId": "neither-does-this"}]}]}]}
+
+/* { dg-begin-multiline-output "" }
+In JSON property '/runs/0/graphs/0/edges/0/sourceNodeId':
+   { dg-end-multiline-output "" } */
+/* { dg-begin-multiline-output "" }
+    7 |                                   "sourceNodeId": "this-does-not-exist",
+      |                                                   ^~~~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" }  */
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif
index bc64521..cd7b822 100644
--- a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif
@@ -1,3 +1,6 @@
+/* { dg-additional-options "-fdiagnostics-add-output=experimental-html:file=3.11.6-embedded-links.sarif.html,javascript=no" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:file=3.11.6-embedded-links.sarif.roundtrip.sarif" } */
+
 {"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
  "version": "2.1.0",
  "runs": [{"tool": {"driver": {"name": "hand-written"}},
@@ -16,10 +19,24 @@ hand-written: warning: 002: Prohibited term used in [para\[0\]\\spans\[2\](1).
                        
                        /* With the fix from https://github.com/oasis-tcs/sarif-spec/issues/656 */
                        {"message": {"text": "003: Prohibited term used in [para\\[0\\]\\\\spans\\[2\\]](1)."},
-	   	        "locations": []}
+	   	        "locations": []},
 /* { dg-begin-multiline-output "" }
 hand-written: warning: 003: Prohibited term used in para[0]\spans[2].
    { dg-end-multiline-output "" } */
 
+                       {"message": {"text": "004: This is a [link](http://www.example.com)."},
+	   	        "locations": []}
+/* { dg-begin-multiline-output "" }
+hand-written: warning: 004: This is a link.
+   { dg-end-multiline-output "" } */
+
 ]}]}
 
+/* Use a Python script to verify various properties about the generated
+   .html file:
+   { dg-final { run-html-pytest 3.11.6-embedded-links.sarif "2.1.0-valid/embedded-links-check-html.py" } } */
+
+/* Use a Python script to verify various properties about the *generated*
+   .sarif file:
+   { dg-final { run-sarif-pytest 3.11.6-embedded-links.sarif.roundtrip "2.1.0-valid/embedded-links-check-sarif-roundtrip.py" } } */
+
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py
new file mode 100644
index 0000000..845a7af
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py
@@ -0,0 +1,28 @@
+from htmltest import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def html_tree():
+    return html_tree_from_env()
+
+def test_generated_html(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    head = root.find('xhtml:head', ns)
+    assert head is not None
+
+    # Get "warning: 004: This is a link."
+    diag = get_diag_by_index(html_tree, 3)
+
+    msg = get_message_within_diag(diag)
+    assert msg is not None
+    
+    assert_tag(msg[0], 'strong')
+    assert msg[0].text == 'warning: '
+    assert msg[0].tail == ' 004: This is a '
+    assert_tag(msg[1], 'a')
+    assert msg[1].text == 'link'
+    assert msg[1].get('href') == 'http://www.example.com'
+    assert msg[1].tail == '.'
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py
new file mode 100644
index 0000000..171339e
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py
@@ -0,0 +1,13 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_roundtrip_of_url_in_generated_sarif(sarif):
+    # Get "warning: 004: This is a link."
+    result = get_result_by_index(sarif, 3)
+    assert result['level'] == 'warning'
+    assert result['message']['text'] == "004: This is a [link](http://www.example.com)."
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-html.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-html.py
new file mode 100644
index 0000000..355c84b
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-html.py
@@ -0,0 +1,46 @@
+from htmltest import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def html_tree():
+    return html_tree_from_env()
+
+def test_result_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    diag_list = body.find('xhtml:div', ns)
+    assert diag_list is not None
+    assert diag_list.attrib['class'] == 'gcc-diagnostic-list'
+
+    diag = diag_list.find('xhtml:div', ns)
+    assert diag is not None
+
+    message = diag.find("./xhtml:div[@class='gcc-message']", ns)
+    assert message.attrib['id'] == 'gcc-diag-0-message'
+
+    assert message[0].tag == make_tag('strong')
+    assert message[0].tail == ' this is a placeholder error, with graphs'
+
+    graph = diag.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'foo'
+
+def test_run_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    graph = body.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'Optimization Passes'
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py
new file mode 100644
index 0000000..4bb7535
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py
@@ -0,0 +1,55 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_basics(sarif):
+    schema = sarif['$schema']
+    assert schema == "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json"
+
+    version = sarif['version']
+    assert version == "2.1.0"
+
+def test_result_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+    results = run['results']
+
+    assert len(results) == 1
+    
+    result = results[0]
+    assert result['level'] == 'error'
+    assert result['message']['text'] == "this is a placeholder error, with graphs"
+
+    assert len(result['graphs']) == 2
+
+    assert result['graphs'][0]['description']['text'] == 'foo'
+
+    assert len(result['graphs'][0]['nodes']) == 2
+    assert result['graphs'][0]['nodes'][0]['id'] == 'a'
+    assert result['graphs'][0]['nodes'][1]['id'] == 'b'
+    assert result['graphs'][0]['nodes'][1]['properties']['/placeholder-prefix/color'] == 'red'
+    assert len(result['graphs'][0]['nodes'][1]['children']) == 1
+    assert result['graphs'][0]['nodes'][1]['children'][0]['id'] == 'c'
+    assert result['graphs'][0]['nodes'][1]['children'][0]['label']['text'] == 'I am a node label'
+
+    assert len(result['graphs'][0]['edges']) == 1
+    result['graphs'][0]['edges'][0]['id'] == 'my-edge'
+    assert result['graphs'][0]['edges'][0]['label']['text'] == 'I am an edge label'
+    assert result['graphs'][0]['edges'][0]['sourceNodeId'] == 'a'
+    assert result['graphs'][0]['edges'][0]['targetNodeId'] == 'c'
+    
+    assert result['graphs'][1]['description']['text'] == 'bar'
+
+def test_run_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+
+    assert len(run['graphs']) == 1
+
+    assert run['graphs'][0]['description']['text'] == 'Optimization Passes'
+    assert run['graphs'][0]['nodes'][0]['id'] == 'all_lowering_passes'
+    assert run['graphs'][0]['edges'][0]['id'] == 'edge0'
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs.sarif
new file mode 100644
index 0000000..da236ba
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs.sarif
@@ -0,0 +1,2445 @@
+/* Test a replay of a .sarif file generated from GCC testsuite.
+
+   The dg directives were stripped out from the generated .sarif
+   to avoid confusing DejaGnu for this test.   */
+/* { dg-additional-options "-fdiagnostics-add-output=experimental-html:file=graphs.sarif.html,javascript=no" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:file=graphs.roundtrip.sarif" } */
+
+{"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [{"tool": {"driver": {"name": "GNU C23",
+                               "fullName": "GNU C23 (GCC) version 16.0.0 20250702 (experimental) (x86_64-pc-linux-gnu)",
+                               "version": "16.0.0 20250702 (experimental)",
+                               "informationUri": "https://gcc.gnu.org/gcc-16/",
+                               "rules": []},
+                    "extensions": [{"name": "diagnostic_plugin_test_graphs",
+                                    "fullName": "./diagnostic_plugin_test_graphs.so"}]},
+           "invocations": [{"arguments": ["/home/david/gcc-newgit-gcc16/build/gcc/cc1",
+                                          "-quiet",
+                                          "-iprefix",
+                                          "/usr/local/lib/gcc/x86_64-pc-linux-gnu/16.0.0/",
+                                          "-isystem",
+                                          "/home/david/gcc-newgit-gcc16/build/gcc/include",
+                                          "-isystem",
+                                          "/home/david/gcc-newgit-gcc16/build/gcc/include-fixed",
+                                          "-iplugindir=/home/david/gcc-newgit-gcc16/build/gcc/plugin",
+                                          "/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c",
+                                          "-iplugindir=/home/david/gcc-newgit-gcc16/build/gcc/plugin",
+                                          "-quiet",
+                                          "-dumpbase",
+                                          "diagnostic-test-graphs-sarif.c",
+                                          "-dumpbase-ext",
+                                          ".c",
+                                          "-mtune=generic",
+                                          "-march=x86-64",
+                                          "-fdiagnostics-color=never",
+                                          "-fdiagnostics-urls=never",
+                                          "-fno-diagnostics-show-caret",
+                                          "-fno-diagnostics-show-line-numbers",
+                                          "-fdiagnostics-path-format=separate-events",
+                                          "-fdiagnostics-text-art-charset=none",
+                                          "-fno-diagnostics-show-event-links",
+                                          "-fplugin=./diagnostic_plugin_test_graphs.so",
+                                          "-fdiagnostics-add-output=sarif",
+                                          "-o",
+                                          "diagnostic-test-graphs-sarif.s"],
+                            "workingDirectory": {"uri": "/home/david/gcc-newgit-gcc16/build/gcc/testsuite/gcc"},
+                            "startTimeUtc": "2025-07-09T22:43:31Z",
+                            "executionSuccessful": false,
+                            "toolExecutionNotifications": [],
+                            "endTimeUtc": "2025-07-09T22:43:31Z"}],
+           "artifacts": [{"location": {"uri": "/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c"},
+                          "sourceLanguage": "c",
+                          "roles": ["analysisTarget"]}],
+           "results": [{"ruleId": "error",
+                        "level": "error",
+                        "message": {"text": "this is a placeholder error, with graphs"},
+                        "locations": [{"physicalLocation": {"artifactLocation": {"uri": "/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c"},
+                                                            "region": {"startLine": 8,
+                                                                       "startColumn": 3,
+                                                                       "endColumn": 10},
+                                                            "contextRegion": {"startLine": 8,
+                                                                              "snippet": {"text": "  here ();"}}},
+                                       "logicalLocations": [{"index": 0,
+                                                             "fullyQualifiedName": "test_graphs"}]}],
+                        "graphs": [{"description": {"text": "foo"},
+                                    "nodes": [{"id": "a"},
+                                              {"id": "b",
+                                               "properties": {"/placeholder-prefix/color": "red"},
+                                               "children": [{"id": "c",
+                                                             "label": {"text": "I am a node label"}}]}],
+                                    "edges": [{"id": "my-edge",
+                                               "label": {"text": "I am an edge label"},
+                                               "sourceNodeId": "a",
+                                               "targetNodeId": "c"}]},
+                                   {"description": {"text": "bar"},
+                                    "nodes": [{"id": "a"},
+                                              {"id": "b",
+                                               "properties": {"/placeholder-prefix/color": "red"},
+                                               "children": [{"id": "c",
+                                                             "label": {"text": "I am a node label"}}]}],
+                                    "edges": [{"id": "my-edge",
+                                               "label": {"text": "I am an edge label"},
+                                               "sourceNodeId": "a",
+                                               "targetNodeId": "c"}]}]}],
+           "logicalLocations": [{"name": "test_graphs",
+                                 "fullyQualifiedName": "test_graphs",
+                                 "decoratedName": "test_graphs",
+                                 "kind": "function",
+                                 "index": 0}],
+           "graphs": [{"description": {"text": "Optimization Passes"},
+                       "nodes": [{"id": "all_lowering_passes",
+                                  "label": {"text": "all_lowering_passes"},
+                                  "children": [{"id": "*warn_unused_result_0x101ef3d0",
+                                                "label": {"text": "*warn_unused_result"}},
+                                               {"id": "*diagnose_omp_blocks_0x101ef430",
+                                                "label": {"text": "*diagnose_omp_blocks"}},
+                                               {"id": "*diagnose_tm_blocks_0x101ef490",
+                                                "label": {"text": "*diagnose_tm_blocks"}},
+                                               {"id": "13_omp_oacc_kernels_decompose",
+                                                "label": {"text": "13_omp_oacc_kernels_decompose"}},
+                                               {"id": "14_omplower",
+                                                "label": {"text": "14_omplower"}},
+                                               {"id": "15_lower",
+                                                "label": {"text": "15_lower"}},
+                                               {"id": "16_tmlower",
+                                                "label": {"text": "16_tmlower"}},
+                                               {"id": "17_ehopt",
+                                                "label": {"text": "17_ehopt"}},
+                                               {"id": "18_eh",
+                                                "label": {"text": "18_eh"}},
+                                               {"id": "19_coro-lower-builtins",
+                                                "label": {"text": "19_coro-lower-builtins"}},
+                                               {"id": "20_cfg",
+                                                "label": {"text": "20_cfg"}},
+                                               {"id": "*warn_function_return_0x101ef7f0",
+                                                "label": {"text": "*warn_function_return"}},
+                                               {"id": "21_coro-early-expand-ifns",
+                                                "label": {"text": "21_coro-early-expand-ifns"}},
+                                               {"id": "22_ompexp",
+                                                "label": {"text": "22_ompexp"}},
+                                               {"id": "*build_cgraph_edges_0x101ef910",
+                                                "label": {"text": "*build_cgraph_edges"}}]},
+                                 {"id": "all_small_ipa_passes",
+                                  "label": {"text": "all_small_ipa_passes"},
+                                  "children": [{"id": "23_afdo_offline",
+                                                "label": {"text": "23_afdo_offline"}},
+                                               {"id": "*free_lang_data_0x101ef9d0",
+                                                "label": {"text": "*free_lang_data"}},
+                                               {"id": "24_visibility",
+                                                "label": {"text": "24_visibility"}},
+                                               {"id": "25_strubm",
+                                                "label": {"text": "25_strubm"}},
+                                               {"id": "26_build_ssa_passes",
+                                                "label": {"text": "26_build_ssa_passes"}},
+                                               {"id": "27_fixup_cfg",
+                                                "label": {"text": "27_fixup_cfg"}},
+                                               {"id": "28_ssa",
+                                                "label": {"text": "28_ssa"}},
+                                               {"id": "376_test_graph_emission",
+                                                "label": {"text": "376_test_graph_emission"}},
+                                               {"id": "29_walloca",
+                                                "label": {"text": "29_walloca"}},
+                                               {"id": "30_warn-printf",
+                                                "label": {"text": "30_warn-printf"}},
+                                               {"id": "*nonnullcmp_0x101efce0",
+                                                "label": {"text": "*nonnullcmp"}},
+                                               {"id": "31_early_uninit",
+                                                "label": {"text": "31_early_uninit"}},
+                                               {"id": "32_waccess",
+                                                "label": {"text": "32_waccess"}},
+                                               {"id": "33_ubsan",
+                                                "label": {"text": "33_ubsan"}},
+                                               {"id": "34_nothrow",
+                                                "label": {"text": "34_nothrow"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f0020",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "35_opt_local_passes",
+                                                "label": {"text": "35_opt_local_passes"}},
+                                               {"id": "36_fixup_cfg",
+                                                "label": {"text": "36_fixup_cfg"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f0140",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "37_local-fnsummary",
+                                                "label": {"text": "37_local-fnsummary"}},
+                                               {"id": "38_einline",
+                                                "label": {"text": "38_einline"}},
+                                               {"id": "*infinite-recursion_0x101f0260",
+                                                "label": {"text": "*infinite-recursion"}},
+                                               {"id": "39_early_optimizations",
+                                                "label": {"text": "39_early_optimizations"}},
+                                               {"id": "*remove_cgraph_callee_edges_0x101f0340",
+                                                "label": {"text": "*remove_cgraph_callee_edges"}},
+                                               {"id": "40_early_objsz",
+                                                "label": {"text": "40_early_objsz"}},
+                                               {"id": "41_ccp",
+                                                "label": {"text": "41_ccp"}},
+                                               {"id": "42_forwprop",
+                                                "label": {"text": "42_forwprop"}},
+                                               {"id": "43_ethread",
+                                                "label": {"text": "43_ethread"}},
+                                               {"id": "44_esra",
+                                                "label": {"text": "44_esra"}},
+                                               {"id": "45_ealias",
+                                                "label": {"text": "45_ealias"}},
+                                               {"id": "46_phiprop",
+                                                "label": {"text": "46_phiprop"}},
+                                               {"id": "47_fre",
+                                                "label": {"text": "47_fre"}},
+                                               {"id": "48_evrp",
+                                                "label": {"text": "48_evrp"}},
+                                               {"id": "49_mergephi",
+                                                "label": {"text": "49_mergephi"}},
+                                               {"id": "50_dse",
+                                                "label": {"text": "50_dse"}},
+                                               {"id": "51_cddce",
+                                                "label": {"text": "51_cddce"}},
+                                               {"id": "52_phiopt",
+                                                "label": {"text": "52_phiopt"}},
+                                               {"id": "53_tailr",
+                                                "label": {"text": "53_tailr"}},
+                                               {"id": "54_iftoswitch",
+                                                "label": {"text": "54_iftoswitch"}},
+                                               {"id": "55_switchconv",
+                                                "label": {"text": "55_switchconv"}},
+                                               {"id": "56_ehcleanup",
+                                                "label": {"text": "56_ehcleanup"}},
+                                               {"id": "57_sccopy",
+                                                "label": {"text": "57_sccopy"}},
+                                               {"id": "58_profile_estimate",
+                                                "label": {"text": "58_profile_estimate"}},
+                                               {"id": "59_local-pure-const",
+                                                "label": {"text": "59_local-pure-const"}},
+                                               {"id": "60_modref",
+                                                "label": {"text": "60_modref"}},
+                                               {"id": "61_fnsplit",
+                                                "label": {"text": "61_fnsplit"}},
+                                               {"id": "*strip_predict_hints_0x101f0c60",
+                                                "label": {"text": "*strip_predict_hints"}},
+                                               {"id": "62_release_ssa",
+                                                "label": {"text": "62_release_ssa"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f0d30",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "63_local-fnsummary",
+                                                "label": {"text": "63_local-fnsummary"}},
+                                               {"id": "64_remove_symbols",
+                                                "label": {"text": "64_remove_symbols"}},
+                                               {"id": "65_strub",
+                                                "label": {"text": "65_strub"}},
+                                               {"id": "66_ipa_oacc",
+                                                "label": {"text": "66_ipa_oacc"}},
+                                               {"id": "67_pta",
+                                                "label": {"text": "67_pta"}},
+                                               {"id": "68_ipa_oacc_kernels",
+                                                "label": {"text": "68_ipa_oacc_kernels"}},
+                                               {"id": "69_oacc_kernels",
+                                                "label": {"text": "69_oacc_kernels"}},
+                                               {"id": "70_ch",
+                                                "label": {"text": "70_ch"}},
+                                               {"id": "71_fre",
+                                                "label": {"text": "71_fre"}},
+                                               {"id": "72_lim",
+                                                "label": {"text": "72_lim"}},
+                                               {"id": "73_dom",
+                                                "label": {"text": "73_dom"}},
+                                               {"id": "74_dce",
+                                                "label": {"text": "74_dce"}},
+                                               {"id": "75_parloops",
+                                                "label": {"text": "75_parloops"}},
+                                               {"id": "76_ompexpssa",
+                                                "label": {"text": "76_ompexpssa"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f1310",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "77_targetclone",
+                                                "label": {"text": "77_targetclone"}},
+                                               {"id": "78_afdo",
+                                                "label": {"text": "78_afdo"}},
+                                               {"id": "79_feedback_fnsplit",
+                                                "label": {"text": "79_feedback_fnsplit"}},
+                                               {"id": "80_profile",
+                                                "label": {"text": "80_profile"}},
+                                               {"id": "81_feedback_fnsplit",
+                                                "label": {"text": "81_feedback_fnsplit"}},
+                                               {"id": "82_free-fnsummary",
+                                                "label": {"text": "82_free-fnsummary"}},
+                                               {"id": "83_increase_alignment",
+                                                "label": {"text": "83_increase_alignment"}},
+                                               {"id": "84_tmipa",
+                                                "label": {"text": "84_tmipa"}},
+                                               {"id": "85_emutls",
+                                                "label": {"text": "85_emutls"}}]},
+                                 {"id": "all_regular_ipa_passes",
+                                  "label": {"text": "all_regular_ipa_passes"},
+                                  "children": [{"id": "86_analyzer",
+                                                "label": {"text": "86_analyzer"}},
+                                               {"id": "87_odr",
+                                                "label": {"text": "87_odr"}},
+                                               {"id": "88_whole-program",
+                                                "label": {"text": "88_whole-program"}},
+                                               {"id": "89_profile_estimate",
+                                                "label": {"text": "89_profile_estimate"}},
+                                               {"id": "90_icf",
+                                                "label": {"text": "90_icf"}},
+                                               {"id": "91_devirt",
+                                                "label": {"text": "91_devirt"}},
+                                               {"id": "92_cdtor",
+                                                "label": {"text": "92_cdtor"}},
+                                               {"id": "93_cp",
+                                                "label": {"text": "93_cp"}},
+                                               {"id": "94_sra",
+                                                "label": {"text": "94_sra"}},
+                                               {"id": "95_fnsummary",
+                                                "label": {"text": "95_fnsummary"}},
+                                               {"id": "96_inline",
+                                                "label": {"text": "96_inline"}},
+                                               {"id": "97_locality-clone",
+                                                "label": {"text": "97_locality-clone"}},
+                                               {"id": "98_pure-const",
+                                                "label": {"text": "98_pure-const"}},
+                                               {"id": "99_modref",
+                                                "label": {"text": "99_modref"}},
+                                               {"id": "100_free-fnsummary",
+                                                "label": {"text": "100_free-fnsummary"}},
+                                               {"id": "101_static-var",
+                                                "label": {"text": "101_static-var"}},
+                                               {"id": "102_single-use",
+                                                "label": {"text": "102_single-use"}},
+                                               {"id": "103_comdats",
+                                                "label": {"text": "103_comdats"}}]},
+                                 {"id": "all_late_ipa_passes",
+                                  "label": {"text": "all_late_ipa_passes"},
+                                  "children": [{"id": "104_pta",
+                                                "label": {"text": "104_pta"}},
+                                               {"id": "105_simdclone",
+                                                "label": {"text": "105_simdclone"}}]},
+                                 {"id": "all_passes",
+                                  "label": {"text": "all_passes"},
+                                  "children": [{"id": "106_fixup_cfg",
+                                                "label": {"text": "106_fixup_cfg"}},
+                                               {"id": "107_ehdisp",
+                                                "label": {"text": "107_ehdisp"}},
+                                               {"id": "108_oaccloops",
+                                                "label": {"text": "108_oaccloops"}},
+                                               {"id": "109_omp_oacc_neuter_broadcast",
+                                                "label": {"text": "109_omp_oacc_neuter_broadcast"}},
+                                               {"id": "110_oaccdevlow",
+                                                "label": {"text": "110_oaccdevlow"}},
+                                               {"id": "111_ompdevlow",
+                                                "label": {"text": "111_ompdevlow"}},
+                                               {"id": "112_omptargetlink",
+                                                "label": {"text": "112_omptargetlink"}},
+                                               {"id": "113_adjust_alignment",
+                                                "label": {"text": "113_adjust_alignment"}},
+                                               {"id": "114_hardcfr",
+                                                "label": {"text": "114_hardcfr"}},
+                                               {"id": "*all_optimizations_0x101f2720",
+                                                "label": {"text": "*all_optimizations"}},
+                                               {"id": "*remove_cgraph_callee_edges_0x101f2780",
+                                                "label": {"text": "*remove_cgraph_callee_edges"}},
+                                               {"id": "*strip_predict_hints_0x101f27e0",
+                                                "label": {"text": "*strip_predict_hints"}},
+                                               {"id": "115_ccp",
+                                                "label": {"text": "115_ccp"}},
+                                               {"id": "116_objsz",
+                                                "label": {"text": "116_objsz"}},
+                                               {"id": "117_post_ipa_warn",
+                                                "label": {"text": "117_post_ipa_warn"}},
+                                               {"id": "118_waccess",
+                                                "label": {"text": "118_waccess"}},
+                                               {"id": "119_rebuild_frequencies",
+                                                "label": {"text": "119_rebuild_frequencies"}},
+                                               {"id": "120_cunrolli",
+                                                "label": {"text": "120_cunrolli"}},
+                                               {"id": "121_backprop",
+                                                "label": {"text": "121_backprop"}},
+                                               {"id": "122_phiprop",
+                                                "label": {"text": "122_phiprop"}},
+                                               {"id": "123_forwprop",
+                                                "label": {"text": "123_forwprop"}},
+                                               {"id": "124_alias",
+                                                "label": {"text": "124_alias"}},
+                                               {"id": "125_retslot",
+                                                "label": {"text": "125_retslot"}},
+                                               {"id": "126_fre",
+                                                "label": {"text": "126_fre"}},
+                                               {"id": "127_mergephi",
+                                                "label": {"text": "127_mergephi"}},
+                                               {"id": "128_threadfull",
+                                                "label": {"text": "128_threadfull"}},
+                                               {"id": "129_vrp",
+                                                "label": {"text": "129_vrp"}},
+                                               {"id": "130_bounds",
+                                                "label": {"text": "130_bounds"}},
+                                               {"id": "131_dse",
+                                                "label": {"text": "131_dse"}},
+                                               {"id": "132_dce",
+                                                "label": {"text": "132_dce"}},
+                                               {"id": "133_stdarg",
+                                                "label": {"text": "133_stdarg"}},
+                                               {"id": "134_cdce",
+                                                "label": {"text": "134_cdce"}},
+                                               {"id": "135_cselim",
+                                                "label": {"text": "135_cselim"}},
+                                               {"id": "136_copyprop",
+                                                "label": {"text": "136_copyprop"}},
+                                               {"id": "137_ifcombine",
+                                                "label": {"text": "137_ifcombine"}},
+                                               {"id": "138_mergephi",
+                                                "label": {"text": "138_mergephi"}},
+                                               {"id": "139_phiopt",
+                                                "label": {"text": "139_phiopt"}},
+                                               {"id": "140_tailr",
+                                                "label": {"text": "140_tailr"}},
+                                               {"id": "141_ch",
+                                                "label": {"text": "141_ch"}},
+                                               {"id": "142_cplxlower",
+                                                "label": {"text": "142_cplxlower"}},
+                                               {"id": "143_bitintlower",
+                                                "label": {"text": "143_bitintlower"}},
+                                               {"id": "144_sra",
+                                                "label": {"text": "144_sra"}},
+                                               {"id": "145_thread",
+                                                "label": {"text": "145_thread"}},
+                                               {"id": "146_dom",
+                                                "label": {"text": "146_dom"}},
+                                               {"id": "147_copyprop",
+                                                "label": {"text": "147_copyprop"}},
+                                               {"id": "148_isolate-paths",
+                                                "label": {"text": "148_isolate-paths"}},
+                                               {"id": "149_reassoc",
+                                                "label": {"text": "149_reassoc"}},
+                                               {"id": "150_dce",
+                                                "label": {"text": "150_dce"}},
+                                               {"id": "151_forwprop",
+                                                "label": {"text": "151_forwprop"}},
+                                               {"id": "152_phiopt",
+                                                "label": {"text": "152_phiopt"}},
+                                               {"id": "153_ccp",
+                                                "label": {"text": "153_ccp"}},
+                                               {"id": "154_pow",
+                                                "label": {"text": "154_pow"}},
+                                               {"id": "155_bswap",
+                                                "label": {"text": "155_bswap"}},
+                                               {"id": "156_laddress",
+                                                "label": {"text": "156_laddress"}},
+                                               {"id": "157_lim",
+                                                "label": {"text": "157_lim"}},
+                                               {"id": "158_walloca",
+                                                "label": {"text": "158_walloca"}},
+                                               {"id": "159_pre",
+                                                "label": {"text": "159_pre"}},
+                                               {"id": "160_sink",
+                                                "label": {"text": "160_sink"}},
+                                               {"id": "161_sancov",
+                                                "label": {"text": "161_sancov"}},
+                                               {"id": "162_asan",
+                                                "label": {"text": "162_asan"}},
+                                               {"id": "163_tsan",
+                                                "label": {"text": "163_tsan"}},
+                                               {"id": "164_dse",
+                                                "label": {"text": "164_dse"}},
+                                               {"id": "165_dce",
+                                                "label": {"text": "165_dce"}},
+                                               {"id": "166_fix_loops",
+                                                "label": {"text": "166_fix_loops"}},
+                                               {"id": "167_loop",
+                                                "label": {"text": "167_loop"}},
+                                               {"id": "168_loopinit",
+                                                "label": {"text": "168_loopinit"}},
+                                               {"id": "169_unswitch",
+                                                "label": {"text": "169_unswitch"}},
+                                               {"id": "170_lsplit",
+                                                "label": {"text": "170_lsplit"}},
+                                               {"id": "171_sccp",
+                                                "label": {"text": "171_sccp"}},
+                                               {"id": "172_lversion",
+                                                "label": {"text": "172_lversion"}},
+                                               {"id": "173_unrolljam",
+                                                "label": {"text": "173_unrolljam"}},
+                                               {"id": "174_cddce",
+                                                "label": {"text": "174_cddce"}},
+                                               {"id": "175_ivcanon",
+                                                "label": {"text": "175_ivcanon"}},
+                                               {"id": "176_ldist",
+                                                "label": {"text": "176_ldist"}},
+                                               {"id": "177_crc",
+                                                "label": {"text": "177_crc"}},
+                                               {"id": "178_linterchange",
+                                                "label": {"text": "178_linterchange"}},
+                                               {"id": "179_copyprop",
+                                                "label": {"text": "179_copyprop"}},
+                                               {"id": "180_graphite0",
+                                                "label": {"text": "180_graphite0"}},
+                                               {"id": "181_graphite",
+                                                "label": {"text": "181_graphite"}},
+                                               {"id": "182_lim",
+                                                "label": {"text": "182_lim"}},
+                                               {"id": "183_copyprop",
+                                                "label": {"text": "183_copyprop"}},
+                                               {"id": "184_dce",
+                                                "label": {"text": "184_dce"}},
+                                               {"id": "185_parloops",
+                                                "label": {"text": "185_parloops"}},
+                                               {"id": "186_ompexpssa",
+                                                "label": {"text": "186_ompexpssa"}},
+                                               {"id": "187_ch_vect",
+                                                "label": {"text": "187_ch_vect"}},
+                                               {"id": "188_ifcvt",
+                                                "label": {"text": "188_ifcvt"}},
+                                               {"id": "189_vect",
+                                                "label": {"text": "189_vect"}},
+                                               {"id": "190_dce",
+                                                "label": {"text": "190_dce"}},
+                                               {"id": "191_pcom",
+                                                "label": {"text": "191_pcom"}},
+                                               {"id": "192_cunroll",
+                                                "label": {"text": "192_cunroll"}},
+                                               {"id": "*pre_slp_scalar_cleanup_0x101f4880",
+                                                "label": {"text": "*pre_slp_scalar_cleanup"}},
+                                               {"id": "193_fre",
+                                                "label": {"text": "193_fre"}},
+                                               {"id": "194_dse",
+                                                "label": {"text": "194_dse"}},
+                                               {"id": "195_slp",
+                                                "label": {"text": "195_slp"}},
+                                               {"id": "196_aprefetch",
+                                                "label": {"text": "196_aprefetch"}},
+                                               {"id": "197_ivopts",
+                                                "label": {"text": "197_ivopts"}},
+                                               {"id": "198_lim",
+                                                "label": {"text": "198_lim"}},
+                                               {"id": "199_loopdone",
+                                                "label": {"text": "199_loopdone"}},
+                                               {"id": "200_no_loop",
+                                                "label": {"text": "200_no_loop"}},
+                                               {"id": "201_slp",
+                                                "label": {"text": "201_slp"}},
+                                               {"id": "202_simduid",
+                                                "label": {"text": "202_simduid"}},
+                                               {"id": "203_veclower2",
+                                                "label": {"text": "203_veclower2"}},
+                                               {"id": "204_switchlower",
+                                                "label": {"text": "204_switchlower"}},
+                                               {"id": "205_sincos",
+                                                "label": {"text": "205_sincos"}},
+                                               {"id": "206_recip",
+                                                "label": {"text": "206_recip"}},
+                                               {"id": "207_reassoc",
+                                                "label": {"text": "207_reassoc"}},
+                                               {"id": "208_slsr",
+                                                "label": {"text": "208_slsr"}},
+                                               {"id": "209_split-paths",
+                                                "label": {"text": "209_split-paths"}},
+                                               {"id": "210_tracer",
+                                                "label": {"text": "210_tracer"}},
+                                               {"id": "211_fre",
+                                                "label": {"text": "211_fre"}},
+                                               {"id": "212_thread",
+                                                "label": {"text": "212_thread"}},
+                                               {"id": "213_dom",
+                                                "label": {"text": "213_dom"}},
+                                               {"id": "214_strlen",
+                                                "label": {"text": "214_strlen"}},
+                                               {"id": "215_threadfull",
+                                                "label": {"text": "215_threadfull"}},
+                                               {"id": "216_vrp",
+                                                "label": {"text": "216_vrp"}},
+                                               {"id": "217_ccp",
+                                                "label": {"text": "217_ccp"}},
+                                               {"id": "218_wrestrict",
+                                                "label": {"text": "218_wrestrict"}},
+                                               {"id": "219_dse",
+                                                "label": {"text": "219_dse"}},
+                                               {"id": "220_dce",
+                                                "label": {"text": "220_dce"}},
+                                               {"id": "221_forwprop",
+                                                "label": {"text": "221_forwprop"}},
+                                               {"id": "222_sink",
+                                                "label": {"text": "222_sink"}},
+                                               {"id": "223_phiopt",
+                                                "label": {"text": "223_phiopt"}},
+                                               {"id": "224_fab",
+                                                "label": {"text": "224_fab"}},
+                                               {"id": "225_widening_mul",
+                                                "label": {"text": "225_widening_mul"}},
+                                               {"id": "226_store-merging",
+                                                "label": {"text": "226_store-merging"}},
+                                               {"id": "227_cddce",
+                                                "label": {"text": "227_cddce"}},
+                                               {"id": "228_sccopy",
+                                                "label": {"text": "228_sccopy"}},
+                                               {"id": "229_tailc",
+                                                "label": {"text": "229_tailc"}},
+                                               {"id": "230_crited",
+                                                "label": {"text": "230_crited"}},
+                                               {"id": "231_uninit",
+                                                "label": {"text": "231_uninit"}},
+                                               {"id": "232_local-pure-const",
+                                                "label": {"text": "232_local-pure-const"}},
+                                               {"id": "233_modref",
+                                                "label": {"text": "233_modref"}},
+                                               {"id": "234_uncprop",
+                                                "label": {"text": "234_uncprop"}},
+                                               {"id": "*all_optimizations_g_0x101f5af0",
+                                                "label": {"text": "*all_optimizations_g"}},
+                                               {"id": "*remove_cgraph_callee_edges_0x101f5b50",
+                                                "label": {"text": "*remove_cgraph_callee_edges"}},
+                                               {"id": "*strip_predict_hints_0x101f5bb0",
+                                                "label": {"text": "*strip_predict_hints"}},
+                                               {"id": "235_cplxlower",
+                                                "label": {"text": "235_cplxlower"}},
+                                               {"id": "236_bitintlower",
+                                                "label": {"text": "236_bitintlower"}},
+                                               {"id": "237_veclower2",
+                                                "label": {"text": "237_veclower2"}},
+                                               {"id": "238_switchlower",
+                                                "label": {"text": "238_switchlower"}},
+                                               {"id": "239_ccp",
+                                                "label": {"text": "239_ccp"}},
+                                               {"id": "240_post_ipa_warn",
+                                                "label": {"text": "240_post_ipa_warn"}},
+                                               {"id": "241_objsz",
+                                                "label": {"text": "241_objsz"}},
+                                               {"id": "242_fab",
+                                                "label": {"text": "242_fab"}},
+                                               {"id": "243_strlen",
+                                                "label": {"text": "243_strlen"}},
+                                               {"id": "244_copyprop",
+                                                "label": {"text": "244_copyprop"}},
+                                               {"id": "245_dce",
+                                                "label": {"text": "245_dce"}},
+                                               {"id": "246_rebuild_frequencies",
+                                                "label": {"text": "246_rebuild_frequencies"}},
+                                               {"id": "247_sancov",
+                                                "label": {"text": "247_sancov"}},
+                                               {"id": "248_asan",
+                                                "label": {"text": "248_asan"}},
+                                               {"id": "249_tsan",
+                                                "label": {"text": "249_tsan"}},
+                                               {"id": "250_crited",
+                                                "label": {"text": "250_crited"}},
+                                               {"id": "251_uninit",
+                                                "label": {"text": "251_uninit"}},
+                                               {"id": "252_uncprop",
+                                                "label": {"text": "252_uncprop"}},
+                                               {"id": "253_assumptions",
+                                                "label": {"text": "253_assumptions"}},
+                                               {"id": "*tminit_0x101f6370",
+                                                "label": {"text": "*tminit"}},
+                                               {"id": "254_tmmark",
+                                                "label": {"text": "254_tmmark"}},
+                                               {"id": "255_tmmemopt",
+                                                "label": {"text": "255_tmmemopt"}},
+                                               {"id": "256_tmedge",
+                                                "label": {"text": "256_tmedge"}},
+                                               {"id": "257_simduid",
+                                                "label": {"text": "257_simduid"}},
+                                               {"id": "258_vtable-verify",
+                                                "label": {"text": "258_vtable-verify"}},
+                                               {"id": "259_lower_vaarg",
+                                                "label": {"text": "259_lower_vaarg"}},
+                                               {"id": "260_veclower",
+                                                "label": {"text": "260_veclower"}},
+                                               {"id": "261_cplxlower0",
+                                                "label": {"text": "261_cplxlower0"}},
+                                               {"id": "262_bitintlower0",
+                                                "label": {"text": "262_bitintlower0"}},
+                                               {"id": "263_sancov_O0",
+                                                "label": {"text": "263_sancov_O0"}},
+                                               {"id": "264_switchlower_O0",
+                                                "label": {"text": "264_switchlower_O0"}},
+                                               {"id": "265_asan0",
+                                                "label": {"text": "265_asan0"}},
+                                               {"id": "266_tsan0",
+                                                "label": {"text": "266_tsan0"}},
+                                               {"id": "267_musttail",
+                                                "label": {"text": "267_musttail"}},
+                                               {"id": "268_sanopt",
+                                                "label": {"text": "268_sanopt"}},
+                                               {"id": "269_ehcleanup",
+                                                "label": {"text": "269_ehcleanup"}},
+                                               {"id": "270_resx",
+                                                "label": {"text": "270_resx"}},
+                                               {"id": "271_nrv",
+                                                "label": {"text": "271_nrv"}},
+                                               {"id": "272_isel",
+                                                "label": {"text": "272_isel"}},
+                                               {"id": "273_hardcbr",
+                                                "label": {"text": "273_hardcbr"}},
+                                               {"id": "274_hardcmp",
+                                                "label": {"text": "274_hardcmp"}},
+                                               {"id": "275_waccess",
+                                                "label": {"text": "275_waccess"}},
+                                               {"id": "276_optimized",
+                                                "label": {"text": "276_optimized"}},
+                                               {"id": "*warn_function_noreturn_0x101f6dd0",
+                                                "label": {"text": "*warn_function_noreturn"}},
+                                               {"id": "277_expand",
+                                                "label": {"text": "277_expand"}},
+                                               {"id": "*rest_of_compilation_0x101f6e90",
+                                                "label": {"text": "*rest_of_compilation"}},
+                                               {"id": "278_vregs",
+                                                "label": {"text": "278_vregs"}},
+                                               {"id": "279_into_cfglayout",
+                                                "label": {"text": "279_into_cfglayout"}},
+                                               {"id": "280_jump",
+                                                "label": {"text": "280_jump"}},
+                                               {"id": "281_subreg1",
+                                                "label": {"text": "281_subreg1"}},
+                                               {"id": "282_dfinit",
+                                                "label": {"text": "282_dfinit"}},
+                                               {"id": "283_cse1",
+                                                "label": {"text": "283_cse1"}},
+                                               {"id": "284_fwprop1",
+                                                "label": {"text": "284_fwprop1"}},
+                                               {"id": "285_cprop",
+                                                "label": {"text": "285_cprop"}},
+                                               {"id": "286_rtl pre",
+                                                "label": {"text": "286_rtl pre"}},
+                                               {"id": "287_hoist",
+                                                "label": {"text": "287_hoist"}},
+                                               {"id": "288_hardreg_pre",
+                                                "label": {"text": "288_hardreg_pre"}},
+                                               {"id": "289_cprop",
+                                                "label": {"text": "289_cprop"}},
+                                               {"id": "290_store_motion",
+                                                "label": {"text": "290_store_motion"}},
+                                               {"id": "291_cse_local",
+                                                "label": {"text": "291_cse_local"}},
+                                               {"id": "292_ce1",
+                                                "label": {"text": "292_ce1"}},
+                                               {"id": "293_apx_nfcvt",
+                                                "label": {"text": "293_apx_nfcvt"}},
+                                               {"id": "294_reginfo",
+                                                "label": {"text": "294_reginfo"}},
+                                               {"id": "295_loop2",
+                                                "label": {"text": "295_loop2"}},
+                                               {"id": "296_loop2_init",
+                                                "label": {"text": "296_loop2_init"}},
+                                               {"id": "297_loop2_invariant",
+                                                "label": {"text": "297_loop2_invariant"}},
+                                               {"id": "298_loop2_unroll",
+                                                "label": {"text": "298_loop2_unroll"}},
+                                               {"id": "299_loop2_doloop",
+                                                "label": {"text": "299_loop2_doloop"}},
+                                               {"id": "300_loop2_done",
+                                                "label": {"text": "300_loop2_done"}},
+                                               {"id": "301_subreg2",
+                                                "label": {"text": "301_subreg2"}},
+                                               {"id": "302_web",
+                                                "label": {"text": "302_web"}},
+                                               {"id": "303_cprop",
+                                                "label": {"text": "303_cprop"}},
+                                               {"id": "304_stv",
+                                                "label": {"text": "304_stv"}},
+                                               {"id": "305_cse2",
+                                                "label": {"text": "305_cse2"}},
+                                               {"id": "306_dse1",
+                                                "label": {"text": "306_dse1"}},
+                                               {"id": "307_fwprop2",
+                                                "label": {"text": "307_fwprop2"}},
+                                               {"id": "308_auto_inc_dec",
+                                                "label": {"text": "308_auto_inc_dec"}},
+                                               {"id": "309_init-regs",
+                                                "label": {"text": "309_init-regs"}},
+                                               {"id": "310_ud_dce",
+                                                "label": {"text": "310_ud_dce"}},
+                                               {"id": "311_ext_dce",
+                                                "label": {"text": "311_ext_dce"}},
+                                               {"id": "312_combine",
+                                                "label": {"text": "312_combine"}},
+                                               {"id": "313_late_combine",
+                                                "label": {"text": "313_late_combine"}},
+                                               {"id": "314_rpad",
+                                                "label": {"text": "314_rpad"}},
+                                               {"id": "315_rrvl",
+                                                "label": {"text": "315_rrvl"}},
+                                               {"id": "316_stv",
+                                                "label": {"text": "316_stv"}},
+                                               {"id": "317_ce2",
+                                                "label": {"text": "317_ce2"}},
+                                               {"id": "318_jump_after_combine",
+                                                "label": {"text": "318_jump_after_combine"}},
+                                               {"id": "319_bbpart",
+                                                "label": {"text": "319_bbpart"}},
+                                               {"id": "320_outof_cfglayout",
+                                                "label": {"text": "320_outof_cfglayout"}},
+                                               {"id": "321_split1",
+                                                "label": {"text": "321_split1"}},
+                                               {"id": "322_subreg3",
+                                                "label": {"text": "322_subreg3"}},
+                                               {"id": "323_no-opt dfinit",
+                                                "label": {"text": "323_no-opt dfinit"}},
+                                               {"id": "*stack_ptr_mod_0x101f8050",
+                                                "label": {"text": "*stack_ptr_mod"}},
+                                               {"id": "324_mode_sw",
+                                                "label": {"text": "324_mode_sw"}},
+                                               {"id": "325_asmcons",
+                                                "label": {"text": "325_asmcons"}},
+                                               {"id": "326_sms",
+                                                "label": {"text": "326_sms"}},
+                                               {"id": "327_lr_shrinkage",
+                                                "label": {"text": "327_lr_shrinkage"}},
+                                               {"id": "328_sched1",
+                                                "label": {"text": "328_sched1"}},
+                                               {"id": "329_avoid_store_forwarding",
+                                                "label": {"text": "329_avoid_store_forwarding"}},
+                                               {"id": "330_early_remat",
+                                                "label": {"text": "330_early_remat"}},
+                                               {"id": "331_ira",
+                                                "label": {"text": "331_ira"}},
+                                               {"id": "332_reload",
+                                                "label": {"text": "332_reload"}},
+                                               {"id": "*all-postreload_0x101f8410",
+                                                "label": {"text": "*all-postreload"}},
+                                               {"id": "333_postreload",
+                                                "label": {"text": "333_postreload"}},
+                                               {"id": "334_vzeroupper",
+                                                "label": {"text": "334_vzeroupper"}},
+                                               {"id": "335_late_combine",
+                                                "label": {"text": "335_late_combine"}},
+                                               {"id": "336_gcse2",
+                                                "label": {"text": "336_gcse2"}},
+                                               {"id": "337_split2",
+                                                "label": {"text": "337_split2"}},
+                                               {"id": "338_ree",
+                                                "label": {"text": "338_ree"}},
+                                               {"id": "339_cmpelim",
+                                                "label": {"text": "339_cmpelim"}},
+                                               {"id": "340_pro_and_epilogue",
+                                                "label": {"text": "340_pro_and_epilogue"}},
+                                               {"id": "341_dse2",
+                                                "label": {"text": "341_dse2"}},
+                                               {"id": "342_csa",
+                                                "label": {"text": "342_csa"}},
+                                               {"id": "343_jump2",
+                                                "label": {"text": "343_jump2"}},
+                                               {"id": "344_compgotos",
+                                                "label": {"text": "344_compgotos"}},
+                                               {"id": "345_sched_fusion",
+                                                "label": {"text": "345_sched_fusion"}},
+                                               {"id": "346_peephole2",
+                                                "label": {"text": "346_peephole2"}},
+                                               {"id": "347_ce3",
+                                                "label": {"text": "347_ce3"}},
+                                               {"id": "348_rnreg",
+                                                "label": {"text": "348_rnreg"}},
+                                               {"id": "349_fold_mem_offsets",
+                                                "label": {"text": "349_fold_mem_offsets"}},
+                                               {"id": "350_cprop_hardreg",
+                                                "label": {"text": "350_cprop_hardreg"}},
+                                               {"id": "351_rtl_dce",
+                                                "label": {"text": "351_rtl_dce"}},
+                                               {"id": "352_bbro",
+                                                "label": {"text": "352_bbro"}},
+                                               {"id": "*leaf_regs_0x101f8bf0",
+                                                "label": {"text": "*leaf_regs"}},
+                                               {"id": "353_split3",
+                                                "label": {"text": "353_split3"}},
+                                               {"id": "354_sched2",
+                                                "label": {"text": "354_sched2"}},
+                                               {"id": "*stack_regs_0x101f8d10",
+                                                "label": {"text": "*stack_regs"}},
+                                               {"id": "355_split4",
+                                                "label": {"text": "355_split4"}},
+                                               {"id": "356_stack",
+                                                "label": {"text": "356_stack"}},
+                                               {"id": "357_late_pro_and_epilogue",
+                                                "label": {"text": "357_late_pro_and_epilogue"}},
+                                               {"id": "*all-late_compilation_0x101f8e90",
+                                                "label": {"text": "*all-late_compilation"}},
+                                               {"id": "358_zero_call_used_regs",
+                                                "label": {"text": "358_zero_call_used_regs"}},
+                                               {"id": "359_alignments",
+                                                "label": {"text": "359_alignments"}},
+                                               {"id": "360_vartrack",
+                                                "label": {"text": "360_vartrack"}},
+                                               {"id": "*free_cfg_0x101f9010",
+                                                "label": {"text": "*free_cfg"}},
+                                               {"id": "361_mach",
+                                                "label": {"text": "361_mach"}},
+                                               {"id": "362_barriers",
+                                                "label": {"text": "362_barriers"}},
+                                               {"id": "363_dbr",
+                                                "label": {"text": "363_dbr"}},
+                                               {"id": "364_split5",
+                                                "label": {"text": "364_split5"}},
+                                               {"id": "365_eh_ranges",
+                                                "label": {"text": "365_eh_ranges"}},
+                                               {"id": "366_endbr_and_patchable_area",
+                                                "label": {"text": "366_endbr_and_patchable_area"}},
+                                               {"id": "367_align_tight_loops",
+                                                "label": {"text": "367_align_tight_loops"}},
+                                               {"id": "368_shorten",
+                                                "label": {"text": "368_shorten"}},
+                                               {"id": "369_nothrow",
+                                                "label": {"text": "369_nothrow"}},
+                                               {"id": "370_dwarf2",
+                                                "label": {"text": "370_dwarf2"}},
+                                               {"id": "371_final",
+                                                "label": {"text": "371_final"}},
+                                               {"id": "372_dfinish",
+                                                "label": {"text": "372_dfinish"}},
+                                               {"id": "*clean_state_0x101f9500",
+                                                "label": {"text": "*clean_state"}}]}],
+                       "edges": [{"id": "edge0",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "22_ompexp",
+                                  "targetNodeId": "*build_cgraph_edges_0x101ef910"},
+                                 {"id": "edge1",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "21_coro-early-expand-ifns",
+                                  "targetNodeId": "22_ompexp"},
+                                 {"id": "edge2",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*warn_function_return_0x101ef7f0",
+                                  "targetNodeId": "21_coro-early-expand-ifns"},
+                                 {"id": "edge3",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "20_cfg",
+                                  "targetNodeId": "*warn_function_return_0x101ef7f0"},
+                                 {"id": "edge4",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "19_coro-lower-builtins",
+                                  "targetNodeId": "20_cfg"},
+                                 {"id": "edge5",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "18_eh",
+                                  "targetNodeId": "19_coro-lower-builtins"},
+                                 {"id": "edge6",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "17_ehopt",
+                                  "targetNodeId": "18_eh"},
+                                 {"id": "edge7",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "16_tmlower",
+                                  "targetNodeId": "17_ehopt"},
+                                 {"id": "edge8",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "15_lower",
+                                  "targetNodeId": "16_tmlower"},
+                                 {"id": "edge9",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "14_omplower",
+                                  "targetNodeId": "15_lower"},
+                                 {"id": "edge10",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "13_omp_oacc_kernels_decompose",
+                                  "targetNodeId": "14_omplower"},
+                                 {"id": "edge11",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*diagnose_tm_blocks_0x101ef490",
+                                  "targetNodeId": "13_omp_oacc_kernels_decompose"},
+                                 {"id": "edge12",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*diagnose_omp_blocks_0x101ef430",
+                                  "targetNodeId": "*diagnose_tm_blocks_0x101ef490"},
+                                 {"id": "edge13",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*warn_unused_result_0x101ef3d0",
+                                  "targetNodeId": "*diagnose_omp_blocks_0x101ef430"},
+                                 {"id": "edge14",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "34_nothrow",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f0020"},
+                                 {"id": "edge15",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "33_ubsan",
+                                  "targetNodeId": "34_nothrow"},
+                                 {"id": "edge16",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "32_waccess",
+                                  "targetNodeId": "33_ubsan"},
+                                 {"id": "edge17",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "31_early_uninit",
+                                  "targetNodeId": "32_waccess"},
+                                 {"id": "edge18",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*nonnullcmp_0x101efce0",
+                                  "targetNodeId": "31_early_uninit"},
+                                 {"id": "edge19",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "30_warn-printf",
+                                  "targetNodeId": "*nonnullcmp_0x101efce0"},
+                                 {"id": "edge20",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "29_walloca",
+                                  "targetNodeId": "30_warn-printf"},
+                                 {"id": "edge21",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "376_test_graph_emission",
+                                  "targetNodeId": "29_walloca"},
+                                 {"id": "edge22",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "28_ssa",
+                                  "targetNodeId": "376_test_graph_emission"},
+                                 {"id": "edge23",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "27_fixup_cfg",
+                                  "targetNodeId": "28_ssa"},
+                                 {"id": "edge24",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "26_build_ssa_passes",
+                                  "targetNodeId": "27_fixup_cfg"},
+                                 {"id": "edge25",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "61_fnsplit",
+                                  "targetNodeId": "*strip_predict_hints_0x101f0c60"},
+                                 {"id": "edge26",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "60_modref",
+                                  "targetNodeId": "61_fnsplit"},
+                                 {"id": "edge27",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "59_local-pure-const",
+                                  "targetNodeId": "60_modref"},
+                                 {"id": "edge28",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "58_profile_estimate",
+                                  "targetNodeId": "59_local-pure-const"},
+                                 {"id": "edge29",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "57_sccopy",
+                                  "targetNodeId": "58_profile_estimate"},
+                                 {"id": "edge30",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "56_ehcleanup",
+                                  "targetNodeId": "57_sccopy"},
+                                 {"id": "edge31",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "55_switchconv",
+                                  "targetNodeId": "56_ehcleanup"},
+                                 {"id": "edge32",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "54_iftoswitch",
+                                  "targetNodeId": "55_switchconv"},
+                                 {"id": "edge33",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "53_tailr",
+                                  "targetNodeId": "54_iftoswitch"},
+                                 {"id": "edge34",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "52_phiopt",
+                                  "targetNodeId": "53_tailr"},
+                                 {"id": "edge35",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "51_cddce",
+                                  "targetNodeId": "52_phiopt"},
+                                 {"id": "edge36",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "50_dse",
+                                  "targetNodeId": "51_cddce"},
+                                 {"id": "edge37",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "49_mergephi",
+                                  "targetNodeId": "50_dse"},
+                                 {"id": "edge38",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "48_evrp",
+                                  "targetNodeId": "49_mergephi"},
+                                 {"id": "edge39",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "47_fre",
+                                  "targetNodeId": "48_evrp"},
+                                 {"id": "edge40",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "46_phiprop",
+                                  "targetNodeId": "47_fre"},
+                                 {"id": "edge41",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "45_ealias",
+                                  "targetNodeId": "46_phiprop"},
+                                 {"id": "edge42",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "44_esra",
+                                  "targetNodeId": "45_ealias"},
+                                 {"id": "edge43",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "43_ethread",
+                                  "targetNodeId": "44_esra"},
+                                 {"id": "edge44",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "42_forwprop",
+                                  "targetNodeId": "43_ethread"},
+                                 {"id": "edge45",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "41_ccp",
+                                  "targetNodeId": "42_forwprop"},
+                                 {"id": "edge46",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "40_early_objsz",
+                                  "targetNodeId": "41_ccp"},
+                                 {"id": "edge47",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*remove_cgraph_callee_edges_0x101f0340",
+                                  "targetNodeId": "40_early_objsz"},
+                                 {"id": "edge48",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "39_early_optimizations",
+                                  "targetNodeId": "*remove_cgraph_callee_edges_0x101f0340"},
+                                 {"id": "edge49",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*rebuild_cgraph_edges_0x101f0d30",
+                                  "targetNodeId": "63_local-fnsummary"},
+                                 {"id": "edge50",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "62_release_ssa",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f0d30"},
+                                 {"id": "edge51",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "39_early_optimizations",
+                                  "targetNodeId": "62_release_ssa"},
+                                 {"id": "edge52",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*infinite-recursion_0x101f0260",
+                                  "targetNodeId": "39_early_optimizations"},
+                                 {"id": "edge53",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "38_einline",
+                                  "targetNodeId": "*infinite-recursion_0x101f0260"},
+                                 {"id": "edge54",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "37_local-fnsummary",
+                                  "targetNodeId": "38_einline"},
+                                 {"id": "edge55",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*rebuild_cgraph_edges_0x101f0140",
+                                  "targetNodeId": "37_local-fnsummary"},
+                                 {"id": "edge56",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "36_fixup_cfg",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f0140"},
+                                 {"id": "edge57",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "35_opt_local_passes",
+                                  "targetNodeId": "36_fixup_cfg"},
+                                 {"id": "edge58",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "76_ompexpssa",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f1310"},
+                                 {"id": "edge59",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "75_parloops",
+                                  "targetNodeId": "76_ompexpssa"},
+                                 {"id": "edge60",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "74_dce",
+                                  "targetNodeId": "75_parloops"},
+                                 {"id": "edge61",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "73_dom",
+                                  "targetNodeId": "74_dce"},
+                                 {"id": "edge62",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "72_lim",
+                                  "targetNodeId": "73_dom"},
+                                 {"id": "edge63",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "71_fre",
+                                  "targetNodeId": "72_lim"},
+                                 {"id": "edge64",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "70_ch",
+                                  "targetNodeId": "71_fre"},
+                                 {"id": "edge65",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "69_oacc_kernels",
+                                  "targetNodeId": "70_ch"},
+                                 {"id": "edge66",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "68_ipa_oacc_kernels",
+                                  "targetNodeId": "69_oacc_kernels"},
+                                 {"id": "edge67",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "67_pta",
+                                  "targetNodeId": "68_ipa_oacc_kernels"},
+                                 {"id": "edge68",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "66_ipa_oacc",
+                                  "targetNodeId": "67_pta"},
+                                 {"id": "edge69",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "78_afdo",
+                                  "targetNodeId": "79_feedback_fnsplit"},
+                                 {"id": "edge70",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "80_profile",
+                                  "targetNodeId": "81_feedback_fnsplit"},
+                                 {"id": "edge71",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "84_tmipa",
+                                  "targetNodeId": "85_emutls"},
+                                 {"id": "edge72",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "83_increase_alignment",
+                                  "targetNodeId": "84_tmipa"},
+                                 {"id": "edge73",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "82_free-fnsummary",
+                                  "targetNodeId": "83_increase_alignment"},
+                                 {"id": "edge74",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "80_profile",
+                                  "targetNodeId": "82_free-fnsummary"},
+                                 {"id": "edge75",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "78_afdo",
+                                  "targetNodeId": "80_profile"},
+                                 {"id": "edge76",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "77_targetclone",
+                                  "targetNodeId": "78_afdo"},
+                                 {"id": "edge77",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "66_ipa_oacc",
+                                  "targetNodeId": "77_targetclone"},
+                                 {"id": "edge78",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "65_strub",
+                                  "targetNodeId": "66_ipa_oacc"},
+                                 {"id": "edge79",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "64_remove_symbols",
+                                  "targetNodeId": "65_strub"},
+                                 {"id": "edge80",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "35_opt_local_passes",
+                                  "targetNodeId": "64_remove_symbols"},
+                                 {"id": "edge81",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "26_build_ssa_passes",
+                                  "targetNodeId": "35_opt_local_passes"},
+                                 {"id": "edge82",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "25_strubm",
+                                  "targetNodeId": "26_build_ssa_passes"},
+                                 {"id": "edge83",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "24_visibility",
+                                  "targetNodeId": "25_strubm"},
+                                 {"id": "edge84",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*free_lang_data_0x101ef9d0",
+                                  "targetNodeId": "24_visibility"},
+                                 {"id": "edge85",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "23_afdo_offline",
+                                  "targetNodeId": "*free_lang_data_0x101ef9d0"},
+                                 {"id": "edge86",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "102_single-use",
+                                  "targetNodeId": "103_comdats"},
+                                 {"id": "edge87",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "101_static-var",
+                                  "targetNodeId": "102_single-use"},
+                                 {"id": "edge88",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "100_free-fnsummary",
+                                  "targetNodeId": "101_static-var"},
+                                 {"id": "edge89",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "99_modref",
+                                  "targetNodeId": "100_free-fnsummary"},
+                                 {"id": "edge90",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "98_pure-const",
+                                  "targetNodeId": "99_modref"},
+                                 {"id": "edge91",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "97_locality-clone",
+                                  "targetNodeId": "98_pure-const"},
+                                 {"id": "edge92",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "96_inline",
+                                  "targetNodeId": "97_locality-clone"},
+                                 {"id": "edge93",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "95_fnsummary",
+                                  "targetNodeId": "96_inline"},
+                                 {"id": "edge94",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "94_sra",
+                                  "targetNodeId": "95_fnsummary"},
+                                 {"id": "edge95",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "93_cp",
+                                  "targetNodeId": "94_sra"},
+                                 {"id": "edge96",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "92_cdtor",
+                                  "targetNodeId": "93_cp"},
+                                 {"id": "edge97",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "91_devirt",
+                                  "targetNodeId": "92_cdtor"},
+                                 {"id": "edge98",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "90_icf",
+                                  "targetNodeId": "91_devirt"},
+                                 {"id": "edge99",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "89_profile_estimate",
+                                  "targetNodeId": "90_icf"},
+                                 {"id": "edge100",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "88_whole-program",
+                                  "targetNodeId": "89_profile_estimate"},
+                                 {"id": "edge101",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "87_odr",
+                                  "targetNodeId": "88_whole-program"},
+                                 {"id": "edge102",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "86_analyzer",
+                                  "targetNodeId": "87_odr"},
+                                 {"id": "edge103",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "104_pta",
+                                  "targetNodeId": "105_simdclone"},
+                                 {"id": "edge104",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "183_copyprop",
+                                  "targetNodeId": "184_dce"},
+                                 {"id": "edge105",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "182_lim",
+                                  "targetNodeId": "183_copyprop"},
+                                 {"id": "edge106",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "181_graphite",
+                                  "targetNodeId": "182_lim"},
+                                 {"id": "edge107",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "180_graphite0",
+                                  "targetNodeId": "181_graphite"},
+                                 {"id": "edge108",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "189_vect",
+                                  "targetNodeId": "190_dce"},
+                                 {"id": "edge109",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "193_fre",
+                                  "targetNodeId": "194_dse"},
+                                 {"id": "edge110",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*pre_slp_scalar_cleanup_0x101f4880",
+                                  "targetNodeId": "193_fre"},
+                                 {"id": "edge111",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "198_lim",
+                                  "targetNodeId": "199_loopdone"},
+                                 {"id": "edge112",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "197_ivopts",
+                                  "targetNodeId": "198_lim"},
+                                 {"id": "edge113",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "196_aprefetch",
+                                  "targetNodeId": "197_ivopts"},
+                                 {"id": "edge114",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "195_slp",
+                                  "targetNodeId": "196_aprefetch"},
+                                 {"id": "edge115",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*pre_slp_scalar_cleanup_0x101f4880",
+                                  "targetNodeId": "195_slp"},
+                                 {"id": "edge116",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "192_cunroll",
+                                  "targetNodeId": "*pre_slp_scalar_cleanup_0x101f4880"},
+                                 {"id": "edge117",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "191_pcom",
+                                  "targetNodeId": "192_cunroll"},
+                                 {"id": "edge118",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "189_vect",
+                                  "targetNodeId": "191_pcom"},
+                                 {"id": "edge119",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "188_ifcvt",
+                                  "targetNodeId": "189_vect"},
+                                 {"id": "edge120",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "187_ch_vect",
+                                  "targetNodeId": "188_ifcvt"},
+                                 {"id": "edge121",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "186_ompexpssa",
+                                  "targetNodeId": "187_ch_vect"},
+                                 {"id": "edge122",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "185_parloops",
+                                  "targetNodeId": "186_ompexpssa"},
+                                 {"id": "edge123",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "180_graphite0",
+                                  "targetNodeId": "185_parloops"},
+                                 {"id": "edge124",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "179_copyprop",
+                                  "targetNodeId": "180_graphite0"},
+                                 {"id": "edge125",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "178_linterchange",
+                                  "targetNodeId": "179_copyprop"},
+                                 {"id": "edge126",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "177_crc",
+                                  "targetNodeId": "178_linterchange"},
+                                 {"id": "edge127",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "176_ldist",
+                                  "targetNodeId": "177_crc"},
+                                 {"id": "edge128",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "175_ivcanon",
+                                  "targetNodeId": "176_ldist"},
+                                 {"id": "edge129",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "174_cddce",
+                                  "targetNodeId": "175_ivcanon"},
+                                 {"id": "edge130",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "173_unrolljam",
+                                  "targetNodeId": "174_cddce"},
+                                 {"id": "edge131",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "172_lversion",
+                                  "targetNodeId": "173_unrolljam"},
+                                 {"id": "edge132",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "171_sccp",
+                                  "targetNodeId": "172_lversion"},
+                                 {"id": "edge133",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "170_lsplit",
+                                  "targetNodeId": "171_sccp"},
+                                 {"id": "edge134",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "169_unswitch",
+                                  "targetNodeId": "170_lsplit"},
+                                 {"id": "edge135",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "168_loopinit",
+                                  "targetNodeId": "169_unswitch"},
+                                 {"id": "edge136",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "167_loop",
+                                  "targetNodeId": "168_loopinit"},
+                                 {"id": "edge137",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "200_no_loop",
+                                  "targetNodeId": "201_slp"},
+                                 {"id": "edge138",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "233_modref",
+                                  "targetNodeId": "234_uncprop"},
+                                 {"id": "edge139",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "232_local-pure-const",
+                                  "targetNodeId": "233_modref"},
+                                 {"id": "edge140",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "231_uninit",
+                                  "targetNodeId": "232_local-pure-const"},
+                                 {"id": "edge141",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "230_crited",
+                                  "targetNodeId": "231_uninit"},
+                                 {"id": "edge142",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "229_tailc",
+                                  "targetNodeId": "230_crited"},
+                                 {"id": "edge143",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "228_sccopy",
+                                  "targetNodeId": "229_tailc"},
+                                 {"id": "edge144",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "227_cddce",
+                                  "targetNodeId": "228_sccopy"},
+                                 {"id": "edge145",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "226_store-merging",
+                                  "targetNodeId": "227_cddce"},
+                                 {"id": "edge146",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "225_widening_mul",
+                                  "targetNodeId": "226_store-merging"},
+                                 {"id": "edge147",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "224_fab",
+                                  "targetNodeId": "225_widening_mul"},
+                                 {"id": "edge148",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "223_phiopt",
+                                  "targetNodeId": "224_fab"},
+                                 {"id": "edge149",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "222_sink",
+                                  "targetNodeId": "223_phiopt"},
+                                 {"id": "edge150",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "221_forwprop",
+                                  "targetNodeId": "222_sink"},
+                                 {"id": "edge151",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "220_dce",
+                                  "targetNodeId": "221_forwprop"},
+                                 {"id": "edge152",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "219_dse",
+                                  "targetNodeId": "220_dce"},
+                                 {"id": "edge153",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "218_wrestrict",
+                                  "targetNodeId": "219_dse"},
+                                 {"id": "edge154",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "217_ccp",
+                                  "targetNodeId": "218_wrestrict"},
+                                 {"id": "edge155",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "216_vrp",
+                                  "targetNodeId": "217_ccp"},
+                                 {"id": "edge156",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "215_threadfull",
+                                  "targetNodeId": "216_vrp"},
+                                 {"id": "edge157",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "214_strlen",
+                                  "targetNodeId": "215_threadfull"},
+                                 {"id": "edge158",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "213_dom",
+                                  "targetNodeId": "214_strlen"},
+                                 {"id": "edge159",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "212_thread",
+                                  "targetNodeId": "213_dom"},
+                                 {"id": "edge160",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "211_fre",
+                                  "targetNodeId": "212_thread"},
+                                 {"id": "edge161",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "210_tracer",
+                                  "targetNodeId": "211_fre"},
+                                 {"id": "edge162",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "209_split-paths",
+                                  "targetNodeId": "210_tracer"},
+                                 {"id": "edge163",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "208_slsr",
+                                  "targetNodeId": "209_split-paths"},
+                                 {"id": "edge164",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "207_reassoc",
+                                  "targetNodeId": "208_slsr"},
+                                 {"id": "edge165",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "206_recip",
+                                  "targetNodeId": "207_reassoc"},
+                                 {"id": "edge166",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "205_sincos",
+                                  "targetNodeId": "206_recip"},
+                                 {"id": "edge167",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "204_switchlower",
+                                  "targetNodeId": "205_sincos"},
+                                 {"id": "edge168",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "203_veclower2",
+                                  "targetNodeId": "204_switchlower"},
+                                 {"id": "edge169",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "202_simduid",
+                                  "targetNodeId": "203_veclower2"},
+                                 {"id": "edge170",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "200_no_loop",
+                                  "targetNodeId": "202_simduid"},
+                                 {"id": "edge171",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "167_loop",
+                                  "targetNodeId": "200_no_loop"},
+                                 {"id": "edge172",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "166_fix_loops",
+                                  "targetNodeId": "167_loop"},
+                                 {"id": "edge173",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "165_dce",
+                                  "targetNodeId": "166_fix_loops"},
+                                 {"id": "edge174",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "164_dse",
+                                  "targetNodeId": "165_dce"},
+                                 {"id": "edge175",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "163_tsan",
+                                  "targetNodeId": "164_dse"},
+                                 {"id": "edge176",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "162_asan",
+                                  "targetNodeId": "163_tsan"},
+                                 {"id": "edge177",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "161_sancov",
+                                  "targetNodeId": "162_asan"},
+                                 {"id": "edge178",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "160_sink",
+                                  "targetNodeId": "161_sancov"},
+                                 {"id": "edge179",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "159_pre",
+                                  "targetNodeId": "160_sink"},
+                                 {"id": "edge180",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "158_walloca",
+                                  "targetNodeId": "159_pre"},
+                                 {"id": "edge181",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "157_lim",
+                                  "targetNodeId": "158_walloca"},
+                                 {"id": "edge182",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "156_laddress",
+                                  "targetNodeId": "157_lim"},
+                                 {"id": "edge183",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "155_bswap",
+                                  "targetNodeId": "156_laddress"},
+                                 {"id": "edge184",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "154_pow",
+                                  "targetNodeId": "155_bswap"},
+                                 {"id": "edge185",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "153_ccp",
+                                  "targetNodeId": "154_pow"},
+                                 {"id": "edge186",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "152_phiopt",
+                                  "targetNodeId": "153_ccp"},
+                                 {"id": "edge187",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "151_forwprop",
+                                  "targetNodeId": "152_phiopt"},
+                                 {"id": "edge188",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "150_dce",
+                                  "targetNodeId": "151_forwprop"},
+                                 {"id": "edge189",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "149_reassoc",
+                                  "targetNodeId": "150_dce"},
+                                 {"id": "edge190",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "148_isolate-paths",
+                                  "targetNodeId": "149_reassoc"},
+                                 {"id": "edge191",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "147_copyprop",
+                                  "targetNodeId": "148_isolate-paths"},
+                                 {"id": "edge192",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "146_dom",
+                                  "targetNodeId": "147_copyprop"},
+                                 {"id": "edge193",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "145_thread",
+                                  "targetNodeId": "146_dom"},
+                                 {"id": "edge194",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "144_sra",
+                                  "targetNodeId": "145_thread"},
+                                 {"id": "edge195",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "143_bitintlower",
+                                  "targetNodeId": "144_sra"},
+                                 {"id": "edge196",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "142_cplxlower",
+                                  "targetNodeId": "143_bitintlower"},
+                                 {"id": "edge197",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "141_ch",
+                                  "targetNodeId": "142_cplxlower"},
+                                 {"id": "edge198",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "140_tailr",
+                                  "targetNodeId": "141_ch"},
+                                 {"id": "edge199",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "139_phiopt",
+                                  "targetNodeId": "140_tailr"},
+                                 {"id": "edge200",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "138_mergephi",
+                                  "targetNodeId": "139_phiopt"},
+                                 {"id": "edge201",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "137_ifcombine",
+                                  "targetNodeId": "138_mergephi"},
+                                 {"id": "edge202",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "136_copyprop",
+                                  "targetNodeId": "137_ifcombine"},
+                                 {"id": "edge203",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "135_cselim",
+                                  "targetNodeId": "136_copyprop"},
+                                 {"id": "edge204",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "134_cdce",
+                                  "targetNodeId": "135_cselim"},
+                                 {"id": "edge205",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "133_stdarg",
+                                  "targetNodeId": "134_cdce"},
+                                 {"id": "edge206",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "132_dce",
+                                  "targetNodeId": "133_stdarg"},
+                                 {"id": "edge207",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "131_dse",
+                                  "targetNodeId": "132_dce"},
+                                 {"id": "edge208",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "130_bounds",
+                                  "targetNodeId": "131_dse"},
+                                 {"id": "edge209",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "129_vrp",
+                                  "targetNodeId": "130_bounds"},
+                                 {"id": "edge210",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "128_threadfull",
+                                  "targetNodeId": "129_vrp"},
+                                 {"id": "edge211",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "127_mergephi",
+                                  "targetNodeId": "128_threadfull"},
+                                 {"id": "edge212",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "126_fre",
+                                  "targetNodeId": "127_mergephi"},
+                                 {"id": "edge213",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "125_retslot",
+                                  "targetNodeId": "126_fre"},
+                                 {"id": "edge214",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "124_alias",
+                                  "targetNodeId": "125_retslot"},
+                                 {"id": "edge215",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "123_forwprop",
+                                  "targetNodeId": "124_alias"},
+                                 {"id": "edge216",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "122_phiprop",
+                                  "targetNodeId": "123_forwprop"},
+                                 {"id": "edge217",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "121_backprop",
+                                  "targetNodeId": "122_phiprop"},
+                                 {"id": "edge218",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "120_cunrolli",
+                                  "targetNodeId": "121_backprop"},
+                                 {"id": "edge219",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "119_rebuild_frequencies",
+                                  "targetNodeId": "120_cunrolli"},
+                                 {"id": "edge220",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "118_waccess",
+                                  "targetNodeId": "119_rebuild_frequencies"},
+                                 {"id": "edge221",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "117_post_ipa_warn",
+                                  "targetNodeId": "118_waccess"},
+                                 {"id": "edge222",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "116_objsz",
+                                  "targetNodeId": "117_post_ipa_warn"},
+                                 {"id": "edge223",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "115_ccp",
+                                  "targetNodeId": "116_objsz"},
+                                 {"id": "edge224",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*strip_predict_hints_0x101f27e0",
+                                  "targetNodeId": "115_ccp"},
+                                 {"id": "edge225",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*remove_cgraph_callee_edges_0x101f2780",
+                                  "targetNodeId": "*strip_predict_hints_0x101f27e0"},
+                                 {"id": "edge226",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all_optimizations_0x101f2720",
+                                  "targetNodeId": "*remove_cgraph_callee_edges_0x101f2780"},
+                                 {"id": "edge227",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "251_uninit",
+                                  "targetNodeId": "252_uncprop"},
+                                 {"id": "edge228",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "250_crited",
+                                  "targetNodeId": "251_uninit"},
+                                 {"id": "edge229",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "249_tsan",
+                                  "targetNodeId": "250_crited"},
+                                 {"id": "edge230",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "248_asan",
+                                  "targetNodeId": "249_tsan"},
+                                 {"id": "edge231",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "247_sancov",
+                                  "targetNodeId": "248_asan"},
+                                 {"id": "edge232",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "246_rebuild_frequencies",
+                                  "targetNodeId": "247_sancov"},
+                                 {"id": "edge233",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "245_dce",
+                                  "targetNodeId": "246_rebuild_frequencies"},
+                                 {"id": "edge234",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "244_copyprop",
+                                  "targetNodeId": "245_dce"},
+                                 {"id": "edge235",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "243_strlen",
+                                  "targetNodeId": "244_copyprop"},
+                                 {"id": "edge236",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "242_fab",
+                                  "targetNodeId": "243_strlen"},
+                                 {"id": "edge237",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "241_objsz",
+                                  "targetNodeId": "242_fab"},
+                                 {"id": "edge238",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "240_post_ipa_warn",
+                                  "targetNodeId": "241_objsz"},
+                                 {"id": "edge239",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "239_ccp",
+                                  "targetNodeId": "240_post_ipa_warn"},
+                                 {"id": "edge240",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "238_switchlower",
+                                  "targetNodeId": "239_ccp"},
+                                 {"id": "edge241",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "237_veclower2",
+                                  "targetNodeId": "238_switchlower"},
+                                 {"id": "edge242",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "236_bitintlower",
+                                  "targetNodeId": "237_veclower2"},
+                                 {"id": "edge243",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "235_cplxlower",
+                                  "targetNodeId": "236_bitintlower"},
+                                 {"id": "edge244",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*strip_predict_hints_0x101f5bb0",
+                                  "targetNodeId": "235_cplxlower"},
+                                 {"id": "edge245",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*remove_cgraph_callee_edges_0x101f5b50",
+                                  "targetNodeId": "*strip_predict_hints_0x101f5bb0"},
+                                 {"id": "edge246",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all_optimizations_g_0x101f5af0",
+                                  "targetNodeId": "*remove_cgraph_callee_edges_0x101f5b50"},
+                                 {"id": "edge247",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "255_tmmemopt",
+                                  "targetNodeId": "256_tmedge"},
+                                 {"id": "edge248",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "254_tmmark",
+                                  "targetNodeId": "255_tmmemopt"},
+                                 {"id": "edge249",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*tminit_0x101f6370",
+                                  "targetNodeId": "254_tmmark"},
+                                 {"id": "edge250",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "299_loop2_doloop",
+                                  "targetNodeId": "300_loop2_done"},
+                                 {"id": "edge251",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "298_loop2_unroll",
+                                  "targetNodeId": "299_loop2_doloop"},
+                                 {"id": "edge252",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "297_loop2_invariant",
+                                  "targetNodeId": "298_loop2_unroll"},
+                                 {"id": "edge253",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "296_loop2_init",
+                                  "targetNodeId": "297_loop2_invariant"},
+                                 {"id": "edge254",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "295_loop2",
+                                  "targetNodeId": "296_loop2_init"},
+                                 {"id": "edge255",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "355_split4",
+                                  "targetNodeId": "356_stack"},
+                                 {"id": "edge256",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*stack_regs_0x101f8d10",
+                                  "targetNodeId": "355_split4"},
+                                 {"id": "edge257",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "354_sched2",
+                                  "targetNodeId": "*stack_regs_0x101f8d10"},
+                                 {"id": "edge258",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "353_split3",
+                                  "targetNodeId": "354_sched2"},
+                                 {"id": "edge259",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*leaf_regs_0x101f8bf0",
+                                  "targetNodeId": "353_split3"},
+                                 {"id": "edge260",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "352_bbro",
+                                  "targetNodeId": "*leaf_regs_0x101f8bf0"},
+                                 {"id": "edge261",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "351_rtl_dce",
+                                  "targetNodeId": "352_bbro"},
+                                 {"id": "edge262",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "350_cprop_hardreg",
+                                  "targetNodeId": "351_rtl_dce"},
+                                 {"id": "edge263",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "349_fold_mem_offsets",
+                                  "targetNodeId": "350_cprop_hardreg"},
+                                 {"id": "edge264",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "348_rnreg",
+                                  "targetNodeId": "349_fold_mem_offsets"},
+                                 {"id": "edge265",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "347_ce3",
+                                  "targetNodeId": "348_rnreg"},
+                                 {"id": "edge266",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "346_peephole2",
+                                  "targetNodeId": "347_ce3"},
+                                 {"id": "edge267",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "345_sched_fusion",
+                                  "targetNodeId": "346_peephole2"},
+                                 {"id": "edge268",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "344_compgotos",
+                                  "targetNodeId": "345_sched_fusion"},
+                                 {"id": "edge269",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "343_jump2",
+                                  "targetNodeId": "344_compgotos"},
+                                 {"id": "edge270",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "342_csa",
+                                  "targetNodeId": "343_jump2"},
+                                 {"id": "edge271",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "341_dse2",
+                                  "targetNodeId": "342_csa"},
+                                 {"id": "edge272",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "340_pro_and_epilogue",
+                                  "targetNodeId": "341_dse2"},
+                                 {"id": "edge273",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "339_cmpelim",
+                                  "targetNodeId": "340_pro_and_epilogue"},
+                                 {"id": "edge274",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "338_ree",
+                                  "targetNodeId": "339_cmpelim"},
+                                 {"id": "edge275",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "337_split2",
+                                  "targetNodeId": "338_ree"},
+                                 {"id": "edge276",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "336_gcse2",
+                                  "targetNodeId": "337_split2"},
+                                 {"id": "edge277",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "335_late_combine",
+                                  "targetNodeId": "336_gcse2"},
+                                 {"id": "edge278",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "334_vzeroupper",
+                                  "targetNodeId": "335_late_combine"},
+                                 {"id": "edge279",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "333_postreload",
+                                  "targetNodeId": "334_vzeroupper"},
+                                 {"id": "edge280",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all-postreload_0x101f8410",
+                                  "targetNodeId": "333_postreload"},
+                                 {"id": "edge281",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "370_dwarf2",
+                                  "targetNodeId": "371_final"},
+                                 {"id": "edge282",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "369_nothrow",
+                                  "targetNodeId": "370_dwarf2"},
+                                 {"id": "edge283",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "368_shorten",
+                                  "targetNodeId": "369_nothrow"},
+                                 {"id": "edge284",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "367_align_tight_loops",
+                                  "targetNodeId": "368_shorten"},
+                                 {"id": "edge285",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "366_endbr_and_patchable_area",
+                                  "targetNodeId": "367_align_tight_loops"},
+                                 {"id": "edge286",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "365_eh_ranges",
+                                  "targetNodeId": "366_endbr_and_patchable_area"},
+                                 {"id": "edge287",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "364_split5",
+                                  "targetNodeId": "365_eh_ranges"},
+                                 {"id": "edge288",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "363_dbr",
+                                  "targetNodeId": "364_split5"},
+                                 {"id": "edge289",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "362_barriers",
+                                  "targetNodeId": "363_dbr"},
+                                 {"id": "edge290",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "361_mach",
+                                  "targetNodeId": "362_barriers"},
+                                 {"id": "edge291",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*free_cfg_0x101f9010",
+                                  "targetNodeId": "361_mach"},
+                                 {"id": "edge292",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "360_vartrack",
+                                  "targetNodeId": "*free_cfg_0x101f9010"},
+                                 {"id": "edge293",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "359_alignments",
+                                  "targetNodeId": "360_vartrack"},
+                                 {"id": "edge294",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "358_zero_call_used_regs",
+                                  "targetNodeId": "359_alignments"},
+                                 {"id": "edge295",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all-late_compilation_0x101f8e90",
+                                  "targetNodeId": "358_zero_call_used_regs"},
+                                 {"id": "edge296",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all-late_compilation_0x101f8e90",
+                                  "targetNodeId": "372_dfinish"},
+                                 {"id": "edge297",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "357_late_pro_and_epilogue",
+                                  "targetNodeId": "*all-late_compilation_0x101f8e90"},
+                                 {"id": "edge298",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all-postreload_0x101f8410",
+                                  "targetNodeId": "357_late_pro_and_epilogue"},
+                                 {"id": "edge299",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "332_reload",
+                                  "targetNodeId": "*all-postreload_0x101f8410"},
+                                 {"id": "edge300",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "331_ira",
+                                  "targetNodeId": "332_reload"},
+                                 {"id": "edge301",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "330_early_remat",
+                                  "targetNodeId": "331_ira"},
+                                 {"id": "edge302",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "329_avoid_store_forwarding",
+                                  "targetNodeId": "330_early_remat"},
+                                 {"id": "edge303",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "328_sched1",
+                                  "targetNodeId": "329_avoid_store_forwarding"},
+                                 {"id": "edge304",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "327_lr_shrinkage",
+                                  "targetNodeId": "328_sched1"},
+                                 {"id": "edge305",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "326_sms",
+                                  "targetNodeId": "327_lr_shrinkage"},
+                                 {"id": "edge306",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "325_asmcons",
+                                  "targetNodeId": "326_sms"},
+                                 {"id": "edge307",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "324_mode_sw",
+                                  "targetNodeId": "325_asmcons"},
+                                 {"id": "edge308",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*stack_ptr_mod_0x101f8050",
+                                  "targetNodeId": "324_mode_sw"},
+                                 {"id": "edge309",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "323_no-opt dfinit",
+                                  "targetNodeId": "*stack_ptr_mod_0x101f8050"},
+                                 {"id": "edge310",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "322_subreg3",
+                                  "targetNodeId": "323_no-opt dfinit"},
+                                 {"id": "edge311",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "321_split1",
+                                  "targetNodeId": "322_subreg3"},
+                                 {"id": "edge312",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "320_outof_cfglayout",
+                                  "targetNodeId": "321_split1"},
+                                 {"id": "edge313",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "319_bbpart",
+                                  "targetNodeId": "320_outof_cfglayout"},
+                                 {"id": "edge314",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "318_jump_after_combine",
+                                  "targetNodeId": "319_bbpart"},
+                                 {"id": "edge315",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "317_ce2",
+                                  "targetNodeId": "318_jump_after_combine"},
+                                 {"id": "edge316",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "316_stv",
+                                  "targetNodeId": "317_ce2"},
+                                 {"id": "edge317",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "315_rrvl",
+                                  "targetNodeId": "316_stv"},
+                                 {"id": "edge318",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "314_rpad",
+                                  "targetNodeId": "315_rrvl"},
+                                 {"id": "edge319",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "313_late_combine",
+                                  "targetNodeId": "314_rpad"},
+                                 {"id": "edge320",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "312_combine",
+                                  "targetNodeId": "313_late_combine"},
+                                 {"id": "edge321",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "311_ext_dce",
+                                  "targetNodeId": "312_combine"},
+                                 {"id": "edge322",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "310_ud_dce",
+                                  "targetNodeId": "311_ext_dce"},
+                                 {"id": "edge323",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "309_init-regs",
+                                  "targetNodeId": "310_ud_dce"},
+                                 {"id": "edge324",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "308_auto_inc_dec",
+                                  "targetNodeId": "309_init-regs"},
+                                 {"id": "edge325",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "307_fwprop2",
+                                  "targetNodeId": "308_auto_inc_dec"},
+                                 {"id": "edge326",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "306_dse1",
+                                  "targetNodeId": "307_fwprop2"},
+                                 {"id": "edge327",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "305_cse2",
+                                  "targetNodeId": "306_dse1"},
+                                 {"id": "edge328",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "304_stv",
+                                  "targetNodeId": "305_cse2"},
+                                 {"id": "edge329",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "303_cprop",
+                                  "targetNodeId": "304_stv"},
+                                 {"id": "edge330",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "302_web",
+                                  "targetNodeId": "303_cprop"},
+                                 {"id": "edge331",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "301_subreg2",
+                                  "targetNodeId": "302_web"},
+                                 {"id": "edge332",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "295_loop2",
+                                  "targetNodeId": "301_subreg2"},
+                                 {"id": "edge333",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "294_reginfo",
+                                  "targetNodeId": "295_loop2"},
+                                 {"id": "edge334",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "293_apx_nfcvt",
+                                  "targetNodeId": "294_reginfo"},
+                                 {"id": "edge335",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "292_ce1",
+                                  "targetNodeId": "293_apx_nfcvt"},
+                                 {"id": "edge336",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "291_cse_local",
+                                  "targetNodeId": "292_ce1"},
+                                 {"id": "edge337",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "290_store_motion",
+                                  "targetNodeId": "291_cse_local"},
+                                 {"id": "edge338",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "289_cprop",
+                                  "targetNodeId": "290_store_motion"},
+                                 {"id": "edge339",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "288_hardreg_pre",
+                                  "targetNodeId": "289_cprop"},
+                                 {"id": "edge340",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "287_hoist",
+                                  "targetNodeId": "288_hardreg_pre"},
+                                 {"id": "edge341",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "286_rtl pre",
+                                  "targetNodeId": "287_hoist"},
+                                 {"id": "edge342",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "285_cprop",
+                                  "targetNodeId": "286_rtl pre"},
+                                 {"id": "edge343",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "284_fwprop1",
+                                  "targetNodeId": "285_cprop"},
+                                 {"id": "edge344",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "283_cse1",
+                                  "targetNodeId": "284_fwprop1"},
+                                 {"id": "edge345",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "282_dfinit",
+                                  "targetNodeId": "283_cse1"},
+                                 {"id": "edge346",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "281_subreg1",
+                                  "targetNodeId": "282_dfinit"},
+                                 {"id": "edge347",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "280_jump",
+                                  "targetNodeId": "281_subreg1"},
+                                 {"id": "edge348",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "279_into_cfglayout",
+                                  "targetNodeId": "280_jump"},
+                                 {"id": "edge349",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "278_vregs",
+                                  "targetNodeId": "279_into_cfglayout"},
+                                 {"id": "edge350",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*rest_of_compilation_0x101f6e90",
+                                  "targetNodeId": "278_vregs"},
+                                 {"id": "edge351",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*rest_of_compilation_0x101f6e90",
+                                  "targetNodeId": "*clean_state_0x101f9500"},
+                                 {"id": "edge352",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "277_expand",
+                                  "targetNodeId": "*rest_of_compilation_0x101f6e90"},
+                                 {"id": "edge353",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*warn_function_noreturn_0x101f6dd0",
+                                  "targetNodeId": "277_expand"},
+                                 {"id": "edge354",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "276_optimized",
+                                  "targetNodeId": "*warn_function_noreturn_0x101f6dd0"},
+                                 {"id": "edge355",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "275_waccess",
+                                  "targetNodeId": "276_optimized"},
+                                 {"id": "edge356",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "274_hardcmp",
+                                  "targetNodeId": "275_waccess"},
+                                 {"id": "edge357",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "273_hardcbr",
+                                  "targetNodeId": "274_hardcmp"},
+                                 {"id": "edge358",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "272_isel",
+                                  "targetNodeId": "273_hardcbr"},
+                                 {"id": "edge359",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "271_nrv",
+                                  "targetNodeId": "272_isel"},
+                                 {"id": "edge360",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "270_resx",
+                                  "targetNodeId": "271_nrv"},
+                                 {"id": "edge361",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "269_ehcleanup",
+                                  "targetNodeId": "270_resx"},
+                                 {"id": "edge362",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "268_sanopt",
+                                  "targetNodeId": "269_ehcleanup"},
+                                 {"id": "edge363",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "267_musttail",
+                                  "targetNodeId": "268_sanopt"},
+                                 {"id": "edge364",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "266_tsan0",
+                                  "targetNodeId": "267_musttail"},
+                                 {"id": "edge365",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "265_asan0",
+                                  "targetNodeId": "266_tsan0"},
+                                 {"id": "edge366",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "264_switchlower_O0",
+                                  "targetNodeId": "265_asan0"},
+                                 {"id": "edge367",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "263_sancov_O0",
+                                  "targetNodeId": "264_switchlower_O0"},
+                                 {"id": "edge368",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "262_bitintlower0",
+                                  "targetNodeId": "263_sancov_O0"},
+                                 {"id": "edge369",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "261_cplxlower0",
+                                  "targetNodeId": "262_bitintlower0"},
+                                 {"id": "edge370",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "260_veclower",
+                                  "targetNodeId": "261_cplxlower0"},
+                                 {"id": "edge371",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "259_lower_vaarg",
+                                  "targetNodeId": "260_veclower"},
+                                 {"id": "edge372",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "258_vtable-verify",
+                                  "targetNodeId": "259_lower_vaarg"},
+                                 {"id": "edge373",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "257_simduid",
+                                  "targetNodeId": "258_vtable-verify"},
+                                 {"id": "edge374",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*tminit_0x101f6370",
+                                  "targetNodeId": "257_simduid"},
+                                 {"id": "edge375",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "253_assumptions",
+                                  "targetNodeId": "*tminit_0x101f6370"},
+                                 {"id": "edge376",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all_optimizations_g_0x101f5af0",
+                                  "targetNodeId": "253_assumptions"},
+                                 {"id": "edge377",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all_optimizations_0x101f2720",
+                                  "targetNodeId": "*all_optimizations_g_0x101f5af0"},
+                                 {"id": "edge378",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "114_hardcfr",
+                                  "targetNodeId": "*all_optimizations_0x101f2720"},
+                                 {"id": "edge379",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "113_adjust_alignment",
+                                  "targetNodeId": "114_hardcfr"},
+                                 {"id": "edge380",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "112_omptargetlink",
+                                  "targetNodeId": "113_adjust_alignment"},
+                                 {"id": "edge381",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "111_ompdevlow",
+                                  "targetNodeId": "112_omptargetlink"},
+                                 {"id": "edge382",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "110_oaccdevlow",
+                                  "targetNodeId": "111_ompdevlow"},
+                                 {"id": "edge383",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "109_omp_oacc_neuter_broadcast",
+                                  "targetNodeId": "110_oaccdevlow"},
+                                 {"id": "edge384",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "108_oaccloops",
+                                  "targetNodeId": "109_omp_oacc_neuter_broadcast"},
+                                 {"id": "edge385",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "107_ehdisp",
+                                  "targetNodeId": "108_oaccloops"},
+                                 {"id": "edge386",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "106_fixup_cfg",
+                                  "targetNodeId": "107_ehdisp"}]}]}]}
+
+/* { dg-begin-multiline-output "" }
+In function 'test_graphs':
+/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c:8:3: error: this is a placeholder error, with graphs
+   { dg-end-multiline-output "" } */
+
+/* Use a Python script to verify various properties about the generated
+   .html file:
+   { dg-final { run-html-pytest graphs.sarif "2.1.0-valid/graphs-check-html.py" } } */
+
+/* Use a Python script to verify various properties about the *generated*
+   .sarif file:
+   { dg-final { run-sarif-pytest graphs.roundtrip "2.1.0-valid/graphs-check-sarif-roundtrip.py" } } */
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/nested-diagnostics-1.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/nested-diagnostics-1.sarif
new file mode 100644
index 0000000..b924012
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/nested-diagnostics-1.sarif
@@ -0,0 +1,164 @@
+/* Test a replay of a .sarif file generated from GCC testsuite.
+
+   The dg directives were stripped out from the generated .sarif
+   to avoid confusing DejaGnu for this test.   */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:file=nested-diagnostics-1.roundtrip.sarif" } */
+/* { dg-additional-options "-fdiagnostics-add-output=experimental-html:file=nested-diagnostics-1.sarif.html,javascript=no" } */
+
+{"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [{"tool": {"driver": {"name": "GNU C23",
+                               "fullName": "GNU C23 (GCC) version 16.0.0 20250723 (experimental) (x86_64-pc-linux-gnu)",
+                               "version": "16.0.0 20250723 (experimental)",
+                               "informationUri": "https://gcc.gnu.org/gcc-16/",
+                               "rules": []},
+                    "extensions": [{"name": "diagnostic_plugin_test_nesting",
+                                    "fullName": "./diagnostic_plugin_test_nesting.so"}]},
+           "invocations": [{"executionSuccessful": false,
+                            "toolExecutionNotifications": []}],
+           "artifacts": [{"location": {"uri": "diagnostic-test-nesting-sarif.c"},
+                          "sourceLanguage": "c",
+                          "contents": {"text": "\n\nextern void foo (void);\n\nvoid test_nesting (void)\n{\n  foo ();\n}\n"},
+                          "roles": ["analysisTarget"]}],
+           "results": [{"ruleId": "error",
+                        "level": "error",
+                        "message": {"text": "top-level error"},
+                        "locations": [{"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                            "region": {"startLine": 8,
+                                                                       "startColumn": 3,
+                                                                       "endColumn": 9},
+                                                            "contextRegion": {"startLine": 8,
+                                                                              "snippet": {"text": "  foo ();\n"}}},
+                                       "logicalLocations": [{"index": 0,
+                                                             "fullyQualifiedName": "test_nesting"}]}],
+                        "relatedLocations": [{"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "child 0"},
+                                              "properties": {"nestingLevel": 1}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 0 0"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 0 1"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 0 2"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "child 1"},
+                                              "properties": {"nestingLevel": 1}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 1 0"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 1 1"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 1 2"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "child 2"},
+                                              "properties": {"nestingLevel": 1}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 2 0"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 2 1"},
+                                              "properties": {"nestingLevel": 2}},
+                                             {"physicalLocation": {"artifactLocation": {"uri": "diagnostic-test-nesting-sarif.c"},
+                                                                   "region": {"startLine": 8,
+                                                                              "startColumn": 3,
+                                                                              "endColumn": 9},
+                                                                   "contextRegion": {"startLine": 8,
+                                                                                     "snippet": {"text": "  foo ();\n"}}},
+                                              "message": {"text": "grandchild 2 2"},
+                                              "properties": {"nestingLevel": 2}}]}],
+           "logicalLocations": [{"name": "test_nesting",
+                                 "fullyQualifiedName": "test_nesting",
+                                 "decoratedName": "test_nesting",
+                                 "kind": "function",
+                                 "index": 0}]}]}
+
+/* For now, we don't have a way of enabling showing the nesting
+   on the default text output.  However we do test the nesting
+   in the SARIF and HTML outputs below.  */
+/* { dg-begin-multiline-output "" }
+In function 'test_nesting':
+diagnostic-test-nesting-sarif.c:8:3: error: top-level error
+    8 | }
+      |   ^     
+diagnostic-test-nesting-sarif.c:8:3: note: child 0
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 0 0
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 0 1
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 0 2
+diagnostic-test-nesting-sarif.c:8:3: note: child 1
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 1 0
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 1 1
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 1 2
+diagnostic-test-nesting-sarif.c:8:3: note: child 2
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 2 0
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 2 1
+diagnostic-test-nesting-sarif.c:8:3: note: grandchild 2 2
+   { dg-end-multiline-output "" } */
+
+/* Use a Python script to verify various properties about the *generated*
+   .sarif file:
+   { dg-final { run-sarif-pytest nested-diagnostics-1.roundtrip "../gcc.dg/plugin/diagnostic-test-nesting-sarif.py" } } */
+
+/* Use a Python script to verify various properties about the generated
+   .html file:
+   { dg-final { run-html-pytest nested-diagnostics-1.sarif "../gcc.dg/plugin/diagnostic-test-nesting-html.py" } } */
diff --git a/gcc/text-art/style.cc b/gcc/text-art/style.cc
index ffc75b6..40886d6 100644
--- a/gcc/text-art/style.cc
+++ b/gcc/text-art/style.cc
@@ -29,7 +29,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "text-art/selftests.h"
 #include "text-art/types.h"
 #include "color-macros.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 
 using namespace text_art;
 
diff --git a/gcc/toplev.cc b/gcc/toplev.cc
index 00a8ccb..d349d83 100644
--- a/gcc/toplev.cc
+++ b/gcc/toplev.cc
@@ -81,7 +81,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "ipa-utils.h"
 #include "gcse.h"
 #include "omp-offload.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
+#include "diagnostics/file-cache.h"
 #include "tree-pass.h"
 #include "dumpfile.h"
 #include "ipa-fnsummary.h"
@@ -230,7 +231,8 @@ announce_function (tree decl)
 		 identifier_to_locale (lang_hooks.decl_printable_name (decl, 2)));
       fflush (stderr);
       pp_needs_newline (global_dc->get_reference_printer ()) = true;
-      diagnostic_set_last_function (global_dc, (diagnostic_info *) NULL);
+      diagnostic_set_last_function (global_dc,
+				    (diagnostics::diagnostic_info *) nullptr);
     }
 }
 
@@ -1017,7 +1019,7 @@ open_auxiliary_file (const char *ext)
 /* Alternative diagnostics callback for reentered ICE reporting.  */
 
 static void
-internal_error_reentered (diagnostic_context *, const char *, va_list *)
+internal_error_reentered (diagnostics::context *, const char *, va_list *)
 {
   /* Flush the dump file if emergency_dump_function itself caused an ICE.  */
   if (dump_file)
@@ -1027,7 +1029,7 @@ internal_error_reentered (diagnostic_context *, const char *, va_list *)
 /* Auxiliary callback for the diagnostics code.  */
 
 static void
-internal_error_function (diagnostic_context *, const char *, va_list *)
+internal_error_function (diagnostics::context *, const char *, va_list *)
 {
   global_dc->set_internal_error_callback (internal_error_reentered);
   warn_if_plugins ();
@@ -1068,13 +1070,14 @@ general_init (const char *argv0, bool init_signals, unique_argv original_argv)
 
   global_dc->set_original_argv (std::move (original_argv));
 
-  global_dc->m_source_printing.enabled
+  auto &source_printing_opts = global_dc->get_source_printing_options ();
+  source_printing_opts.enabled
     = global_options_init.x_flag_diagnostics_show_caret;
-  global_dc->m_source_printing.show_event_links_p
+  source_printing_opts.show_event_links_p
     = global_options_init.x_flag_diagnostics_show_event_links;
-  global_dc->m_source_printing.show_labels_p
+  source_printing_opts.show_labels_p
     = global_options_init.x_flag_diagnostics_show_labels;
-  global_dc->m_source_printing.show_line_numbers_p
+  source_printing_opts.show_line_numbers_p
     = global_options_init.x_flag_diagnostics_show_line_numbers;
   global_dc->set_show_cwe (global_options_init.x_flag_diagnostics_show_cwe);
   global_dc->set_show_rules (global_options_init.x_flag_diagnostics_show_rules);
@@ -1085,7 +1088,7 @@ general_init (const char *argv0, bool init_signals, unique_argv original_argv)
     (global_options_init.x_flag_diagnostics_show_path_depths);
   global_dc->set_show_option_requested
     (global_options_init.x_flag_diagnostics_show_option);
-  global_dc->m_source_printing.min_margin_width
+  source_printing_opts.min_margin_width
     = global_options_init.x_diagnostics_minimum_margin_width;
   global_dc->m_show_column
     = global_options_init.x_flag_show_column;
@@ -1093,10 +1096,10 @@ general_init (const char *argv0, bool init_signals, unique_argv original_argv)
     (global_options_init.x_flag_diagnostics_show_highlight_colors);
   global_dc->set_internal_error_callback (internal_error_function);
   const unsigned lang_mask = lang_hooks.option_lang_mask ();
-  global_dc->set_option_manager
-    (std::make_unique<compiler_diagnostic_option_manager> (*global_dc,
-							   lang_mask,
-							   &global_options),
+  global_dc->set_option_id_manager
+    (std::make_unique<compiler_diagnostic_option_id_manager> (*global_dc,
+							      lang_mask,
+							      &global_options),
      lang_mask);
   global_dc->push_owned_urlifier (make_gcc_urlifier (lang_mask));
 
@@ -1285,7 +1288,7 @@ process_options ()
   input_location = saved_location;
 
   if (flag_diagnostics_generate_patch)
-    global_dc->create_edit_context ();
+    global_dc->initialize_fixits_change_set ();
 
   /* Avoid any informative notes in the second run of -fcompare-debug.  */
   if (flag_compare_debug)
@@ -1748,11 +1751,13 @@ process_options ()
       if (warn_coverage_mismatch
 	  && option_unspecified_p (OPT_Wcoverage_mismatch))
 	diagnostic_classify_diagnostic (global_dc, OPT_Wcoverage_mismatch,
-					DK_ERROR, UNKNOWN_LOCATION);
+					diagnostics::kind::error,
+					UNKNOWN_LOCATION);
       if (warn_coverage_invalid_linenum
 	  && option_unspecified_p (OPT_Wcoverage_invalid_line_number))
 	diagnostic_classify_diagnostic (global_dc, OPT_Wcoverage_invalid_line_number,
-					DK_ERROR, UNKNOWN_LOCATION);
+					diagnostics::kind::error,
+					UNKNOWN_LOCATION);
     }
 
   /* Save the current optimization options.  */
@@ -1815,6 +1820,10 @@ backend_init_target (void)
 static void
 backend_init (void)
 {
+#if CHECKING_P
+  verify_reg_names_in_constraints ();
+#endif
+
   init_emit_once ();
 
   init_rtlanal ();
@@ -2385,11 +2394,11 @@ toplev::main (int argc, char **argv)
      emit some diagnostics here.  */
   invoke_plugin_callbacks (PLUGIN_FINISH, NULL);
 
-  if (auto edit_context_ptr = global_dc->get_edit_context ())
+  if (auto change_set_ptr = global_dc->get_fixits_change_set ())
     {
       pretty_printer pp;
       pp_show_color (&pp) = pp_show_color (global_dc->get_reference_printer ());
-      edit_context_ptr->print_diff (&pp, true);
+      change_set_ptr->print_diff (&pp, true);
       pp_flush (&pp);
     }
 
diff --git a/gcc/tree-call-cdce.cc b/gcc/tree-call-cdce.cc
index 649c1e2..3edea75 100644
--- a/gcc/tree-call-cdce.cc
+++ b/gcc/tree-call-cdce.cc
@@ -36,6 +36,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "builtins.h"
 #include "internal-fn.h"
 #include "tree-dfa.h"
+#include "tree-eh.h"
 
 
 /* This pass serves two closely-related purposes:
@@ -1222,8 +1223,20 @@ use_internal_fn (gcall *call)
     {
       /* Skip the call if LHS == LHS.  If we reach here, EDOM is the only
 	 valid errno value and it is used iff the result is NaN.  */
-      conds.quick_push (gimple_build_cond (EQ_EXPR, lhs, lhs,
-					   NULL_TREE, NULL_TREE));
+      /* In the case of non call exceptions, with signaling NaNs, EQ_EXPR
+	 can throw an exception and that can't be part of the GIMPLE_COND. */
+      if (flag_exceptions
+	  && cfun->can_throw_non_call_exceptions
+	  && operation_could_trap_p (EQ_EXPR, true, false, NULL_TREE))
+	{
+	  tree b = make_ssa_name (boolean_type_node);
+	  conds.quick_push (gimple_build_assign (b, EQ_EXPR, lhs, lhs));
+	  conds.quick_push (gimple_build_cond (NE_EXPR, b, boolean_false_node,
+					       NULL_TREE, NULL_TREE));
+	}
+      else
+	conds.quick_push (gimple_build_cond (EQ_EXPR, lhs, lhs,
+					     NULL_TREE, NULL_TREE));
       nconds++;
 
       /* Try replacing the original call with a direct assignment to
diff --git a/gcc/tree-cfg.cc b/gcc/tree-cfg.cc
index 72763fd..275fc43 100644
--- a/gcc/tree-cfg.cc
+++ b/gcc/tree-cfg.cc
@@ -114,43 +114,6 @@ struct replace_decls_d
   tree to_context;
 };
 
-/* Hash table to store last discriminator assigned for each locus.  */
-struct locus_discrim_map
-{
-  int location_line;
-  int discriminator;
-};
-
-/* Hashtable helpers.  */
-
-struct locus_discrim_hasher : free_ptr_hash <locus_discrim_map>
-{
-  static inline hashval_t hash (const locus_discrim_map *);
-  static inline bool equal (const locus_discrim_map *,
-			    const locus_discrim_map *);
-};
-
-/* Trivial hash function for a location_t.  ITEM is a pointer to
-   a hash table entry that maps a location_t to a discriminator.  */
-
-inline hashval_t
-locus_discrim_hasher::hash (const locus_discrim_map *item)
-{
-  return item->location_line;
-}
-
-/* Equality function for the locus-to-discriminator map.  A and B
-   point to the two hash table entries to compare.  */
-
-inline bool
-locus_discrim_hasher::equal (const locus_discrim_map *a,
-			     const locus_discrim_map *b)
-{
-  return a->location_line == b->location_line;
-}
-
-static hash_table<locus_discrim_hasher> *discriminator_per_locus;
-
 /* Basic blocks and flowgraphs.  */
 static void make_blocks (gimple_seq);
 
@@ -168,7 +131,6 @@ static edge gimple_try_redirect_by_replacing_jump (edge, basic_block);
 static inline bool stmt_starts_bb_p (gimple *, gimple *);
 static bool gimple_verify_flow_info (void);
 static void gimple_make_forwarder_block (edge);
-static gimple *first_non_label_nondebug_stmt (basic_block);
 static bool verify_gimple_transaction (gtransaction *);
 static bool call_can_make_abnormal_goto (gimple *);
 
@@ -247,12 +209,9 @@ build_gimple_cfg (gimple_seq seq)
   group_case_labels ();
 
   /* Create the edges of the flowgraph.  */
-  discriminator_per_locus = new hash_table<locus_discrim_hasher> (13);
   make_edges ();
   assign_discriminators ();
   cleanup_dead_labels ();
-  delete discriminator_per_locus;
-  discriminator_per_locus = NULL;
 }
 
 /* Look for ANNOTATE calls with loop annotation kind in BB; if found, remove
@@ -1120,77 +1079,41 @@ gimple_find_sub_bbs (gimple_seq seq, gimple_stmt_iterator *gsi)
   return true;
 }
 
-/* Find the next available discriminator value for LOCUS.  The
-   discriminator distinguishes among several basic blocks that
-   share a common locus, allowing for more accurate sample-based
-   profiling.  */
-
-static int
-next_discriminator_for_locus (int line)
+/* Auto-profile needs discriminator to distinguish statements with same line
+   number (file name is ignored) which are in different basic block.  This
+   map keeps track of current discriminator for a given line number.  */
+struct discrim_entry
 {
-  struct locus_discrim_map item;
-  struct locus_discrim_map **slot;
-
-  item.location_line = line;
-  item.discriminator = 0;
-  slot = discriminator_per_locus->find_slot_with_hash (&item, line, INSERT);
-  gcc_assert (slot);
-  if (*slot == HTAB_EMPTY_ENTRY)
-    {
-      *slot = XNEW (struct locus_discrim_map);
-      gcc_assert (*slot);
-      (*slot)->location_line = line;
-      (*slot)->discriminator = 0;
-    }
-  (*slot)->discriminator++;
-  return (*slot)->discriminator;
-}
-
-/* Return TRUE if LOCUS1 and LOCUS2 refer to the same source line.  */
-
-static bool
-same_line_p (location_t locus1, expanded_location *from, location_t locus2)
-{
-  expanded_location to;
-
-  if (locus1 == locus2)
-    return true;
-
-  to = expand_location (locus2);
-
-  if (from->line != to.line)
-    return false;
-  if (from->file == to.file)
-    return true;
-  return (from->file != NULL
-          && to.file != NULL
-          && filename_cmp (from->file, to.file) == 0);
-}
+  /* ID of basic block we saw line number last time.  */
+  unsigned int bb_id;
+  /* Discriminator we used.  */
+  unsigned int discrim;
+};
 
-/* Assign a unique discriminator value to all statements in block bb that
-   have the same line number as locus. */
+/* Return updated LOC with discriminator for use in basic block BB_ID.
+   MAP keeps track of current values.  */
 
-static void
-assign_discriminator (location_t locus, basic_block bb)
+location_t
+assign_discriminator (location_t loc, unsigned int bb_id,
+		      hash_map<int_hash <int64_t, -1, -2>, discrim_entry> &map)
 {
-  gimple_stmt_iterator gsi;
-  int discriminator;
-
-  if (locus == UNKNOWN_LOCATION)
-    return;
-
-  expanded_location locus_e = expand_location (locus);
-
-  discriminator = next_discriminator_for_locus (locus_e.line);
-
-  for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+  bool existed;
+  discrim_entry &e = map.get_or_insert (LOCATION_LINE (loc), &existed);
+  gcc_checking_assert (!has_discriminator (loc));
+  if (!existed)
     {
-      gimple *stmt = gsi_stmt (gsi);
-      location_t stmt_locus = gimple_location (stmt);
-      if (same_line_p (locus, &locus_e, stmt_locus))
-	gimple_set_location (stmt,
-	    location_with_discriminator (stmt_locus, discriminator));
+      e.bb_id = bb_id;
+      e.discrim = 0;
+      return loc;
     }
+  if (e.bb_id != bb_id)
+    {
+      e.bb_id = bb_id;
+      e.discrim++;
+    }
+  if (e.discrim)
+    return location_with_discriminator (loc, e.discrim);
+  return loc;
 }
 
 /* Assign discriminators to statement locations.  */
@@ -1198,92 +1121,54 @@ assign_discriminator (location_t locus, basic_block bb)
 static void
 assign_discriminators (void)
 {
+  hash_map<int_hash <int64_t, -1, -2>, discrim_entry> map (13);
+  unsigned int bb_id = 0;
   basic_block bb;
-
   FOR_EACH_BB_FN (bb, cfun)
     {
-      edge e;
-      edge_iterator ei;
-      gimple_stmt_iterator gsi;
-      location_t curr_locus = UNKNOWN_LOCATION;
-      expanded_location curr_locus_e = {};
-      int curr_discr = 0;
-
+      location_t prev_loc = UNKNOWN_LOCATION, prev_replacement = UNKNOWN_LOCATION;
       /* Traverse the basic block, if two function calls within a basic block
 	are mapped to the same line, assign a new discriminator because a call
 	stmt could be a split point of a basic block.  */
-      for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+      for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
+	   !gsi_end_p (gsi); gsi_next (&gsi))
 	{
 	  gimple *stmt = gsi_stmt (gsi);
-
-	  /* Don't allow debug stmts to affect discriminators, but
-	     allow them to take discriminators when they're on the
-	     same line as the preceding nondebug stmt.  */
-	  if (is_gimple_debug (stmt))
-	    {
-	      if (curr_locus != UNKNOWN_LOCATION
-		  && same_line_p (curr_locus, &curr_locus_e,
-				  gimple_location (stmt)))
-		{
-		  location_t loc = gimple_location (stmt);
-		  location_t dloc = location_with_discriminator (loc,
-								 curr_discr);
-		  gimple_set_location (stmt, dloc);
-		}
-	      continue;
-	    }
-	  if (curr_locus == UNKNOWN_LOCATION)
-	    {
-	      curr_locus = gimple_location (stmt);
-	      curr_locus_e = expand_location (curr_locus);
-	    }
-	  else if (!same_line_p (curr_locus, &curr_locus_e, gimple_location (stmt)))
-	    {
-	      curr_locus = gimple_location (stmt);
-	      curr_locus_e = expand_location (curr_locus);
-	      curr_discr = 0;
-	    }
-	  else if (curr_discr != 0)
+	  location_t loc = gimple_location (stmt);
+	  if (loc == UNKNOWN_LOCATION)
+	    continue;
+	  if (loc == prev_loc)
+	    gimple_set_location (stmt, prev_replacement);
+	  else
 	    {
-	      location_t loc = gimple_location (stmt);
-	      location_t dloc = location_with_discriminator (loc, curr_discr);
-	      gimple_set_location (stmt, dloc);
+	      prev_loc = loc;
+	      prev_replacement = assign_discriminator (loc, bb_id, map);
+	      gimple_set_location (stmt, prev_replacement);
 	    }
-	  /* Allocate a new discriminator for CALL stmt.  */
+	  /* Break basic blocks after each call.  This is requires so each
+	     call site has unque discriminator.
+	     More correctly, we can break after each statement that can possibly
+	     terinate execution of the basic block, but for auto-profile this
+	     precision is probably not useful.  */
 	  if (gimple_code (stmt) == GIMPLE_CALL)
-	    curr_discr = next_discriminator_for_locus (curr_locus_e.line);
-	}
-
-      gimple *last = last_nondebug_stmt (bb);
-      location_t locus = last ? gimple_location (last) : UNKNOWN_LOCATION;
-      if (locus == UNKNOWN_LOCATION)
-	continue;
-
-      expanded_location locus_e = expand_location (locus);
-
-      FOR_EACH_EDGE (e, ei, bb->succs)
-	{
-	  gimple *first = first_non_label_nondebug_stmt (e->dest);
-	  gimple *last = last_nondebug_stmt (e->dest);
-
-	  gimple *stmt_on_same_line = NULL;
-	  if (first && same_line_p (locus, &locus_e,
-				     gimple_location (first)))
-	    stmt_on_same_line = first;
-	  else if (last && same_line_p (locus, &locus_e,
-					gimple_location (last)))
-	    stmt_on_same_line = last;
-
-	  if (stmt_on_same_line)
 	    {
-	      if (has_discriminator (gimple_location (stmt_on_same_line))
-		  && !has_discriminator (locus))
-		assign_discriminator (locus, bb);
-	      else
-		assign_discriminator (locus, e->dest);
+	      prev_loc = UNKNOWN_LOCATION;
+	      bb_id++;
 	    }
 	}
+      /* IF basic block has multiple sucessors, consdier every edge as a separate
+	 block.  */
+      if (!single_succ_p (bb))
+	bb_id++;
+      for (edge e : bb->succs)
+	if (e->goto_locus != UNKNOWN_LOCATION)
+	  {
+	    e->goto_locus = assign_discriminator (e->goto_locus, bb_id, map);
+	    bb_id++;
+	  }
+      bb_id++;
     }
+
 }
 
 /* Create the edges for a GIMPLE_COND starting at block BB.  */
@@ -2948,16 +2833,6 @@ first_stmt (basic_block bb)
   return stmt;
 }
 
-/* Return the first non-label/non-debug statement in basic block BB.  */
-
-static gimple *
-first_non_label_nondebug_stmt (basic_block bb)
-{
-  gimple_stmt_iterator i;
-  i = gsi_start_nondebug_after_labels_bb (bb);
-  return !gsi_end_p (i) ? gsi_stmt (i) : NULL;
-}
-
 /* Return the last statement in basic block BB.  */
 
 gimple *
@@ -4623,6 +4498,14 @@ verify_gimple_assign_single (gassign *stmt)
       return true;
     }
 
+  /* LHS can't be a constant or an address expression. */
+  if (CONSTANT_CLASS_P (lhs)|| TREE_CODE (lhs) == ADDR_EXPR)
+    {
+      error ("invalid LHS (%qs) for assignment: %qs",
+	     get_tree_code_name (TREE_CODE (lhs)), code_name);
+      return true;
+    }
+
   if (gimple_clobber_p (stmt)
       && !(DECL_P (lhs) || TREE_CODE (lhs) == MEM_REF))
     {
@@ -4745,6 +4628,11 @@ verify_gimple_assign_single (gassign *stmt)
 
 	  if (CONSTRUCTOR_NELTS (rhs1) == 0)
 	    return res;
+	  if (!is_gimple_reg (lhs))
+	    {
+	      error ("non-register as LHS with vector constructor");
+	      return true;
+	    }
 	  /* For vector CONSTRUCTORs we require that either it is empty
 	     CONSTRUCTOR, or it is a CONSTRUCTOR of smaller vector elements
 	     (then the element count must be correct to cover the whole
@@ -9961,6 +9849,8 @@ do_warn_unused_result (gimple_seq seq)
 	    break;
 	  if (gimple_call_internal_p (g))
 	    break;
+	  if (warning_suppressed_p (g, OPT_Wunused_result))
+	    break;
 
 	  /* This is a naked call, as opposed to a GIMPLE_CALL with an
 	     LHS.  All calls whose value is ignored should be
diff --git a/gcc/tree-cfgcleanup.cc b/gcc/tree-cfgcleanup.cc
index a34a51e..5aaa18d 100644
--- a/gcc/tree-cfgcleanup.cc
+++ b/gcc/tree-cfgcleanup.cc
@@ -47,6 +47,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-into-ssa.h"
 #include "tree-cfgcleanup.h"
 #include "gimple-pretty-print.h"
+#include "target.h"
 
 
 /* The set of blocks in that at least one of the following changes happened:
@@ -1569,6 +1570,29 @@ execute_cleanup_cfg_post_optimizing (void)
   cleanup_dead_labels ();
   if (group_case_labels ())
     todo |= TODO_cleanup_cfg;
+
+  basic_block bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
+  gimple_stmt_iterator gsi = gsi_start_nondebug_after_labels_bb (bb);
+  /* If the first (and only) bb and the only non debug
+     statement is __builtin_unreachable call, then replace it with a trap
+     so the function is at least one instruction in size.  */
+  if (!gsi_end_p (gsi)
+      && gimple_call_builtin_p (gsi_stmt (gsi), BUILT_IN_UNREACHABLE))
+    {
+      if (targetm.have_trap ())
+	{
+	  gimple_call_set_fndecl (gsi_stmt (gsi), builtin_decl_implicit (BUILT_IN_UNREACHABLE_TRAP));
+	  update_stmt (gsi_stmt (gsi));
+	}
+      /* If the target does not have a trap, convert it into an infinite loop. */
+      else
+	{
+	  gsi_remove (&gsi, true);
+	  make_single_succ_edge (bb, bb, EDGE_FALLTHRU);
+	  fix_loop_structure (NULL);
+	}
+    }
+
   if ((flag_compare_debug_opt || flag_compare_debug)
       && flag_dump_final_insns)
     {
diff --git a/gcc/tree-diagnostic-client-data-hooks.cc b/gcc/tree-diagnostic-client-data-hooks.cc
index 84bd4b5..77eb292 100644
--- a/gcc/tree-diagnostic-client-data-hooks.cc
+++ b/gcc/tree-diagnostic-client-data-hooks.cc
@@ -1,4 +1,4 @@
-/* Implementation of diagnostic_client_data_hooks for the compilers
+/* Implementation of diagnostics::client_data_hooks for the compilers
    (e.g. with knowledge of "tree", lang_hooks, and timevars).
    Copyright (C) 2022-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
@@ -26,18 +26,18 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree.h"
 #include "diagnostic.h"
 #include "tree-logical-location.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/sarif-sink.h"
 #include "langhooks.h"
 #include "plugin.h"
 #include "timevar.h"
 
-/* Concrete class for supplying a diagnostic_context with information
+/* Concrete class for supplying a diagnostics::context with information
    about a specific plugin within the client, when the client is the
    compiler (i.e. a GCC plugin).  */
 
 class compiler_diagnostic_client_plugin_info
-  : public diagnostic_client_plugin_info
+  : public diagnostics::client_plugin_info
 {
 public:
   compiler_diagnostic_client_plugin_info (const plugin_name_args *args)
@@ -67,7 +67,7 @@ private:
 /* Concrete subclass of client_version_info for use by compilers proper,
    (i.e. using lang_hooks, and with knowledge of GCC plugins).  */
 
-class compiler_version_info : public client_version_info
+class compiler_version_info : public diagnostics::client_version_info
 {
 public:
   const char *get_tool_name () const final override
@@ -112,23 +112,26 @@ private:
   }
 };
 
-/* Subclass of diagnostic_client_data_hooks for use by compilers proper
+/* Subclass of diagnostics::client_data_hooks for use by compilers proper
    i.e. with knowledge of "tree", access to langhooks, timevars etc.  */
 
-class compiler_data_hooks : public diagnostic_client_data_hooks
+class compiler_data_hooks : public diagnostics::client_data_hooks
 {
 public:
-  const client_version_info *get_any_version_info () const final override
+  const diagnostics::client_version_info *
+  get_any_version_info () const final override
   {
     return &m_version_info;
   }
 
-  const logical_location_manager *get_logical_location_manager () const final override
+  const diagnostics::logical_locations::manager *
+  get_logical_location_manager () const final override
   {
     return &m_logical_location_manager;
   }
 
-  logical_location_manager::key get_current_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_current_logical_location () const final override
   {
     return m_logical_location_manager.key_from_tree (current_function_decl);
   }
@@ -140,13 +143,13 @@ public:
   }
 
   void
-  add_sarif_invocation_properties (sarif_object &invocation_obj)
+  add_sarif_invocation_properties (diagnostics::sarif_object &invocation_obj)
     const final override
   {
     if (g_timer)
       if (auto timereport_val = g_timer->make_json ())
 	{
-	  sarif_property_bag &bag_obj
+	  auto &bag_obj
 	    = invocation_obj.get_or_create_properties ();
 	  bag_obj.set ("gcc/timeReport", std::move (timereport_val));
 
@@ -167,7 +170,7 @@ private:
 /* Create a compiler_data_hooks (so that the class can be local
    to this file).  */
 
-std::unique_ptr<diagnostic_client_data_hooks>
+std::unique_ptr<diagnostics::client_data_hooks>
 make_compiler_data_hooks ()
 {
   return std::make_unique<compiler_data_hooks> ();
diff --git a/gcc/tree-diagnostic.cc b/gcc/tree-diagnostic.cc
index 57c6715..3761fc0 100644
--- a/gcc/tree-diagnostic.cc
+++ b/gcc/tree-diagnostic.cc
@@ -27,29 +27,29 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-pretty-print.h"
 #include "gimple-pretty-print.h"
 #include "tree-diagnostic.h"
-#include "diagnostic-client-data-hooks.h"
+#include "diagnostics/client-data-hooks.h"
 #include "langhooks.h"
 #include "intl.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 /* Prints out, if necessary, the name of the current function
    that caused an error.  */
 void
-diagnostic_report_current_function (diagnostic_text_output_format &text_output,
-				    const diagnostic_info *diagnostic)
+diagnostic_report_current_function (diagnostics::text_sink &text_output,
+				    const diagnostics::diagnostic_info *diag)
 {
-  location_t loc = diagnostic_location (diagnostic);
+  location_t loc = diagnostic_location (diag);
   text_output.report_current_module (loc);
-  lang_hooks.print_error_function (text_output, LOCATION_FILE (loc), diagnostic);
+  lang_hooks.print_error_function (text_output, LOCATION_FILE (loc), diag);
 }
 
 static void
-default_tree_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-				      const diagnostic_info *diagnostic)
+default_tree_diagnostic_text_starter (diagnostics::text_sink &text_output,
+				      const diagnostics::diagnostic_info *diag)
 {
   pretty_printer *const pp = text_output.get_printer ();
-  diagnostic_report_current_function (text_output, diagnostic);
-  pp_set_prefix (pp, text_output.build_prefix (*diagnostic));
+  diagnostic_report_current_function (text_output, diag);
+  pp_set_prefix (pp, text_output.build_prefix (*diag));
 }
 
 /* Default tree printer.   Handles declarations only.  */
@@ -110,8 +110,8 @@ default_tree_printer (pretty_printer *pp, text_info *text, const char *spec,
    to the DIAGNOSTIC location.  */
 
 static void
-set_inlining_locations (diagnostic_context *,
-			diagnostic_info *diagnostic)
+set_inlining_locations (diagnostics::context *,
+			diagnostics::diagnostic_info *diagnostic)
 {
   location_t loc = diagnostic_location (diagnostic);
   tree block = LOCATION_BLOCK (loc);
@@ -169,16 +169,16 @@ set_inlining_locations (diagnostic_context *,
   /* Set if all locations are in a system header.  */
   diagnostic->m_iinfo.m_allsyslocs = nsyslocs == ilocs.length ();
 
-  if (tree *ao = pp_ti_abstract_origin (&diagnostic->message))
+  if (tree *ao = pp_ti_abstract_origin (&diagnostic->m_message))
     *ao = (tree)diagnostic->m_iinfo.m_ao;
 }
 
 /* Sets CONTEXT to use language independent diagnostics.  */
 void
-tree_diagnostics_defaults (diagnostic_context *context)
+tree_diagnostics_defaults (diagnostics::context *context)
 {
-  diagnostic_text_starter (context) = default_tree_diagnostic_text_starter;
-  diagnostic_text_finalizer (context) = default_diagnostic_text_finalizer;
+  diagnostics::text_starter (context) = default_tree_diagnostic_text_starter;
+  diagnostics::text_finalizer (context) = diagnostics::default_text_finalizer;
   context->set_format_decoder (default_tree_printer);
   context->set_set_locations_callback (set_inlining_locations);
   context->set_client_data_hooks (make_compiler_data_hooks ());
diff --git a/gcc/tree-diagnostic.h b/gcc/tree-diagnostic.h
index b138b8b..9315d35 100644
--- a/gcc/tree-diagnostic.h
+++ b/gcc/tree-diagnostic.h
@@ -48,10 +48,10 @@ along with GCC; see the file COPYING3.  If not see
        ? diagnostic_abstract_origin (DI)		\
        : current_function_decl)
 
-void diagnostic_report_current_function (diagnostic_context *,
-					 const diagnostic_info *);
+void diagnostic_report_current_function (diagnostics::context *,
+					 const diagnostics::diagnostic_info *);
 
-void tree_diagnostics_defaults (diagnostic_context *context);
+void tree_diagnostics_defaults (diagnostics::context *);
 bool default_tree_printer (pretty_printer *, text_info *, const char *,
 			   int, bool, bool, bool, bool *, pp_token_list &);
 
diff --git a/gcc/tree-if-conv.cc b/gcc/tree-if-conv.cc
index 636361e..b7ce072 100644
--- a/gcc/tree-if-conv.cc
+++ b/gcc/tree-if-conv.cc
@@ -494,6 +494,10 @@ fold_or_predicates (location_t loc, tree c1, tree c2)
 static tree
 fold_build_cond_expr (tree type, tree cond, tree rhs, tree lhs)
 {
+  /* Short cut the case where both rhs and lhs are the same. */
+  if (operand_equal_p (rhs, lhs))
+    return rhs;
+
   /* If COND is comparison r != 0 and r has boolean type, convert COND
      to SSA_NAME to accept by vect bool pattern.  */
   if (TREE_CODE (cond) == NE_EXPR)
@@ -2110,6 +2114,194 @@ gen_phi_arg_condition (gphi *phi, ifcvt_arg_entry_t &arg,
   return cond;
 }
 
+/* Find the operand which is different between ARG0_OP and ARG1_OP.
+   Returns the operand num where the difference is.
+   Set NEWARG0 and NEWARG1 from the different argument.
+   Returns -1 if none is found.
+   If ARG0_OP/ARG1_OP is commutative also try swapping the
+   two commutative operands and return the operand number where
+   the difference happens in ARG0_OP. */
+
+static int
+find_different_opnum (const gimple_match_op &arg0_op,
+		      const gimple_match_op &arg1_op,
+		      tree *new_arg0, tree *new_arg1)
+{
+  unsigned opnum = -1;
+  unsigned first;
+  first = first_commutative_argument (arg1_op.code, arg1_op.type);
+  for (unsigned i = 0; i < arg0_op.num_ops; i++)
+    {
+      if (!operand_equal_for_phi_arg_p (arg0_op.ops[i],
+					arg1_op.ops[i]))
+	{
+	  /* Can handle only one non equal operand. */
+	  if (opnum != -1u)
+	    {
+	      /* Though if opnum is right before i and opnum is equal
+		 to the first communtative argument, handle communtative
+		 specially. */
+	      if (i == opnum + 1 && opnum == first)
+		goto commutative;
+	      return -1;
+	    }
+	  opnum = i;
+	}
+  }
+  /* If all operands are equal only do this is there was single
+     operand.  */
+  if (opnum == -1u)
+    {
+      if (arg0_op.num_ops != 1)
+	return -1;
+      opnum = 0;
+    }
+  *new_arg0 = arg0_op.ops[opnum];
+  *new_arg1 = arg1_op.ops[opnum];
+  return opnum;
+
+/* Handle commutative operations. */
+commutative:
+  gcc_assert (first != -1u);
+
+  /* Check the rest of the arguments to make sure they are the same. */
+  for (unsigned i = first + 2; i < arg0_op.num_ops; i++)
+    if (!operand_equal_for_phi_arg_p (arg0_op.ops[i],
+				      arg1_op.ops[i]))
+      return -1;
+
+  /* If the arg0[first+1] and arg1[first] are the same
+     then the one which is different is arg0[first] and arg1[first+1]
+     return first since this is based on arg0.  */
+  if (operand_equal_for_phi_arg_p (arg0_op.ops[first + 1],
+				   arg1_op.ops[first]))
+    {
+       *new_arg0 = arg0_op.ops[first];
+       *new_arg1 = arg1_op.ops[first + 1];
+       return first;
+    }
+  /* If the arg0[first] and arg1[first+1] are the same
+     then the one which is different is arg0[first+1] and arg1[first]
+     return first+1 since this is based on arg0.  */
+  if (operand_equal_for_phi_arg_p (arg0_op.ops[first],
+				   arg1_op.ops[first + 1]))
+    {
+       *new_arg0 = arg0_op.ops[first + 1];
+       *new_arg1 = arg1_op.ops[first];
+       return first + 1;
+    }
+  return -1;
+}
+
+/* Factors out an operation from *ARG0 and *ARG1 and
+   create the new statement at GSI. *RES is the
+   result of that new statement. Update *ARG0 and *ARG1
+   and *RES to the new values if the factoring happened.
+   Loops until all of the factoring is completed.  */
+
+static void
+factor_out_operators (tree *res, gimple_stmt_iterator *gsi,
+		      tree *arg0, tree *arg1, gphi *phi)
+{
+  gimple_match_op arg0_op, arg1_op;
+  bool repeated = false;
+
+again:
+  if (TREE_CODE (*arg0) != SSA_NAME || TREE_CODE (*arg1) != SSA_NAME)
+    return;
+
+  if (operand_equal_p (*arg0, *arg1))
+    return;
+
+  /* If either args have > 1 use, then this transformation actually
+     increases the number of expressions evaluated at runtime.  */
+  if (repeated
+      ? (!has_zero_uses (*arg0) || !has_zero_uses (*arg1))
+      : (!has_single_use (*arg0) || !has_single_use (*arg1)))
+    return;
+
+  gimple *arg0_def_stmt = SSA_NAME_DEF_STMT (*arg0);
+  if (!gimple_extract_op (arg0_def_stmt, &arg0_op))
+    return;
+
+  /* At this point there should be no ssa names occuring in abnormals.  */
+  gcc_assert (!arg0_op.operands_occurs_in_abnormal_phi ());
+
+  gimple *arg1_def_stmt = SSA_NAME_DEF_STMT (*arg1);
+  if (!gimple_extract_op (arg1_def_stmt, &arg1_op))
+    return;
+
+  /* At this point there should be no ssa names occuring in abnormals.  */
+  gcc_assert (!arg1_op.operands_occurs_in_abnormal_phi ());
+
+  /* No factoring can happen if the codes are different
+     or the number operands.  */
+  if (arg1_op.code != arg0_op.code
+      || arg1_op.num_ops != arg0_op.num_ops)
+    return;
+
+  tree new_arg0, new_arg1;
+  int opnum = find_different_opnum (arg0_op, arg1_op, &new_arg0, &new_arg1);
+  if (opnum == -1)
+    return;
+
+  if (!types_compatible_p (TREE_TYPE (new_arg0), TREE_TYPE (new_arg1)))
+    return;
+  tree new_res = make_ssa_name (TREE_TYPE (new_arg0), NULL);
+
+  /* Create the operation stmt if possible and insert it.  */
+
+  gimple_match_op new_op = arg0_op;
+  new_op.ops[opnum] = new_res;
+  gimple_seq seq = NULL;
+  tree result = *res;
+  result = maybe_push_res_to_seq (&new_op, &seq, result);
+
+  /* If we can't create the new statement, release the temp name
+     and return back.  */
+  if (!result)
+    {
+      release_ssa_name (new_res);
+      return;
+    }
+  gsi_insert_seq_before (gsi, seq, GSI_CONTINUE_LINKING);
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "PHI ");
+      print_generic_expr (dump_file, gimple_phi_result (phi));
+      fprintf (dump_file,
+	       " changed to factor operation out from COND_EXPR.\n");
+      fprintf (dump_file, "New stmt with OPERATION that defines ");
+      print_generic_expr (dump_file, result);
+      fprintf (dump_file, ".\n");
+    }
+
+  /* Remove the old operation(s) that has single use.  */
+  gimple_stmt_iterator gsi_for_def;
+
+  gsi_for_def = gsi_for_stmt (arg0_def_stmt);
+  gsi_remove (&gsi_for_def, true);
+  release_defs (arg0_def_stmt);
+  gsi_for_def = gsi_for_stmt (arg1_def_stmt);
+  gsi_remove (&gsi_for_def, true);
+  release_defs (arg1_def_stmt);
+
+  /* Update the arguments and try again.  */
+  *arg0 = new_arg0;
+  *arg1 = new_arg1;
+  *res = new_res;
+
+  /* Update the phi node too. */
+  gimple_phi_set_result (phi, new_res);
+  gimple_phi_arg (phi, 0)->def = new_arg0;
+  gimple_phi_arg (phi, 0)->def = new_arg1;
+  update_stmt (phi);
+
+  repeated = true;
+  goto again;
+}
+
 /* Create the smallest nested conditional possible.  On pre-order we record
    which conditionals are live, and on post-order rewrite the chain by removing
    already active conditions.
@@ -2289,6 +2481,11 @@ predicate_scalar_phi (gphi *phi, gimple_stmt_iterator *gsi, bool loop_versioned)
 	  arg0 = gimple_phi_arg_def (phi, 0);
 	  arg1 = gimple_phi_arg_def (phi, 1);
 	}
+
+      /* Factor out operand if possible. This can only be done easily
+	 for PHI with 2 elements.  */
+      factor_out_operators (&res, gsi, &arg0, &arg1, phi);
+
       if (is_cond_scalar_reduction (phi, &reduc, arg0, arg1,
 				    &op0, &op1, false, &has_nop,
 				    &nop_reduc))
@@ -3004,12 +3201,10 @@ combine_blocks (class loop *loop, bool loop_versioned)
 
   /* Reset flow-sensitive info before predicating stmts or PHIs we
      might fold.  */
-  bool *predicated = XNEWVEC (bool, orig_loop_num_nodes);
   for (i = 0; i < orig_loop_num_nodes; i++)
     {
       bb = ifc_bbs[i];
-      predicated[i] = is_predicated (bb);
-      if (predicated[i])
+      if (is_predicated (bb))
 	{
 	  for (auto gsi = gsi_start_phis (bb);
 	       !gsi_end_p (gsi); gsi_next (&gsi))
@@ -3211,7 +3406,6 @@ combine_blocks (class loop *loop, bool loop_versioned)
 
   free (ifc_bbs);
   ifc_bbs = NULL;
-  free (predicated);
 }
 
 /* Version LOOP before if-converting it; the original loop
diff --git a/gcc/tree-inline.cc b/gcc/tree-inline.cc
index 7e0ac69..08e6421 100644
--- a/gcc/tree-inline.cc
+++ b/gcc/tree-inline.cc
@@ -1460,10 +1460,7 @@ copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
 		   || OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_DEPEND))
 	{
 	  tree t = OMP_CLAUSE_DECL (*tp);
-	  if (t
-	      && TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (t && OMP_ITERATOR_DECL_P (t))
 	    {
 	      *walk_subtrees = 0;
 	      OMP_CLAUSE_DECL (*tp) = copy_node (t);
@@ -2888,11 +2885,9 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
   profile_count::adjust_for_ipa_scaling (&num, &den);
 
   ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
-    ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (count,
-				ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
+    ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
   EXIT_BLOCK_PTR_FOR_FN (cfun)->count =
-    EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (count,
-				ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
+    EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
   if (src_cfun->eh)
     init_eh_for_function ();
 
diff --git a/gcc/tree-logical-location.cc b/gcc/tree-logical-location.cc
index 1d0cc58..19bccd1 100644
--- a/gcc/tree-logical-location.cc
+++ b/gcc/tree-logical-location.cc
@@ -27,6 +27,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "langhooks.h"
 #include "intl.h"
 
+using namespace diagnostics::logical_locations;
+
 static void
 assert_valid_tree (const_tree node)
 {
@@ -35,7 +37,8 @@ assert_valid_tree (const_tree node)
   gcc_assert (TREE_CODE (node) != TRANSLATION_UNIT_DECL);
 }
 
-/* class tree_logical_location_manager : public logical_location_manager.  */
+/* class tree_logical_location_manager
+   : public diagnostics::logical_locations::manager.  */
 
 const char *
 tree_logical_location_manager::get_short_name (key k) const
@@ -81,7 +84,7 @@ tree_logical_location_manager::get_internal_name (key k) const
   return NULL;
 }
 
-enum logical_location_kind
+enum kind
 tree_logical_location_manager::get_kind (key k) const
 {
   tree node = tree_from_key (k);
@@ -90,18 +93,18 @@ tree_logical_location_manager::get_kind (key k) const
   switch (TREE_CODE (node))
     {
     default:
-      return logical_location_kind::unknown;
+      return kind::unknown;
     case FUNCTION_DECL:
-      return logical_location_kind::function;
+      return kind::function;
     case PARM_DECL:
-      return logical_location_kind::parameter;
+      return kind::parameter;
     case VAR_DECL:
-      return logical_location_kind::variable;
+      return kind::variable;
     case NAMESPACE_DECL:
-      return logical_location_kind::namespace_;
+      return kind::namespace_;
 
     case RECORD_TYPE:
-      return logical_location_kind::type;
+      return kind::type;
     }
 }
 
@@ -123,7 +126,7 @@ tree_logical_location_manager::get_name_for_path_output (key k) const
   return label_text ();
 }
 
-logical_location
+key
 tree_logical_location_manager::get_parent (key k) const
 {
   tree node = tree_from_key (k);
@@ -132,16 +135,16 @@ tree_logical_location_manager::get_parent (key k) const
   if (DECL_P (node))
     {
       if (!DECL_CONTEXT (node))
-	return logical_location ();
+	return key ();
       if (TREE_CODE (DECL_CONTEXT (node)) == TRANSLATION_UNIT_DECL)
-	return logical_location ();
+	return key ();
       return key_from_tree (DECL_CONTEXT (node));
     }
   else if (TYPE_P (node))
     {
       if (!TYPE_CONTEXT (node))
-	return logical_location ();
+	return key ();
       return key_from_tree (TYPE_CONTEXT (node));
     }
-  return logical_location ();
+  return key ();
 }
diff --git a/gcc/tree-logical-location.h b/gcc/tree-logical-location.h
index 0d57888..2a7de37 100644
--- a/gcc/tree-logical-location.h
+++ b/gcc/tree-logical-location.h
@@ -21,29 +21,33 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_TREE_LOGICAL_LOCATION_H
 #define GCC_TREE_LOGICAL_LOCATION_H
 
-#include "logical-location.h"
+#include "diagnostics/logical-locations.h"
 
-/* A subclass of logical_location_manager in which the keys are
-   "tree".
+/* A subclass of diagnostics::logical_locations::manager in which the
+   keys are "tree".
    Note that there is no integration with the garbage collector,
-   and so logical_location instances can only be short-lived.  */
-class tree_logical_location_manager : public logical_location_manager
+   and so key instances can only be short-lived.  */
+class tree_logical_location_manager
+  : public diagnostics::logical_locations::manager
 {
 public:
+  using key = diagnostics::logical_locations::key;
+  using kind = diagnostics::logical_locations::kind;
+
   const char *get_short_name (key) const final override;
   const char *get_name_with_scope (key) const final override;
   const char *get_internal_name (key) const final override;
-  enum logical_location_kind get_kind (key) const final override;
+  kind get_kind (key) const final override;
   label_text get_name_for_path_output (key) const final override;
   key get_parent (key) const final override;
 
-  static tree tree_from_key (logical_location k)
+  static tree tree_from_key (key k)
   {
     return const_cast<tree> (k.cast_to<const_tree> ());
   }
-  static logical_location key_from_tree (tree node)
+  static key key_from_tree (tree node)
   {
-    return logical_location::from_ptr (node);
+    return key::from_ptr (node);
   }
 };
 
diff --git a/gcc/tree-object-size.cc b/gcc/tree-object-size.cc
index a551f2b..8545eff 100644
--- a/gcc/tree-object-size.cc
+++ b/gcc/tree-object-size.cc
@@ -851,21 +851,17 @@ addr_object_size (struct object_size_info *osi, const_tree ptr,
 
 /* Compute __builtin_object_size for a CALL to .ACCESS_WITH_SIZE,
    OBJECT_SIZE_TYPE is the second argument from __builtin_object_size.
-   The 2nd, 3rd, and the 4th parameters of the call determine the size of
+
+   The 2nd, 3rd, and 4th parameters of the call determine the size of
    the CALL:
 
    2nd argument REF_TO_SIZE: The reference to the size of the object,
-   3rd argument CLASS_OF_SIZE: The size referenced by the REF_TO_SIZE represents
-     0: the number of bytes;
-     1: the number of the elements of the object type;
-   4th argument TYPE_OF_SIZE: A constant 0 with its TYPE being the same as the TYPE
-    of the object referenced by REF_TO_SIZE
-   6th argument: A constant 0 with the pointer TYPE to the original flexible
-     array type or pointer field type.
-
-   The size of the element can be retrived from the TYPE of the 6th argument
-   of the call, which is the pointer to the original flexible array type or
-   the type of the original pointer field.  */
+   3rd argument TYPE_OF_SIZE + ACCESS_MODE: An integer constant with a pointer
+     TYPE.
+     The pointee TYPE of this pointer TYPE is the TYPE of the object referenced
+	by REF_TO_SIZE.
+
+   4th argument:  The TYPE_SIZE_UNIT of the element TYPE of the array.  */
 
 static tree
 access_with_size_object_size (const gcall *call, int object_size_type)
@@ -873,17 +869,14 @@ access_with_size_object_size (const gcall *call, int object_size_type)
   /* If not for dynamic object size, return.  */
   if ((object_size_type & OST_DYNAMIC) == 0)
     return size_unknown (object_size_type);
-
   gcc_assert (gimple_call_internal_p (call, IFN_ACCESS_WITH_SIZE));
-  /* The type of the 6th argument type is the pointer TYPE to the original
-     flexible array type or to the original pointer type.  */
-  tree pointer_to_array_type = TREE_TYPE (gimple_call_arg (call, 5));
-  gcc_assert (POINTER_TYPE_P (pointer_to_array_type));
-  tree element_type = TREE_TYPE (TREE_TYPE (pointer_to_array_type));
-  tree element_size = TYPE_SIZE_UNIT (element_type);
+
   tree ref_to_size = gimple_call_arg (call, 1);
-  unsigned int class_of_size = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
-  tree type = TREE_TYPE (gimple_call_arg (call, 3));
+  tree type = TREE_TYPE (TREE_TYPE (gimple_call_arg (call, 2)));
+
+  /* The 4th argument is the TYPE_SIZE_UNIT for the element of the original
+     flexible array.  */
+  tree element_size = gimple_call_arg (call, 3);
 
   tree size = fold_build2 (MEM_REF, type, ref_to_size,
 			   build_int_cst (ptr_type_node, 0));
@@ -897,12 +890,9 @@ access_with_size_object_size (const gcall *call, int object_size_type)
 			build_zero_cst (type), size);
   }
 
-  if (class_of_size == 1)
-    size = size_binop (MULT_EXPR,
-		       fold_convert (sizetype, size),
-		       fold_convert (sizetype, element_size));
-  else
-    size = fold_convert (sizetype, size);
+  size = size_binop (MULT_EXPR,
+		     fold_convert (sizetype, size),
+		     fold_convert (sizetype, element_size));
 
   if (!todo)
     todo = TODO_update_ssa_only_virtuals;
@@ -1944,17 +1934,6 @@ collect_object_sizes_for (struct object_size_info *osi, tree var)
             if (TREE_CODE (rhs) == SSA_NAME
                 && POINTER_TYPE_P (TREE_TYPE (rhs)))
 	      reexamine = merge_object_sizes (osi, var, rhs);
-	    /* Handle the following stmt #2 to propagate the size from the
-	       stmt #1 to #3:
-		1  _1 = .ACCESS_WITH_SIZE (_3, _4, 1, 0, -1, 0B);
-		2  _5 = *_1;
-		3  _6 = __builtin_dynamic_object_size (_5, 1);
-	     */
-	    else if (TREE_CODE (rhs) == MEM_REF
-		     && POINTER_TYPE_P (TREE_TYPE (rhs))
-		     && TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME
-		     && integer_zerop (TREE_OPERAND (rhs, 1)))
-	      reexamine = merge_object_sizes (osi, var, TREE_OPERAND (rhs, 0));
             else
               expr_object_size (osi, var, rhs);
           }
diff --git a/gcc/tree-pretty-print.cc b/gcc/tree-pretty-print.cc
index fadafd6..50d0851 100644
--- a/gcc/tree-pretty-print.cc
+++ b/gcc/tree-pretty-print.cc
@@ -902,9 +902,7 @@ dump_omp_clause (pretty_printer *pp, tree clause, int spc, dump_flags_t flags)
       pp_string (pp, "affinity(");
       {
 	tree t = OMP_CLAUSE_DECL (clause);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    dump_omp_iterators (pp, TREE_PURPOSE (t), spc, flags);
 	    pp_colon (pp);
@@ -944,9 +942,7 @@ dump_omp_clause (pretty_printer *pp, tree clause, int spc, dump_flags_t flags)
 	}
       {
 	tree t = OMP_CLAUSE_DECL (clause);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    dump_omp_iterators (pp, TREE_PURPOSE (t), spc, flags);
 	    pp_colon (pp);
diff --git a/gcc/tree-scalar-evolution.cc b/gcc/tree-scalar-evolution.cc
index 43311e5..3b3748a 100644
--- a/gcc/tree-scalar-evolution.cc
+++ b/gcc/tree-scalar-evolution.cc
@@ -670,6 +670,17 @@ scev_dfs::add_to_evolution_1 (tree chrec_before, tree to_add, gimple *at_stmt)
 	  to_add = chrec_convert (type, to_add, at_stmt);
 	  right = chrec_convert_rhs (type, right, at_stmt);
 	  right = chrec_fold_plus (chrec_type (right), right, to_add);
+	  /* When we have an evolution in a non-wrapping type and
+	     in the process of accumulating CHREC_RIGHT there was
+	     overflow this indicates in the association that happened
+	     in building the CHREC clearly involved UB.  Avoid this.
+	     In building a CHREC we basically turn (a + INCR1) + INCR2
+	     into a + (INCR1 + INCR2) which is not always valid.
+	     Note this check only catches few invalid cases.  */
+	  if ((INTEGRAL_TYPE_P (type) && ! TYPE_OVERFLOW_WRAPS (type))
+	      && TREE_CODE (right) == INTEGER_CST
+	      && TREE_OVERFLOW (right))
+	    return chrec_dont_know;
 	  return build_polynomial_chrec (var, left, right);
 	}
       else
@@ -3088,7 +3099,7 @@ iv_can_overflow_p (class loop *loop, tree type, tree base, tree step)
   type_max = wi::max_value (type);
 
   /* Just sanity check that we don't see values out of the range of the type.
-     In this case the arithmetics bellow would overflow.  */
+     In this case the arithmetics below would overflow.  */
   gcc_checking_assert (wi::ge_p (base_min, type_min, sgn)
 		       && wi::le_p (base_max, type_max, sgn));
 
diff --git a/gcc/tree-sra.cc b/gcc/tree-sra.cc
index 23236fc..032f277 100644
--- a/gcc/tree-sra.cc
+++ b/gcc/tree-sra.cc
@@ -2504,6 +2504,12 @@ sort_and_splice_var_accesses (tree var)
 		}
 	      unscalarizable_region = true;
 	    }
+	  /* If there the same place is accessed with two incompatible
+	     aggregate types, trying to base total scalarization on either of
+	     them can be wrong.  */
+	  if (!first_scalar && !types_compatible_p (access->type, ac2->type))
+	    bitmap_set_bit (cannot_scalarize_away_bitmap,
+			    DECL_UID (access->base));
 
 	  if (grp_same_access_path
 	      && (!ac2->grp_same_access_path
@@ -2889,7 +2895,10 @@ analyze_access_subtree (struct access *root, struct access *parent,
 
   for (child = root->first_child; child; child = child->next_sibling)
     {
-      hole |= covered_to < child->offset;
+      if (totally)
+	covered_to = child->offset;
+      else
+	hole |= covered_to < child->offset;
       sth_created |= analyze_access_subtree (child, root,
 					     allow_replacements && !scalar
 					     && !root->grp_partial_lhs,
@@ -2900,6 +2909,8 @@ analyze_access_subtree (struct access *root, struct access *parent,
 	covered_to += child->size;
       else
 	hole = true;
+      if (totally && !hole)
+	covered_to = limit;
     }
 
   if (allow_replacements && scalar && !root->first_child
@@ -2972,7 +2983,7 @@ analyze_access_subtree (struct access *root, struct access *parent,
 	root->grp_total_scalarization = 0;
     }
 
-  if (!hole || totally)
+  if (!hole)
     root->grp_covered = 1;
   else if (root->grp_write || comes_initialized_p (root->base))
     root->grp_unscalarized_data = 1; /* not covered and written to */
diff --git a/gcc/tree-ssa-alias.cc b/gcc/tree-ssa-alias.cc
index 9dd1780..9b028e0 100644
--- a/gcc/tree-ssa-alias.cc
+++ b/gcc/tree-ssa-alias.cc
@@ -901,7 +901,9 @@ ao_ref_init_from_ptr_and_range (ao_ref *ref, tree ptr,
   if (TREE_CODE (ptr) == ADDR_EXPR)
     {
       ref->base = get_addr_base_and_unit_offset (TREE_OPERAND (ptr, 0), &t);
-      if (ref->base)
+      if (ref->base
+	  && coeffs_in_range_p (t, -HOST_WIDE_INT_MAX / BITS_PER_UNIT,
+				HOST_WIDE_INT_MAX / BITS_PER_UNIT))
 	ref->offset = BITS_PER_UNIT * t;
       else
 	{
@@ -4376,7 +4378,7 @@ ao_compare::compare_ao_refs (ao_ref *ref1, ao_ref *ref2,
       i++;
     }
 
-  /* For variable accesses we can not rely on offset match bellow.
+  /* For variable accesses we can not rely on offset match below.
      We know that paths are struturally same, so only check that
      starts of TBAA paths did not diverge.  */
   if (!known_eq (ref1->size, ref1->max_size)
diff --git a/gcc/tree-ssa-dse.cc b/gcc/tree-ssa-dse.cc
index 5ac4280..51a5723 100644
--- a/gcc/tree-ssa-dse.cc
+++ b/gcc/tree-ssa-dse.cc
@@ -181,10 +181,10 @@ initialize_ao_ref_for_dse (gimple *stmt, ao_ref *write, bool may_def_ok = false)
 	       can provide a may-def variant.  */
 	    if (may_def_ok)
 	      {
-		ao_ref_init_from_ptr_and_size (
-		  write, gimple_call_arg (stmt, 0),
-		  TYPE_SIZE_UNIT (
-		    TREE_TYPE (gimple_call_arg (stmt, stored_value_index))));
+		ao_ref_init_from_ptr_and_range (
+		  write, gimple_call_arg (stmt, 0), true, 0, -1,
+		  tree_to_poly_int64 (TYPE_SIZE (
+		    TREE_TYPE (gimple_call_arg (stmt, stored_value_index)))));
 		return true;
 	      }
 	    break;
diff --git a/gcc/tree-ssa-live.cc b/gcc/tree-ssa-live.cc
index 5b8bfd0..5e08913 100644
--- a/gcc/tree-ssa-live.cc
+++ b/gcc/tree-ssa-live.cc
@@ -702,7 +702,10 @@ dump_scope_block (FILE *file, int indent, tree scope, dump_flags_t flags)
   if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (scope)) != UNKNOWN_LOCATION)
     {
       expanded_location s = expand_location (BLOCK_SOURCE_LOCATION (scope));
-      fprintf (file, " %s:%i", s.file, s.line);
+      fprintf (file, " %s:%i:%i", s.file, s.line, s.column);
+      if (has_discriminator (BLOCK_SOURCE_LOCATION (scope)))
+	fprintf (file, " discrim %i",
+		 get_discriminator_from_loc (BLOCK_SOURCE_LOCATION (scope)));
     }
   if (BLOCK_ABSTRACT_ORIGIN (scope))
     {
diff --git a/gcc/tree-ssa-loop-ivopts.cc b/gcc/tree-ssa-loop-ivopts.cc
index 544a946..2fe2655 100644
--- a/gcc/tree-ssa-loop-ivopts.cc
+++ b/gcc/tree-ssa-loop-ivopts.cc
@@ -132,6 +132,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-vectorizer.h"
 #include "dbgcnt.h"
 #include "cfganal.h"
+#include "gimple-fold.h"
 
 /* For lang_hooks.types.type_for_mode.  */
 #include "langhooks.h"
@@ -147,7 +148,7 @@ along with GCC; see the file COPYING3.  If not see
    The average trip count is computed from profile data if it
    exists. */
 
-static inline HOST_WIDE_INT
+static inline unsigned HOST_WIDE_INT
 avg_loop_niter (class loop *loop)
 {
   HOST_WIDE_INT niter = estimated_stmt_executions_int (loop);
@@ -4213,7 +4214,9 @@ adjust_setup_cost (struct ivopts_data *data, int64_t cost,
     return cost;
   else if (optimize_loop_for_speed_p (data->current_loop))
     {
-      int64_t niters = (int64_t) avg_loop_niter (data->current_loop);
+      uint64_t niters = avg_loop_niter (data->current_loop);
+      if (niters > (uint64_t) cost)
+	return (round_up_p && cost != 0) ? 1 : 0;
       return (cost + (round_up_p ? niters - 1 : 0)) / niters;
     }
   else
@@ -7250,7 +7253,24 @@ create_new_iv (struct ivopts_data *data, struct iv_cand *cand)
 
   base = unshare_expr (cand->iv->base);
 
-  create_iv (base, PLUS_EXPR, unshare_expr (cand->iv->step),
+  /* The step computation could invoke UB when the loop does not iterate.
+     Avoid inserting it on the preheader in its native form but rewrite
+     it to a well-defined form.  This also helps masking SCEV issues
+     which freely re-associates the IV computations when building up
+     CHRECs without much regard for signed overflow invoking UB.  */
+  gimple_seq stmts = NULL;
+  tree step = force_gimple_operand (unshare_expr (cand->iv->step), &stmts,
+				    true, NULL_TREE);
+  if (stmts)
+    {
+      for (auto gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
+	if (gimple_needing_rewrite_undefined (gsi_stmt (gsi)))
+	  rewrite_to_defined_unconditional (&gsi);
+      gsi_insert_seq_on_edge_immediate
+	(loop_preheader_edge (data->current_loop), stmts);
+    }
+
+  create_iv (base, PLUS_EXPR, step,
 	     cand->var_before, data->current_loop,
 	     &incr_pos, after, &cand->var_before, &cand->var_after);
 }
@@ -7277,7 +7297,7 @@ create_new_ivs (struct ivopts_data *data, class iv_ca *set)
       if (data->loop_loc != UNKNOWN_LOCATION)
 	fprintf (dump_file, " at %s:%d", LOCATION_FILE (data->loop_loc),
 		 LOCATION_LINE (data->loop_loc));
-      fprintf (dump_file, ", " HOST_WIDE_INT_PRINT_DEC " avg niters",
+      fprintf (dump_file, ", " HOST_WIDE_INT_PRINT_UNSIGNED " avg niters",
 	       avg_loop_niter (data->current_loop));
       fprintf (dump_file, ", %lu IVs:\n", bitmap_count_bits (set->cands));
       EXECUTE_IF_SET_IN_BITMAP (set->cands, 0, i, bi)
diff --git a/gcc/tree-ssa-loop-niter.cc b/gcc/tree-ssa-loop-niter.cc
index 9ce8813..6e13086 100644
--- a/gcc/tree-ssa-loop-niter.cc
+++ b/gcc/tree-ssa-loop-niter.cc
@@ -4701,7 +4701,7 @@ maybe_lower_iteration_bound (class loop *loop)
 
      TODO: Due to the way record_estimate choose estimates to store, the bounds
      will be always nb_iterations_upper_bound-1.  We can change this to record
-     also statements not dominating the loop latch and update the walk bellow
+     also statements not dominating the loop latch and update the walk below
      to the shortest path algorithm.  */
   for (elt = loop->bounds; elt; elt = elt->next)
     {
diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc
index 4cfcc42..ca98205 100644
--- a/gcc/tree-ssa-math-opts.cc
+++ b/gcc/tree-ssa-math-opts.cc
@@ -4064,6 +4064,7 @@ arith_overflow_check_p (gimple *stmt, gimple *cast_stmt, gimple *&use_stmt,
 extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
 extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
 extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
+extern bool gimple_unsigned_integer_sat_mul (tree, tree*, tree (*)(tree));
 
 extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
 extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
@@ -4217,6 +4218,30 @@ match_unsigned_saturation_sub (gimple_stmt_iterator *gsi, gassign *stmt)
 }
 
 /*
+ * Try to match saturation unsigned mul.
+ *   _1 = (unsigned int) a_6(D);
+ *   _2 = (unsigned int) b_7(D);
+ *   x_8 = _1 * _2;
+ *   overflow_9 = x_8 > 255;
+ *   _3 = (unsigned char) overflow_9;
+ *   _4 = -_3;
+ *   _5 = (unsigned char) x_8;
+ *   _10 = _4 | _5;
+ *   =>
+ *   _10 = .SAT_SUB (a_6, b_7);  */
+
+static void
+match_unsigned_saturation_mul (gimple_stmt_iterator *gsi, gassign *stmt)
+{
+  tree ops[2];
+  tree lhs = gimple_assign_lhs (stmt);
+
+  if (gimple_unsigned_integer_sat_mul (lhs, ops, NULL))
+    build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_MUL, lhs,
+						    ops[0], ops[1]);
+}
+
+/*
  * Try to match saturation unsigned sub.
  *  <bb 2> [local count: 1073741824]:
  *  if (x_2(D) > y_3(D))
@@ -6469,6 +6494,7 @@ math_opts_dom_walker::after_dom_children (basic_block bb)
 	      break;
 
 	    case NOP_EXPR:
+	      match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
 	      match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
 	      match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
 	      break;
diff --git a/gcc/tree-ssa-operands.cc b/gcc/tree-ssa-operands.cc
index 4eb9673..a5970ac 100644
--- a/gcc/tree-ssa-operands.cc
+++ b/gcc/tree-ssa-operands.cc
@@ -721,7 +721,7 @@ operands_scanner::get_asm_stmt_operands (gasm *stmt)
       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
       oconstraints[i] = constraint;
       parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-	                       &allows_reg, &is_inout);
+			       &allows_reg, &is_inout, nullptr);
 
       /* This should have been split in gimplify_asm_expr.  */
       gcc_assert (!allows_reg || !is_inout);
@@ -740,7 +740,7 @@ operands_scanner::get_asm_stmt_operands (gasm *stmt)
       tree link = gimple_asm_input_op (stmt, i);
       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
       parse_input_constraint (&constraint, 0, 0, noutputs, 0, oconstraints,
-	                      &allows_mem, &allows_reg);
+			      &allows_mem, &allows_reg, nullptr);
 
       /* Memory operands are addressable.  Note that STMT needs the
 	 address of this operand.  */
diff --git a/gcc/tree-ssa-pre.cc b/gcc/tree-ssa-pre.cc
index f6c531e..9933173 100644
--- a/gcc/tree-ssa-pre.cc
+++ b/gcc/tree-ssa-pre.cc
@@ -2773,16 +2773,17 @@ find_or_generate_expression (basic_block block, tree op, gimple_seq *stmts)
   bitmap exprset = value_expressions[lookfor];
   bitmap_iterator bi;
   unsigned int i;
-  EXECUTE_IF_SET_IN_BITMAP (exprset, 0, i, bi)
-    {
-      pre_expr temp = expression_for_id (i);
-      /* We cannot insert random REFERENCE expressions at arbitrary
-	 places.  We can insert NARYs which eventually re-materializes
-	 its operand values.  */
-      if (temp->kind == NARY)
-	return create_expression_by_pieces (block, temp, stmts,
-					    TREE_TYPE (op));
-    }
+  if (exprset)
+    EXECUTE_IF_SET_IN_BITMAP (exprset, 0, i, bi)
+      {
+	pre_expr temp = expression_for_id (i);
+	/* We cannot insert random REFERENCE expressions at arbitrary
+	   places.  We can insert NARYs which eventually re-materializes
+	   its operand values.  */
+	if (temp->kind == NARY)
+	  return create_expression_by_pieces (block, temp, stmts,
+					      TREE_TYPE (op));
+      }
 
   /* Defer.  */
   return NULL_TREE;
diff --git a/gcc/tree-ssa-reassoc.cc b/gcc/tree-ssa-reassoc.cc
index 3c38f3d..c140f76 100644
--- a/gcc/tree-ssa-reassoc.cc
+++ b/gcc/tree-ssa-reassoc.cc
@@ -7167,9 +7167,10 @@ reassociate_bb (basic_block bb)
 
 		  /* If the target support FMA, rank_ops_for_fma will detect if
 		     the chain has fmas and rearrange the ops if so.  */
-		  if (direct_internal_fn_supported_p (IFN_FMA,
-						      TREE_TYPE (lhs),
-						      opt_type)
+		  if (!reassoc_insert_powi_p
+		      && direct_internal_fn_supported_p (IFN_FMA,
+							 TREE_TYPE (lhs),
+							 opt_type)
 		      && (rhs_code == PLUS_EXPR || rhs_code == MINUS_EXPR))
 		    {
 		      mult_num = rank_ops_for_fma (&ops);
@@ -7200,7 +7201,8 @@ reassociate_bb (basic_block bb)
 			 to make sure the ones that get the double
 			 binary op are chosen wisely.  */
 		      int len = ops.length ();
-		      if (len >= 3
+		      if (!reassoc_insert_powi_p
+			  && len >= 3
 			  && (!has_fma
 			      /* width > 1 means ranking ops results in better
 				 parallelism.  Check current value to avoid
diff --git a/gcc/tree-ssa-sccvn.cc b/gcc/tree-ssa-sccvn.cc
index 9cdbf3d..00315d1 100644
--- a/gcc/tree-ssa-sccvn.cc
+++ b/gcc/tree-ssa-sccvn.cc
@@ -717,7 +717,7 @@ vn_reference_compute_hash (const vn_reference_t vr1)
   hashval_t result;
   int i;
   vn_reference_op_t vro;
-  poly_int64 off = -1;
+  poly_offset_int off = -1;
   bool deref = false;
 
   FOR_EACH_VEC_ELT (vr1->operands, i, vro)
@@ -736,7 +736,7 @@ vn_reference_compute_hash (const vn_reference_t vr1)
 	{
 	  if (maybe_ne (off, -1)
 	      && maybe_ne (off, 0))
-	    hstate.add_poly_int (off);
+	    hstate.add_poly_hwi (off.force_shwi ());
 	  off = -1;
 	  if (deref
 	      && vro->opcode == ADDR_EXPR)
@@ -850,7 +850,7 @@ vn_reference_eq (const_vn_reference_t const vr1, const_vn_reference_t const vr2)
   j = 0;
   do
     {
-      poly_int64 off1 = 0, off2 = 0;
+      poly_offset_int off1 = 0, off2 = 0;
       vn_reference_op_t vro1, vro2;
       vn_reference_op_s tem1, tem2;
       bool deref1 = false, deref2 = false;
@@ -1219,7 +1219,7 @@ ao_ref_init_from_vn_reference (ao_ref *ref,
 		  offset = 0;
 		}
 	      else
-		offset += pop->off * BITS_PER_UNIT;
+		offset += poly_offset_int (pop->off) * BITS_PER_UNIT;
 	      op0_p = NULL;
 	      break;
 	    }
@@ -1270,7 +1270,7 @@ ao_ref_init_from_vn_reference (ao_ref *ref,
 	  if (maybe_eq (op->off, -1))
 	    max_size = -1;
 	  else
-	    offset += op->off * BITS_PER_UNIT;
+	    offset += poly_offset_int (op->off) * BITS_PER_UNIT;
 	  break;
 
 	case REALPART_EXPR:
@@ -2809,7 +2809,8 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
          we find a VN result with exactly the same value as the
 	 possible clobber.  In this case we can ignore the clobber
 	 and return the found value.  */
-      if (is_gimple_reg_type (TREE_TYPE (lhs))
+      if (!gimple_has_volatile_ops (def_stmt)
+	  && is_gimple_reg_type (TREE_TYPE (lhs))
 	  && types_compatible_p (TREE_TYPE (lhs), vr->type)
 	  && (ref->ref || data->orig_ref.ref)
 	  && !data->mask
@@ -3093,7 +3094,8 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
   else if (is_gimple_reg_type (vr->type)
 	   && gimple_assign_single_p (def_stmt)
 	   && gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR
-	   && CONSTRUCTOR_NELTS (gimple_assign_rhs1 (def_stmt)) == 0)
+	   && CONSTRUCTOR_NELTS (gimple_assign_rhs1 (def_stmt)) == 0
+	   && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt)))
     {
       tree base2;
       poly_int64 offset2, size2, maxsize2;
@@ -3149,6 +3151,7 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
 	   && !reverse_storage_order_for_component_p (vr->operands)
 	   && !contains_storage_order_barrier_p (vr->operands)
 	   && gimple_assign_single_p (def_stmt)
+	   && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
 	   && CHAR_BIT == 8
 	   && BITS_PER_UNIT == 8
 	   && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
@@ -3307,6 +3310,7 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
 	   && !reverse_storage_order_for_component_p (vr->operands)
 	   && !contains_storage_order_barrier_p (vr->operands)
 	   && gimple_assign_single_p (def_stmt)
+	   && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
 	   && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
     {
       tree lhs = gimple_assign_lhs (def_stmt);
@@ -3518,6 +3522,7 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
      the copy kills ref.  */
   else if (data->vn_walk_kind == VN_WALKREWRITE
 	   && gimple_assign_single_p (def_stmt)
+	   && !gimple_has_volatile_ops (def_stmt)
 	   && (DECL_P (gimple_assign_rhs1 (def_stmt))
 	       || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == MEM_REF
 	       || handled_component_p (gimple_assign_rhs1 (def_stmt))))
@@ -3544,8 +3549,95 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
 
       /* Find the common base of ref and the lhs.  lhs_ops already
          contains valueized operands for the lhs.  */
+      poly_int64 extra_off = 0;
       i = vr->operands.length () - 1;
       j = lhs_ops.length () - 1;
+
+      /* The base should be always equal due to the above check.  */
+      if (! vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
+	return (void *)-1;
+      i--, j--;
+
+      /* The 2nd component should always exist and be a MEM_REF.  */
+      if (!(i >= 0 && j >= 0))
+	;
+      else if (vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
+	i--, j--;
+      else if (vr->operands[i].opcode == MEM_REF
+	       && lhs_ops[j].opcode == MEM_REF
+	       && known_ne (lhs_ops[j].off, -1)
+	       && known_ne (vr->operands[i].off, -1))
+	{
+	  bool found = false;
+	  /* When we ge a mismatch at a MEM_REF that is not the sole component
+	     try finding a match in one of the outer components and continue
+	     stripping there.  This happens when addresses of components get
+	     forwarded into dereferences.  */
+	  if (j > 0)
+	    {
+	      int temi = i - 1;
+	      extra_off = vr->operands[i].off;
+	      while (temi >= 0
+		     && known_ne (vr->operands[temi].off, -1))
+		{
+		  if (vr->operands[temi].type
+		      && lhs_ops[j].type
+		      && (TYPE_MAIN_VARIANT (vr->operands[temi].type)
+			  == TYPE_MAIN_VARIANT (lhs_ops[j].type)))
+		    {
+		      i = temi;
+		      /* Strip the component that was type matched to
+			 the MEM_REF.  */
+		      extra_off += vr->operands[i].off - lhs_ops[j].off;
+		      i--, j--;
+		      /* Strip further equal components.  */
+		      found = true;
+		      break;
+		    }
+		  extra_off += vr->operands[temi].off;
+		  temi--;
+		}
+	    }
+	  if (!found && i > 0)
+	    {
+	      int temj = j - 1;
+	      extra_off = -lhs_ops[j].off;
+	      while (temj >= 0
+		     && known_ne (lhs_ops[temj].off, -1))
+		{
+		  if (vr->operands[i].type
+		      && lhs_ops[temj].type
+		      && (TYPE_MAIN_VARIANT (vr->operands[i].type)
+			  == TYPE_MAIN_VARIANT (lhs_ops[temj].type)))
+		    {
+		      j = temj;
+		      /* Strip the component that was type matched to
+			 the MEM_REF.  */
+		      extra_off += vr->operands[i].off - lhs_ops[j].off;
+		      i--, j--;
+		      /* Strip further equal components.  */
+		      found = true;
+		      break;
+		    }
+		  extra_off += -lhs_ops[temj].off;
+		  temj--;
+		}
+	    }
+	  /* When the LHS is already at the outermost level simply
+	     adjust for any offset difference.  Further lookups
+	     will fail when there's too gross of a type compatibility
+	     issue.  */
+	  if (!found && j == 0)
+	    {
+	      extra_off = vr->operands[i].off - lhs_ops[j].off;
+	      i--, j--;
+	    }
+	}
+      else
+	return (void *)-1;
+
+      /* Strip further common components, attempting to consume lhs_ops
+	 in full.  */
       while (j >= 0 && i >= 0
 	     && vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
 	{
@@ -3553,27 +3645,6 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
 	  j--;
 	}
 
-      /* ???  The innermost op should always be a MEM_REF and we already
-         checked that the assignment to the lhs kills vr.  Thus for
-	 aggregate copies using char[] types the vn_reference_op_eq
-	 may fail when comparing types for compatibility.  But we really
-	 don't care here - further lookups with the rewritten operands
-	 will simply fail if we messed up types too badly.  */
-      poly_int64 extra_off = 0;
-      if (j == 0 && i >= 0
-	  && lhs_ops[0].opcode == MEM_REF
-	  && maybe_ne (lhs_ops[0].off, -1))
-	{
-	  if (known_eq (lhs_ops[0].off, vr->operands[i].off))
-	    i--, j--;
-	  else if (vr->operands[i].opcode == MEM_REF
-		   && maybe_ne (vr->operands[i].off, -1))
-	    {
-	      extra_off = vr->operands[i].off - lhs_ops[0].off;
-	      i--, j--;
-	    }
-	}
-
       /* i now points to the first additional op.
 	 ???  LHS may not be completely contained in VR, one or more
 	 VIEW_CONVERT_EXPRs could be in its way.  We could at least
diff --git a/gcc/tree-ssa-sink.cc b/gcc/tree-ssa-sink.cc
index 959e0d5..2244e89f 100644
--- a/gcc/tree-ssa-sink.cc
+++ b/gcc/tree-ssa-sink.cc
@@ -356,37 +356,54 @@ statement_sink_location (gimple *stmt, basic_block frombb,
 
   use = NULL;
 
-  /* If stmt is a store the one and only use needs to be the VOP
-     merging PHI node.  */
+  /* If stmt is a store the one and only use needs to be a VUSE on
+     the live path.  */
   if (virtual_operand_p (DEF_FROM_PTR (def_p)))
     {
+      tree lhs = gimple_get_lhs (stmt);
+      ao_ref ref;
+      ao_ref_init (&ref, lhs);
       FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
 	{
 	  gimple *use_stmt = USE_STMT (use_p);
 
 	  /* A killing definition is not a use.  */
-	  if ((gimple_has_lhs (use_stmt)
-	       && operand_equal_p (gimple_get_lhs (stmt),
-				   gimple_get_lhs (use_stmt), 0))
-	      || stmt_kills_ref_p (use_stmt, gimple_get_lhs (stmt)))
+	  if (gimple_vdef (use_stmt)
+	      && ((gimple_has_lhs (use_stmt)
+		   && operand_equal_p (lhs,
+				       gimple_get_lhs (use_stmt), 0))
+		  || stmt_kills_ref_p (use_stmt, &ref)))
 	    {
 	      /* If use_stmt is or might be a nop assignment then USE_STMT
 	         acts as a use as well as definition.  */
 	      if (stmt != use_stmt
-		  && ref_maybe_used_by_stmt_p (use_stmt,
-					       gimple_get_lhs (stmt)))
-		return false;
+		  && ref_maybe_used_by_stmt_p (use_stmt, &ref))
+		{
+		  if (use && use != use_stmt)
+		    return false;
+		  use = use_stmt;
+		}
 	      continue;
 	    }
 
-	  if (gimple_code (use_stmt) != GIMPLE_PHI)
-	    return false;
-
-	  if (use
-	      && use != use_stmt)
-	    return false;
+	  if (is_a <gphi *> (use_stmt)
+	      || ref_maybe_used_by_stmt_p (use_stmt, &ref))
+	    {
+	      if (use && use != use_stmt)
+		return false;
+	      use = use_stmt;
+	      continue;
+	    }
 
-	  use = use_stmt;
+	  if (gimple_vdef (use_stmt))
+	    {
+	      if (stmt_may_clobber_ref_p_1 (use_stmt, &ref, false))
+		return false;
+	      /* We do not look past VDEFs, so treat them as sink location.  */
+	      if (use && use != use_stmt)
+		return false;
+	      use = use_stmt;
+	    }
 	}
       if (!use)
 	return false;
@@ -448,18 +465,26 @@ statement_sink_location (gimple *stmt, basic_block frombb,
 	  break;
 	}
       use = USE_STMT (one_use);
+    }
 
-      if (gimple_code (use) != GIMPLE_PHI)
-	{
-	  sinkbb = select_best_block (frombb, gimple_bb (use), stmt);
+  if (gimple_code (use) != GIMPLE_PHI)
+    {
+      sinkbb = select_best_block (frombb, gimple_bb (use), stmt);
 
-	  if (sinkbb == frombb)
-	    return false;
+      if (sinkbb == frombb)
+	return false;
 
-	  *togsi = gsi_after_labels (sinkbb);
+      /* The SSA update for sinking of stores cannot insert PHIs, the
+	 sink location has to lead to exit without crossing any CFG
+	 merge points to paths not dominated by the sink location.  */
+      if (gimple_vdef (stmt)
+	  && (!single_succ_p (sinkbb)
+	      || single_succ (sinkbb)->index != EXIT_BLOCK))
+	return false;
 
-	  return true;
-	}
+      *togsi = gsi_after_labels (sinkbb);
+
+      return true;
     }
 
   sinkbb = find_bb_for_arg (as_a <gphi *> (use), DEF_FROM_PTR (def_p));
diff --git a/gcc/tree-ssa-structalias.cc b/gcc/tree-ssa-structalias.cc
index deca44a..0035e50 100644
--- a/gcc/tree-ssa-structalias.cc
+++ b/gcc/tree-ssa-structalias.cc
@@ -48,6 +48,9 @@
 #include "ipa-modref.h"
 #include "attr-fnspec.h"
 
+#include "tree-ssa-structalias.h"
+#include "pta-andersen.h"
+
 /* The idea behind this analyzer is to generate set constraints from the
    program, then solve the resulting constraints in order to generate the
    points-to sets.
@@ -201,172 +204,357 @@
 
    And probably more.  */
 
-static bool use_field_sensitive = true;
-static int in_ipa_mode = 0;
-
-/* Used for predecessor bitmaps. */
-static bitmap_obstack predbitmap_obstack;
+namespace pointer_analysis {
 
 /* Used for points-to sets.  */
-static bitmap_obstack pta_obstack;
+bitmap_obstack pta_obstack;
 
-/* Used for oldsolution members of variables. */
-static bitmap_obstack oldpta_obstack;
+/* Used for oldsolution members of variables.  */
+bitmap_obstack oldpta_obstack;
 
-/* Used for per-solver-iteration bitmaps.  */
-static bitmap_obstack iteration_obstack;
+/* Table of variable info structures for constraint variables.
+   Indexed directly by variable info id.  */
+vec<varinfo_t> varmap;
 
-static unsigned int create_variable_info_for (tree, const char *, bool);
-typedef struct constraint_graph *constraint_graph_t;
-static void unify_nodes (constraint_graph_t, unsigned int, unsigned int, bool);
+/* List of constraints that we use to build the constraint graph from.  */
+vec<constraint_t> constraints;
+
+/* Map from trees to variable infos.  */
+static hash_map<tree, varinfo_t> *vi_for_tree;
 
-struct constraint;
-typedef struct constraint *constraint_t;
+/* The representative variable for a variable.  The points-to solution for a
+   var can be found in its rep.  Trivially, a var can be its own rep.
 
+   The solver provides this array once it is done solving.  */
+unsigned int *var_rep;
 
-#define EXECUTE_IF_IN_NONNULL_BITMAP(a, b, c, d)	\
-  if (a)						\
-    EXECUTE_IF_SET_IN_BITMAP (a, b, c, d)
+struct constraint_stats stats;
 
-static struct constraint_stats
+/* Find the first varinfo in the same variable as START that overlaps with
+   OFFSET.  Return NULL if we can't find one.  */
+
+varinfo_t
+first_vi_for_offset (varinfo_t start, unsigned HOST_WIDE_INT offset)
 {
-  unsigned int total_vars;
-  unsigned int nonpointer_vars;
-  unsigned int unified_vars_static;
-  unsigned int unified_vars_dynamic;
-  unsigned int iterations;
-  unsigned int num_edges;
-  unsigned int num_implicit_edges;
-  unsigned int num_avoided_edges;
-  unsigned int points_to_sets_created;
-} stats;
-
-struct variable_info
+  /* If the offset is outside of the variable, bail out.  */
+  if (offset >= start->fullsize)
+    return NULL;
+
+  /* If we cannot reach offset from start, lookup the first field
+     and start from there.  */
+  if (start->offset > offset)
+    start = get_varinfo (start->head);
+
+  while (start)
+    {
+      /* We may not find a variable in the field list with the actual
+	 offset when we have glommed a structure to a variable.
+	 In that case, however, offset should still be within the size
+	 of the variable.  */
+      if (offset >= start->offset
+	  && (offset - start->offset) < start->size)
+	return start;
+
+      start = vi_next (start);
+    }
+
+  return NULL;
+}
+
+/* Find the first varinfo in the same variable as START that overlaps with
+   OFFSET.  If there is no such varinfo the varinfo directly preceding
+   OFFSET is returned.  */
+
+varinfo_t
+first_or_preceding_vi_for_offset (varinfo_t start,
+				  unsigned HOST_WIDE_INT offset)
 {
-  /* ID of this variable  */
-  unsigned int id;
+  /* If we cannot reach offset from start, lookup the first field
+     and start from there.  */
+  if (start->offset > offset)
+    start = get_varinfo (start->head);
 
-  /* True if this is a variable created by the constraint analysis, such as
-     heap variables and constraints we had to break up.  */
-  unsigned int is_artificial_var : 1;
+  /* We may not find a variable in the field list with the actual
+     offset when we have glommed a structure to a variable.
+     In that case, however, offset should still be within the size
+     of the variable.
+     If we got beyond the offset we look for return the field
+     directly preceding offset which may be the last field.  */
+  while (start->next
+	 && offset >= start->offset
+	 && !((offset - start->offset) < start->size))
+    start = vi_next (start);
 
-  /* True if this is a special variable whose solution set should not be
-     changed.  */
-  unsigned int is_special_var : 1;
+  return start;
+}
 
-  /* True for variables whose size is not known or variable.  */
-  unsigned int is_unknown_size_var : 1;
+/* Print out constraint C to FILE.  */
 
-  /* True for (sub-)fields that represent a whole variable.  */
-  unsigned int is_full_var : 1;
+void
+dump_constraint (FILE *file, constraint_t c)
+{
+  if (c->lhs.type == ADDRESSOF)
+    fprintf (file, "&");
+  else if (c->lhs.type == DEREF)
+    fprintf (file, "*");
+  if (dump_file)
+    fprintf (file, "%s", get_varinfo (c->lhs.var)->name);
+  else
+    fprintf (file, "V%d", c->lhs.var);
+  if (c->lhs.offset == UNKNOWN_OFFSET)
+    fprintf (file, " + UNKNOWN");
+  else if (c->lhs.offset != 0)
+    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->lhs.offset);
+  fprintf (file, " = ");
+  if (c->rhs.type == ADDRESSOF)
+    fprintf (file, "&");
+  else if (c->rhs.type == DEREF)
+    fprintf (file, "*");
+  if (dump_file)
+    fprintf (file, "%s", get_varinfo (c->rhs.var)->name);
+  else
+    fprintf (file, "V%d", c->rhs.var);
+  if (c->rhs.offset == UNKNOWN_OFFSET)
+    fprintf (file, " + UNKNOWN");
+  else if (c->rhs.offset != 0)
+    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->rhs.offset);
+}
 
-  /* True if this is a heap variable.  */
-  unsigned int is_heap_var : 1;
+/* Print out constraint C to stderr.  */
 
-  /* True if this is a register variable.  */
-  unsigned int is_reg_var : 1;
+DEBUG_FUNCTION void
+debug_constraint (constraint_t c)
+{
+  dump_constraint (stderr, c);
+  fprintf (stderr, "\n");
+}
 
-  /* True if this field may contain pointers.  */
-  unsigned int may_have_pointers : 1;
+/* Print out all constraints to FILE.  */
 
-  /* True if this field has only restrict qualified pointers.  */
-  unsigned int only_restrict_pointers : 1;
+void
+dump_constraints (FILE *file, int from)
+{
+  int i;
+  constraint_t c;
+  for (i = from; constraints.iterate (i, &c); i++)
+    if (c)
+      {
+	dump_constraint (file, c);
+	fprintf (file, "\n");
+      }
+}
 
-  /* True if this represents a heap var created for a restrict qualified
-     pointer.  */
-  unsigned int is_restrict_var : 1;
+/* Print out all constraints to stderr.  */
 
-  /* True if this represents a global variable.  */
-  unsigned int is_global_var : 1;
+DEBUG_FUNCTION void
+debug_constraints (void)
+{
+  dump_constraints (stderr, 0);
+}
 
-  /* True if this represents a module escape point for IPA analysis.  */
-  unsigned int is_ipa_escape_point : 1;
+/* Print out the points-to solution for VAR to FILE.  */
 
-  /* True if this represents a IPA function info.  */
-  unsigned int is_fn_info : 1;
+void
+dump_solution_for_var (FILE *file, unsigned int var)
+{
+  varinfo_t vi = get_varinfo (var);
+  unsigned int i;
+  bitmap_iterator bi;
 
-  /* True if this appears as RHS in a ADDRESSOF constraint.  */
-  unsigned int address_taken : 1;
+  /* Dump the solution for unified vars anyway, this avoids difficulties
+     in scanning dumps in the testsuite.  */
+  fprintf (file, "%s = { ", vi->name);
+  vi = get_varinfo (var_rep[var]);
+  EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
+    fprintf (file, "%s ", get_varinfo (i)->name);
+  fprintf (file, "}");
 
-  /* ???  Store somewhere better.  */
-  unsigned short ruid;
+  /* But note when the variable was unified.  */
+  if (vi->id != var)
+    fprintf (file, " same as %s", vi->name);
 
-  /* The ID of the variable for the next field in this structure
-     or zero for the last field in this structure.  */
-  unsigned next;
+  fprintf (file, "\n");
+}
 
-  /* The ID of the variable for the first field in this structure.  */
-  unsigned head;
+/* Print the points-to solution for VAR to stderr.  */
 
-  /* Offset of this variable, in bits, from the base variable  */
-  unsigned HOST_WIDE_INT offset;
+DEBUG_FUNCTION void
+debug_solution_for_var (unsigned int var)
+{
+  dump_solution_for_var (stderr, var);
+}
 
-  /* Size of the variable, in bits.  */
-  unsigned HOST_WIDE_INT size;
+/* Dump stats information to OUTFILE.  */
 
-  /* Full size of the base variable, in bits.  */
-  unsigned HOST_WIDE_INT fullsize;
+void
+dump_sa_stats (FILE *outfile)
+{
+  fprintf (outfile, "Points-to Stats:\n");
+  fprintf (outfile, "Total vars:               %d\n", stats.total_vars);
+  fprintf (outfile, "Non-pointer vars:          %d\n",
+	   stats.nonpointer_vars);
+  fprintf (outfile, "Statically unified vars:  %d\n",
+	   stats.unified_vars_static);
+  fprintf (outfile, "Dynamically unified vars: %d\n",
+	   stats.unified_vars_dynamic);
+  fprintf (outfile, "Iterations:               %d\n", stats.iterations);
+  fprintf (outfile, "Number of edges:          %d\n", stats.num_edges);
+  fprintf (outfile, "Number of implicit edges: %d\n",
+	   stats.num_implicit_edges);
+  fprintf (outfile, "Number of avoided edges: %d\n",
+	   stats.num_avoided_edges);
+}
 
-  /* In IPA mode the shadow UID in case the variable needs to be duplicated in
-     the final points-to solution because it reaches its containing
-     function recursively.  Zero if none is needed.  */
-  unsigned int shadow_var_uid;
+/* Dump points-to information to OUTFILE.  */
 
-  /* Name of this variable */
-  const char *name;
+void
+dump_sa_points_to_info (FILE *outfile)
+{
+  fprintf (outfile, "\nPoints-to sets\n\n");
 
-  /* Tree that this variable is associated with.  */
-  tree decl;
+  for (unsigned i = 1; i < varmap.length (); i++)
+    {
+      varinfo_t vi = get_varinfo (i);
+      if (!vi->may_have_pointers)
+	continue;
+      dump_solution_for_var (outfile, i);
+    }
+}
 
-  /* Points-to set for this variable.  */
-  bitmap solution;
 
-  /* Old points-to set for this variable.  */
-  bitmap oldsolution;
-};
-typedef struct variable_info *varinfo_t;
+/* Debug points-to information to stderr.  */
 
-static varinfo_t first_vi_for_offset (varinfo_t, unsigned HOST_WIDE_INT);
-static varinfo_t first_or_preceding_vi_for_offset (varinfo_t,
-						   unsigned HOST_WIDE_INT);
-static varinfo_t lookup_vi_for_tree (tree);
-static inline bool type_can_have_subvars (const_tree);
-static void make_param_constraints (varinfo_t);
+DEBUG_FUNCTION void
+debug_sa_points_to_info (void)
+{
+  dump_sa_points_to_info (stderr);
+}
 
-/* Pool of variable info structures.  */
-static object_allocator<variable_info> variable_info_pool
-  ("Variable info pool");
+/* Dump varinfo VI to FILE.  */
 
-/* Map varinfo to final pt_solution.  */
-static hash_map<varinfo_t, pt_solution *> *final_solutions;
-struct obstack final_solutions_obstack;
+void
+dump_varinfo (FILE *file, varinfo_t vi)
+{
+  if (vi == NULL)
+    return;
 
-/* Table of variable info structures for constraint variables.
-   Indexed directly by variable info id.  */
-static vec<varinfo_t> varmap;
+  fprintf (file, "%u: %s\n", vi->id, vi->name);
 
-/* Return the varmap element N */
+  const char *sep = " ";
+  if (vi->is_artificial_var)
+    fprintf (file, "%sartificial", sep);
+  if (vi->is_special_var)
+    fprintf (file, "%sspecial", sep);
+  if (vi->is_unknown_size_var)
+    fprintf (file, "%sunknown-size", sep);
+  if (vi->is_full_var)
+    fprintf (file, "%sfull", sep);
+  if (vi->is_heap_var)
+    fprintf (file, "%sheap", sep);
+  if (vi->may_have_pointers)
+    fprintf (file, "%smay-have-pointers", sep);
+  if (vi->only_restrict_pointers)
+    fprintf (file, "%sonly-restrict-pointers", sep);
+  if (vi->is_restrict_var)
+    fprintf (file, "%sis-restrict-var", sep);
+  if (vi->is_global_var)
+    fprintf (file, "%sglobal", sep);
+  if (vi->is_ipa_escape_point)
+    fprintf (file, "%sipa-escape-point", sep);
+  if (vi->is_fn_info)
+    fprintf (file, "%sfn-info", sep);
+  if (vi->ruid)
+    fprintf (file, "%srestrict-uid:%u", sep, vi->ruid);
+  if (vi->next)
+    fprintf (file, "%snext:%u", sep, vi->next);
+  if (vi->head != vi->id)
+    fprintf (file, "%shead:%u", sep, vi->head);
+  if (vi->offset)
+    fprintf (file, "%soffset:" HOST_WIDE_INT_PRINT_DEC, sep, vi->offset);
+  if (vi->size != ~HOST_WIDE_INT_0U)
+    fprintf (file, "%ssize:" HOST_WIDE_INT_PRINT_DEC, sep, vi->size);
+  if (vi->fullsize != ~HOST_WIDE_INT_0U && vi->fullsize != vi->size)
+    fprintf (file, "%sfullsize:" HOST_WIDE_INT_PRINT_DEC, sep,
+	     vi->fullsize);
+  fprintf (file, "\n");
 
-static inline varinfo_t
-get_varinfo (unsigned int n)
+  if (vi->solution && !bitmap_empty_p (vi->solution))
+    {
+      bitmap_iterator bi;
+      unsigned i;
+      fprintf (file, " solution: {");
+      EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
+	fprintf (file, " %u", i);
+      fprintf (file, " }\n");
+    }
+
+  if (vi->oldsolution && !bitmap_empty_p (vi->oldsolution)
+      && !bitmap_equal_p (vi->solution, vi->oldsolution))
+    {
+      bitmap_iterator bi;
+      unsigned i;
+      fprintf (file, " oldsolution: {");
+      EXECUTE_IF_SET_IN_BITMAP (vi->oldsolution, 0, i, bi)
+	fprintf (file, " %u", i);
+      fprintf (file, " }\n");
+    }
+}
+
+/* Dump varinfo VI to stderr.  */
+
+DEBUG_FUNCTION void
+debug_varinfo (varinfo_t vi)
+{
+  dump_varinfo (stderr, vi);
+}
+
+/* Dump varmap to FILE.  */
+
+void
+dump_varmap (FILE *file)
 {
-  return varmap[n];
+  if (varmap.length () == 0)
+    return;
+
+  fprintf (file, "variables:\n");
+
+  for (unsigned int i = 0; i < varmap.length (); ++i)
+    {
+      varinfo_t vi = get_varinfo (i);
+      dump_varinfo (file, vi);
+    }
+
+  fprintf (file, "\n");
 }
 
-/* Return the next variable in the list of sub-variables of VI
-   or NULL if VI is the last sub-variable.  */
+/* Dump varmap to stderr.  */
 
-static inline varinfo_t
-vi_next (varinfo_t vi)
+DEBUG_FUNCTION void
+debug_varmap (void)
 {
-  return get_varinfo (vi->next);
+  dump_varmap (stderr);
 }
 
-/* Static IDs for the special variables.  Variable ID zero is unused
-   and used as terminator for the sub-variable chain.  */
-enum { nothing_id = 1, anything_id = 2, string_id = 3,
-       escaped_id = 4, nonlocal_id = 5, escaped_return_id = 6,
-       storedanything_id = 7, integer_id = 8 };
+} // namespace pointer_analysis
+
+
+using namespace pointer_analysis;
+
+static bool use_field_sensitive = true;
+static int in_ipa_mode = 0;
+
+static unsigned int create_variable_info_for (tree, const char *, bool);
+static varinfo_t lookup_vi_for_tree (tree);
+static inline bool type_can_have_subvars (const_tree);
+static void make_param_constraints (varinfo_t);
+
+/* Pool of variable info structures.  */
+static object_allocator<variable_info> variable_info_pool
+  ("Variable info pool");
+
+/* Map varinfo to final pt_solution.  */
+static hash_map<varinfo_t, pt_solution *> *final_solutions;
+static struct obstack final_solutions_obstack;
 
 /* Return a new variable info structure consisting for a variable
    named NAME, and using constraint graph node NODE.  Append it
@@ -502,166 +690,15 @@ get_call_clobber_vi (gcall *call)
 }
 
 
-enum constraint_expr_type {SCALAR, DEREF, ADDRESSOF};
-
-/* An expression that appears in a constraint.  */
-
-struct constraint_expr
-{
-  /* Constraint type.  */
-  constraint_expr_type type;
-
-  /* Variable we are referring to in the constraint.  */
-  unsigned int var;
-
-  /* Offset, in bits, of this constraint from the beginning of
-     variables it ends up referring to.
-
-     IOW, in a deref constraint, we would deref, get the result set,
-     then add OFFSET to each member.   */
-  HOST_WIDE_INT offset;
-};
-
-/* Use 0x8000... as special unknown offset.  */
-#define UNKNOWN_OFFSET HOST_WIDE_INT_MIN
-
-typedef struct constraint_expr ce_s;
 static void get_constraint_for_1 (tree, vec<ce_s> *, bool, bool);
 static void get_constraint_for (tree, vec<ce_s> *);
 static void get_constraint_for_rhs (tree, vec<ce_s> *);
 static void do_deref (vec<ce_s> *);
 
-/* Our set constraints are made up of two constraint expressions, one
-   LHS, and one RHS.
-
-   As described in the introduction, our set constraints each represent an
-   operation between set valued variables.
-*/
-struct constraint
-{
-  struct constraint_expr lhs;
-  struct constraint_expr rhs;
-};
+/* Allocator for 'constraints' vector.  */
 
-/* List of constraints that we use to build the constraint graph from.  */
-
-static vec<constraint_t> constraints;
 static object_allocator<constraint> constraint_pool ("Constraint pool");
 
-/* The constraint graph is represented as an array of bitmaps
-   containing successor nodes.  */
-
-struct constraint_graph
-{
-  /* Size of this graph, which may be different than the number of
-     nodes in the variable map.  */
-  unsigned int size;
-
-  /* Explicit successors of each node. */
-  bitmap *succs;
-
-  /* Implicit predecessors of each node (Used for variable
-     substitution). */
-  bitmap *implicit_preds;
-
-  /* Explicit predecessors of each node (Used for variable substitution).  */
-  bitmap *preds;
-
-  /* Indirect cycle representatives, or -1 if the node has no indirect
-     cycles.  */
-  int *indirect_cycles;
-
-  /* Representative node for a node.  rep[a] == a unless the node has
-     been unified. */
-  unsigned int *rep;
-
-  /* Equivalence class representative for a label.  This is used for
-     variable substitution.  */
-  int *eq_rep;
-
-  /* Pointer equivalence label for a node.  All nodes with the same
-     pointer equivalence label can be unified together at some point
-     (either during constraint optimization or after the constraint
-     graph is built).  */
-  unsigned int *pe;
-
-  /* Pointer equivalence representative for a label.  This is used to
-     handle nodes that are pointer equivalent but not location
-     equivalent.  We can unite these once the addressof constraints
-     are transformed into initial points-to sets.  */
-  int *pe_rep;
-
-  /* Pointer equivalence label for each node, used during variable
-     substitution.  */
-  unsigned int *pointer_label;
-
-  /* Location equivalence label for each node, used during location
-     equivalence finding.  */
-  unsigned int *loc_label;
-
-  /* Pointed-by set for each node, used during location equivalence
-     finding.  This is pointed-by rather than pointed-to, because it
-     is constructed using the predecessor graph.  */
-  bitmap *pointed_by;
-
-  /* Points to sets for pointer equivalence.  This is *not* the actual
-     points-to sets for nodes.  */
-  bitmap *points_to;
-
-  /* Bitmap of nodes where the bit is set if the node is a direct
-     node.  Used for variable substitution.  */
-  sbitmap direct_nodes;
-
-  /* Bitmap of nodes where the bit is set if the node is address
-     taken.  Used for variable substitution.  */
-  bitmap address_taken;
-
-  /* Vector of complex constraints for each graph node.  Complex
-     constraints are those involving dereferences or offsets that are
-     not 0.  */
-  vec<constraint_t> *complex;
-};
-
-static constraint_graph_t graph;
-
-/* During variable substitution and the offline version of indirect
-   cycle finding, we create nodes to represent dereferences and
-   address taken constraints.  These represent where these start and
-   end.  */
-#define FIRST_REF_NODE (varmap).length ()
-#define LAST_REF_NODE (FIRST_REF_NODE + (FIRST_REF_NODE - 1))
-
-/* Return the representative node for NODE, if NODE has been unioned
-   with another NODE.
-   This function performs path compression along the way to finding
-   the representative.  */
-
-static unsigned int
-find (unsigned int node)
-{
-  gcc_checking_assert (node < graph->size);
-  if (graph->rep[node] != node)
-    return graph->rep[node] = find (graph->rep[node]);
-  return node;
-}
-
-/* Union the TO and FROM nodes to the TO nodes.
-   Note that at some point in the future, we may want to do
-   union-by-rank, in which case we are going to have to return the
-   node we unified to.  */
-
-static bool
-unite (unsigned int to, unsigned int from)
-{
-  gcc_checking_assert (to < graph->size && from < graph->size);
-  if (to != from && graph->rep[from] != to)
-    {
-      graph->rep[from] = to;
-      return true;
-    }
-  return false;
-}
-
 /* Create a new constraint consisting of LHS and RHS expressions.  */
 
 static constraint_t
@@ -674,2312 +711,6 @@ new_constraint (const struct constraint_expr lhs,
   return ret;
 }
 
-/* Print out constraint C to FILE.  */
-
-static void
-dump_constraint (FILE *file, constraint_t c)
-{
-  if (c->lhs.type == ADDRESSOF)
-    fprintf (file, "&");
-  else if (c->lhs.type == DEREF)
-    fprintf (file, "*");
-  if (dump_file)
-    fprintf (file, "%s", get_varinfo (c->lhs.var)->name);
-  else
-    fprintf (file, "V%d", c->lhs.var);
-  if (c->lhs.offset == UNKNOWN_OFFSET)
-    fprintf (file, " + UNKNOWN");
-  else if (c->lhs.offset != 0)
-    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->lhs.offset);
-  fprintf (file, " = ");
-  if (c->rhs.type == ADDRESSOF)
-    fprintf (file, "&");
-  else if (c->rhs.type == DEREF)
-    fprintf (file, "*");
-  if (dump_file)
-    fprintf (file, "%s", get_varinfo (c->rhs.var)->name);
-  else
-    fprintf (file, "V%d", c->rhs.var);
-  if (c->rhs.offset == UNKNOWN_OFFSET)
-    fprintf (file, " + UNKNOWN");
-  else if (c->rhs.offset != 0)
-    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->rhs.offset);
-}
-
-
-void debug_constraint (constraint_t);
-void debug_constraints (void);
-void debug_constraint_graph (void);
-void debug_solution_for_var (unsigned int);
-void debug_sa_points_to_info (void);
-void debug_varinfo (varinfo_t);
-void debug_varmap (void);
-
-/* Print out constraint C to stderr.  */
-
-DEBUG_FUNCTION void
-debug_constraint (constraint_t c)
-{
-  dump_constraint (stderr, c);
-  fprintf (stderr, "\n");
-}
-
-/* Print out all constraints to FILE */
-
-static void
-dump_constraints (FILE *file, int from)
-{
-  int i;
-  constraint_t c;
-  for (i = from; constraints.iterate (i, &c); i++)
-    if (c)
-      {
-	dump_constraint (file, c);
-	fprintf (file, "\n");
-      }
-}
-
-/* Print out all constraints to stderr.  */
-
-DEBUG_FUNCTION void
-debug_constraints (void)
-{
-  dump_constraints (stderr, 0);
-}
-
-/* Print the constraint graph in dot format.  */
-
-static void
-dump_constraint_graph (FILE *file)
-{
-  unsigned int i;
-
-  /* Only print the graph if it has already been initialized:  */
-  if (!graph)
-    return;
-
-  /* Prints the header of the dot file:  */
-  fprintf (file, "strict digraph {\n");
-  fprintf (file, "  node [\n    shape = box\n  ]\n");
-  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
-  fprintf (file, "\n  // List of nodes and complex constraints in "
-	   "the constraint graph:\n");
-
-  /* The next lines print the nodes in the graph together with the
-     complex constraints attached to them.  */
-  for (i = 1; i < graph->size; i++)
-    {
-      if (i == FIRST_REF_NODE)
-	continue;
-      if (find (i) != i)
-	continue;
-      if (i < FIRST_REF_NODE)
-	fprintf (file, "\"%s\"", get_varinfo (i)->name);
-      else
-	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-      if (graph->complex[i].exists ())
-	{
-	  unsigned j;
-	  constraint_t c;
-	  fprintf (file, " [label=\"\\N\\n");
-	  for (j = 0; graph->complex[i].iterate (j, &c); ++j)
-	    {
-	      dump_constraint (file, c);
-	      fprintf (file, "\\l");
-	    }
-	  fprintf (file, "\"]");
-	}
-      fprintf (file, ";\n");
-    }
-
-  /* Go over the edges.  */
-  fprintf (file, "\n  // Edges in the constraint graph:\n");
-  for (i = 1; i < graph->size; i++)
-    {
-      unsigned j;
-      bitmap_iterator bi;
-      if (find (i) != i)
-	continue;
-      EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i], 0, j, bi)
-	{
-	  unsigned to = find (j);
-	  if (i == to)
-	    continue;
-	  if (i < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-	  fprintf (file, " -> ");
-	  if (to < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (to)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (to - FIRST_REF_NODE)->name);
-	  fprintf (file, ";\n");
-	}
-    }
-
-  /* Prints the tail of the dot file.  */
-  fprintf (file, "}\n");
-}
-
-/* Print out the constraint graph to stderr.  */
-
-DEBUG_FUNCTION void
-debug_constraint_graph (void)
-{
-  dump_constraint_graph (stderr);
-}
-
-/* SOLVER FUNCTIONS
-
-   The solver is a simple worklist solver, that works on the following
-   algorithm:
-
-   sbitmap changed_nodes = all zeroes;
-   changed_count = 0;
-   For each node that is not already collapsed:
-       changed_count++;
-       set bit in changed nodes
-
-   while (changed_count > 0)
-   {
-     compute topological ordering for constraint graph
-
-     find and collapse cycles in the constraint graph (updating
-     changed if necessary)
-
-     for each node (n) in the graph in topological order:
-       changed_count--;
-
-       Process each complex constraint associated with the node,
-       updating changed if necessary.
-
-       For each outgoing edge from n, propagate the solution from n to
-       the destination of the edge, updating changed as necessary.
-
-   }  */
-
-/* Return true if two constraint expressions A and B are equal.  */
-
-static bool
-constraint_expr_equal (struct constraint_expr a, struct constraint_expr b)
-{
-  return a.type == b.type && a.var == b.var && a.offset == b.offset;
-}
-
-/* Return true if constraint expression A is less than constraint expression
-   B.  This is just arbitrary, but consistent, in order to give them an
-   ordering.  */
-
-static bool
-constraint_expr_less (struct constraint_expr a, struct constraint_expr b)
-{
-  if (a.type == b.type)
-    {
-      if (a.var == b.var)
-	return a.offset < b.offset;
-      else
-	return a.var < b.var;
-    }
-  else
-    return a.type < b.type;
-}
-
-/* Return true if constraint A is less than constraint B.  This is just
-   arbitrary, but consistent, in order to give them an ordering.  */
-
-static bool
-constraint_less (const constraint_t &a, const constraint_t &b)
-{
-  if (constraint_expr_less (a->lhs, b->lhs))
-    return true;
-  else if (constraint_expr_less (b->lhs, a->lhs))
-    return false;
-  else
-    return constraint_expr_less (a->rhs, b->rhs);
-}
-
-/* Return true if two constraints A and B are equal.  */
-
-static bool
-constraint_equal (const constraint &a, const constraint &b)
-{
-  return constraint_expr_equal (a.lhs, b.lhs)
-    && constraint_expr_equal (a.rhs, b.rhs);
-}
-
-
-/* Find a constraint LOOKFOR in the sorted constraint vector VEC */
-
-static constraint_t
-constraint_vec_find (vec<constraint_t> vec,
-		     constraint &lookfor)
-{
-  unsigned int place;
-  constraint_t found;
-
-  if (!vec.exists ())
-    return NULL;
-
-  place = vec.lower_bound (&lookfor, constraint_less);
-  if (place >= vec.length ())
-    return NULL;
-  found = vec[place];
-  if (!constraint_equal (*found, lookfor))
-    return NULL;
-  return found;
-}
-
-/* Union two constraint vectors, TO and FROM.  Put the result in TO.
-   Returns true of TO set is changed.  */
-
-static bool
-constraint_set_union (vec<constraint_t> *to,
-		      vec<constraint_t> *from)
-{
-  int i;
-  constraint_t c;
-  bool any_change = false;
-
-  FOR_EACH_VEC_ELT (*from, i, c)
-    {
-      if (constraint_vec_find (*to, *c) == NULL)
-	{
-	  unsigned int place = to->lower_bound (c, constraint_less);
-	  to->safe_insert (place, c);
-          any_change = true;
-	}
-    }
-  return any_change;
-}
-
-/* Expands the solution in SET to all sub-fields of variables included.  */
-
-static bitmap
-solution_set_expand (bitmap set, bitmap *expanded)
-{
-  bitmap_iterator bi;
-  unsigned j;
-
-  if (*expanded)
-    return *expanded;
-
-  *expanded = BITMAP_ALLOC (&iteration_obstack);
-
-  /* In a first pass expand variables, once for each head to avoid
-     quadratic behavior, to include all sub-fields.  */
-  unsigned prev_head = 0;
-  EXECUTE_IF_SET_IN_BITMAP (set, 0, j, bi)
-    {
-      varinfo_t v = get_varinfo (j);
-      if (v->is_artificial_var
-	  || v->is_full_var)
-	continue;
-      if (v->head != prev_head)
-	{
-	  varinfo_t head = get_varinfo (v->head);
-	  unsigned num = 1;
-	  for (varinfo_t n = vi_next (head); n != NULL; n = vi_next (n))
-	    {
-	      if (n->id != head->id + num)
-		{
-		  /* Usually sub variables are adjacent but since we
-		     create pointed-to restrict representatives there
-		     can be gaps as well.  */
-		  bitmap_set_range (*expanded, head->id, num);
-		  head = n;
-		  num = 1;
-		}
-	      else
-		num++;
-	    }
-
-	  bitmap_set_range (*expanded, head->id, num);
-	  prev_head = v->head;
-	}
-    }
-
-  /* And finally set the rest of the bits from SET in an efficient way.  */
-  bitmap_ior_into (*expanded, set);
-
-  return *expanded;
-}
-
-/* Union solution sets TO and DELTA, and add INC to each member of DELTA in the
-   process.  */
-
-static bool
-set_union_with_increment  (bitmap to, bitmap delta, HOST_WIDE_INT inc,
-			   bitmap *expanded_delta)
-{
-  bool changed = false;
-  bitmap_iterator bi;
-  unsigned int i;
-
-  /* If the solution of DELTA contains anything it is good enough to transfer
-     this to TO.  */
-  if (bitmap_bit_p (delta, anything_id))
-    return bitmap_set_bit (to, anything_id);
-
-  /* If the offset is unknown we have to expand the solution to
-     all subfields.  */
-  if (inc == UNKNOWN_OFFSET)
-    {
-      delta = solution_set_expand (delta, expanded_delta);
-      changed |= bitmap_ior_into (to, delta);
-      return changed;
-    }
-
-  /* For non-zero offset union the offsetted solution into the destination.  */
-  EXECUTE_IF_SET_IN_BITMAP (delta, 0, i, bi)
-    {
-      varinfo_t vi = get_varinfo (i);
-
-      /* If this is a variable with just one field just set its bit
-         in the result.  */
-      if (vi->is_artificial_var
-	  || vi->is_unknown_size_var
-	  || vi->is_full_var)
-	changed |= bitmap_set_bit (to, i);
-      else
-	{
-	  HOST_WIDE_INT fieldoffset = vi->offset + inc;
-	  unsigned HOST_WIDE_INT size = vi->size;
-
-	  /* If the offset makes the pointer point to before the
-	     variable use offset zero for the field lookup.  */
-	  if (fieldoffset < 0)
-	    vi = get_varinfo (vi->head);
-	  else
-	    vi = first_or_preceding_vi_for_offset (vi, fieldoffset);
-
-	  do
-	    {
-	      changed |= bitmap_set_bit (to, vi->id);
-	      if (vi->is_full_var
-		  || vi->next == 0)
-		break;
-
-	      /* We have to include all fields that overlap the current field
-	         shifted by inc.  */
-	      vi = vi_next (vi);
-	    }
-	  while (vi->offset < fieldoffset + size);
-	}
-    }
-
-  return changed;
-}
-
-/* Insert constraint C into the list of complex constraints for graph
-   node VAR.  */
-
-static void
-insert_into_complex (constraint_graph_t graph,
-		     unsigned int var, constraint_t c)
-{
-  vec<constraint_t> complex = graph->complex[var];
-  unsigned int place = complex.lower_bound (c, constraint_less);
-
-  /* Only insert constraints that do not already exist.  */
-  if (place >= complex.length ()
-      || !constraint_equal (*c, *complex[place]))
-    graph->complex[var].safe_insert (place, c);
-}
-
-
-/* Condense two variable nodes into a single variable node, by moving
-   all associated info from FROM to TO. Returns true if TO node's
-   constraint set changes after the merge.  */
-
-static bool
-merge_node_constraints (constraint_graph_t graph, unsigned int to,
-			unsigned int from)
-{
-  unsigned int i;
-  constraint_t c;
-  bool any_change = false;
-
-  gcc_checking_assert (find (from) == to);
-
-  /* Move all complex constraints from src node into to node  */
-  FOR_EACH_VEC_ELT (graph->complex[from], i, c)
-    {
-      /* In complex constraints for node FROM, we may have either
-	 a = *FROM, and *FROM = a, or an offseted constraint which are
-	 always added to the rhs node's constraints.  */
-
-      if (c->rhs.type == DEREF)
-	c->rhs.var = to;
-      else if (c->lhs.type == DEREF)
-	c->lhs.var = to;
-      else
-	c->rhs.var = to;
-
-    }
-  any_change = constraint_set_union (&graph->complex[to],
-				     &graph->complex[from]);
-  graph->complex[from].release ();
-  return any_change;
-}
-
-
-/* Remove edges involving NODE from GRAPH.  */
-
-static void
-clear_edges_for_node (constraint_graph_t graph, unsigned int node)
-{
-  if (graph->succs[node])
-    BITMAP_FREE (graph->succs[node]);
-}
-
-/* Merge GRAPH nodes FROM and TO into node TO.  */
-
-static void
-merge_graph_nodes (constraint_graph_t graph, unsigned int to,
-		   unsigned int from)
-{
-  if (graph->indirect_cycles[from] != -1)
-    {
-      /* If we have indirect cycles with the from node, and we have
-	 none on the to node, the to node has indirect cycles from the
-	 from node now that they are unified.
-	 If indirect cycles exist on both, unify the nodes that they
-	 are in a cycle with, since we know they are in a cycle with
-	 each other.  */
-      if (graph->indirect_cycles[to] == -1)
-	graph->indirect_cycles[to] = graph->indirect_cycles[from];
-    }
-
-  /* Merge all the successor edges.  */
-  if (graph->succs[from])
-    {
-      if (!graph->succs[to])
-	graph->succs[to] = BITMAP_ALLOC (&pta_obstack);
-      bitmap_ior_into (graph->succs[to],
-		       graph->succs[from]);
-    }
-
-  clear_edges_for_node (graph, from);
-}
-
-
-/* Add an indirect graph edge to GRAPH, going from TO to FROM if
-   it doesn't exist in the graph already.  */
-
-static void
-add_implicit_graph_edge (constraint_graph_t graph, unsigned int to,
-			 unsigned int from)
-{
-  if (to == from)
-    return;
-
-  if (!graph->implicit_preds[to])
-    graph->implicit_preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
-
-  if (bitmap_set_bit (graph->implicit_preds[to], from))
-    stats.num_implicit_edges++;
-}
-
-/* Add a predecessor graph edge to GRAPH, going from TO to FROM if
-   it doesn't exist in the graph already.
-   Return false if the edge already existed, true otherwise.  */
-
-static void
-add_pred_graph_edge (constraint_graph_t graph, unsigned int to,
-		     unsigned int from)
-{
-  if (!graph->preds[to])
-    graph->preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
-  bitmap_set_bit (graph->preds[to], from);
-}
-
-/* Add a graph edge to GRAPH, going from FROM to TO if
-   it doesn't exist in the graph already.
-   Return false if the edge already existed, true otherwise.  */
-
-static bool
-add_graph_edge (constraint_graph_t graph, unsigned int to,
-		unsigned int from)
-{
-  if (to == from)
-    {
-      return false;
-    }
-  else
-    {
-      bool r = false;
-
-      if (!graph->succs[from])
-	graph->succs[from] = BITMAP_ALLOC (&pta_obstack);
-
-      /* The graph solving process does not avoid "triangles", thus
-	 there can be multiple paths from a node to another involving
-	 intermediate other nodes.  That causes extra copying which is
-	 most difficult to avoid when the intermediate node is ESCAPED
-	 because there are no edges added from ESCAPED.  Avoid
-	 adding the direct edge FROM -> TO when we have FROM -> ESCAPED
-	 and TO contains ESCAPED.
-	 ???  Note this is only a heuristic, it does not prevent the
-	 situation from occuring.  The heuristic helps PR38474 and
-	 PR99912 significantly.  */
-      if (to < FIRST_REF_NODE
-	  && bitmap_bit_p (graph->succs[from], find (escaped_id))
-	  && bitmap_bit_p (get_varinfo (find (to))->solution, escaped_id))
-	{
-	  stats.num_avoided_edges++;
-	  return false;
-	}
-
-      if (bitmap_set_bit (graph->succs[from], to))
-	{
-	  r = true;
-	  if (to < FIRST_REF_NODE && from < FIRST_REF_NODE)
-	    stats.num_edges++;
-	}
-      return r;
-    }
-}
-
-
-/* Initialize the constraint graph structure to contain SIZE nodes.  */
-
-static void
-init_graph (unsigned int size)
-{
-  unsigned int j;
-
-  graph = XCNEW (struct constraint_graph);
-  graph->size = size;
-  graph->succs = XCNEWVEC (bitmap, graph->size);
-  graph->indirect_cycles = XNEWVEC (int, graph->size);
-  graph->rep = XNEWVEC (unsigned int, graph->size);
-  /* ??? Macros do not support template types with multiple arguments,
-     so we use a typedef to work around it.  */
-  typedef vec<constraint_t> vec_constraint_t_heap;
-  graph->complex = XCNEWVEC (vec_constraint_t_heap, size);
-  graph->pe = XCNEWVEC (unsigned int, graph->size);
-  graph->pe_rep = XNEWVEC (int, graph->size);
-
-  for (j = 0; j < graph->size; j++)
-    {
-      graph->rep[j] = j;
-      graph->pe_rep[j] = -1;
-      graph->indirect_cycles[j] = -1;
-    }
-}
-
-/* Build the constraint graph, adding only predecessor edges right now.  */
-
-static void
-build_pred_graph (void)
-{
-  int i;
-  constraint_t c;
-  unsigned int j;
-
-  graph->implicit_preds = XCNEWVEC (bitmap, graph->size);
-  graph->preds = XCNEWVEC (bitmap, graph->size);
-  graph->pointer_label = XCNEWVEC (unsigned int, graph->size);
-  graph->loc_label = XCNEWVEC (unsigned int, graph->size);
-  graph->pointed_by = XCNEWVEC (bitmap, graph->size);
-  graph->points_to = XCNEWVEC (bitmap, graph->size);
-  graph->eq_rep = XNEWVEC (int, graph->size);
-  graph->direct_nodes = sbitmap_alloc (graph->size);
-  graph->address_taken = BITMAP_ALLOC (&predbitmap_obstack);
-  bitmap_clear (graph->direct_nodes);
-
-  for (j = 1; j < FIRST_REF_NODE; j++)
-    {
-      if (!get_varinfo (j)->is_special_var)
-	bitmap_set_bit (graph->direct_nodes, j);
-    }
-
-  for (j = 0; j < graph->size; j++)
-    graph->eq_rep[j] = -1;
-
-  for (j = 0; j < varmap.length (); j++)
-    graph->indirect_cycles[j] = -1;
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      struct constraint_expr lhs = c->lhs;
-      struct constraint_expr rhs = c->rhs;
-      unsigned int lhsvar = lhs.var;
-      unsigned int rhsvar = rhs.var;
-
-      if (lhs.type == DEREF)
-	{
-	  /* *x = y.  */
-	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
-	    {
-	      if (lhs.var == anything_id)
-		add_pred_graph_edge (graph, storedanything_id, rhsvar);
-	      else
-		add_pred_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
-	    }
-	}
-      else if (rhs.type == DEREF)
-	{
-	  /* x = *y */
-	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
-	    add_pred_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
-	  else
-	    bitmap_clear_bit (graph->direct_nodes, lhsvar);
-	}
-      else if (rhs.type == ADDRESSOF)
-	{
-	  varinfo_t v;
-
-	  /* x = &y */
-	  if (graph->points_to[lhsvar] == NULL)
-	    graph->points_to[lhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
-	  bitmap_set_bit (graph->points_to[lhsvar], rhsvar);
-
-	  if (graph->pointed_by[rhsvar] == NULL)
-	    graph->pointed_by[rhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
-	  bitmap_set_bit (graph->pointed_by[rhsvar], lhsvar);
-
-	  /* Implicitly, *x = y */
-	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
-
-	  /* All related variables are no longer direct nodes.  */
-	  bitmap_clear_bit (graph->direct_nodes, rhsvar);
-          v = get_varinfo (rhsvar);
-          if (!v->is_full_var)
-            {
-              v = get_varinfo (v->head);
-              do
-                {
-                  bitmap_clear_bit (graph->direct_nodes, v->id);
-                  v = vi_next (v);
-                }
-              while (v != NULL);
-            }
-	  bitmap_set_bit (graph->address_taken, rhsvar);
-	}
-      else if (lhsvar > anything_id
-	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
-	{
-	  /* x = y */
-	  add_pred_graph_edge (graph, lhsvar, rhsvar);
-	  /* Implicitly, *x = *y */
-	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar,
-				   FIRST_REF_NODE + rhsvar);
-	}
-      else if (lhs.offset != 0 || rhs.offset != 0)
-	{
-	  if (rhs.offset != 0)
-	    bitmap_clear_bit (graph->direct_nodes, lhs.var);
-	  else if (lhs.offset != 0)
-	    bitmap_clear_bit (graph->direct_nodes, rhs.var);
-	}
-    }
-}
-
-/* Build the constraint graph, adding successor edges.  */
-
-static void
-build_succ_graph (void)
-{
-  unsigned i, t;
-  constraint_t c;
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      struct constraint_expr lhs;
-      struct constraint_expr rhs;
-      unsigned int lhsvar;
-      unsigned int rhsvar;
-
-      if (!c)
-	continue;
-
-      lhs = c->lhs;
-      rhs = c->rhs;
-      lhsvar = find (lhs.var);
-      rhsvar = find (rhs.var);
-
-      if (lhs.type == DEREF)
-	{
-	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
-	    {
-	      if (lhs.var == anything_id)
-		add_graph_edge (graph, storedanything_id, rhsvar);
-	      else
-		add_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
-	    }
-	}
-      else if (rhs.type == DEREF)
-	{
-	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
-	    add_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
-	}
-      else if (rhs.type == ADDRESSOF)
-	{
-	  /* x = &y */
-	  gcc_checking_assert (find (rhs.var) == rhs.var);
-	  bitmap_set_bit (get_varinfo (lhsvar)->solution, rhsvar);
-	}
-      else if (lhsvar > anything_id
-	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
-	{
-	  add_graph_edge (graph, lhsvar, rhsvar);
-	}
-    }
-
-  /* Add edges from STOREDANYTHING to all nodes that can receive pointers.  */
-  t = find (storedanything_id);
-  for (i = integer_id + 1; i < FIRST_REF_NODE; ++i)
-    {
-      if (get_varinfo (i)->may_have_pointers)
-	add_graph_edge (graph, find (i), t);
-    }
-
-  /* Everything stored to ANYTHING also potentially escapes.  */
-  add_graph_edge (graph, find (escaped_id), t);
-}
-
-
-/* Changed variables on the last iteration.  */
-static bitmap changed;
-
-/* Strongly Connected Component visitation info.  */
-
-class scc_info
-{
-public:
-  scc_info (size_t size);
-  ~scc_info ();
-
-  auto_sbitmap visited;
-  auto_sbitmap deleted;
-  unsigned int *dfs;
-  unsigned int *node_mapping;
-  int current_index;
-  auto_vec<unsigned> scc_stack;
-};
-
-
-/* Recursive routine to find strongly connected components in GRAPH.
-   SI is the SCC info to store the information in, and N is the id of current
-   graph node we are processing.
-
-   This is Tarjan's strongly connected component finding algorithm, as
-   modified by Nuutila to keep only non-root nodes on the stack.
-   The algorithm can be found in "On finding the strongly connected
-   connected components in a directed graph" by Esko Nuutila and Eljas
-   Soisalon-Soininen, in Information Processing Letters volume 49,
-   number 1, pages 9-14.  */
-
-static void
-scc_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
-{
-  unsigned int i;
-  bitmap_iterator bi;
-  unsigned int my_dfs;
-
-  bitmap_set_bit (si->visited, n);
-  si->dfs[n] = si->current_index ++;
-  my_dfs = si->dfs[n];
-
-  /* Visit all the successors.  */
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[n], 0, i, bi)
-    {
-      unsigned int w;
-
-      if (i > LAST_REF_NODE)
-	break;
-
-      w = find (i);
-      if (bitmap_bit_p (si->deleted, w))
-	continue;
-
-      if (!bitmap_bit_p (si->visited, w))
-	scc_visit (graph, si, w);
-
-      unsigned int t = find (w);
-      gcc_checking_assert (find (n) == n);
-      if (si->dfs[t] < si->dfs[n])
-	si->dfs[n] = si->dfs[t];
-    }
-
-  /* See if any components have been identified.  */
-  if (si->dfs[n] == my_dfs)
-    {
-      if (si->scc_stack.length () > 0
-	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
-	{
-	  bitmap scc = BITMAP_ALLOC (NULL);
-	  unsigned int lowest_node;
-	  bitmap_iterator bi;
-
-	  bitmap_set_bit (scc, n);
-
-	  while (si->scc_stack.length () != 0
-		 && si->dfs[si->scc_stack.last ()] >= my_dfs)
-	    {
-	      unsigned int w = si->scc_stack.pop ();
-
-	      bitmap_set_bit (scc, w);
-	    }
-
-	  lowest_node = bitmap_first_set_bit (scc);
-	  gcc_assert (lowest_node < FIRST_REF_NODE);
-
-	  /* Collapse the SCC nodes into a single node, and mark the
-	     indirect cycles.  */
-	  EXECUTE_IF_SET_IN_BITMAP (scc, 0, i, bi)
-	    {
-	      if (i < FIRST_REF_NODE)
-		{
-		  if (unite (lowest_node, i))
-		    unify_nodes (graph, lowest_node, i, false);
-		}
-	      else
-		{
-		  unite (lowest_node, i);
-		  graph->indirect_cycles[i - FIRST_REF_NODE] = lowest_node;
-		}
-	    }
-	  bitmap_set_bit (si->deleted, lowest_node);
-	}
-      else
-	bitmap_set_bit (si->deleted, n);
-    }
-  else
-    si->scc_stack.safe_push (n);
-}
-
-/* Unify node FROM into node TO, updating the changed count if
-   necessary when UPDATE_CHANGED is true.  */
-
-static void
-unify_nodes (constraint_graph_t graph, unsigned int to, unsigned int from,
-	     bool update_changed)
-{
-  gcc_checking_assert (to != from && find (to) == to);
-
-  if (dump_file && (dump_flags & TDF_DETAILS))
-    fprintf (dump_file, "Unifying %s to %s\n",
-	     get_varinfo (from)->name,
-	     get_varinfo (to)->name);
-
-  if (update_changed)
-    stats.unified_vars_dynamic++;
-  else
-    stats.unified_vars_static++;
-
-  merge_graph_nodes (graph, to, from);
-  if (merge_node_constraints (graph, to, from))
-    {
-      if (update_changed)
-	bitmap_set_bit (changed, to);
-    }
-
-  /* Mark TO as changed if FROM was changed. If TO was already marked
-     as changed, decrease the changed count.  */
-
-  if (update_changed
-      && bitmap_clear_bit (changed, from))
-    bitmap_set_bit (changed, to);
-  varinfo_t fromvi = get_varinfo (from);
-  if (fromvi->solution)
-    {
-      /* If the solution changes because of the merging, we need to mark
-	 the variable as changed.  */
-      varinfo_t tovi = get_varinfo (to);
-      if (bitmap_ior_into (tovi->solution, fromvi->solution))
-	{
-	  if (update_changed)
-	    bitmap_set_bit (changed, to);
-	}
-
-      BITMAP_FREE (fromvi->solution);
-      if (fromvi->oldsolution)
-	BITMAP_FREE (fromvi->oldsolution);
-
-      if (stats.iterations > 0
-	  && tovi->oldsolution)
-	BITMAP_FREE (tovi->oldsolution);
-    }
-  if (graph->succs[to])
-    bitmap_clear_bit (graph->succs[to], to);
-}
-
-/* Add a copy edge FROM -> TO, optimizing special cases.  Returns TRUE
-   if the solution of TO changed.  */
-
-static bool
-solve_add_graph_edge (constraint_graph_t graph, unsigned int to,
-		      unsigned int from)
-{
-  /* Adding edges from the special vars is pointless.
-     They don't have sets that can change.  */
-  if (get_varinfo (from)->is_special_var)
-    return bitmap_ior_into (get_varinfo (to)->solution,
-			    get_varinfo (from)->solution);
-  /* Merging the solution from ESCAPED needlessly increases
-     the set.  Use ESCAPED as representative instead.  */
-  else if (from == find (escaped_id))
-    return bitmap_set_bit (get_varinfo (to)->solution, escaped_id);
-  else if (get_varinfo (from)->may_have_pointers
-	   && add_graph_edge (graph, to, from))
-    return bitmap_ior_into (get_varinfo (to)->solution,
-			    get_varinfo (from)->solution);
-  return false;
-}
-
-/* Process a constraint C that represents x = *(y + off), using DELTA as the
-   starting solution for y.  */
-
-static void
-do_sd_constraint (constraint_graph_t graph, constraint_t c,
-		  bitmap delta, bitmap *expanded_delta)
-{
-  unsigned int lhs = c->lhs.var;
-  bool flag = false;
-  bitmap sol = get_varinfo (lhs)->solution;
-  unsigned int j;
-  bitmap_iterator bi;
-  HOST_WIDE_INT roffset = c->rhs.offset;
-
-  /* Our IL does not allow this.  */
-  gcc_checking_assert (c->lhs.offset == 0);
-
-  /* If the solution of Y contains anything it is good enough to transfer
-     this to the LHS.  */
-  if (bitmap_bit_p (delta, anything_id))
-    {
-      flag |= bitmap_set_bit (sol, anything_id);
-      goto done;
-    }
-
-  /* If we do not know at with offset the rhs is dereferenced compute
-     the reachability set of DELTA, conservatively assuming it is
-     dereferenced at all valid offsets.  */
-  if (roffset == UNKNOWN_OFFSET)
-    {
-      delta = solution_set_expand (delta, expanded_delta);
-      /* No further offset processing is necessary.  */
-      roffset = 0;
-    }
-
-  /* For each variable j in delta (Sol(y)), add
-     an edge in the graph from j to x, and union Sol(j) into Sol(x).  */
-  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
-    {
-      varinfo_t v = get_varinfo (j);
-      HOST_WIDE_INT fieldoffset = v->offset + roffset;
-      unsigned HOST_WIDE_INT size = v->size;
-      unsigned int t;
-
-      if (v->is_full_var)
-	;
-      else if (roffset != 0)
-	{
-	  if (fieldoffset < 0)
-	    v = get_varinfo (v->head);
-	  else
-	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
-	}
-
-      /* We have to include all fields that overlap the current field
-	 shifted by roffset.  */
-      do
-	{
-	  t = find (v->id);
-
-	  flag |= solve_add_graph_edge (graph, lhs, t);
-
-	  if (v->is_full_var
-	      || v->next == 0)
-	    break;
-
-	  v = vi_next (v);
-	}
-      while (v->offset < fieldoffset + size);
-    }
-
-done:
-  /* If the LHS solution changed, mark the var as changed.  */
-  if (flag)
-    bitmap_set_bit (changed, lhs);
-}
-
-/* Process a constraint C that represents *(x + off) = y using DELTA
-   as the starting solution for x.  */
-
-static void
-do_ds_constraint (constraint_t c, bitmap delta, bitmap *expanded_delta)
-{
-  unsigned int rhs = c->rhs.var;
-  bitmap sol = get_varinfo (rhs)->solution;
-  unsigned int j;
-  bitmap_iterator bi;
-  HOST_WIDE_INT loff = c->lhs.offset;
-  bool escaped_p = false;
-
-  /* Our IL does not allow this.  */
-  gcc_checking_assert (c->rhs.offset == 0);
-
-  /* If the solution of y contains ANYTHING simply use the ANYTHING
-     solution.  This avoids needlessly increasing the points-to sets.  */
-  if (bitmap_bit_p (sol, anything_id))
-    sol = get_varinfo (find (anything_id))->solution;
-
-  /* If the solution for x contains ANYTHING we have to merge the
-     solution of y into all pointer variables which we do via
-     STOREDANYTHING.  */
-  if (bitmap_bit_p (delta, anything_id))
-    {
-      unsigned t = find (storedanything_id);
-      if (solve_add_graph_edge (graph, t, rhs))
-	bitmap_set_bit (changed, t);
-      return;
-    }
-
-  /* If we do not know at with offset the rhs is dereferenced compute
-     the reachability set of DELTA, conservatively assuming it is
-     dereferenced at all valid offsets.  */
-  if (loff == UNKNOWN_OFFSET)
-    {
-      delta = solution_set_expand (delta, expanded_delta);
-      loff = 0;
-    }
-
-  /* For each member j of delta (Sol(x)), add an edge from y to j and
-     union Sol(y) into Sol(j) */
-  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
-    {
-      varinfo_t v = get_varinfo (j);
-      unsigned int t;
-      HOST_WIDE_INT fieldoffset = v->offset + loff;
-      unsigned HOST_WIDE_INT size = v->size;
-
-      if (v->is_full_var)
-	;
-      else if (loff != 0)
-	{
-	  if (fieldoffset < 0)
-	    v = get_varinfo (v->head);
-	  else
-	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
-	}
-
-      /* We have to include all fields that overlap the current field
-	 shifted by loff.  */
-      do
-	{
-	  if (v->may_have_pointers)
-	    {
-	      /* If v is a global variable then this is an escape point.  */
-	      if (v->is_global_var
-		  && !escaped_p)
-		{
-		  t = find (escaped_id);
-		  if (add_graph_edge (graph, t, rhs)
-		      && bitmap_ior_into (get_varinfo (t)->solution, sol))
-		    bitmap_set_bit (changed, t);
-		  /* Enough to let rhs escape once.  */
-		  escaped_p = true;
-		}
-
-	      if (v->is_special_var)
-		break;
-
-	      t = find (v->id);
-
-	      if (solve_add_graph_edge  (graph, t, rhs))
-		bitmap_set_bit (changed, t);
-	    }
-
-	  if (v->is_full_var
-	      || v->next == 0)
-	    break;
-
-	  v = vi_next (v);
-	}
-      while (v->offset < fieldoffset + size);
-    }
-}
-
-/* Handle a non-simple (simple meaning requires no iteration),
-   constraint (IE *x = &y, x = *y, *x = y, and x = y with offsets involved).  */
-
-static void
-do_complex_constraint (constraint_graph_t graph, constraint_t c, bitmap delta,
-		       bitmap *expanded_delta)
-{
-  if (c->lhs.type == DEREF)
-    {
-      if (c->rhs.type == ADDRESSOF)
-	{
-	  gcc_unreachable ();
-	}
-      else
-	{
-	  /* *x = y */
-	  do_ds_constraint (c, delta, expanded_delta);
-	}
-    }
-  else if (c->rhs.type == DEREF)
-    {
-      /* x = *y */
-      if (!(get_varinfo (c->lhs.var)->is_special_var))
-	do_sd_constraint (graph, c, delta, expanded_delta);
-    }
-  else
-    {
-      bitmap tmp;
-      bool flag = false;
-
-      gcc_checking_assert (c->rhs.type == SCALAR && c->lhs.type == SCALAR
-			   && c->rhs.offset != 0 && c->lhs.offset == 0);
-      tmp = get_varinfo (c->lhs.var)->solution;
-
-      flag = set_union_with_increment (tmp, delta, c->rhs.offset,
-				       expanded_delta);
-
-      if (flag)
-	bitmap_set_bit (changed, c->lhs.var);
-    }
-}
-
-/* Initialize and return a new SCC info structure.  */
-
-scc_info::scc_info (size_t size) :
-  visited (size), deleted (size), current_index (0), scc_stack (1)
-{
-  bitmap_clear (visited);
-  bitmap_clear (deleted);
-  node_mapping = XNEWVEC (unsigned int, size);
-  dfs = XCNEWVEC (unsigned int, size);
-
-  for (size_t i = 0; i < size; i++)
-    node_mapping[i] = i;
-}
-
-/* Free an SCC info structure pointed to by SI */
-
-scc_info::~scc_info ()
-{
-  free (node_mapping);
-  free (dfs);
-}
-
-
-/* Find indirect cycles in GRAPH that occur, using strongly connected
-   components, and note them in the indirect cycles map.
-
-   This technique comes from Ben Hardekopf and Calvin Lin,
-   "It Pays to be Lazy: Fast and Accurate Pointer Analysis for Millions of
-   Lines of Code", submitted to PLDI 2007.  */
-
-static void
-find_indirect_cycles (constraint_graph_t graph)
-{
-  unsigned int i;
-  unsigned int size = graph->size;
-  scc_info si (size);
-
-  for (i = 0; i < MIN (LAST_REF_NODE, size); i ++ )
-    if (!bitmap_bit_p (si.visited, i) && find (i) == i)
-      scc_visit (graph, &si, i);
-}
-
-/* Visit the graph in topological order starting at node N, and store the
-   order in TOPO_ORDER using VISITED to indicate visited nodes.  */
-
-static void
-topo_visit (constraint_graph_t graph, vec<unsigned> &topo_order,
-	    sbitmap visited, unsigned int n)
-{
-  bitmap_iterator bi;
-  unsigned int j;
-
-  bitmap_set_bit (visited, n);
-
-  if (graph->succs[n])
-    EXECUTE_IF_SET_IN_BITMAP (graph->succs[n], 0, j, bi)
-      {
-	unsigned k = find (j);
-	if (!bitmap_bit_p (visited, k))
-	  topo_visit (graph, topo_order, visited, k);
-      }
-
-  /* Also consider copy with offset complex constraints as implicit edges.  */
-  for (auto c : graph->complex[n])
-    {
-      /* Constraints are ordered so that SCALAR = SCALAR appear first.  */
-      if (c->lhs.type != SCALAR || c->rhs.type != SCALAR)
-	break;
-      gcc_checking_assert (c->rhs.var == n);
-      unsigned k = find (c->lhs.var);
-      if (!bitmap_bit_p (visited, k))
-	topo_visit (graph, topo_order, visited, k);
-    }
-
-  topo_order.quick_push (n);
-}
-
-/* Compute a topological ordering for GRAPH, and return the result.  */
-
-static auto_vec<unsigned>
-compute_topo_order (constraint_graph_t graph)
-{
-  unsigned int i;
-  unsigned int size = graph->size;
-
-  auto_sbitmap visited (size);
-  bitmap_clear (visited);
-
-  /* For the heuristic in add_graph_edge to work optimally make sure to
-     first visit the connected component of the graph containing
-     ESCAPED.  Do this by extracting the connected component
-     with ESCAPED and append that to all other components as solve_graph
-     pops from the order.  */
-  auto_vec<unsigned> tail (size);
-  topo_visit (graph, tail, visited, find (escaped_id));
-
-  auto_vec<unsigned> topo_order (size);
-
-  for (i = 0; i != size; ++i)
-    if (!bitmap_bit_p (visited, i) && find (i) == i)
-      topo_visit (graph, topo_order, visited, i);
-
-  topo_order.splice (tail);
-  return topo_order;
-}
-
-/* Structure used to for hash value numbering of pointer equivalence
-   classes.  */
-
-typedef struct equiv_class_label
-{
-  hashval_t hashcode;
-  unsigned int equivalence_class;
-  bitmap labels;
-} *equiv_class_label_t;
-typedef const struct equiv_class_label *const_equiv_class_label_t;
-
-/* Equiv_class_label hashtable helpers.  */
-
-struct equiv_class_hasher : nofree_ptr_hash <equiv_class_label>
-{
-  static inline hashval_t hash (const equiv_class_label *);
-  static inline bool equal (const equiv_class_label *,
-			    const equiv_class_label *);
-};
-
-/* Hash function for a equiv_class_label_t */
-
-inline hashval_t
-equiv_class_hasher::hash (const equiv_class_label *ecl)
-{
-  return ecl->hashcode;
-}
-
-/* Equality function for two equiv_class_label_t's.  */
-
-inline bool
-equiv_class_hasher::equal (const equiv_class_label *eql1,
-			   const equiv_class_label *eql2)
-{
-  return (eql1->hashcode == eql2->hashcode
-	  && bitmap_equal_p (eql1->labels, eql2->labels));
-}
-
-/* A hashtable for mapping a bitmap of labels->pointer equivalence
-   classes.  */
-static hash_table<equiv_class_hasher> *pointer_equiv_class_table;
-
-/* A hashtable for mapping a bitmap of labels->location equivalence
-   classes.  */
-static hash_table<equiv_class_hasher> *location_equiv_class_table;
-
-struct obstack equiv_class_obstack;
-
-/* Lookup a equivalence class in TABLE by the bitmap of LABELS with
-   hash HAS it contains.  Sets *REF_LABELS to the bitmap LABELS
-   is equivalent to.  */
-
-static equiv_class_label *
-equiv_class_lookup_or_add (hash_table<equiv_class_hasher> *table,
-			   bitmap labels)
-{
-  equiv_class_label **slot;
-  equiv_class_label ecl;
-
-  ecl.labels = labels;
-  ecl.hashcode = bitmap_hash (labels);
-  slot = table->find_slot (&ecl, INSERT);
-  if (!*slot)
-    {
-      *slot = XOBNEW (&equiv_class_obstack, struct equiv_class_label);
-      (*slot)->labels = labels;
-      (*slot)->hashcode = ecl.hashcode;
-      (*slot)->equivalence_class = 0;
-    }
-
-  return *slot;
-}
-
-/* Perform offline variable substitution.
-
-   This is a worst case quadratic time way of identifying variables
-   that must have equivalent points-to sets, including those caused by
-   static cycles, and single entry subgraphs, in the constraint graph.
-
-   The technique is described in "Exploiting Pointer and Location
-   Equivalence to Optimize Pointer Analysis. In the 14th International
-   Static Analysis Symposium (SAS), August 2007."  It is known as the
-   "HU" algorithm, and is equivalent to value numbering the collapsed
-   constraint graph including evaluating unions.
-
-   The general method of finding equivalence classes is as follows:
-   Add fake nodes (REF nodes) and edges for *a = b and a = *b constraints.
-   Initialize all non-REF nodes to be direct nodes.
-   For each constraint a = a U {b}, we set pts(a) = pts(a) u {fresh
-   variable}
-   For each constraint containing the dereference, we also do the same
-   thing.
-
-   We then compute SCC's in the graph and unify nodes in the same SCC,
-   including pts sets.
-
-   For each non-collapsed node x:
-    Visit all unvisited explicit incoming edges.
-    Ignoring all non-pointers, set pts(x) = Union of pts(a) for y
-    where y->x.
-    Lookup the equivalence class for pts(x).
-     If we found one, equivalence_class(x) = found class.
-     Otherwise, equivalence_class(x) = new class, and new_class is
-    added to the lookup table.
-
-   All direct nodes with the same equivalence class can be replaced
-   with a single representative node.
-   All unlabeled nodes (label == 0) are not pointers and all edges
-   involving them can be eliminated.
-   We perform these optimizations during rewrite_constraints
-
-   In addition to pointer equivalence class finding, we also perform
-   location equivalence class finding.  This is the set of variables
-   that always appear together in points-to sets.  We use this to
-   compress the size of the points-to sets.  */
-
-/* Current maximum pointer equivalence class id.  */
-static int pointer_equiv_class;
-
-/* Current maximum location equivalence class id.  */
-static int location_equiv_class;
-
-/* Recursive routine to find strongly connected components in GRAPH,
-   and label it's nodes with DFS numbers.  */
-
-static void
-condense_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
-{
-  unsigned int i;
-  bitmap_iterator bi;
-  unsigned int my_dfs;
-
-  gcc_checking_assert (si->node_mapping[n] == n);
-  bitmap_set_bit (si->visited, n);
-  si->dfs[n] = si->current_index ++;
-  my_dfs = si->dfs[n];
-
-  /* Visit all the successors.  */
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
-    {
-      unsigned int w = si->node_mapping[i];
-
-      if (bitmap_bit_p (si->deleted, w))
-	continue;
-
-      if (!bitmap_bit_p (si->visited, w))
-	condense_visit (graph, si, w);
-
-      unsigned int t = si->node_mapping[w];
-      gcc_checking_assert (si->node_mapping[n] == n);
-      if (si->dfs[t] < si->dfs[n])
-	si->dfs[n] = si->dfs[t];
-    }
-
-  /* Visit all the implicit predecessors.  */
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->implicit_preds[n], 0, i, bi)
-    {
-      unsigned int w = si->node_mapping[i];
-
-      if (bitmap_bit_p (si->deleted, w))
-	continue;
-
-      if (!bitmap_bit_p (si->visited, w))
-	condense_visit (graph, si, w);
-
-      unsigned int t = si->node_mapping[w];
-      gcc_assert (si->node_mapping[n] == n);
-      if (si->dfs[t] < si->dfs[n])
-	si->dfs[n] = si->dfs[t];
-    }
-
-  /* See if any components have been identified.  */
-  if (si->dfs[n] == my_dfs)
-    {
-      if (si->scc_stack.length () != 0
-	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
-	{
-	  /* Find the first node of the SCC and do non-bitmap work.  */
-	  bool direct_p = true;
-	  unsigned first = si->scc_stack.length ();
-	  do
-	    {
-	      --first;
-	      unsigned int w = si->scc_stack[first];
-	      si->node_mapping[w] = n;
-	      if (!bitmap_bit_p (graph->direct_nodes, w))
-		direct_p = false;
-	    }
-	  while (first > 0
-		 && si->dfs[si->scc_stack[first - 1]] >= my_dfs);
-	  if (!direct_p)
-	    bitmap_clear_bit (graph->direct_nodes, n);
-
-	  /* Want to reduce to node n, push that first.  */
-	  si->scc_stack.reserve (1);
-	  si->scc_stack.quick_push (si->scc_stack[first]);
-	  si->scc_stack[first] = n;
-
-	  unsigned scc_size = si->scc_stack.length () - first;
-	  unsigned split = scc_size / 2;
-	  unsigned carry = scc_size - split * 2;
-	  while (split > 0)
-	    {
-	      for (unsigned i = 0; i < split; ++i)
-		{
-		  unsigned a = si->scc_stack[first + i];
-		  unsigned b = si->scc_stack[first + split + carry + i];
-
-		  /* Unify our nodes.  */
-		  if (graph->preds[b])
-		    {
-		      if (!graph->preds[a])
-			std::swap (graph->preds[a], graph->preds[b]);
-		      else
-			bitmap_ior_into_and_free (graph->preds[a],
-						  &graph->preds[b]);
-		    }
-		  if (graph->implicit_preds[b])
-		    {
-		      if (!graph->implicit_preds[a])
-			std::swap (graph->implicit_preds[a],
-				   graph->implicit_preds[b]);
-		      else
-			bitmap_ior_into_and_free (graph->implicit_preds[a],
-						  &graph->implicit_preds[b]);
-		    }
-		  if (graph->points_to[b])
-		    {
-		      if (!graph->points_to[a])
-			std::swap (graph->points_to[a], graph->points_to[b]);
-		      else
-			bitmap_ior_into_and_free (graph->points_to[a],
-						  &graph->points_to[b]);
-		    }
-		}
-	      unsigned remain = split + carry;
-	      split = remain / 2;
-	      carry = remain - split * 2;
-	    }
-	  /* Actually pop the SCC.  */
-	  si->scc_stack.truncate (first);
-	}
-      bitmap_set_bit (si->deleted, n);
-    }
-  else
-    si->scc_stack.safe_push (n);
-}
-
-/* Label pointer equivalences.
-
-   This performs a value numbering of the constraint graph to
-   discover which variables will always have the same points-to sets
-   under the current set of constraints.
-
-   The way it value numbers is to store the set of points-to bits
-   generated by the constraints and graph edges.  This is just used as a
-   hash and equality comparison.  The *actual set of points-to bits* is
-   completely irrelevant, in that we don't care about being able to
-   extract them later.
-
-   The equality values (currently bitmaps) just have to satisfy a few
-   constraints, the main ones being:
-   1. The combining operation must be order independent.
-   2. The end result of a given set of operations must be unique iff the
-      combination of input values is unique
-   3. Hashable.  */
-
-static void
-label_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
-{
-  unsigned int i, first_pred;
-  bitmap_iterator bi;
-
-  bitmap_set_bit (si->visited, n);
-
-  /* Label and union our incoming edges's points to sets.  */
-  first_pred = -1U;
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
-    {
-      unsigned int w = si->node_mapping[i];
-      if (!bitmap_bit_p (si->visited, w))
-	label_visit (graph, si, w);
-
-      /* Skip unused edges  */
-      if (w == n || graph->pointer_label[w] == 0)
-	continue;
-
-      if (graph->points_to[w])
-	{
-	  if (!graph->points_to[n])
-	    {
-	      if (first_pred == -1U)
-		first_pred = w;
-	      else
-		{
-		  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
-		  bitmap_ior (graph->points_to[n],
-			      graph->points_to[first_pred],
-			      graph->points_to[w]);
-		}
-	    }
-	  else
-	    bitmap_ior_into (graph->points_to[n], graph->points_to[w]);
-	}
-    }
-
-  /* Indirect nodes get fresh variables and a new pointer equiv class.  */
-  if (!bitmap_bit_p (graph->direct_nodes, n))
-    {
-      if (!graph->points_to[n])
-	{
-	  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
-	  if (first_pred != -1U)
-	    bitmap_copy (graph->points_to[n], graph->points_to[first_pred]);
-	}
-      bitmap_set_bit (graph->points_to[n], FIRST_REF_NODE + n);
-      graph->pointer_label[n] = pointer_equiv_class++;
-      equiv_class_label_t ecl;
-      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
-				       graph->points_to[n]);
-      ecl->equivalence_class = graph->pointer_label[n];
-      return;
-    }
-
-  /* If there was only a single non-empty predecessor the pointer equiv
-     class is the same.  */
-  if (!graph->points_to[n])
-    {
-      if (first_pred != -1U)
-	{
-	  graph->pointer_label[n] = graph->pointer_label[first_pred];
-	  graph->points_to[n] = graph->points_to[first_pred];
-	}
-      return;
-    }
-
-  if (!bitmap_empty_p (graph->points_to[n]))
-    {
-      equiv_class_label_t ecl;
-      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
-				       graph->points_to[n]);
-      if (ecl->equivalence_class == 0)
-	ecl->equivalence_class = pointer_equiv_class++;
-      else
-	{
-	  BITMAP_FREE (graph->points_to[n]);
-	  graph->points_to[n] = ecl->labels;
-	}
-      graph->pointer_label[n] = ecl->equivalence_class;
-    }
-}
-
-/* Print the pred graph in dot format.  */
-
-static void
-dump_pred_graph (class scc_info *si, FILE *file)
-{
-  unsigned int i;
-
-  /* Only print the graph if it has already been initialized:  */
-  if (!graph)
-    return;
-
-  /* Prints the header of the dot file:  */
-  fprintf (file, "strict digraph {\n");
-  fprintf (file, "  node [\n    shape = box\n  ]\n");
-  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
-  fprintf (file, "\n  // List of nodes and complex constraints in "
-	   "the constraint graph:\n");
-
-  /* The next lines print the nodes in the graph together with the
-     complex constraints attached to them.  */
-  for (i = 1; i < graph->size; i++)
-    {
-      if (i == FIRST_REF_NODE)
-	continue;
-      if (si->node_mapping[i] != i)
-	continue;
-      if (i < FIRST_REF_NODE)
-	fprintf (file, "\"%s\"", get_varinfo (i)->name);
-      else
-	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-      if (graph->points_to[i]
-	  && !bitmap_empty_p (graph->points_to[i]))
-	{
-	  if (i < FIRST_REF_NODE)
-	    fprintf (file, "[label=\"%s = {", get_varinfo (i)->name);
-	  else
-	    fprintf (file, "[label=\"*%s = {",
-		     get_varinfo (i - FIRST_REF_NODE)->name);
-	  unsigned j;
-	  bitmap_iterator bi;
-	  EXECUTE_IF_SET_IN_BITMAP (graph->points_to[i], 0, j, bi)
-	    fprintf (file, " %d", j);
-	  fprintf (file, " }\"]");
-	}
-      fprintf (file, ";\n");
-    }
-
-  /* Go over the edges.  */
-  fprintf (file, "\n  // Edges in the constraint graph:\n");
-  for (i = 1; i < graph->size; i++)
-    {
-      unsigned j;
-      bitmap_iterator bi;
-      if (si->node_mapping[i] != i)
-	continue;
-      EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[i], 0, j, bi)
-	{
-	  unsigned from = si->node_mapping[j];
-	  if (from < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (from)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (from - FIRST_REF_NODE)->name);
-	  fprintf (file, " -> ");
-	  if (i < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-	  fprintf (file, ";\n");
-	}
-    }
-
-  /* Prints the tail of the dot file.  */
-  fprintf (file, "}\n");
-}
-
-/* Perform offline variable substitution, discovering equivalence
-   classes, and eliminating non-pointer variables.  */
-
-static class scc_info *
-perform_var_substitution (constraint_graph_t graph)
-{
-  unsigned int i;
-  unsigned int size = graph->size;
-  scc_info *si = new scc_info (size);
-
-  bitmap_obstack_initialize (&iteration_obstack);
-  gcc_obstack_init (&equiv_class_obstack);
-  pointer_equiv_class_table = new hash_table<equiv_class_hasher> (511);
-  location_equiv_class_table
-    = new hash_table<equiv_class_hasher> (511);
-  pointer_equiv_class = 1;
-  location_equiv_class = 1;
-
-  /* Condense the nodes, which means to find SCC's, count incoming
-     predecessors, and unite nodes in SCC's.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
-      condense_visit (graph, si, si->node_mapping[i]);
-
-  if (dump_file && (dump_flags & TDF_GRAPH))
-    {
-      fprintf (dump_file, "\n\n// The constraint graph before var-substitution "
-	       "in dot format:\n");
-      dump_pred_graph (si, dump_file);
-      fprintf (dump_file, "\n\n");
-    }
-
-  bitmap_clear (si->visited);
-  /* Actually the label the nodes for pointer equivalences  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
-      label_visit (graph, si, si->node_mapping[i]);
-
-  /* Calculate location equivalence labels.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      bitmap pointed_by;
-      bitmap_iterator bi;
-      unsigned int j;
-
-      if (!graph->pointed_by[i])
-	continue;
-      pointed_by = BITMAP_ALLOC (&iteration_obstack);
-
-      /* Translate the pointed-by mapping for pointer equivalence
-	 labels.  */
-      EXECUTE_IF_SET_IN_BITMAP (graph->pointed_by[i], 0, j, bi)
-	{
-	  bitmap_set_bit (pointed_by,
-			  graph->pointer_label[si->node_mapping[j]]);
-	}
-      /* The original pointed_by is now dead.  */
-      BITMAP_FREE (graph->pointed_by[i]);
-
-      /* Look up the location equivalence label if one exists, or make
-	 one otherwise.  */
-      equiv_class_label_t ecl;
-      ecl = equiv_class_lookup_or_add (location_equiv_class_table, pointed_by);
-      if (ecl->equivalence_class == 0)
-	ecl->equivalence_class = location_equiv_class++;
-      else
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    fprintf (dump_file, "Found location equivalence for node %s\n",
-		     get_varinfo (i)->name);
-	  BITMAP_FREE (pointed_by);
-	}
-      graph->loc_label[i] = ecl->equivalence_class;
-
-    }
-
-  if (dump_file && (dump_flags & TDF_DETAILS))
-    for (i = 1; i < FIRST_REF_NODE; i++)
-      {
-	unsigned j = si->node_mapping[i];
-	if (j != i)
-	  {
-	    fprintf (dump_file, "%s node id %d ",
-		     bitmap_bit_p (graph->direct_nodes, i)
-		     ? "Direct" : "Indirect", i);
-	    if (i < FIRST_REF_NODE)
-	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
-	    else
-	      fprintf (dump_file, "\"*%s\"",
-		       get_varinfo (i - FIRST_REF_NODE)->name);
-	    fprintf (dump_file, " mapped to SCC leader node id %d ", j);
-	    if (j < FIRST_REF_NODE)
-	      fprintf (dump_file, "\"%s\"\n", get_varinfo (j)->name);
-	    else
-	      fprintf (dump_file, "\"*%s\"\n",
-		       get_varinfo (j - FIRST_REF_NODE)->name);
-	  }
-	else
-	  {
-	    fprintf (dump_file,
-		     "Equivalence classes for %s node id %d ",
-		     bitmap_bit_p (graph->direct_nodes, i)
-		     ? "direct" : "indirect", i);
-	    if (i < FIRST_REF_NODE)
-	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
-	    else
-	      fprintf (dump_file, "\"*%s\"",
-		       get_varinfo (i - FIRST_REF_NODE)->name);
-	    fprintf (dump_file,
-		     ": pointer %d, location %d\n",
-		     graph->pointer_label[i], graph->loc_label[i]);
-	  }
-      }
-
-  /* Quickly eliminate our non-pointer variables.  */
-
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      unsigned int node = si->node_mapping[i];
-
-      if (graph->pointer_label[node] == 0)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    fprintf (dump_file,
-		     "%s is a non-pointer variable, eliminating edges.\n",
-		     get_varinfo (node)->name);
-	  stats.nonpointer_vars++;
-	  clear_edges_for_node (graph, node);
-	}
-    }
-
-  return si;
-}
-
-/* Free information that was only necessary for variable
-   substitution.  */
-
-static void
-free_var_substitution_info (class scc_info *si)
-{
-  delete si;
-  free (graph->pointer_label);
-  free (graph->loc_label);
-  free (graph->pointed_by);
-  free (graph->points_to);
-  free (graph->eq_rep);
-  sbitmap_free (graph->direct_nodes);
-  delete pointer_equiv_class_table;
-  pointer_equiv_class_table = NULL;
-  delete location_equiv_class_table;
-  location_equiv_class_table = NULL;
-  obstack_free (&equiv_class_obstack, NULL);
-  bitmap_obstack_release (&iteration_obstack);
-}
-
-/* Return an existing node that is equivalent to NODE, which has
-   equivalence class LABEL, if one exists.  Return NODE otherwise.  */
-
-static unsigned int
-find_equivalent_node (constraint_graph_t graph,
-		      unsigned int node, unsigned int label)
-{
-  /* If the address version of this variable is unused, we can
-     substitute it for anything else with the same label.
-     Otherwise, we know the pointers are equivalent, but not the
-     locations, and we can unite them later.  */
-
-  if (!bitmap_bit_p (graph->address_taken, node))
-    {
-      gcc_checking_assert (label < graph->size);
-
-      if (graph->eq_rep[label] != -1)
-	{
-	  /* Unify the two variables since we know they are equivalent.  */
-	  if (unite (graph->eq_rep[label], node))
-	    unify_nodes (graph, graph->eq_rep[label], node, false);
-	  return graph->eq_rep[label];
-	}
-      else
-	{
-	  graph->eq_rep[label] = node;
-	  graph->pe_rep[label] = node;
-	}
-    }
-  else
-    {
-      gcc_checking_assert (label < graph->size);
-      graph->pe[node] = label;
-      if (graph->pe_rep[label] == -1)
-	graph->pe_rep[label] = node;
-    }
-
-  return node;
-}
-
-/* Unite pointer equivalent but not location equivalent nodes in
-   GRAPH.  This may only be performed once variable substitution is
-   finished.  */
-
-static void
-unite_pointer_equivalences (constraint_graph_t graph)
-{
-  unsigned int i;
-
-  /* Go through the pointer equivalences and unite them to their
-     representative, if they aren't already.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      unsigned int label = graph->pe[i];
-      if (label)
-	{
-	  int label_rep = graph->pe_rep[label];
-
-	  if (label_rep == -1)
-	    continue;
-
-	  label_rep = find (label_rep);
-	  if (label_rep >= 0 && unite (label_rep, find (i)))
-	    unify_nodes (graph, label_rep, i, false);
-	}
-    }
-}
-
-/* Move complex constraints to the GRAPH nodes they belong to.  */
-
-static void
-move_complex_constraints (constraint_graph_t graph)
-{
-  int i;
-  constraint_t c;
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      if (c)
-	{
-	  struct constraint_expr lhs = c->lhs;
-	  struct constraint_expr rhs = c->rhs;
-
-	  if (lhs.type == DEREF)
-	    {
-	      insert_into_complex (graph, lhs.var, c);
-	    }
-	  else if (rhs.type == DEREF)
-	    {
-	      if (!(get_varinfo (lhs.var)->is_special_var))
-		insert_into_complex (graph, rhs.var, c);
-	    }
-	  else if (rhs.type != ADDRESSOF && lhs.var > anything_id
-		   && (lhs.offset != 0 || rhs.offset != 0))
-	    {
-	      insert_into_complex (graph, rhs.var, c);
-	    }
-	}
-    }
-}
-
-
-/* Optimize and rewrite complex constraints while performing
-   collapsing of equivalent nodes.  SI is the SCC_INFO that is the
-   result of perform_variable_substitution.  */
-
-static void
-rewrite_constraints (constraint_graph_t graph,
-		     class scc_info *si)
-{
-  int i;
-  constraint_t c;
-
-  if (flag_checking)
-    {
-      for (unsigned int j = 0; j < graph->size; j++)
-	gcc_assert (find (j) == j);
-    }
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      struct constraint_expr lhs = c->lhs;
-      struct constraint_expr rhs = c->rhs;
-      unsigned int lhsvar = find (lhs.var);
-      unsigned int rhsvar = find (rhs.var);
-      unsigned int lhsnode, rhsnode;
-      unsigned int lhslabel, rhslabel;
-
-      lhsnode = si->node_mapping[lhsvar];
-      rhsnode = si->node_mapping[rhsvar];
-      lhslabel = graph->pointer_label[lhsnode];
-      rhslabel = graph->pointer_label[rhsnode];
-
-      /* See if it is really a non-pointer variable, and if so, ignore
-	 the constraint.  */
-      if (lhslabel == 0)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    {
-
-	      fprintf (dump_file, "%s is a non-pointer variable, "
-		       "ignoring constraint:",
-		       get_varinfo (lhs.var)->name);
-	      dump_constraint (dump_file, c);
-	      fprintf (dump_file, "\n");
-	    }
-	  constraints[i] = NULL;
-	  continue;
-	}
-
-      if (rhslabel == 0)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    {
-
-	      fprintf (dump_file, "%s is a non-pointer variable, "
-		       "ignoring constraint:",
-		       get_varinfo (rhs.var)->name);
-	      dump_constraint (dump_file, c);
-	      fprintf (dump_file, "\n");
-	    }
-	  constraints[i] = NULL;
-	  continue;
-	}
-
-      lhsvar = find_equivalent_node (graph, lhsvar, lhslabel);
-      rhsvar = find_equivalent_node (graph, rhsvar, rhslabel);
-      c->lhs.var = lhsvar;
-      c->rhs.var = rhsvar;
-    }
-}
-
-/* Eliminate indirect cycles involving NODE.  Return true if NODE was
-   part of an SCC, false otherwise.  */
-
-static bool
-eliminate_indirect_cycles (unsigned int node)
-{
-  if (graph->indirect_cycles[node] != -1
-      && !bitmap_empty_p (get_varinfo (node)->solution))
-    {
-      unsigned int i;
-      auto_vec<unsigned> queue;
-      int queuepos;
-      unsigned int to = find (graph->indirect_cycles[node]);
-      bitmap_iterator bi;
-
-      /* We can't touch the solution set and call unify_nodes
-	 at the same time, because unify_nodes is going to do
-	 bitmap unions into it. */
-
-      EXECUTE_IF_SET_IN_BITMAP (get_varinfo (node)->solution, 0, i, bi)
-	{
-	  if (find (i) == i && i != to)
-	    {
-	      if (unite (to, i))
-		queue.safe_push (i);
-	    }
-	}
-
-      for (queuepos = 0;
-	   queue.iterate (queuepos, &i);
-	   queuepos++)
-	{
-	  unify_nodes (graph, to, i, true);
-	}
-      return true;
-    }
-  return false;
-}
-
-/* Solve the constraint graph GRAPH using our worklist solver.
-   This is based on the PW* family of solvers from the "Efficient Field
-   Sensitive Pointer Analysis for C" paper.
-   It works by iterating over all the graph nodes, processing the complex
-   constraints and propagating the copy constraints, until everything stops
-   changed.  This corresponds to steps 6-8 in the solving list given above.  */
-
-static void
-solve_graph (constraint_graph_t graph)
-{
-  unsigned int size = graph->size;
-  unsigned int i;
-  bitmap pts;
-
-  changed = BITMAP_ALLOC (NULL);
-
-  /* Mark all initial non-collapsed nodes as changed.  */
-  for (i = 1; i < size; i++)
-    {
-      varinfo_t ivi = get_varinfo (i);
-      if (find (i) == i && !bitmap_empty_p (ivi->solution)
-	  && ((graph->succs[i] && !bitmap_empty_p (graph->succs[i]))
-	      || graph->complex[i].length () > 0))
-	bitmap_set_bit (changed, i);
-    }
-
-  /* Allocate a bitmap to be used to store the changed bits.  */
-  pts = BITMAP_ALLOC (&pta_obstack);
-
-  while (!bitmap_empty_p (changed))
-    {
-      unsigned int i;
-      stats.iterations++;
-
-      bitmap_obstack_initialize (&iteration_obstack);
-
-      auto_vec<unsigned> topo_order = compute_topo_order (graph);
-      while (topo_order.length () != 0)
-	{
-	  i = topo_order.pop ();
-
-	  /* If this variable is not a representative, skip it.  */
-	  if (find (i) != i)
-	    continue;
-
-	  /* In certain indirect cycle cases, we may merge this
-	     variable to another.  */
-	  if (eliminate_indirect_cycles (i) && find (i) != i)
-	    continue;
-
-	  /* If the node has changed, we need to process the
-	     complex constraints and outgoing edges again.  For complex
-	     constraints that modify i itself, like the common group of
-	       callarg = callarg + UNKNOWN;
-	       callarg = *callarg + UNKNOWN;
-	       *callarg = callescape;
-	     make sure to iterate immediately because that maximizes
-	     cache reuse and expands the graph quickest, leading to
-	     better visitation order in the next iteration.  */
-	  while (bitmap_clear_bit (changed, i))
-	    {
-	      bitmap solution;
-	      vec<constraint_t> &complex = graph->complex[i];
-	      varinfo_t vi = get_varinfo (i);
-	      bool solution_empty;
-
-	      /* Compute the changed set of solution bits.  If anything
-	         is in the solution just propagate that.  */
-	      if (bitmap_bit_p (vi->solution, anything_id))
-		{
-		  /* If anything is also in the old solution there is
-		     nothing to do.
-		     ???  But we shouldn't ended up with "changed" set ...  */
-		  if (vi->oldsolution
-		      && bitmap_bit_p (vi->oldsolution, anything_id))
-		    break;
-		  bitmap_copy (pts, get_varinfo (find (anything_id))->solution);
-		}
-	      else if (vi->oldsolution)
-		bitmap_and_compl (pts, vi->solution, vi->oldsolution);
-	      else
-		bitmap_copy (pts, vi->solution);
-
-	      if (bitmap_empty_p (pts))
-		break;
-
-	      if (vi->oldsolution)
-		bitmap_ior_into (vi->oldsolution, pts);
-	      else
-		{
-		  vi->oldsolution = BITMAP_ALLOC (&oldpta_obstack);
-		  bitmap_copy (vi->oldsolution, pts);
-		}
-
-	      solution = vi->solution;
-	      solution_empty = bitmap_empty_p (solution);
-
-	      /* Process the complex constraints */
-	      hash_set<constraint_t> *cvisited = nullptr;
-	      if (flag_checking)
-		cvisited = new hash_set<constraint_t>;
-	      bitmap expanded_pts = NULL;
-	      for (unsigned j = 0; j < complex.length (); ++j)
-		{
-		  constraint_t c = complex[j];
-		  /* At unification time only the directly involved nodes
-		     will get their complex constraints updated.  Update
-		     our complex constraints now but keep the constraint
-		     vector sorted and clear of duplicates.  Also make
-		     sure to evaluate each prevailing constraint only once.  */
-		  unsigned int new_lhs = find (c->lhs.var);
-		  unsigned int new_rhs = find (c->rhs.var);
-		  if (c->lhs.var != new_lhs || c->rhs.var != new_rhs)
-		    {
-		      constraint tem = *c;
-		      tem.lhs.var = new_lhs;
-		      tem.rhs.var = new_rhs;
-		      unsigned int place
-			= complex.lower_bound (&tem, constraint_less);
-		      c->lhs.var = new_lhs;
-		      c->rhs.var = new_rhs;
-		      if (place != j)
-			{
-			  complex.ordered_remove (j);
-			  if (j < place)
-			    --place;
-			  if (place < complex.length ())
-			    {
-			      if (constraint_equal (*complex[place], *c))
-				{
-				  j--;
-				  continue;
-				}
-			      else
-				complex.safe_insert (place, c);
-			    }
-			  else
-			    complex.quick_push (c);
-			  if (place > j)
-			    {
-			      j--;
-			      continue;
-			    }
-			}
-		    }
-
-		  /* The only complex constraint that can change our
-		     solution to non-empty, given an empty solution,
-		     is a constraint where the lhs side is receiving
-		     some set from elsewhere.  */
-		  if (cvisited && cvisited->add (c))
-		    gcc_unreachable ();
-		  if (!solution_empty || c->lhs.type != DEREF)
-		    do_complex_constraint (graph, c, pts, &expanded_pts);
-		}
-	      if (cvisited)
-		{
-		  /* When checking, verify the order of constraints is
-		     maintained and each constraint is evaluated exactly
-		     once.  */
-		  for (unsigned j = 1; j < complex.length (); ++j)
-		    gcc_assert (constraint_less (complex[j-1], complex[j]));
-		  gcc_assert (cvisited->elements () == complex.length ());
-		  delete cvisited;
-		}
-	      BITMAP_FREE (expanded_pts);
-
-	      solution_empty = bitmap_empty_p (solution);
-
-	      if (!solution_empty)
-		{
-		  bitmap_iterator bi;
-		  unsigned eff_escaped_id = find (escaped_id);
-		  unsigned j;
-
-		  /* Propagate solution to all successors.  */
-		  unsigned to_remove = ~0U;
-		  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i],
-						0, j, bi)
-		    {
-		      if (to_remove != ~0U)
-			{
-			  bitmap_clear_bit (graph->succs[i], to_remove);
-			  to_remove = ~0U;
-			}
-		      unsigned int to = find (j);
-		      if (to != j)
-			{
-			  /* Update the succ graph, avoiding duplicate
-			     work.  */
-			  to_remove = j;
-			  if (! bitmap_set_bit (graph->succs[i], to))
-			    continue;
-			  /* We eventually end up processing 'to' twice
-			     as it is undefined whether bitmap iteration
-			     iterates over bits set during iteration.
-			     Play safe instead of doing tricks.  */
-			}
-		      /* Don't try to propagate to ourselves.  */
-		      if (to == i)
-			{
-			  to_remove = j;
-			  continue;
-			}
-		      /* Early node unification can lead to edges from
-			 escaped - remove them.  */
-		      if (i == eff_escaped_id)
-			{
-			  to_remove = j;
-			  if (bitmap_set_bit (get_varinfo (to)->solution,
-					      escaped_id))
-			    bitmap_set_bit (changed, to);
-			  continue;
-			}
-
-		      if (bitmap_ior_into (get_varinfo (to)->solution, pts))
-			bitmap_set_bit (changed, to);
-		    }
-		  if (to_remove != ~0U)
-		    bitmap_clear_bit (graph->succs[i], to_remove);
-		}
-	    }
-	}
-      bitmap_obstack_release (&iteration_obstack);
-    }
-
-  BITMAP_FREE (pts);
-  BITMAP_FREE (changed);
-  bitmap_obstack_release (&oldpta_obstack);
-}
-
-/* Map from trees to variable infos.  */
-static hash_map<tree, varinfo_t> *vi_for_tree;
-
-
 /* Insert ID as the variable id for tree T in the vi_for_tree map.  */
 
 static void
@@ -3003,7 +734,7 @@ lookup_vi_for_tree (tree t)
   return *slot;
 }
 
-/* Return a printable name for DECL  */
+/* Return a printable name for DECL.  */
 
 static const char *
 alias_get_name (tree decl)
@@ -3191,10 +922,10 @@ process_constraint (constraint_t t)
   if (!get_varinfo (lhs.var)->may_have_pointers)
     return;
 
-  /* This can happen in our IR with things like n->a = *p */
+  /* This can happen in our IR with things like n->a = *p.  */
   if (rhs.type == DEREF && lhs.type == DEREF && rhs.var != anything_id)
     {
-      /* Split into tmp = *rhs, *lhs = tmp */
+      /* Split into tmp = *rhs, *lhs = tmp.  */
       struct constraint_expr tmplhs;
       tmplhs = new_scalar_tmp_constraint_exp ("doubledereftmp", true);
       process_constraint (new_constraint (tmplhs, rhs));
@@ -3202,7 +933,7 @@ process_constraint (constraint_t t)
     }
   else if ((rhs.type != SCALAR || rhs.offset != 0) && lhs.type == DEREF)
     {
-      /* Split into tmp = &rhs, *lhs = tmp */
+      /* Split into tmp = &rhs, *lhs = tmp.  */
       struct constraint_expr tmplhs;
       tmplhs = new_scalar_tmp_constraint_exp ("derefaddrtmp", true);
       process_constraint (new_constraint (tmplhs, rhs));
@@ -3370,7 +1101,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
   tree forzero;
 
   /* Some people like to do cute things like take the address of
-     &0->a.b */
+     &0->a.b.  */
   forzero = t;
   while (handled_component_p (forzero)
 	 || INDIRECT_REF_P (forzero)
@@ -3444,7 +1175,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
     {
       /* In languages like C, you can access one past the end of an
 	 array.  You aren't allowed to dereference it, so we can
-	 ignore this constraint. When we handle pointer subtraction,
+	 ignore this constraint.  When we handle pointer subtraction,
 	 we may have to do something cute here.  */
 
       if (maybe_lt (poly_uint64 (bitpos), get_varinfo (result.var)->fullsize)
@@ -3481,7 +1212,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
 	      results->safe_push (cexpr);
 	    }
 	  else if (results->length () == 0)
-	    /* Assert that we found *some* field there. The user couldn't be
+	    /* Assert that we found *some* field there.  The user couldn't be
 	       accessing *only* padding.  */
 	    /* Still the user could access one past the end of an array
 	       embedded in a struct resulting in accessing *only* padding.  */
@@ -3535,7 +1266,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
 /* Dereference the constraint expression CONS, and return the result.
    DEREF (ADDRESSOF) = SCALAR
    DEREF (SCALAR) = DEREF
-   DEREF (DEREF) = (temp = DEREF1; result = DEREF(temp))
+   DEREF (DEREF) = (temp = DEREF1; result = DEREF (temp))
    This is needed so that we can handle dereferencing DEREF constraints.  */
 
 static void
@@ -3594,7 +1325,7 @@ get_constraint_for_1 (tree t, vec<ce_s> *results, bool address_p,
      point to anything by itself.  That is, of course, unless it is an
      integer constant being treated as a pointer, in which case, we
      will return that this is really the addressof anything.  This
-     happens below, since it will fall into the default case. The only
+     happens below, since it will fall into the default case.  The only
      case we know something about an integer treated like a pointer is
      when it is the NULL pointer, and then we just say it points to
      NULL.
@@ -3690,7 +1421,10 @@ get_constraint_for_1 (tree t, vec<ce_s> *results, bool address_p,
 		    size = -1;
 		  for (; curr; curr = vi_next (curr))
 		    {
-		      if (curr->offset - vi->offset < size)
+		      /* The start of the access might happen anywhere
+			 within vi, so conservatively assume it was
+			 at its end.  */
+		      if (curr->offset - (vi->offset + vi->size - 1) < size)
 			{
 			  cs.var = curr->id;
 			  results->safe_push (cs);
@@ -3742,7 +1476,7 @@ get_constraint_for_1 (tree t, vec<ce_s> *results, bool address_p,
 		  tmp.truncate (0);
 		}
 	      /* We do not know whether the constructor was complete,
-	         so technically we have to add &NOTHING or &ANYTHING
+		 so technically we have to add &NOTHING or &ANYTHING
 		 like we do for an empty constructor as well.  */
 	      return;
 	    }
@@ -4141,7 +1875,7 @@ get_function_part_constraint (varinfo_t fi, unsigned part)
 /* Produce constraints for argument ARG of call STMT with eaf flags
    FLAGS.  RESULTS is array holding constraints for return value.
    CALLESCAPE_ID is variable where call loocal escapes are added.
-   WRITES_GLOVEL_MEMORY is true if callee may write global memory. */
+   WRITES_GLOVEL_MEMORY is true if callee may write global memory.  */
 
 static void
 handle_call_arg (gcall *stmt, tree arg, vec<ce_s> *results, int flags,
@@ -4507,7 +2241,7 @@ handle_lhs_call (gcall *stmt, tree lhs, int flags, vec<ce_s> &rhsc,
       rhsc.truncate (0);
       vi = make_heapvar ("HEAP", true);
       /* We are marking allocated storage local, we deal with it becoming
-         global by escaping and setting of vars_contains_escaped_heap.  */
+	 global by escaping and setting of vars_contains_escaped_heap.  */
       DECL_EXTERNAL (vi->decl) = 0;
       vi->is_global_var = 0;
       /* If this is not a real malloc call assume the memory was
@@ -4695,8 +2429,8 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	}
       case BUILT_IN_STACK_SAVE:
       case BUILT_IN_STACK_RESTORE:
-        /* Nothing interesting happens.  */
-        return true;
+	/* Nothing interesting happens.  */
+	return true;
       case BUILT_IN_ALLOCA:
       case BUILT_IN_ALLOCA_WITH_ALIGN:
       case BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX:
@@ -4720,7 +2454,7 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	  return true;
 	}
       case BUILT_IN_POSIX_MEMALIGN:
-        {
+	{
 	  tree ptrptr = gimple_call_arg (t, 0);
 	  get_constraint_for (ptrptr, &lhsc);
 	  do_deref (&lhsc);
@@ -4803,7 +2537,7 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	  }
 	break;
       /* String / character search functions return a pointer into the
-         source string or NULL.  */
+	 source string or NULL.  */
       case BUILT_IN_INDEX:
       case BUILT_IN_STRCHR:
       case BUILT_IN_STRRCHR:
@@ -4824,7 +2558,7 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	  }
 	return true;
       /* Pure functions that return something not based on any object and
-         that use the memory pointed to by their arguments (but not
+	 that use the memory pointed to by their arguments (but not
 	 transitively).  */
       case BUILT_IN_STRCMP:
       case BUILT_IN_STRCMP_EQ:
@@ -5189,7 +2923,7 @@ find_func_aliases (struct function *fn, gimple *origt)
 	}
     }
   /* In IPA mode, we need to generate constraints to pass call
-     arguments through their calls.   There are two cases,
+     arguments through their calls.  There are two cases,
      either a GIMPLE_CALL returning a value, or just a plain
      GIMPLE_CALL when we are not.
 
@@ -5334,7 +3068,7 @@ find_func_aliases (struct function *fn, gimple *origt)
 	  constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
 	  oconstraints[i] = constraint;
 	  parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-				   &allows_reg, &is_inout);
+				   &allows_reg, &is_inout, nullptr);
 
 	  /* A memory constraint makes the address of the operand escape.  */
 	  if (!allows_reg && allows_mem)
@@ -5367,7 +3101,7 @@ find_func_aliases (struct function *fn, gimple *origt)
 	  constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
 
 	  parse_input_constraint (&constraint, 0, 0, noutputs, 0, oconstraints,
-				  &allows_mem, &allows_reg);
+				  &allows_mem, &allows_reg, nullptr);
 
 	  /* A memory constraint makes the address of the operand escape.  */
 	  if (!allows_reg && allows_mem)
@@ -5442,7 +3176,7 @@ find_func_clobbers (struct function *fn, gimple *origt)
 	  || (TREE_CODE (tem) == MEM_REF
 	      && !(TREE_CODE (TREE_OPERAND (tem, 0)) == ADDR_EXPR
 		   && auto_var_in_fn_p
-		        (TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
+			(TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
 	{
 	  struct constraint_expr lhsc, *rhsp;
 	  unsigned i;
@@ -5471,7 +3205,7 @@ find_func_clobbers (struct function *fn, gimple *origt)
 	  || (TREE_CODE (tem) == MEM_REF
 	      && !(TREE_CODE (TREE_OPERAND (tem, 0)) == ADDR_EXPR
 		   && auto_var_in_fn_p
-		        (TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
+			(TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
 	{
 	  struct constraint_expr lhs, *rhsp;
 	  unsigned i;
@@ -5783,65 +3517,6 @@ find_func_clobbers (struct function *fn, gimple *origt)
 }
 
 
-/* Find the first varinfo in the same variable as START that overlaps with
-   OFFSET.  Return NULL if we can't find one.  */
-
-static varinfo_t
-first_vi_for_offset (varinfo_t start, unsigned HOST_WIDE_INT offset)
-{
-  /* If the offset is outside of the variable, bail out.  */
-  if (offset >= start->fullsize)
-    return NULL;
-
-  /* If we cannot reach offset from start, lookup the first field
-     and start from there.  */
-  if (start->offset > offset)
-    start = get_varinfo (start->head);
-
-  while (start)
-    {
-      /* We may not find a variable in the field list with the actual
-	 offset when we have glommed a structure to a variable.
-	 In that case, however, offset should still be within the size
-	 of the variable. */
-      if (offset >= start->offset
-	  && (offset - start->offset) < start->size)
-	return start;
-
-      start = vi_next (start);
-    }
-
-  return NULL;
-}
-
-/* Find the first varinfo in the same variable as START that overlaps with
-   OFFSET.  If there is no such varinfo the varinfo directly preceding
-   OFFSET is returned.  */
-
-static varinfo_t
-first_or_preceding_vi_for_offset (varinfo_t start,
-				  unsigned HOST_WIDE_INT offset)
-{
-  /* If we cannot reach offset from start, lookup the first field
-     and start from there.  */
-  if (start->offset > offset)
-    start = get_varinfo (start->head);
-
-  /* We may not find a variable in the field list with the actual
-     offset when we have glommed a structure to a variable.
-     In that case, however, offset should still be within the size
-     of the variable.
-     If we got beyond the offset we look for return the field
-     directly preceding offset which may be the last field.  */
-  while (start->next
-	 && offset >= start->offset
-	 && !((offset - start->offset) < start->size))
-    start = vi_next (start);
-
-  return start;
-}
-
-
 /* This structure is used during pushing fields onto the fieldstack
    to track the offset of the field, since bitpos_of_field gives it
    relative to its immediate containing type, and we want it relative
@@ -5868,7 +3543,7 @@ struct fieldoff
 typedef struct fieldoff fieldoff_s;
 
 
-/* qsort comparison function for two fieldoff's PA and PB */
+/* qsort comparison function for two fieldoff's PA and PB.  */
 
 static int
 fieldoff_compare (const void *pa, const void *pb)
@@ -6596,38 +4271,6 @@ create_variable_info_for (tree decl, const char *name, bool add_id)
   return id;
 }
 
-/* Print out the points-to solution for VAR to FILE.  */
-
-static void
-dump_solution_for_var (FILE *file, unsigned int var)
-{
-  varinfo_t vi = get_varinfo (var);
-  unsigned int i;
-  bitmap_iterator bi;
-
-  /* Dump the solution for unified vars anyway, this avoids difficulties
-     in scanning dumps in the testsuite.  */
-  fprintf (file, "%s = { ", vi->name);
-  vi = get_varinfo (find (var));
-  EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
-    fprintf (file, "%s ", get_varinfo (i)->name);
-  fprintf (file, "}");
-
-  /* But note when the variable was unified.  */
-  if (vi->id != var)
-    fprintf (file, " same as %s", vi->name);
-
-  fprintf (file, "\n");
-}
-
-/* Print the points-to solution for VAR to stderr.  */
-
-DEBUG_FUNCTION void
-debug_solution_for_var (unsigned int var)
-{
-  dump_solution_for_var (stderr, var);
-}
-
 /* Register the constraints for function parameter related VI.  */
 
 static void
@@ -6715,7 +4358,7 @@ struct shared_bitmap_hasher : free_ptr_hash <shared_bitmap_info>
 			    const shared_bitmap_info *);
 };
 
-/* Hash function for a shared_bitmap_info_t */
+/* Hash function for a shared_bitmap_info_t.  */
 
 inline hashval_t
 shared_bitmap_hasher::hash (const shared_bitmap_info *bi)
@@ -6723,7 +4366,7 @@ shared_bitmap_hasher::hash (const shared_bitmap_info *bi)
   return bi->hashcode;
 }
 
-/* Equality function for two shared_bitmap_info_t's. */
+/* Equality function for two shared_bitmap_info_t's.  */
 
 inline bool
 shared_bitmap_hasher::equal (const shared_bitmap_info *sbi1,
@@ -6781,8 +4424,8 @@ set_uids_in_ptset (bitmap into, bitmap from, struct pt_solution *pt,
 {
   unsigned int i;
   bitmap_iterator bi;
-  varinfo_t escaped_vi = get_varinfo (find (escaped_id));
-  varinfo_t escaped_return_vi = get_varinfo (find (escaped_return_id));
+  varinfo_t escaped_vi = get_varinfo (var_rep[escaped_id]);
+  varinfo_t escaped_return_vi = get_varinfo (var_rep[escaped_return_id]);
   bool everything_escaped
     = escaped_vi->solution && bitmap_bit_p (escaped_vi->solution, anything_id);
 
@@ -6880,7 +4523,7 @@ find_what_var_points_to (tree fndecl, varinfo_t orig_vi)
 
   /* This variable may have been collapsed, let's get the real
      variable.  */
-  vi = get_varinfo (find (orig_vi->id));
+  vi = get_varinfo (var_rep[orig_vi->id]);
 
   /* See if we have already computed the solution and return it.  */
   pt_solution **slot = &final_solutions->get_or_insert (vi);
@@ -6907,7 +4550,7 @@ find_what_var_points_to (tree fndecl, varinfo_t orig_vi)
 	      else
 		pt->escaped = 1;
 	      /* Expand some special vars of ESCAPED in-place here.  */
-	      varinfo_t evi = get_varinfo (find (escaped_id));
+	      varinfo_t evi = get_varinfo (var_rep[escaped_id]);
 	      if (bitmap_bit_p (evi->solution, nonlocal_id))
 		pt->nonlocal = 1;
 	    }
@@ -7134,7 +4777,7 @@ pt_solution_includes_global (struct pt_solution *pt, bool escaped_local_p)
       || pt->nonlocal
       || pt->vars_contains_nonlocal
       /* The following is a hack to make the malloc escape hack work.
-         In reality we'd need different sets for escaped-through-return
+	 In reality we'd need different sets for escaped-through-return
 	 and escaped-to-callees and passes would need to be updated.  */
       || pt->vars_contains_escaped_heap)
     return true;
@@ -7270,52 +4913,6 @@ pt_solutions_intersect (struct pt_solution *pt1, struct pt_solution *pt2)
   return res;
 }
 
-/* Dump stats information to OUTFILE.  */
-
-static void
-dump_sa_stats (FILE *outfile)
-{
-  fprintf (outfile, "Points-to Stats:\n");
-  fprintf (outfile, "Total vars:               %d\n", stats.total_vars);
-  fprintf (outfile, "Non-pointer vars:          %d\n",
-	   stats.nonpointer_vars);
-  fprintf (outfile, "Statically unified vars:  %d\n",
-	   stats.unified_vars_static);
-  fprintf (outfile, "Dynamically unified vars: %d\n",
-	   stats.unified_vars_dynamic);
-  fprintf (outfile, "Iterations:               %d\n", stats.iterations);
-  fprintf (outfile, "Number of edges:          %d\n", stats.num_edges);
-  fprintf (outfile, "Number of implicit edges: %d\n",
-	   stats.num_implicit_edges);
-  fprintf (outfile, "Number of avoided edges: %d\n",
-	   stats.num_avoided_edges);
-}
-
-/* Dump points-to information to OUTFILE.  */
-
-static void
-dump_sa_points_to_info (FILE *outfile)
-{
-  fprintf (outfile, "\nPoints-to sets\n\n");
-
-  for (unsigned i = 1; i < varmap.length (); i++)
-    {
-      varinfo_t vi = get_varinfo (i);
-      if (!vi->may_have_pointers)
-	continue;
-      dump_solution_for_var (outfile, i);
-    }
-}
-
-
-/* Debug points-to information to stderr.  */
-
-DEBUG_FUNCTION void
-debug_sa_points_to_info (void)
-{
-  dump_sa_points_to_info (stderr);
-}
-
 
 /* Initialize the always-existing constraint variables for NULL
    ANYTHING, READONLY, and INTEGER */
@@ -7508,7 +5105,7 @@ init_base_vars (void)
   process_constraint (new_constraint (lhs, rhs));
 }
 
-/* Initialize things necessary to perform PTA */
+/* Initialize things necessary to perform PTA.  */
 
 static void
 init_alias_vars (void)
@@ -7517,7 +5114,6 @@ init_alias_vars (void)
 
   bitmap_obstack_initialize (&pta_obstack);
   bitmap_obstack_initialize (&oldpta_obstack);
-  bitmap_obstack_initialize (&predbitmap_obstack);
 
   constraints.create (8);
   varmap.create (8);
@@ -7534,144 +5130,6 @@ init_alias_vars (void)
   gcc_obstack_init (&final_solutions_obstack);
 }
 
-/* Remove the REF and ADDRESS edges from GRAPH, as well as all the
-   predecessor edges.  */
-
-static void
-remove_preds_and_fake_succs (constraint_graph_t graph)
-{
-  unsigned int i;
-
-  /* Clear the implicit ref and address nodes from the successor
-     lists.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      if (graph->succs[i])
-	bitmap_clear_range (graph->succs[i], FIRST_REF_NODE,
-			    FIRST_REF_NODE * 2);
-    }
-
-  /* Free the successor list for the non-ref nodes.  */
-  for (i = FIRST_REF_NODE + 1; i < graph->size; i++)
-    {
-      if (graph->succs[i])
-	BITMAP_FREE (graph->succs[i]);
-    }
-
-  /* Now reallocate the size of the successor list as, and blow away
-     the predecessor bitmaps.  */
-  graph->size = varmap.length ();
-  graph->succs = XRESIZEVEC (bitmap, graph->succs, graph->size);
-
-  free (graph->implicit_preds);
-  graph->implicit_preds = NULL;
-  free (graph->preds);
-  graph->preds = NULL;
-  bitmap_obstack_release (&predbitmap_obstack);
-}
-
-/* Solve the constraint set.  */
-
-static void
-solve_constraints (void)
-{
-  class scc_info *si;
-
-  /* Sort varinfos so that ones that cannot be pointed to are last.
-     This makes bitmaps more efficient.  */
-  unsigned int *map = XNEWVEC (unsigned int, varmap.length ());
-  for (unsigned i = 0; i < integer_id + 1; ++i)
-    map[i] = i;
-  /* Start with address-taken vars, followed by not address-taken vars
-     to move vars never appearing in the points-to solution bitmaps last.  */
-  unsigned j = integer_id + 1;
-  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
-    if (varmap[varmap[i]->head]->address_taken)
-      map[i] = j++;
-  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
-    if (! varmap[varmap[i]->head]->address_taken)
-      map[i] = j++;
-  /* Shuffle varmap according to map.  */
-  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
-    {
-      while (map[varmap[i]->id] != i)
-	std::swap (varmap[i], varmap[map[varmap[i]->id]]);
-      gcc_assert (bitmap_empty_p (varmap[i]->solution));
-      varmap[i]->id = i;
-      varmap[i]->next = map[varmap[i]->next];
-      varmap[i]->head = map[varmap[i]->head];
-    }
-  /* Finally rewrite constraints.  */
-  for (unsigned i = 0; i < constraints.length (); ++i)
-    {
-      constraints[i]->lhs.var = map[constraints[i]->lhs.var];
-      constraints[i]->rhs.var = map[constraints[i]->rhs.var];
-    }
-  free (map);
-
-  if (dump_file)
-    fprintf (dump_file,
-	     "\nCollapsing static cycles and doing variable "
-	     "substitution\n");
-
-  init_graph (varmap.length () * 2);
-
-  if (dump_file)
-    fprintf (dump_file, "Building predecessor graph\n");
-  build_pred_graph ();
-
-  if (dump_file)
-    fprintf (dump_file, "Detecting pointer and location "
-	     "equivalences\n");
-  si = perform_var_substitution (graph);
-
-  if (dump_file)
-    fprintf (dump_file, "Rewriting constraints and unifying "
-	     "variables\n");
-  rewrite_constraints (graph, si);
-
-  build_succ_graph ();
-
-  free_var_substitution_info (si);
-
-  /* Attach complex constraints to graph nodes.  */
-  move_complex_constraints (graph);
-
-  if (dump_file)
-    fprintf (dump_file, "Uniting pointer but not location equivalent "
-	     "variables\n");
-  unite_pointer_equivalences (graph);
-
-  if (dump_file)
-    fprintf (dump_file, "Finding indirect cycles\n");
-  find_indirect_cycles (graph);
-
-  /* Implicit nodes and predecessors are no longer necessary at this
-     point. */
-  remove_preds_and_fake_succs (graph);
-
-  if (dump_file && (dump_flags & TDF_GRAPH))
-    {
-      fprintf (dump_file, "\n\n// The constraint graph before solve-graph "
-	       "in dot format:\n");
-      dump_constraint_graph (dump_file);
-      fprintf (dump_file, "\n\n");
-    }
-
-  if (dump_file)
-    fprintf (dump_file, "Solving graph\n");
-
-  solve_graph (graph);
-
-  if (dump_file && (dump_flags & TDF_GRAPH))
-    {
-      fprintf (dump_file, "\n\n// The constraint graph after solve-graph "
-	       "in dot format:\n");
-      dump_constraint_graph (dump_file);
-      fprintf (dump_file, "\n\n");
-    }
-}
-
 /* Create points-to sets for the current function.  See the comments
    at the start of the file for an algorithmic overview.  */
 
@@ -7842,8 +5300,6 @@ compute_points_to_sets (void)
 static void
 delete_points_to_sets (void)
 {
-  unsigned int i;
-
   delete shared_bitmap_table;
   shared_bitmap_table = NULL;
   if (dump_file && (dump_flags & TDF_STATS))
@@ -7855,16 +5311,7 @@ delete_points_to_sets (void)
   bitmap_obstack_release (&pta_obstack);
   constraints.release ();
 
-  for (i = 0; i < graph->size; i++)
-    graph->complex[i].release ();
-  free (graph->complex);
-
-  free (graph->rep);
-  free (graph->succs);
-  free (graph->pe);
-  free (graph->pe_rep);
-  free (graph->indirect_cycles);
-  free (graph);
+  free (var_rep);
 
   varmap.release ();
   variable_info_pool.release ();
@@ -7911,14 +5358,14 @@ visit_loadstore (gimple *, tree base, tree ref, void *data)
 	  if (! vi)
 	    return false;
 
-	  vi = get_varinfo (find (vi->id));
+	  vi = get_varinfo (var_rep[vi->id]);
 	  if (bitmap_intersect_p (rvars, vi->solution)
 	      || (escaped_p && bitmap_bit_p (vi->solution, escaped_id)))
 	    return false;
 	}
 
       /* Do not overwrite existing cliques (that includes clique, base
-         pairs we just set).  */
+	 pairs we just set).  */
       if (MR_DEPENDENCE_CLIQUE (base) == 0)
 	{
 	  MR_DEPENDENCE_CLIQUE (base) = clique;
@@ -8047,7 +5494,7 @@ compute_dependence_clique (void)
       varinfo_t vi = lookup_vi_for_tree (p);
       if (!vi)
 	continue;
-      vi = get_varinfo (find (vi->id));
+      vi = get_varinfo (var_rep[vi->id]);
       bitmap_iterator bi;
       unsigned j;
       varinfo_t restrict_var = NULL;
@@ -8098,11 +5545,11 @@ compute_dependence_clique (void)
 					      maybe_set_dependence_info);
 	  if (used)
 	    {
-	      /* Add all subvars to the set of restrict pointed-to set. */
+	      /* Add all subvars to the set of restrict pointed-to set.  */
 	      for (unsigned sv = restrict_var->head; sv != 0;
 		   sv = get_varinfo (sv)->next)
 		bitmap_set_bit (rvars, sv);
-	      varinfo_t escaped = get_varinfo (find (escaped_id));
+	      varinfo_t escaped = get_varinfo (var_rep[escaped_id]);
 	      if (bitmap_bit_p (escaped->solution, restrict_var->id))
 		escaped_p = true;
 	    }
@@ -8258,7 +5705,7 @@ struct pt_solution ipa_escaped_pt
   = { true, false, false, false, false, false,
       false, false, false, false, false, NULL };
 
-/* Associate node with varinfo DATA. Worker for
+/* Associate node with varinfo DATA.  Worker for
    cgraph_for_symbol_thunks_and_aliases.  */
 static bool
 associate_varinfo_to_alias (struct cgraph_node *node, void *data)
@@ -8272,111 +5719,6 @@ associate_varinfo_to_alias (struct cgraph_node *node, void *data)
   return false;
 }
 
-/* Dump varinfo VI to FILE.  */
-
-static void
-dump_varinfo (FILE *file, varinfo_t vi)
-{
-  if (vi == NULL)
-    return;
-
-  fprintf (file, "%u: %s\n", vi->id, vi->name);
-
-  const char *sep = " ";
-  if (vi->is_artificial_var)
-    fprintf (file, "%sartificial", sep);
-  if (vi->is_special_var)
-    fprintf (file, "%sspecial", sep);
-  if (vi->is_unknown_size_var)
-    fprintf (file, "%sunknown-size", sep);
-  if (vi->is_full_var)
-    fprintf (file, "%sfull", sep);
-  if (vi->is_heap_var)
-    fprintf (file, "%sheap", sep);
-  if (vi->may_have_pointers)
-    fprintf (file, "%smay-have-pointers", sep);
-  if (vi->only_restrict_pointers)
-    fprintf (file, "%sonly-restrict-pointers", sep);
-  if (vi->is_restrict_var)
-    fprintf (file, "%sis-restrict-var", sep);
-  if (vi->is_global_var)
-    fprintf (file, "%sglobal", sep);
-  if (vi->is_ipa_escape_point)
-    fprintf (file, "%sipa-escape-point", sep);
-  if (vi->is_fn_info)
-    fprintf (file, "%sfn-info", sep);
-  if (vi->ruid)
-    fprintf (file, "%srestrict-uid:%u", sep, vi->ruid);
-  if (vi->next)
-    fprintf (file, "%snext:%u", sep, vi->next);
-  if (vi->head != vi->id)
-    fprintf (file, "%shead:%u", sep, vi->head);
-  if (vi->offset)
-    fprintf (file, "%soffset:" HOST_WIDE_INT_PRINT_DEC, sep, vi->offset);
-  if (vi->size != ~HOST_WIDE_INT_0U)
-    fprintf (file, "%ssize:" HOST_WIDE_INT_PRINT_DEC, sep, vi->size);
-  if (vi->fullsize != ~HOST_WIDE_INT_0U && vi->fullsize != vi->size)
-    fprintf (file, "%sfullsize:" HOST_WIDE_INT_PRINT_DEC, sep,
-	     vi->fullsize);
-  fprintf (file, "\n");
-
-  if (vi->solution && !bitmap_empty_p (vi->solution))
-    {
-      bitmap_iterator bi;
-      unsigned i;
-      fprintf (file, " solution: {");
-      EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
-	fprintf (file, " %u", i);
-      fprintf (file, " }\n");
-    }
-
-  if (vi->oldsolution && !bitmap_empty_p (vi->oldsolution)
-      && !bitmap_equal_p (vi->solution, vi->oldsolution))
-    {
-      bitmap_iterator bi;
-      unsigned i;
-      fprintf (file, " oldsolution: {");
-      EXECUTE_IF_SET_IN_BITMAP (vi->oldsolution, 0, i, bi)
-	fprintf (file, " %u", i);
-      fprintf (file, " }\n");
-    }
-}
-
-/* Dump varinfo VI to stderr.  */
-
-DEBUG_FUNCTION void
-debug_varinfo (varinfo_t vi)
-{
-  dump_varinfo (stderr, vi);
-}
-
-/* Dump varmap to FILE.  */
-
-static void
-dump_varmap (FILE *file)
-{
-  if (varmap.length () == 0)
-    return;
-
-  fprintf (file, "variables:\n");
-
-  for (unsigned int i = 0; i < varmap.length (); ++i)
-    {
-      varinfo_t vi = get_varinfo (i);
-      dump_varinfo (file, vi);
-    }
-
-  fprintf (file, "\n");
-}
-
-/* Dump varmap to stderr.  */
-
-DEBUG_FUNCTION void
-debug_varmap (void)
-{
-  dump_varmap (stderr);
-}
-
 /* Compute whether node is refered to non-locally.  Worker for
    cgraph_for_symbol_thunks_and_aliases.  */
 static bool
@@ -8490,7 +5832,7 @@ ipa_pta_execute (void)
       varinfo_t vi = get_vi_for_tree (var->decl);
 
       /* For the purpose of IPA PTA unit-local globals are not
-         escape points.  */
+	 escape points.  */
       bool nonlocal_p = (DECL_EXTERNAL (var->decl)
 			 || TREE_PUBLIC (var->decl)
 			 || var->used_from_other_partition
@@ -8589,7 +5931,7 @@ ipa_pta_execute (void)
 	for (varinfo_t ai = first_vi_for_offset (fi, fi_parm_base);
 	     ai; ai = vi_next (ai))
 	  {
-	    varinfo_t vi = get_varinfo (find (ai->id));
+	    varinfo_t vi = get_varinfo (var_rep[ai->id]);
 	    bitmap_iterator bi;
 	    unsigned j;
 	    EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, j, bi)
@@ -8605,12 +5947,12 @@ ipa_pta_execute (void)
 	      }
 	  }
       /* As well as global variables which are another way of passing
-         arguments to recursive invocations.  */
+	 arguments to recursive invocations.  */
       else if (fi->is_global_var)
 	{
 	  for (varinfo_t ai = fi; ai; ai = vi_next (ai))
 	    {
-	      varinfo_t vi = get_varinfo (find (ai->id));
+	      varinfo_t vi = get_varinfo (var_rep[ai->id]);
 	      bitmap_iterator bi;
 	      unsigned j;
 	      EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, j, bi)
@@ -8705,10 +6047,10 @@ ipa_pta_execute (void)
 		{
 		  *gimple_call_clobber_set (stmt)
 		     = find_what_var_points_to
-		         (node->decl, first_vi_for_offset (fi, fi_clobbers));
+			 (node->decl, first_vi_for_offset (fi, fi_clobbers));
 		  *gimple_call_use_set (stmt)
 		     = find_what_var_points_to
-		         (node->decl, first_vi_for_offset (fi, fi_uses));
+			 (node->decl, first_vi_for_offset (fi, fi_uses));
 		}
 	      /* Handle direct calls to external functions.  */
 	      else if (decl && (!fi || fi->decl))
@@ -8735,7 +6077,8 @@ ipa_pta_execute (void)
 		    }
 
 		  pt = gimple_call_clobber_set (stmt);
-		  if (gimple_call_flags (stmt) & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
+		  if (gimple_call_flags (stmt) &
+		      (ECF_CONST|ECF_PURE|ECF_NOVOPS))
 		    memset (pt, 0, sizeof (struct pt_solution));
 		  else if ((vi = lookup_call_clobber_vi (stmt)) != NULL)
 		    {
@@ -8760,7 +6103,7 @@ ipa_pta_execute (void)
 		{
 		  /* We need to accumulate all clobbers/uses of all possible
 		     callees.  */
-		  fi = get_varinfo (find (fi->id));
+		  fi = get_varinfo (var_rep[fi->id]);
 		  /* If we cannot constrain the set of functions we'll end up
 		     calling we end up using/clobbering everything.  */
 		  if (bitmap_bit_p (fi->solution, anything_id)
@@ -8820,7 +6163,7 @@ ipa_pta_execute (void)
       fn->gimple_df->ipa_pta = true;
 
       /* We have to re-set the final-solution cache after each function
-         because what is a "global" is dependent on function context.  */
+	 because what is a "global" is dependent on function context.  */
       final_solutions->empty ();
       obstack_free (&final_solutions_obstack, NULL);
       gcc_obstack_init (&final_solutions_obstack);
diff --git a/gcc/tree-ssa-structalias.h b/gcc/tree-ssa-structalias.h
new file mode 100644
index 0000000..4104bad
--- /dev/null
+++ b/gcc/tree-ssa-structalias.h
@@ -0,0 +1,217 @@
+/* Tree based points-to analysis
+   Copyright (C) 2005-2025 Free Software Foundation, Inc.
+   Contributed by Daniel Berlin <dberlin@dberlin.org>
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+/* NOTE: This file declares the internal interface of the points-to analyzer.
+   Outward-facing function declarations can be found in tree-ssa-alias.h.  */
+
+#ifndef TREE_SSA_STRUCTALIAS_H
+#define TREE_SSA_STRUCTALIAS_H
+
+namespace pointer_analysis {
+
+enum constraint_expr_type {SCALAR, DEREF, ADDRESSOF};
+
+/* Static IDs for the special variables.  Variable ID zero is unused
+   and used as terminator for the sub-variable chain.  */
+enum { nothing_id = 1, anything_id = 2, string_id = 3,
+       escaped_id = 4, nonlocal_id = 5, escaped_return_id = 6,
+       storedanything_id = 7, integer_id = 8 };
+
+/* Use 0x8000... as special unknown offset.  */
+#define UNKNOWN_OFFSET HOST_WIDE_INT_MIN
+
+/* An expression that appears in a constraint.  */
+
+struct constraint_expr
+{
+  /* Constraint type.  */
+  constraint_expr_type type;
+
+  /* Variable we are referring to in the constraint.  */
+  unsigned int var;
+
+  /* Offset, in bits, of this constraint from the beginning of
+     variables it ends up referring to.
+
+     IOW, in a deref constraint, we would deref, get the result set,
+     then add OFFSET to each member.  */
+  HOST_WIDE_INT offset;
+};
+typedef struct constraint_expr ce_s;
+
+/* Our set constraints are made up of two constraint expressions, one
+   LHS, and one RHS.
+
+   As described in the introduction in tree-ssa-structalias.cc, our set
+   constraints each represent an operation between set valued variables.
+*/
+struct constraint
+{
+  struct constraint_expr lhs;
+  struct constraint_expr rhs;
+};
+typedef struct constraint *constraint_t;
+
+struct variable_info
+{
+  /* ID of this variable.  */
+  unsigned int id;
+
+  /* True if this is a variable created by the constraint analysis, such as
+     heap variables and constraints we had to break up.  */
+  unsigned int is_artificial_var : 1;
+
+  /* True if this is a special variable whose solution set should not be
+     changed.  */
+  unsigned int is_special_var : 1;
+
+  /* True for variables whose size is not known or variable.  */
+  unsigned int is_unknown_size_var : 1;
+
+  /* True for (sub-)fields that represent a whole variable.  */
+  unsigned int is_full_var : 1;
+
+  /* True if this is a heap variable.  */
+  unsigned int is_heap_var : 1;
+
+  /* True if this is a register variable.  */
+  unsigned int is_reg_var : 1;
+
+  /* True if this field may contain pointers.  */
+  unsigned int may_have_pointers : 1;
+
+  /* True if this field has only restrict qualified pointers.  */
+  unsigned int only_restrict_pointers : 1;
+
+  /* True if this represents a heap var created for a restrict qualified
+     pointer.  */
+  unsigned int is_restrict_var : 1;
+
+  /* True if this represents a global variable.  */
+  unsigned int is_global_var : 1;
+
+  /* True if this represents a module escape point for IPA analysis.  */
+  unsigned int is_ipa_escape_point : 1;
+
+  /* True if this represents a IPA function info.  */
+  unsigned int is_fn_info : 1;
+
+  /* True if this appears as RHS in a ADDRESSOF constraint.  */
+  unsigned int address_taken : 1;
+
+  /* ???  Store somewhere better.  */
+  unsigned short ruid;
+
+  /* The ID of the variable for the next field in this structure
+     or zero for the last field in this structure.  */
+  unsigned next;
+
+  /* The ID of the variable for the first field in this structure.  */
+  unsigned head;
+
+  /* Offset of this variable, in bits, from the base variable.  */
+  unsigned HOST_WIDE_INT offset;
+
+  /* Size of the variable, in bits.  */
+  unsigned HOST_WIDE_INT size;
+
+  /* Full size of the base variable, in bits.  */
+  unsigned HOST_WIDE_INT fullsize;
+
+  /* In IPA mode the shadow UID in case the variable needs to be duplicated in
+     the final points-to solution because it reaches its containing
+     function recursively.  Zero if none is needed.  */
+  unsigned int shadow_var_uid;
+
+  /* Name of this variable.  */
+  const char *name;
+
+  /* Tree that this variable is associated with.  */
+  tree decl;
+
+  /* Points-to set for this variable.  */
+  bitmap solution;
+
+  /* Old points-to set for this variable.  */
+  bitmap oldsolution;
+};
+typedef struct variable_info *varinfo_t;
+
+struct constraint_stats
+{
+  unsigned int total_vars;
+  unsigned int nonpointer_vars;
+  unsigned int unified_vars_static;
+  unsigned int unified_vars_dynamic;
+  unsigned int iterations;
+  unsigned int num_edges;
+  unsigned int num_implicit_edges;
+  unsigned int num_avoided_edges;
+  unsigned int points_to_sets_created;
+};
+
+extern struct constraint_stats stats;
+
+extern bitmap_obstack pta_obstack;
+extern bitmap_obstack oldpta_obstack;
+
+extern vec<varinfo_t> varmap;
+extern vec<constraint_t> constraints;
+extern unsigned int *var_rep;
+
+
+/* Return the varmap element N.  */
+
+inline varinfo_t
+get_varinfo (unsigned int n)
+{
+  return varmap[n];
+}
+
+/* Return the next variable in the list of sub-variables of VI
+   or NULL if VI is the last sub-variable.  */
+
+inline varinfo_t
+vi_next (varinfo_t vi)
+{
+  return get_varinfo (vi->next);
+}
+
+varinfo_t first_vi_for_offset (varinfo_t start,
+			       unsigned HOST_WIDE_INT offset);
+varinfo_t first_or_preceding_vi_for_offset (varinfo_t start,
+					    unsigned HOST_WIDE_INT offset);
+void dump_constraint (FILE *file, constraint_t c);
+void dump_constraints (FILE *file, int from);
+void dump_solution_for_var (FILE *file, unsigned int var);
+void dump_sa_stats (FILE *outfile);
+void dump_sa_points_to_info (FILE *outfile);
+void dump_varinfo (FILE *file, varinfo_t vi);
+void dump_varmap (FILE *file);
+void debug_constraint (constraint_t);
+void debug_constraints (void);
+void debug_solution_for_var (unsigned int);
+void debug_sa_points_to_info (void);
+void debug_varinfo (varinfo_t);
+void debug_varmap (void);
+
+} // namespace pointer_analysis
+
+#endif /* TREE_SSA_STRUCTALIAS_H  */
diff --git a/gcc/tree-switch-conversion.cc b/gcc/tree-switch-conversion.cc
index d088287..04b357f 100644
--- a/gcc/tree-switch-conversion.cc
+++ b/gcc/tree-switch-conversion.cc
@@ -55,6 +55,7 @@ Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
 #include "hwint.h"
 #include "internal-fn.h"
 #include "diagnostic-core.h"
+#include "output.h"
 
 /* ??? For lang_hooks.types.type_for_mode, but is there a word_mode
    type in the GIMPLE type system that is language-independent?  */
@@ -1033,6 +1034,16 @@ switch_conversion::build_one_array (int num, tree arr_index_type,
       /* The decl is mergable since we don't take the address ever and
 	 just reading from it. */
       DECL_MERGEABLE (decl) = 1;
+
+      /* Increase the alignments as needed. */
+      if (tree_to_uhwi (DECL_SIZE (decl)) > DECL_ALIGN (decl))
+	{
+	  unsigned HOST_WIDE_INT s = tree_to_uhwi (DECL_SIZE (decl));
+	  /* Only support up to the max supported for merging. */
+	  if (s <= MAX_ALIGN_MERGABLE)
+	    SET_DECL_ALIGN (decl, HOST_WIDE_INT_1U << ceil_log2 (s));
+	}
+
       if (offloading_function_p (cfun->decl))
 	DECL_ATTRIBUTES (decl)
 	  = tree_cons (get_identifier ("omp declare target"), NULL_TREE,
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index ee040eb..dc82bf6 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -1501,10 +1501,17 @@ vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info,
       /* We can only use base and misalignment information relative to
 	 an innermost loop if the misalignment stays the same throughout the
 	 execution of the loop.  As above, this is the case if the stride of
-	 the dataref evenly divides by the alignment.  */
-      poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-      step_preserves_misalignment_p
-	= multiple_p (drb->step_alignment * vf, vect_align_c);
+	 the dataref evenly divides by the alignment.  Make sure to check
+	 previous epilogues and the main loop.  */
+      step_preserves_misalignment_p = true;
+      auto lvinfo = loop_vinfo;
+      while (lvinfo)
+	{
+	  poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (lvinfo);
+	  step_preserves_misalignment_p
+	    &= multiple_p (drb->step_alignment * vf, vect_align_c);
+	  lvinfo = LOOP_VINFO_ORIG_LOOP_INFO (lvinfo);
+	}
 
       if (!step_preserves_misalignment_p && dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -1571,6 +1578,7 @@ vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info,
       unsigned int max_alignment;
       tree base = get_base_for_alignment (drb->base_address, &max_alignment);
       if (max_alignment < vect_align_c
+	  || (loop_vinfo && LOOP_VINFO_EPILOGUE_P (loop_vinfo))
 	  || !vect_can_force_dr_alignment_p (base,
 					     vect_align_c * BITS_PER_UNIT))
 	{
@@ -1848,21 +1856,14 @@ vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
 			   stmt_vector_for_cost *prologue_cost_vec)
 {
   stmt_vec_info stmt_info = dr_info->stmt;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  int ncopies;
-
-  if (PURE_SLP_STMT (stmt_info))
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
 
   if (DR_IS_READ (dr_info->dr))
-    vect_get_load_cost (vinfo, stmt_info, NULL, ncopies,
+    vect_get_load_cost (vinfo, stmt_info, NULL, 1,
 			alignment_support_scheme, misalignment, true,
 			inside_cost, outside_cost, prologue_cost_vec,
 			body_cost_vec, false);
   else
-    vect_get_store_cost (vinfo,stmt_info, NULL, ncopies,
+    vect_get_store_cost (vinfo,stmt_info, NULL, 1,
 			 alignment_support_scheme, misalignment, inside_cost,
 			 body_cost_vec);
 
@@ -2910,12 +2911,14 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
      2) there is at least one unsupported misaligned data ref with an unknown
         misalignment, and
      3) all misaligned data refs with a known misalignment are supported, and
-     4) the number of runtime alignment checks is within reason.  */
+     4) the number of runtime alignment checks is within reason.
+     5) the vectorization factor is a constant.  */
 
   do_versioning
     = (optimize_loop_nest_for_speed_p (loop)
        && !loop->inner /* FORNOW */
-       && loop_cost_model (loop) > VECT_COST_MODEL_CHEAP);
+       && loop_cost_model (loop) > VECT_COST_MODEL_CHEAP)
+       && LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ();
 
   if (do_versioning)
     {
@@ -2956,17 +2959,6 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
                   break;
                 }
 
-	      /* At present we don't support versioning for alignment
-		 with variable VF, since there's no guarantee that the
-		 VF is a power of two.  We could relax this if we added
-		 a way of enforcing a power-of-two size.  */
-	      unsigned HOST_WIDE_INT size;
-	      if (!GET_MODE_SIZE (TYPE_MODE (vectype)).is_constant (&size))
-		{
-		  do_versioning = false;
-		  break;
-		}
-
 	      /* Forcing alignment in the first iteration is no good if
 		 we don't keep it across iterations.  For now, just disable
 		 versioning in this case.
@@ -2985,7 +2977,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
                  Construct the mask needed for this test.  For example,
                  GET_MODE_SIZE for the vector mode V4SI is 16 bytes so the
                  mask must be 15 = 0xf. */
-	      int mask = size - 1;
+	      gcc_assert (DR_TARGET_ALIGNMENT (dr_info).is_constant ());
+	      int mask = DR_TARGET_ALIGNMENT (dr_info).to_constant () - 1;
 
 	      /* FORNOW: use the same mask to test all potentially unaligned
 		 references in the loop.  */
@@ -3948,8 +3941,7 @@ vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
     }
   tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
   int misalignment;
-  if (STMT_VINFO_VEC_STMTS (stmt_vinfo).exists ()
-      && ((misalignment = dr_misalignment (dr_info, vectype)), true)
+  if (((misalignment = dr_misalignment (dr_info, vectype)), true)
       && (vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment)
 	  == dr_explicit_realign_optimized))
     {
@@ -4531,10 +4523,13 @@ vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
   info->ifn = gimple_call_internal_fn (call);
   info->decl = NULL_TREE;
   info->base = gimple_call_arg (call, 0);
-  info->offset = gimple_call_arg (call, 1);
-  info->offset_dt = vect_unknown_def_type;
+  info->alias_ptr = gimple_call_arg
+		     (call, internal_fn_alias_ptr_index (info->ifn));
+  info->offset = gimple_call_arg
+		  (call, internal_fn_offset_index (info->ifn));
   info->offset_vectype = NULL_TREE;
-  info->scale = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
+  info->scale = TREE_INT_CST_LOW (gimple_call_arg
+				  (call, internal_fn_scale_index (info->ifn)));
   info->element_type = TREE_TYPE (vectype);
   info->memory_type = TREE_TYPE (DR_REF (dr));
 }
@@ -4659,26 +4654,19 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
   if (off == NULL_TREE)
     off = size_zero_node;
 
-  /* If base is not loop invariant, either off is 0, then we start with just
-     the constant offset in the loop invariant BASE and continue with base
-     as OFF, otherwise give up.
-     We could handle that case by gimplifying the addition of base + off
-     into some SSA_NAME and use that as off, but for now punt.  */
+  /* BASE must be loop invariant.  If it is not invariant, but OFF is, then we
+   * can fix that by swapping BASE and OFF.  */
   if (!expr_invariant_in_loop_p (loop, base))
     {
-      if (!integer_zerop (off))
+      if (!expr_invariant_in_loop_p (loop, off))
 	return false;
-      off = base;
-      base = size_int (pbytepos);
-    }
-  /* Otherwise put base + constant offset into the loop invariant BASE
-     and continue with OFF.  */
-  else
-    {
-      base = fold_convert (sizetype, base);
-      base = size_binop (PLUS_EXPR, base, size_int (pbytepos));
+
+      std::swap (base, off);
     }
 
+  base = fold_convert (sizetype, base);
+  base = size_binop (PLUS_EXPR, base, size_int (pbytepos));
+
   /* OFF at this point may be either a SSA_NAME or some tree expression
      from get_inner_reference.  Try to peel off loop invariants from it
      into BASE as long as possible.  */
@@ -4856,11 +4844,18 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
       offset_vectype = NULL_TREE;
     }
 
+  gcc_checking_assert (expr_invariant_in_loop_p (loop, base));
+  gcc_checking_assert (!expr_invariant_in_loop_p (loop, off));
+
   info->ifn = ifn;
   info->decl = decl;
   info->base = base;
+
+  info->alias_ptr = build_int_cst
+    (reference_alias_ptr_type (DR_REF (dr)),
+     get_object_alignment (DR_REF (dr)));
+
   info->offset = off;
-  info->offset_dt = vect_unknown_def_type;
   info->offset_vectype = offset_vectype;
   info->scale = scale;
   info->element_type = TREE_TYPE (vectype);
@@ -5048,7 +5043,7 @@ vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
 */
 
 opt_result
-vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
+vect_analyze_data_refs (vec_info *vinfo, bool *fatal)
 {
   class loop *loop = NULL;
   unsigned int i;
@@ -5067,7 +5062,6 @@ vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
   FOR_EACH_VEC_ELT (datarefs, i, dr)
     {
       enum { SG_NONE, GATHER, SCATTER } gatherscatter = SG_NONE;
-      poly_uint64 vf;
 
       gcc_assert (DR_REF (dr));
       stmt_vec_info stmt_info = vinfo->lookup_stmt (DR_STMT (dr));
@@ -5259,11 +5253,6 @@ vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
 			     stmt_info->stmt, vectype);
 	}
 
-      /* Adjust the minimal vectorization factor according to the
-	 vector type.  */
-      vf = TYPE_VECTOR_SUBPARTS (vectype);
-      *min_vf = upper_bound (*min_vf, vf);
-
       /* Leave the BB vectorizer to pick the vector type later, based on
 	 the final dataref group size and SLP node size.  */
       if (is_a <loop_vec_info> (vinfo))
@@ -5720,8 +5709,7 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
       standard_iv_increment_position (loop, &incr_gsi, &insert_after);
 
       create_iv (aggr_ptr_init, PLUS_EXPR,
-		 fold_convert (aggr_ptr_type, iv_step),
-		 aggr_ptr, loop, &incr_gsi, insert_after,
+		 iv_step, aggr_ptr, loop, &incr_gsi, insert_after,
 		 &indx_before_incr, &indx_after_incr);
       incr = gsi_stmt (incr_gsi);
 
@@ -5749,7 +5737,7 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
     {
       standard_iv_increment_position (containing_loop, &incr_gsi,
 				      &insert_after);
-      create_iv (aptr, PLUS_EXPR, fold_convert (aggr_ptr_type, DR_STEP (dr)),
+      create_iv (aptr, PLUS_EXPR, DR_STEP (dr),
 		 aggr_ptr, containing_loop, &incr_gsi, insert_after,
 		 &indx_before_incr, &indx_after_incr);
       incr = gsi_stmt (incr_gsi);
@@ -6072,204 +6060,6 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
 }
 
 
-/* Function vect_permute_store_chain.
-
-   Given a chain of interleaved stores in DR_CHAIN of LENGTH that must be
-   a power of 2 or equal to 3, generate interleave_high/low stmts to reorder
-   the data correctly for the stores.  Return the final references for stores
-   in RESULT_CHAIN.
-
-   E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
-   The input is 4 vectors each containing 8 elements.  We assign a number to
-   each element, the input sequence is:
-
-   1st vec:   0  1  2  3  4  5  6  7
-   2nd vec:   8  9 10 11 12 13 14 15
-   3rd vec:  16 17 18 19 20 21 22 23
-   4th vec:  24 25 26 27 28 29 30 31
-
-   The output sequence should be:
-
-   1st vec:  0  8 16 24  1  9 17 25
-   2nd vec:  2 10 18 26  3 11 19 27
-   3rd vec:  4 12 20 28  5 13 21 30
-   4th vec:  6 14 22 30  7 15 23 31
-
-   i.e., we interleave the contents of the four vectors in their order.
-
-   We use interleave_high/low instructions to create such output.  The input of
-   each interleave_high/low operation is two vectors:
-   1st vec    2nd vec
-   0 1 2 3    4 5 6 7
-   the even elements of the result vector are obtained left-to-right from the
-   high/low elements of the first vector.  The odd elements of the result are
-   obtained left-to-right from the high/low elements of the second vector.
-   The output of interleave_high will be:   0 4 1 5
-   and of interleave_low:                   2 6 3 7
-
-
-   The permutation is done in log LENGTH stages.  In each stage interleave_high
-   and interleave_low stmts are created for each pair of vectors in DR_CHAIN,
-   where the first argument is taken from the first half of DR_CHAIN and the
-   second argument from it's second half.
-   In our example,
-
-   I1: interleave_high (1st vec, 3rd vec)
-   I2: interleave_low (1st vec, 3rd vec)
-   I3: interleave_high (2nd vec, 4th vec)
-   I4: interleave_low (2nd vec, 4th vec)
-
-   The output for the first stage is:
-
-   I1:  0 16  1 17  2 18  3 19
-   I2:  4 20  5 21  6 22  7 23
-   I3:  8 24  9 25 10 26 11 27
-   I4: 12 28 13 29 14 30 15 31
-
-   The output of the second stage, i.e. the final result is:
-
-   I1:  0  8 16 24  1  9 17 25
-   I2:  2 10 18 26  3 11 19 27
-   I3:  4 12 20 28  5 13 21 30
-   I4:  6 14 22 30  7 15 23 31.  */
-
-void
-vect_permute_store_chain (vec_info *vinfo, vec<tree> &dr_chain,
-			  unsigned int length,
-			  stmt_vec_info stmt_info,
-			  gimple_stmt_iterator *gsi,
-			  vec<tree> *result_chain)
-{
-  tree vect1, vect2, high, low;
-  gimple *perm_stmt;
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  tree perm_mask_low, perm_mask_high;
-  tree data_ref;
-  tree perm3_mask_low, perm3_mask_high;
-  unsigned int i, j, n, log_length = exact_log2 (length);
-
-  result_chain->quick_grow (length);
-  memcpy (result_chain->address (), dr_chain.address (),
-	  length * sizeof (tree));
-
-  if (length == 3)
-    {
-      /* vect_grouped_store_supported ensures that this is constant.  */
-      unsigned int nelt = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
-      unsigned int j0 = 0, j1 = 0, j2 = 0;
-
-      vec_perm_builder sel (nelt, nelt, 1);
-      sel.quick_grow (nelt);
-      vec_perm_indices indices;
-      for (j = 0; j < 3; j++)
-        {
-	  int nelt0 = ((3 - j) * nelt) % 3;
-	  int nelt1 = ((3 - j) * nelt + 1) % 3;
-	  int nelt2 = ((3 - j) * nelt + 2) % 3;
-
-	  for (i = 0; i < nelt; i++)
-	    {
-	      if (3 * i + nelt0 < nelt)
-		sel[3 * i + nelt0] = j0++;
-	      if (3 * i + nelt1 < nelt)
-		sel[3 * i + nelt1] = nelt + j1++;
-	      if (3 * i + nelt2 < nelt)
-		sel[3 * i + nelt2] = 0;
-	    }
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_low = vect_gen_perm_mask_checked (vectype, indices);
-
-	  for (i = 0; i < nelt; i++)
-	    {
-	      if (3 * i + nelt0 < nelt)
-		sel[3 * i + nelt0] = 3 * i + nelt0;
-	      if (3 * i + nelt1 < nelt)
-		sel[3 * i + nelt1] = 3 * i + nelt1;
-	      if (3 * i + nelt2 < nelt)
-		sel[3 * i + nelt2] = nelt + j2++;
-	    }
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_high = vect_gen_perm_mask_checked (vectype, indices);
-
-	  vect1 = dr_chain[0];
-	  vect2 = dr_chain[1];
-
-	  /* Create interleaving stmt:
-	     low = VEC_PERM_EXPR <vect1, vect2,
-				  {j, nelt, *, j + 1, nelt + j + 1, *,
-				   j + 2, nelt + j + 2, *, ...}>  */
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
-					   vect2, perm3_mask_low);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-
-	  vect1 = data_ref;
-	  vect2 = dr_chain[2];
-	  /* Create interleaving stmt:
-	     low = VEC_PERM_EXPR <vect1, vect2,
-				  {0, 1, nelt + j, 3, 4, nelt + j + 1,
-				   6, 7, nelt + j + 2, ...}>  */
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
-					   vect2, perm3_mask_high);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  (*result_chain)[j] = data_ref;
-	}
-    }
-  else
-    {
-      /* If length is not equal to 3 then only power of 2 is supported.  */
-      gcc_assert (pow2p_hwi (length));
-
-      /* The encoding has 2 interleaved stepped patterns.  */
-      poly_uint64 nelt = TYPE_VECTOR_SUBPARTS (vectype);
-      vec_perm_builder sel (nelt, 2, 3);
-      sel.quick_grow (6);
-      for (i = 0; i < 3; i++)
-	{
-	  sel[i * 2] = i;
-	  sel[i * 2 + 1] = i + nelt;
-	}
-	vec_perm_indices indices (sel, 2, nelt);
-	perm_mask_high = vect_gen_perm_mask_checked (vectype, indices);
-
-	for (i = 0; i < 6; i++)
-	  sel[i] += exact_div (nelt, 2);
-	indices.new_vector (sel, 2, nelt);
-	perm_mask_low = vect_gen_perm_mask_checked (vectype, indices);
-
-	for (i = 0, n = log_length; i < n; i++)
-	  {
-	    for (j = 0; j < length/2; j++)
-	      {
-		vect1 = dr_chain[j];
-		vect2 = dr_chain[j+length/2];
-
-		/* Create interleaving stmt:
-		   high = VEC_PERM_EXPR <vect1, vect2, {0, nelt, 1, nelt+1,
-							...}>  */
-		high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
-		perm_stmt = gimple_build_assign (high, VEC_PERM_EXPR, vect1,
-						 vect2, perm_mask_high);
-		vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-		(*result_chain)[2*j] = high;
-
-		/* Create interleaving stmt:
-		   low = VEC_PERM_EXPR <vect1, vect2,
-					{nelt/2, nelt*3/2, nelt/2+1, nelt*3/2+1,
-					 ...}>  */
-		low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
-		perm_stmt = gimple_build_assign (low, VEC_PERM_EXPR, vect1,
-						 vect2, perm_mask_low);
-		vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-		(*result_chain)[2*j+1] = low;
-	      }
-	    memcpy (dr_chain.address (), result_chain->address (),
-		    length * sizeof (tree));
-	  }
-    }
-}
-
 /* Function vect_setup_realignment
 
    This function is called when vectorizing an unaligned load using
@@ -6700,633 +6490,6 @@ vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
   return IFN_LAST;
 }
 
-/* Function vect_permute_load_chain.
-
-   Given a chain of interleaved loads in DR_CHAIN of LENGTH that must be
-   a power of 2 or equal to 3, generate extract_even/odd stmts to reorder
-   the input data correctly.  Return the final references for loads in
-   RESULT_CHAIN.
-
-   E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
-   The input is 4 vectors each containing 8 elements. We assign a number to each
-   element, the input sequence is:
-
-   1st vec:   0  1  2  3  4  5  6  7
-   2nd vec:   8  9 10 11 12 13 14 15
-   3rd vec:  16 17 18 19 20 21 22 23
-   4th vec:  24 25 26 27 28 29 30 31
-
-   The output sequence should be:
-
-   1st vec:  0 4  8 12 16 20 24 28
-   2nd vec:  1 5  9 13 17 21 25 29
-   3rd vec:  2 6 10 14 18 22 26 30
-   4th vec:  3 7 11 15 19 23 27 31
-
-   i.e., the first output vector should contain the first elements of each
-   interleaving group, etc.
-
-   We use extract_even/odd instructions to create such output.  The input of
-   each extract_even/odd operation is two vectors
-   1st vec    2nd vec
-   0 1 2 3    4 5 6 7
-
-   and the output is the vector of extracted even/odd elements.  The output of
-   extract_even will be:   0 2 4 6
-   and of extract_odd:     1 3 5 7
-
-
-   The permutation is done in log LENGTH stages.  In each stage extract_even
-   and extract_odd stmts are created for each pair of vectors in DR_CHAIN in
-   their order.  In our example,
-
-   E1: extract_even (1st vec, 2nd vec)
-   E2: extract_odd (1st vec, 2nd vec)
-   E3: extract_even (3rd vec, 4th vec)
-   E4: extract_odd (3rd vec, 4th vec)
-
-   The output for the first stage will be:
-
-   E1:  0  2  4  6  8 10 12 14
-   E2:  1  3  5  7  9 11 13 15
-   E3: 16 18 20 22 24 26 28 30
-   E4: 17 19 21 23 25 27 29 31
-
-   In order to proceed and create the correct sequence for the next stage (or
-   for the correct output, if the second stage is the last one, as in our
-   example), we first put the output of extract_even operation and then the
-   output of extract_odd in RESULT_CHAIN (which is then copied to DR_CHAIN).
-   The input for the second stage is:
-
-   1st vec (E1):  0  2  4  6  8 10 12 14
-   2nd vec (E3): 16 18 20 22 24 26 28 30
-   3rd vec (E2):  1  3  5  7  9 11 13 15
-   4th vec (E4): 17 19 21 23 25 27 29 31
-
-   The output of the second stage:
-
-   E1: 0 4  8 12 16 20 24 28
-   E2: 2 6 10 14 18 22 26 30
-   E3: 1 5  9 13 17 21 25 29
-   E4: 3 7 11 15 19 23 27 31
-
-   And RESULT_CHAIN after reordering:
-
-   1st vec (E1):  0 4  8 12 16 20 24 28
-   2nd vec (E3):  1 5  9 13 17 21 25 29
-   3rd vec (E2):  2 6 10 14 18 22 26 30
-   4th vec (E4):  3 7 11 15 19 23 27 31.  */
-
-static void
-vect_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
-			 unsigned int length,
-			 stmt_vec_info stmt_info,
-			 gimple_stmt_iterator *gsi,
-			 vec<tree> *result_chain)
-{
-  tree data_ref, first_vect, second_vect;
-  tree perm_mask_even, perm_mask_odd;
-  tree perm3_mask_low, perm3_mask_high;
-  gimple *perm_stmt;
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  unsigned int i, j, log_length = exact_log2 (length);
-
-  result_chain->quick_grow (length);
-  memcpy (result_chain->address (), dr_chain.address (),
-	  length * sizeof (tree));
-
-  if (length == 3)
-    {
-      /* vect_grouped_load_supported ensures that this is constant.  */
-      unsigned nelt = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
-      unsigned int k;
-
-      vec_perm_builder sel (nelt, nelt, 1);
-      sel.quick_grow (nelt);
-      vec_perm_indices indices;
-      for (k = 0; k < 3; k++)
-	{
-	  for (i = 0; i < nelt; i++)
-	    if (3 * i + k < 2 * nelt)
-	      sel[i] = 3 * i + k;
-	    else
-	      sel[i] = 0;
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_low = vect_gen_perm_mask_checked (vectype, indices);
-
-	  for (i = 0, j = 0; i < nelt; i++)
-	    if (3 * i + k < 2 * nelt)
-	      sel[i] = i;
-	    else
-	      sel[i] = nelt + ((nelt + k) % 3) + 3 * (j++);
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_high = vect_gen_perm_mask_checked (vectype, indices);
-
-	  first_vect = dr_chain[0];
-	  second_vect = dr_chain[1];
-
-	  /* Create interleaving stmt (low part of):
-	     low = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
-							     ...}>  */
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
-					   second_vect, perm3_mask_low);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-
-	  /* Create interleaving stmt (high part of):
-	     high = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
-							      ...}>  */
-	  first_vect = data_ref;
-	  second_vect = dr_chain[2];
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
-					   second_vect, perm3_mask_high);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  (*result_chain)[k] = data_ref;
-	}
-    }
-  else
-    {
-      /* If length is not equal to 3 then only power of 2 is supported.  */
-      gcc_assert (pow2p_hwi (length));
-
-      /* The encoding has a single stepped pattern.  */
-      poly_uint64 nelt = TYPE_VECTOR_SUBPARTS (vectype);
-      vec_perm_builder sel (nelt, 1, 3);
-      sel.quick_grow (3);
-      for (i = 0; i < 3; ++i)
-	sel[i] = i * 2;
-      vec_perm_indices indices (sel, 2, nelt);
-      perm_mask_even = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < 3; ++i)
-	sel[i] = i * 2 + 1;
-      indices.new_vector (sel, 2, nelt);
-      perm_mask_odd = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < log_length; i++)
-	{
-	  for (j = 0; j < length; j += 2)
-	    {
-	      first_vect = dr_chain[j];
-	      second_vect = dr_chain[j+1];
-
-	      /* data_ref = permute_even (first_data_ref, second_data_ref);  */
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_perm_even");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       first_vect, second_vect,
-					       perm_mask_even);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2] = data_ref;
-
-	      /* data_ref = permute_odd (first_data_ref, second_data_ref);  */
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_perm_odd");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       first_vect, second_vect,
-					       perm_mask_odd);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2+length/2] = data_ref;
-	    }
-	  memcpy (dr_chain.address (), result_chain->address (),
-		  length * sizeof (tree));
-	}
-    }
-}
-
-/* Function vect_shift_permute_load_chain.
-
-   Given a chain of loads in DR_CHAIN of LENGTH 2 or 3, generate
-   sequence of stmts to reorder the input data accordingly.
-   Return the final references for loads in RESULT_CHAIN.
-   Return true if successed, false otherwise.
-
-   E.g., LENGTH is 3 and the scalar type is short, i.e., VF is 8.
-   The input is 3 vectors each containing 8 elements.  We assign a
-   number to each element, the input sequence is:
-
-   1st vec:   0  1  2  3  4  5  6  7
-   2nd vec:   8  9 10 11 12 13 14 15
-   3rd vec:  16 17 18 19 20 21 22 23
-
-   The output sequence should be:
-
-   1st vec:  0 3 6  9 12 15 18 21
-   2nd vec:  1 4 7 10 13 16 19 22
-   3rd vec:  2 5 8 11 14 17 20 23
-
-   We use 3 shuffle instructions and 3 * 3 - 1 shifts to create such output.
-
-   First we shuffle all 3 vectors to get correct elements order:
-
-   1st vec:  ( 0  3  6) ( 1  4  7) ( 2  5)
-   2nd vec:  ( 8 11 14) ( 9 12 15) (10 13)
-   3rd vec:  (16 19 22) (17 20 23) (18 21)
-
-   Next we unite and shift vector 3 times:
-
-   1st step:
-     shift right by 6 the concatenation of:
-     "1st vec" and  "2nd vec"
-       ( 0  3  6) ( 1  4  7) |( 2  5) _ ( 8 11 14) ( 9 12 15)| (10 13)
-     "2nd vec" and  "3rd vec"
-       ( 8 11 14) ( 9 12 15) |(10 13) _ (16 19 22) (17 20 23)| (18 21)
-     "3rd vec" and  "1st vec"
-       (16 19 22) (17 20 23) |(18 21) _ ( 0  3  6) ( 1  4  7)| ( 2  5)
-			     | New vectors                   |
-
-     So that now new vectors are:
-
-     1st vec:  ( 2  5) ( 8 11 14) ( 9 12 15)
-     2nd vec:  (10 13) (16 19 22) (17 20 23)
-     3rd vec:  (18 21) ( 0  3  6) ( 1  4  7)
-
-   2nd step:
-     shift right by 5 the concatenation of:
-     "1st vec" and  "3rd vec"
-       ( 2  5) ( 8 11 14) |( 9 12 15) _ (18 21) ( 0  3  6)| ( 1  4  7)
-     "2nd vec" and  "1st vec"
-       (10 13) (16 19 22) |(17 20 23) _ ( 2  5) ( 8 11 14)| ( 9 12 15)
-     "3rd vec" and  "2nd vec"
-       (18 21) ( 0  3  6) |( 1  4  7) _ (10 13) (16 19 22)| (17 20 23)
-			  | New vectors                   |
-
-     So that now new vectors are:
-
-     1st vec:  ( 9 12 15) (18 21) ( 0  3  6)
-     2nd vec:  (17 20 23) ( 2  5) ( 8 11 14)
-     3rd vec:  ( 1  4  7) (10 13) (16 19 22) READY
-
-   3rd step:
-     shift right by 5 the concatenation of:
-     "1st vec" and  "1st vec"
-       ( 9 12 15) (18 21) |( 0  3  6) _ ( 9 12 15) (18 21)| ( 0  3  6)
-     shift right by 3 the concatenation of:
-     "2nd vec" and  "2nd vec"
-               (17 20 23) |( 2  5) ( 8 11 14) _ (17 20 23)| ( 2  5) ( 8 11 14)
-			  | New vectors                   |
-
-     So that now all vectors are READY:
-     1st vec:  ( 0  3  6) ( 9 12 15) (18 21)
-     2nd vec:  ( 2  5) ( 8 11 14) (17 20 23)
-     3rd vec:  ( 1  4  7) (10 13) (16 19 22)
-
-   This algorithm is faster than one in vect_permute_load_chain if:
-     1.  "shift of a concatination" is faster than general permutation.
-	 This is usually so.
-     2.  The TARGET machine can't execute vector instructions in parallel.
-	 This is because each step of the algorithm depends on previous.
-	 The algorithm in vect_permute_load_chain is much more parallel.
-
-   The algorithm is applicable only for LOAD CHAIN LENGTH less than VF.
-*/
-
-static bool
-vect_shift_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
-			       unsigned int length,
-			       stmt_vec_info stmt_info,
-			       gimple_stmt_iterator *gsi,
-			       vec<tree> *result_chain)
-{
-  tree vect[3], vect_shift[3], data_ref, first_vect, second_vect;
-  tree perm2_mask1, perm2_mask2, perm3_mask;
-  tree select_mask, shift1_mask, shift2_mask, shift3_mask, shift4_mask;
-  gimple *perm_stmt;
-
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  machine_mode vmode = TYPE_MODE (vectype);
-  unsigned int i;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-
-  unsigned HOST_WIDE_INT nelt, vf;
-  if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nelt)
-      || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
-    /* Not supported for variable-length vectors.  */
-    return false;
-
-  vec_perm_builder sel (nelt, nelt, 1);
-  sel.quick_grow (nelt);
-
-  result_chain->quick_grow (length);
-  memcpy (result_chain->address (), dr_chain.address (),
-	  length * sizeof (tree));
-
-  if (pow2p_hwi (length) && vf > 4)
-    {
-      unsigned int j, log_length = exact_log2 (length);
-      for (i = 0; i < nelt / 2; ++i)
-	sel[i] = i * 2;
-      for (i = 0; i < nelt / 2; ++i)
-	sel[nelt / 2 + i] = i * 2 + 1;
-      vec_perm_indices indices (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shuffle of 2 fields structure is not \
-			      supported by target\n");
-	  return false;
-	}
-      perm2_mask1 = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < nelt / 2; ++i)
-	sel[i] = i * 2 + 1;
-      for (i = 0; i < nelt / 2; ++i)
-	sel[nelt / 2 + i] = i * 2;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shuffle of 2 fields structure is not \
-			      supported by target\n");
-	  return false;
-	}
-      perm2_mask2 = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {4 5 6 7 8 9 10 11}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = nelt / 2 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift1_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to select vector from 2.
-	 For vector length 8 it is {0 1 2 3 12 13 14 15}.  */
-      for (i = 0; i < nelt / 2; i++)
-	sel[i] = i;
-      for (i = nelt / 2; i < nelt; i++)
-	sel[i] = nelt + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "select is not supported by target\n");
-	  return false;
-	}
-      select_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < log_length; i++)
-	{
-	  for (j = 0; j < length; j += 2)
-	    {
-	      first_vect = dr_chain[j];
-	      second_vect = dr_chain[j + 1];
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       first_vect, first_vect,
-					       perm2_mask1);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      vect[0] = data_ref;
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       second_vect, second_vect,
-					       perm2_mask2);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      vect[1] = data_ref;
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       vect[0], vect[1], shift1_mask);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2 + length/2] = data_ref;
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_select");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       vect[0], vect[1], select_mask);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2] = data_ref;
-	    }
-	  memcpy (dr_chain.address (), result_chain->address (),
-		  length * sizeof (tree));
-	}
-      return true;
-    }
-  if (length == 3 && vf > 2)
-    {
-      unsigned int k = 0, l = 0;
-
-      /* Generating permutation constant to get all elements in rigth order.
-	 For vector length 8 it is {0 3 6 1 4 7 2 5}.  */
-      for (i = 0; i < nelt; i++)
-	{
-	  if (3 * k + (l % 3) >= nelt)
-	    {
-	      k = 0;
-	      l += (3 - (nelt % 3));
-	    }
-	  sel[i] = 3 * k + (l % 3);
-	  k++;
-	}
-      vec_perm_indices indices (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shuffle of 3 fields structure is not \
-			      supported by target\n");
-	  return false;
-	}
-      perm3_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {6 7 8 9 10 11 12 13}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = 2 * (nelt / 3) + (nelt % 3) + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift1_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {5 6 7 8 9 10 11 12}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = 2 * (nelt / 3) + 1 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift2_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {3 4 5 6 7 8 9 10}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = (nelt / 3) + (nelt % 3) / 2 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift3_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {5 6 7 8 9 10 11 12}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = 2 * (nelt / 3) + (nelt % 3) / 2 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift4_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (k = 0; k < 3; k++)
-	{
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					   dr_chain[k], dr_chain[k],
-					   perm3_mask);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  vect[k] = data_ref;
-	}
-
-      for (k = 0; k < 3; k++)
-	{
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift1");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					   vect[k % 3], vect[(k + 1) % 3],
-					   shift1_mask);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  vect_shift[k] = data_ref;
-	}
-
-      for (k = 0; k < 3; k++)
-	{
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift2");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					   vect_shift[(4 - k) % 3],
-					   vect_shift[(3 - k) % 3],
-					   shift2_mask);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  vect[k] = data_ref;
-	}
-
-      (*result_chain)[3 - (nelt % 3)] = vect[2];
-
-      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift3");
-      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[0],
-				       vect[0], shift3_mask);
-      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-      (*result_chain)[nelt % 3] = data_ref;
-
-      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift4");
-      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[1],
-				       vect[1], shift4_mask);
-      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-      (*result_chain)[0] = data_ref;
-      return true;
-    }
-  return false;
-}
-
-/* Function vect_transform_grouped_load.
-
-   Given a chain of input interleaved data-refs (in DR_CHAIN), build statements
-   to perform their permutation and ascribe the result vectorized statements to
-   the scalar statements.
-*/
-
-void
-vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info,
-			     vec<tree> dr_chain,
-			     int size, gimple_stmt_iterator *gsi)
-{
-  machine_mode mode;
-  vec<tree> result_chain = vNULL;
-
-  /* DR_CHAIN contains input data-refs that are a part of the interleaving.
-     RESULT_CHAIN is the output of vect_permute_load_chain, it contains permuted
-     vectors, that are ready for vector computation.  */
-  result_chain.create (size);
-
-  /* If reassociation width for vector type is 2 or greater target machine can
-     execute 2 or more vector instructions in parallel.  Otherwise try to
-     get chain for loads group using vect_shift_permute_load_chain.  */
-  mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
-  if (targetm.sched.reassociation_width (VEC_PERM_EXPR, mode) > 1
-      || pow2p_hwi (size)
-      || !vect_shift_permute_load_chain (vinfo, dr_chain, size, stmt_info,
-					 gsi, &result_chain))
-    vect_permute_load_chain (vinfo, dr_chain,
-			     size, stmt_info, gsi, &result_chain);
-  vect_record_grouped_load_vectors (vinfo, stmt_info, result_chain);
-  result_chain.release ();
-}
-
-/* RESULT_CHAIN contains the output of a group of grouped loads that were
-   generated as part of the vectorization of STMT_INFO.  Assign the statement
-   for each vector to the associated scalar statement.  */
-
-void
-vect_record_grouped_load_vectors (vec_info *, stmt_vec_info stmt_info,
-				  vec<tree> result_chain)
-{
-  stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
-  unsigned int i, gap_count;
-  tree tmp_data_ref;
-
-  /* Put a permuted data-ref in the VECTORIZED_STMT field.
-     Since we scan the chain starting from it's first node, their order
-     corresponds the order of data-refs in RESULT_CHAIN.  */
-  stmt_vec_info next_stmt_info = first_stmt_info;
-  gap_count = 1;
-  FOR_EACH_VEC_ELT (result_chain, i, tmp_data_ref)
-    {
-      if (!next_stmt_info)
-	break;
-
-      /* Skip the gaps.  Loads created for the gaps will be removed by dead
-       code elimination pass later.  No need to check for the first stmt in
-       the group, since it always exists.
-       DR_GROUP_GAP is the number of steps in elements from the previous
-       access (if there is no gap DR_GROUP_GAP is 1).  We skip loads that
-       correspond to the gaps.  */
-      if (next_stmt_info != first_stmt_info
-	  && gap_count < DR_GROUP_GAP (next_stmt_info))
-	{
-	  gap_count++;
-	  continue;
-	}
-
-      /* ???  The following needs cleanup after the removal of
-         DR_GROUP_SAME_DR_STMT.  */
-      if (next_stmt_info)
-        {
-	  gimple *new_stmt = SSA_NAME_DEF_STMT (tmp_data_ref);
-	  /* We assume that if VEC_STMT is not NULL, this is a case of multiple
-	     copies, and we put the new vector statement last.  */
-	  STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt);
-
-	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
-	  gap_count = 1;
-        }
-    }
-}
-
 /* Function vect_force_dr_alignment_p.
 
    Returns whether the alignment of a DECL can be forced to be aligned
@@ -7354,13 +6517,14 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
    alignment.
    If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even
    it is aligned, i.e., check if it is possible to vectorize it with different
-   alignment.  */
+   alignment.  If GS_INFO is passed we are dealing with a gather/scatter.  */
 
 enum dr_alignment_support
 vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
-			       tree vectype, int misalignment)
+			       tree vectype, int misalignment,
+			       gather_scatter_info *gs_info)
 {
-  data_reference *dr = dr_info->dr;
+  data_reference *dr = dr_info ? dr_info->dr : nullptr;
   stmt_vec_info stmt_info = dr_info->stmt;
   machine_mode mode = TYPE_MODE (vectype);
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -7372,14 +6536,6 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
   else if (dr_safe_speculative_read_required (stmt_info))
     return dr_unaligned_unsupported;
 
-  /* For now assume all conditional loads/stores support unaligned
-     access without any special code.  */
-  if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
-    if (gimple_call_internal_p (stmt)
-	&& (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD
-	    || gimple_call_internal_fn (stmt) == IFN_MASK_STORE))
-      return dr_unaligned_supported;
-
   if (loop_vinfo)
     {
       vect_loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -7449,7 +6605,7 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
         }
     } */
 
-  if (DR_IS_READ (dr))
+  if (dr && DR_IS_READ (dr))
     {
       if (can_implement_p (vec_realign_load_optab, mode)
 	  && (!targetm.vectorize.builtin_mask_for_load
@@ -7477,10 +6633,43 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
 
   bool is_packed = false;
   tree type = TREE_TYPE (DR_REF (dr));
+  bool is_gather_scatter = gs_info != nullptr;
   if (misalignment == DR_MISALIGNMENT_UNKNOWN)
-    is_packed = not_size_aligned (DR_REF (dr));
+    {
+      if (!is_gather_scatter || dr != nullptr)
+	is_packed = not_size_aligned (DR_REF (dr));
+      else
+	{
+	  /* Gather-scatter accesses normally perform only component accesses
+	     so alignment is irrelevant for them.  Targets like riscv do care
+	     about scalar alignment in vector accesses, though, so check scalar
+	     alignment here.  We determined the alias pointer as well as the
+	     base alignment during pattern recognition and can re-use it here.
+
+	     As we do not have an analyzed dataref we only know the alignment
+	     of the reference itself and nothing about init, steps, etc.
+	     For now don't try harder to determine misalignment and
+	     just assume it is unknown.  We consider the type packed if its
+	     scalar alignment is lower than the natural alignment of a vector
+	     element's type.  */
+
+	  gcc_assert (!GATHER_SCATTER_LEGACY_P (*gs_info));
+	  gcc_assert (dr == nullptr);
+
+	  tree inner_vectype = TREE_TYPE (vectype);
+
+	  unsigned HOST_WIDE_INT scalar_align
+	    = tree_to_uhwi (gs_info->alias_ptr);
+	  unsigned HOST_WIDE_INT inner_vectype_sz
+	    = tree_to_uhwi (TYPE_SIZE (inner_vectype));
+
+	  bool is_misaligned = scalar_align < inner_vectype_sz;
+	  is_packed = scalar_align > 1 && is_misaligned;
+	}
+    }
   if (targetm.vectorize.support_vector_misalignment (mode, type, misalignment,
-						     is_packed))
+						     is_packed,
+						     is_gather_scatter))
     return dr_unaligned_supported;
 
   /* Unsupported.  */
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 4696943..6c1b26a 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -2857,9 +2857,21 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters,
 	 we set range information to make niters analyzer's life easier.
 	 Note the number of latch iteration value can be TYPE_MAX_VALUE so
 	 we have to represent the vector niter TYPE_MAX_VALUE + 1 / vf.  */
-      if (stmts != NULL && const_vf > 0)
+      if (stmts != NULL && const_vf > 0 && !LOOP_VINFO_EPILOGUE_P (loop_vinfo))
 	{
-	  if (niters_no_overflow)
+	  if (niters_no_overflow
+	      && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
+	    {
+	      int_range<1> vr (type,
+			       wi::one (TYPE_PRECISION (type)),
+			       wi::div_ceil (wi::max_value
+							(TYPE_PRECISION (type),
+							 TYPE_SIGN (type)),
+					     const_vf,
+					     TYPE_SIGN (type)));
+	      set_range_info (niters_vector, vr);
+	    }
+	  else if (niters_no_overflow)
 	    {
 	      int_range<1> vr (type,
 			       wi::one (TYPE_PRECISION (type)),
@@ -3283,7 +3295,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
   bool skip_vector = (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
 		      ? maybe_lt (LOOP_VINFO_INT_NITERS (loop_vinfo),
 				  bound_prolog + bound_epilog)
-		      : (!LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
+		      : (!LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING (loop_vinfo)
 			 || vect_epilogues));
 
   /* Epilog loop must be executed if the number of iterations for epilog
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 2782d61..85f3e90 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -58,6 +58,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-eh.h"
 #include "case-cfn-macros.h"
 #include "langhooks.h"
+#include "opts.h"
 
 /* Loop Vectorization Pass.
 
@@ -167,9 +168,8 @@ static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
    may already be set for general statements (not just data refs).  */
 
 static opt_result
-vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
-			      bool vectype_maybe_set_p,
-			      poly_uint64 *vf)
+vect_determine_vectype_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
+				   bool vectype_maybe_set_p)
 {
   gimple *stmt = stmt_info->stmt;
 
@@ -202,9 +202,6 @@ vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
 	STMT_VINFO_VECTYPE (stmt_info) = stmt_vectype;
     }
 
-  if (nunits_vectype)
-    vect_update_max_nunits (vf, nunits_vectype);
-
   return opt_result::success ();
 }
 
@@ -214,13 +211,12 @@ vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
    or false if something prevented vectorization.  */
 
 static opt_result
-vect_determine_vf_for_stmt (vec_info *vinfo,
-			    stmt_vec_info stmt_info, poly_uint64 *vf)
+vect_determine_vectype_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info)
 {
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
 		     stmt_info->stmt);
-  opt_result res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, false, vf);
+  opt_result res = vect_determine_vectype_for_stmt_1 (vinfo, stmt_info, false);
   if (!res)
     return res;
 
@@ -239,7 +235,7 @@ vect_determine_vf_for_stmt (vec_info *vinfo,
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "==> examining pattern def stmt: %G",
 			     def_stmt_info->stmt);
-	  res = vect_determine_vf_for_stmt_1 (vinfo, def_stmt_info, true, vf);
+	  res = vect_determine_vectype_for_stmt_1 (vinfo, def_stmt_info, true);
 	  if (!res)
 	    return res;
 	}
@@ -248,7 +244,7 @@ vect_determine_vf_for_stmt (vec_info *vinfo,
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "==> examining pattern statement: %G",
 			 stmt_info->stmt);
-      res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, true, vf);
+      res = vect_determine_vectype_for_stmt_1 (vinfo, stmt_info, true);
       if (!res)
 	return res;
     }
@@ -256,45 +252,23 @@ vect_determine_vf_for_stmt (vec_info *vinfo,
   return opt_result::success ();
 }
 
-/* Function vect_determine_vectorization_factor
-
-   Determine the vectorization factor (VF).  VF is the number of data elements
-   that are operated upon in parallel in a single iteration of the vectorized
-   loop.  For example, when vectorizing a loop that operates on 4byte elements,
-   on a target with vector size (VS) 16byte, the VF is set to 4, since 4
-   elements can fit in a single vector register.
-
-   We currently support vectorization of loops in which all types operated upon
-   are of the same size.  Therefore this function currently sets VF according to
-   the size of the types operated upon, and fails if there are multiple sizes
-   in the loop.
+/* Function vect_set_stmts_vectype
 
-   VF is also the factor by which the loop iterations are strip-mined, e.g.:
-   original loop:
-        for (i=0; i<N; i++){
-          a[i] = b[i] + c[i];
-        }
-
-   vectorized loop:
-        for (i=0; i<N; i+=VF){
-          a[i:VF] = b[i:VF] + c[i:VF];
-        }
-*/
+   Set STMT_VINFO_VECTYPE of all stmts.  */
 
 static opt_result
-vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
+vect_set_stmts_vectype (loop_vec_info loop_vinfo)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
   unsigned nbbs = loop->num_nodes;
-  poly_uint64 vectorization_factor = 1;
   tree scalar_type = NULL_TREE;
   gphi *phi;
   tree vectype;
   stmt_vec_info stmt_info;
   unsigned i;
 
-  DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
+  DUMP_VECT_SCOPE ("vect_set_stmts_vectype");
 
   for (i = 0; i < nbbs; i++)
     {
@@ -333,15 +307,6 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
 	      if (dump_enabled_p ())
 		dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n",
 				 vectype);
-
-	      if (dump_enabled_p ())
-		{
-		  dump_printf_loc (MSG_NOTE, vect_location, "nunits = ");
-		  dump_dec (MSG_NOTE, TYPE_VECTOR_SUBPARTS (vectype));
-		  dump_printf (MSG_NOTE, "\n");
-		}
-
-	      vect_update_max_nunits (&vectorization_factor, vectype);
 	    }
 	}
 
@@ -352,25 +317,12 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
 	    continue;
 	  stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
 	  opt_result res
-	    = vect_determine_vf_for_stmt (loop_vinfo,
-					  stmt_info, &vectorization_factor);
+	    = vect_determine_vectype_for_stmt (loop_vinfo, stmt_info);
 	  if (!res)
 	    return res;
         }
     }
 
-  /* TODO: Analyze cost. Decide if worth while to vectorize.  */
-  if (dump_enabled_p ())
-    {
-      dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
-      dump_dec (MSG_NOTE, vectorization_factor);
-      dump_printf (MSG_NOTE, "\n");
-    }
-
-  if (known_le (vectorization_factor, 1U))
-    return opt_result::failure_at (vect_location,
-				   "not vectorized: unsupported data-type\n");
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
   return opt_result::success ();
 }
 
@@ -1967,7 +1919,6 @@ vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
   for (gcond *cond : info->conds)
     {
       stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (cond);
-      STMT_VINFO_TYPE (loop_cond_info) = loop_exit_ctrl_vec_info_type;
       /* Mark the statement as a condition.  */
       STMT_VINFO_DEF_TYPE (loop_cond_info) = vect_condition_def;
     }
@@ -1984,9 +1935,6 @@ vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
 
   if (info->inner_loop_cond)
     {
-      stmt_vec_info inner_loop_cond_info
-	= loop_vinfo->lookup_stmt (info->inner_loop_cond);
-      STMT_VINFO_TYPE (inner_loop_cond_info) = loop_exit_ctrl_vec_info_type;
       /* If we have an estimate on the number of iterations of the inner
 	 loop use that to limit the scale for costing, otherwise use
 	 --param vect-inner-loop-cost-factor literally.  */
@@ -2001,234 +1949,6 @@ vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
 
 
 
-/* Scan the loop stmts and dependent on whether there are any (non-)SLP
-   statements update the vectorization factor.  */
-
-static void
-vect_update_vf_for_slp (loop_vec_info loop_vinfo)
-{
-  class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
-  int nbbs = loop->num_nodes;
-  poly_uint64 vectorization_factor;
-  int i;
-
-  DUMP_VECT_SCOPE ("vect_update_vf_for_slp");
-
-  vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-  gcc_assert (known_ne (vectorization_factor, 0U));
-
-  /* If all the stmts in the loop can be SLPed, we perform only SLP, and
-     vectorization factor of the loop is the unrolling factor required by
-     the SLP instances.  If that unrolling factor is 1, we say, that we
-     perform pure SLP on loop - cross iteration parallelism is not
-     exploited.  */
-  bool only_slp_in_loop = true;
-  for (i = 0; i < nbbs; i++)
-    {
-      basic_block bb = bbs[i];
-      for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-	{
-	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (si.phi ());
-	  if (!stmt_info)
-	    continue;
-	  if ((STMT_VINFO_RELEVANT_P (stmt_info)
-	       || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
-	      && !PURE_SLP_STMT (stmt_info))
-	    /* STMT needs both SLP and loop-based vectorization.  */
-	    only_slp_in_loop = false;
-	}
-      for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-	{
-	  if (is_gimple_debug (gsi_stmt (si)))
-	    continue;
-	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
-	  stmt_info = vect_stmt_to_vectorize (stmt_info);
-	  if ((STMT_VINFO_RELEVANT_P (stmt_info)
-	       || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
-	      && !PURE_SLP_STMT (stmt_info))
-	    /* STMT needs both SLP and loop-based vectorization.  */
-	    only_slp_in_loop = false;
-	}
-    }
-
-  if (only_slp_in_loop)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "Loop contains only SLP stmts\n");
-      vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
-    }
-  else
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "Loop contains SLP and non-SLP stmts\n");
-      /* Both the vectorization factor and unroll factor have the form
-	 GET_MODE_SIZE (loop_vinfo->vector_mode) * X for some rational X,
-	 so they must have a common multiple.  */
-      vectorization_factor
-	= force_common_multiple (vectorization_factor,
-				 LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
-    }
-
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
-  if (dump_enabled_p ())
-    {
-      dump_printf_loc (MSG_NOTE, vect_location,
-		       "Updating vectorization factor to ");
-      dump_dec (MSG_NOTE, vectorization_factor);
-      dump_printf (MSG_NOTE, ".\n");
-    }
-}
-
-/* Return true if STMT_INFO describes a double reduction phi and if
-   the other phi in the reduction is also relevant for vectorization.
-   This rejects cases such as:
-
-      outer1:
-	x_1 = PHI <x_3(outer2), ...>;
-	...
-
-      inner:
-	x_2 = ...;
-	...
-
-      outer2:
-	x_3 = PHI <x_2(inner)>;
-
-   if nothing in x_2 or elsewhere makes x_1 relevant.  */
-
-static bool
-vect_active_double_reduction_p (stmt_vec_info stmt_info)
-{
-  if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
-    return false;
-
-  return STMT_VINFO_RELEVANT_P (STMT_VINFO_REDUC_DEF (stmt_info));
-}
-
-/* Function vect_analyze_loop_operations.
-
-   Scan the loop stmts and make sure they are all vectorizable.  */
-
-static opt_result
-vect_analyze_loop_operations (loop_vec_info loop_vinfo)
-{
-  class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
-  int nbbs = loop->num_nodes;
-  int i;
-  stmt_vec_info stmt_info;
-
-  DUMP_VECT_SCOPE ("vect_analyze_loop_operations");
-
-  for (i = 0; i < nbbs; i++)
-    {
-      basic_block bb = bbs[i];
-
-      for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-        {
-          gphi *phi = si.phi ();
-
-	  stmt_info = loop_vinfo->lookup_stmt (phi);
-          if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location, "examining phi: %G",
-			     (gimple *) phi);
-	  if (virtual_operand_p (gimple_phi_result (phi)))
-	    continue;
-
-	  /* ???  All of the below unconditional FAILs should be in
-	     done earlier after analyzing cycles, possibly when
-	     determining stmt relevancy?  */
-
-          /* Inner-loop loop-closed exit phi in outer-loop vectorization
-             (i.e., a phi in the tail of the outer-loop).  */
-          if (! is_loop_header_bb_p (bb))
-            {
-              /* FORNOW: we currently don't support the case that these phis
-                 are not used in the outerloop (unless it is double reduction,
-                 i.e., this phi is vect_reduction_def), cause this case
-                 requires to actually do something here.  */
-              if (STMT_VINFO_LIVE_P (stmt_info)
-		  && !vect_active_double_reduction_p (stmt_info))
-		return opt_result::failure_at (phi,
-					       "Unsupported loop-closed phi"
-					       " in outer-loop.\n");
-
-              /* If PHI is used in the outer loop, we check that its operand
-                 is defined in the inner loop.  */
-              if (STMT_VINFO_RELEVANT_P (stmt_info))
-                {
-                  tree phi_op;
-
-                  if (gimple_phi_num_args (phi) != 1)
-                    return opt_result::failure_at (phi, "unsupported phi");
-
-                  phi_op = PHI_ARG_DEF (phi, 0);
-		  stmt_vec_info op_def_info = loop_vinfo->lookup_def (phi_op);
-		  if (!op_def_info)
-		    return opt_result::failure_at (phi, "unsupported phi\n");
-
-		  if (STMT_VINFO_RELEVANT (op_def_info) != vect_used_in_outer
-		      && (STMT_VINFO_RELEVANT (op_def_info)
-			  != vect_used_in_outer_by_reduction))
-		    return opt_result::failure_at (phi, "unsupported phi\n");
-
-		  if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
-		       || (STMT_VINFO_DEF_TYPE (stmt_info)
-			   == vect_double_reduction_def))
-		      && ! PURE_SLP_STMT (stmt_info))
-		    return opt_result::failure_at (phi, "unsupported phi\n");
-                }
-
-              continue;
-            }
-
-          gcc_assert (stmt_info);
-
-          if ((STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope
-               || STMT_VINFO_LIVE_P (stmt_info))
-	      && STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def
-	      && STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
-	    /* A scalar-dependence cycle that we don't support.  */
-	    return opt_result::failure_at (phi,
-					   "not vectorized:"
-					   " scalar dependence cycle.\n");
-
-	  if (STMT_VINFO_RELEVANT_P (stmt_info)
-	      && ! PURE_SLP_STMT (stmt_info))
-	    return opt_result::failure_at (phi,
-					   "not vectorized: relevant phi not "
-					   "supported: %G",
-					   static_cast <gimple *> (phi));
-        }
-
-      for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-        {
-	  gimple *stmt = gsi_stmt (si);
-	  if (!gimple_clobber_p (stmt)
-	      && !is_gimple_debug (stmt))
-	    {
-	      bool need_to_vectorize = false;
-	      opt_result res
-		= vect_analyze_stmt (loop_vinfo,
-				     loop_vinfo->lookup_stmt (stmt),
-				     &need_to_vectorize,
-				     NULL, NULL, NULL);
-	      if (!res)
-		return res;
-	    }
-        }
-    } /* bbs */
-
-  return opt_result::success ();
-}
-
 /* Return true if we know that the iteration count is smaller than the
    vectorization factor.  Return false if it isn't, or if we can't be sure
    either way.  */
@@ -2529,78 +2249,6 @@ vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs,
   return opt_result::success ();
 }
 
-/* Look for SLP-only access groups and turn each individual access into its own
-   group.  */
-static void
-vect_dissolve_slp_only_groups (loop_vec_info loop_vinfo)
-{
-  unsigned int i;
-  struct data_reference *dr;
-
-  DUMP_VECT_SCOPE ("vect_dissolve_slp_only_groups");
-
-  vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
-  FOR_EACH_VEC_ELT (datarefs, i, dr)
-    {
-      gcc_assert (DR_REF (dr));
-      stmt_vec_info stmt_info
-	= vect_stmt_to_vectorize (loop_vinfo->lookup_stmt (DR_STMT (dr)));
-
-      /* Check if the load is a part of an interleaving chain.  */
-      if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
-	{
-	  stmt_vec_info first_element = DR_GROUP_FIRST_ELEMENT (stmt_info);
-	  dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_element);
-	  unsigned int group_size = DR_GROUP_SIZE (first_element);
-
-	  /* Check if SLP-only groups.  */
-	  if (!STMT_SLP_TYPE (stmt_info)
-	      && STMT_VINFO_SLP_VECT_ONLY (first_element))
-	    {
-	      /* Dissolve the group.  */
-	      STMT_VINFO_SLP_VECT_ONLY (first_element) = false;
-
-	      stmt_vec_info vinfo = first_element;
-	      while (vinfo)
-		{
-		  stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (vinfo);
-		  DR_GROUP_FIRST_ELEMENT (vinfo) = vinfo;
-		  DR_GROUP_NEXT_ELEMENT (vinfo) = NULL;
-		  DR_GROUP_SIZE (vinfo) = 1;
-		  if (STMT_VINFO_STRIDED_P (first_element)
-		      /* We cannot handle stores with gaps.  */
-		      || DR_IS_WRITE (dr_info->dr))
-		    {
-		      STMT_VINFO_STRIDED_P (vinfo) = true;
-		      DR_GROUP_GAP (vinfo) = 0;
-		    }
-		  else
-		    DR_GROUP_GAP (vinfo) = group_size - 1;
-		  /* Duplicate and adjust alignment info, it needs to
-		     be present on each group leader, see dr_misalignment.  */
-		  if (vinfo != first_element)
-		    {
-		      dr_vec_info *dr_info2 = STMT_VINFO_DR_INFO (vinfo);
-		      dr_info2->target_alignment = dr_info->target_alignment;
-		      int misalignment = dr_info->misalignment;
-		      if (misalignment != DR_MISALIGNMENT_UNKNOWN)
-			{
-			  HOST_WIDE_INT diff
-			    = (TREE_INT_CST_LOW (DR_INIT (dr_info2->dr))
-			       - TREE_INT_CST_LOW (DR_INIT (dr_info->dr)));
-			  unsigned HOST_WIDE_INT align_c
-			    = dr_info->target_alignment.to_constant ();
-			  misalignment = (misalignment + diff) % align_c;
-			}
-		      dr_info2->misalignment = misalignment;
-		    }
-		  vinfo = next;
-		}
-	    }
-	}
-    }
-}
-
 /* Determine if operating on full vectors for LOOP_VINFO might leave
    some scalar iterations still to do.  If so, decide how we should
    handle those scalar iterations.  The possibilities are:
@@ -2721,7 +2369,6 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
   opt_result ok = opt_result::success ();
   int res;
   unsigned int max_vf = MAX_VECTORIZATION_FACTOR;
-  poly_uint64 min_vf = 2;
   loop_vec_info orig_loop_vinfo = NULL;
 
   /* If we are dealing with an epilogue then orig_loop_vinfo points to the
@@ -2768,7 +2415,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
   /* Analyze the data references and also adjust the minimal
      vectorization factor according to the loads and stores.  */
 
-  ok = vect_analyze_data_refs (loop_vinfo, &min_vf, &fatal);
+  ok = vect_analyze_data_refs (loop_vinfo, &fatal);
   if (!ok)
     {
       if (dump_enabled_p ())
@@ -2833,24 +2480,20 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
 			 "bad data dependence.\n");
       return ok;
     }
-  if (max_vf != MAX_VECTORIZATION_FACTOR
-      && maybe_lt (max_vf, min_vf))
-    return opt_result::failure_at (vect_location, "bad data dependence.\n");
   LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) = max_vf;
 
-  ok = vect_determine_vectorization_factor (loop_vinfo);
+  ok = vect_set_stmts_vectype (loop_vinfo);
   if (!ok)
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "can't determine vectorization factor.\n");
+			 "cannot determine vector types.\n");
       return ok;
     }
 
   /* Compute the scalar iteration cost.  */
   vect_compute_single_scalar_iteration_cost (loop_vinfo);
 
-  poly_uint64 saved_vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   bool saved_can_use_partial_vectors_p
     = LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
 
@@ -2866,21 +2509,28 @@ start_over:
     return ok;
 
   /* If there are any SLP instances mark them as pure_slp.  */
-  if (vect_make_slp_decision (loop_vinfo))
-    {
-      /* Find stmts that need to be both vectorized and SLPed.  */
-      vect_detect_hybrid_slp (loop_vinfo);
+  if (!vect_make_slp_decision (loop_vinfo))
+    return opt_result::failure_at (vect_location, "no stmts to vectorize.\n");
 
-      /* Update the vectorization factor based on the SLP decision.  */
-      vect_update_vf_for_slp (loop_vinfo);
-
-      /* Optimize the SLP graph with the vectorization factor fixed.  */
-      vect_optimize_slp (loop_vinfo);
+  if (dump_enabled_p ())
+    dump_printf_loc (MSG_NOTE, vect_location, "Loop contains only SLP stmts\n");
 
-      /* Gather the loads reachable from the SLP graph entries.  */
-      vect_gather_slp_loads (loop_vinfo);
+  /* Determine the vectorization factor from the SLP decision.  */
+  LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+    = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
+  if (dump_enabled_p ())
+    {
+      dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
+      dump_dec (MSG_NOTE, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+      dump_printf (MSG_NOTE, "\n");
     }
 
+  /* Optimize the SLP graph with the vectorization factor fixed.  */
+  vect_optimize_slp (loop_vinfo);
+
+  /* Gather the loads reachable from the SLP graph entries.  */
+  vect_gather_slp_loads (loop_vinfo);
+
   /* We don't expect to have to roll back to anything other than an empty
      set of rgroups.  */
   gcc_assert (LOOP_VINFO_MASKS (loop_vinfo).is_empty ());
@@ -2949,19 +2599,6 @@ start_over:
       goto again;
     }
 
-  /* Dissolve SLP-only groups.  */
-  vect_dissolve_slp_only_groups (loop_vinfo);
-
-  /* Scan all the remaining operations in the loop that we did not catch
-     during SLP build and make sure we fail.  */
-  ok = vect_analyze_loop_operations (loop_vinfo);
-  if (!ok)
-    {
-      ok = opt_result::failure_at (vect_location,
-				   "bad operation or unsupported loop bound\n");
-      goto again;
-    }
-
   /* For now, we don't expect to mix both masking and length approaches for one
      loop, disable it if both are recorded.  */
   if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
@@ -3241,7 +2878,7 @@ again:
 	continue;
       vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
       unsigned int size = DR_GROUP_SIZE (vinfo);
-      tree vectype = STMT_VINFO_VECTYPE (vinfo);
+      tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
       if (vect_store_lanes_supported (vectype, size, false) == IFN_LAST
 	 && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U)
 	 && ! vect_grouped_store_supported (vectype, size))
@@ -3255,7 +2892,7 @@ again:
 	      vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
 	      bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo);
 	      size = DR_GROUP_SIZE (vinfo);
-	      vectype = STMT_VINFO_VECTYPE (vinfo);
+	      vectype = SLP_TREE_VECTYPE (node);
 	      if (vect_load_lanes_supported (vectype, size, false) == IFN_LAST
 		  && ! vect_grouped_load_supported (vectype, single_element_p,
 						    size))
@@ -3271,8 +2908,8 @@ again:
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "re-trying with single-lane SLP\n");
 
-  /* Restore vectorization factor as it were without SLP.  */
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = saved_vectorization_factor;
+  /* Reset the vectorization factor.  */
+  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = 0;
   /* Free the SLP instances.  */
   FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
     vect_free_slp_instance (instance);
@@ -3285,7 +2922,7 @@ again:
 	   !gsi_end_p (si); gsi_next (&si))
 	{
 	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
-	  STMT_SLP_TYPE (stmt_info) = loop_vect;
+	  STMT_SLP_TYPE (stmt_info) = not_vect;
 	  if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
 	      || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
 	    {
@@ -3304,7 +2941,7 @@ again:
 	  if (is_gimple_debug (gsi_stmt (si)))
 	    continue;
 	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
-	  STMT_SLP_TYPE (stmt_info) = loop_vect;
+	  STMT_SLP_TYPE (stmt_info) = not_vect;
 	  if (STMT_VINFO_IN_PATTERN_P (stmt_info))
 	    {
 	      stmt_vec_info pattern_stmt_info
@@ -3313,11 +2950,11 @@ again:
 		STMT_VINFO_IN_PATTERN_P (stmt_info) = false;
 
 	      gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
-	      STMT_SLP_TYPE (pattern_stmt_info) = loop_vect;
+	      STMT_SLP_TYPE (pattern_stmt_info) = not_vect;
 	      for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq);
 		   !gsi_end_p (pi); gsi_next (&pi))
 		STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi)))
-		  = loop_vect;
+		  = not_vect;
 	    }
 	}
     }
@@ -3400,8 +3037,10 @@ vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo,
 }
 
 /* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if ORIG_LOOP_VINFO is
-   not NULL.  Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance
-   MODE_I to the next mode useful to analyze.
+   not NULL.  When MASKED_P is not -1 override the default
+   LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P with it.
+   Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance MODE_I to the next
+   mode useful to analyze.
    Return the loop_vinfo on success and wrapped null on failure.  */
 
 static opt_loop_vec_info
@@ -3409,6 +3048,7 @@ vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
 		     const vect_loop_form_info *loop_form_info,
 		     loop_vec_info orig_loop_vinfo,
 		     const vector_modes &vector_modes, unsigned &mode_i,
+		     int masked_p,
 		     machine_mode &autodetected_vector_mode,
 		     bool &fatal)
 {
@@ -3417,6 +3057,8 @@ vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
 
   machine_mode vector_mode = vector_modes[mode_i];
   loop_vinfo->vector_mode = vector_mode;
+  if (masked_p != -1)
+    loop_vinfo->can_use_partial_vectors_p = masked_p;
   unsigned int suggested_unroll_factor = 1;
   unsigned slp_done_for_suggested_uf = 0;
 
@@ -3600,7 +3242,7 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
       cached_vf_per_mode[last_mode_i] = -1;
       opt_loop_vec_info loop_vinfo
 	= vect_analyze_loop_1 (loop, shared, &loop_form_info,
-			       NULL, vector_modes, mode_i,
+			       NULL, vector_modes, mode_i, -1,
 			       autodetected_vector_mode, fatal);
       if (fatal)
 	break;
@@ -3685,18 +3327,21 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
      array may contain length-agnostic and length-specific modes.  Their
      ordering is not guaranteed, so we could end up picking a mode for the main
      loop that is after the epilogue's optimal mode.  */
+  int masked_p = -1;
   if (!unlimited_cost_model (loop)
-      && first_loop_vinfo->vector_costs->suggested_epilogue_mode () != VOIDmode)
+      && (first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
+	  != VOIDmode))
     {
       vector_modes[0]
-	= first_loop_vinfo->vector_costs->suggested_epilogue_mode ();
+	= first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
       cached_vf_per_mode[0] = 0;
     }
   else
     vector_modes[0] = autodetected_vector_mode;
   mode_i = 0;
 
-  bool supports_partial_vectors = param_vect_partial_vector_usage != 0;
+  bool supports_partial_vectors = (param_vect_partial_vector_usage != 0
+				   || masked_p == 1);
   machine_mode mask_mode;
   if (supports_partial_vectors
       && !partial_vectors_supported_p ()
@@ -3710,6 +3355,10 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
   loop_vec_info orig_loop_vinfo = first_loop_vinfo;
   do
     {
+      /* Let the user override what the target suggests.  */
+      if (OPTION_SET_P (param_vect_partial_vector_usage))
+	masked_p = -1;
+
       while (1)
 	{
 	  /* If the target does not support partial vectors we can shorten the
@@ -3750,7 +3399,7 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
 	  opt_loop_vec_info loop_vinfo
 	    = vect_analyze_loop_1 (loop, shared, &loop_form_info,
 				   orig_loop_vinfo,
-				   vector_modes, mode_i,
+				   vector_modes, mode_i, masked_p,
 				   autodetected_vector_mode, fatal);
 	  if (fatal)
 	    break;
@@ -3781,6 +3430,9 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
 		break;
 	    }
 
+	  /* Revert back to the default from the suggested prefered
+	     epilogue vectorization mode.  */
+	  masked_p = -1;
 	  if (mode_i == vector_modes.length ())
 	    break;
 	}
@@ -3791,13 +3443,14 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
 
       /* When we selected a first vectorized epilogue, see if the target
 	 suggests to have another one.  */
+      masked_p = -1;
       if (!unlimited_cost_model (loop)
 	  && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (orig_loop_vinfo)
-	  && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode ()
+	  && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
 	      != VOIDmode))
 	{
 	  vector_modes[0]
-	    = orig_loop_vinfo->vector_costs->suggested_epilogue_mode ();
+	    = orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
 	  cached_vf_per_mode[0] = 0;
 	  mode_i = 0;
 	}
@@ -5332,7 +4985,7 @@ vect_is_emulated_mixed_dot_prod (stmt_vec_info stmt_info)
 
 static void
 vect_model_reduction_cost (loop_vec_info loop_vinfo,
-			   stmt_vec_info stmt_info, internal_fn reduc_fn,
+			   slp_tree node, internal_fn reduc_fn,
 			   vect_reduction_type reduction_type,
 			   int ncopies, stmt_vector_for_cost *cost_vec)
 {
@@ -5348,9 +5001,10 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
   if (reduction_type == COND_REDUCTION)
     ncopies *= 2;
 
-  vectype = STMT_VINFO_VECTYPE (stmt_info);
+  vectype = SLP_TREE_VECTYPE (node);
   mode = TYPE_MODE (vectype);
-  stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
+  stmt_vec_info orig_stmt_info
+    = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
 
   gimple_match_op op;
   if (!gimple_extract_op (orig_stmt_info->stmt, &op))
@@ -5368,16 +5022,16 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
       if (reduc_fn != IFN_LAST)
 	/* Count one reduction-like operation per vector.  */
 	inside_cost = record_stmt_cost (cost_vec, ncopies, vec_to_scalar,
-					stmt_info, 0, vect_body);
+					node, 0, vect_body);
       else
 	{
 	  /* Use NELEMENTS extracts and NELEMENTS scalar ops.  */
 	  unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
 	  inside_cost = record_stmt_cost (cost_vec, nelements,
-					  vec_to_scalar, stmt_info, 0,
+					  vec_to_scalar, node, 0,
 					  vect_body);
 	  inside_cost += record_stmt_cost (cost_vec, nelements,
-					   scalar_stmt, stmt_info, 0,
+					   scalar_stmt, node, 0,
 					   vect_body);
 	}
     }
@@ -5394,7 +5048,7 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	/* We need the initial reduction value.  */
 	prologue_stmts = 1;
       prologue_cost += record_stmt_cost (cost_vec, prologue_stmts,
-					 scalar_to_vec, stmt_info, 0,
+					 scalar_to_vec, node, 0,
 					 vect_prologue);
     }
 
@@ -5411,24 +5065,24 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	    {
 	      /* An EQ stmt and an COND_EXPR stmt.  */
 	      epilogue_cost += record_stmt_cost (cost_vec, 2,
-						 vector_stmt, stmt_info, 0,
+						 vector_stmt, node, 0,
 						 vect_epilogue);
 	      /* Reduction of the max index and a reduction of the found
 		 values.  */
 	      epilogue_cost += record_stmt_cost (cost_vec, 2,
-						 vec_to_scalar, stmt_info, 0,
+						 vec_to_scalar, node, 0,
 						 vect_epilogue);
 	      /* A broadcast of the max value.  */
 	      epilogue_cost += record_stmt_cost (cost_vec, 1,
-						 scalar_to_vec, stmt_info, 0,
+						 scalar_to_vec, node, 0,
 						 vect_epilogue);
 	    }
 	  else
 	    {
 	      epilogue_cost += record_stmt_cost (cost_vec, 1, vector_stmt,
-						 stmt_info, 0, vect_epilogue);
+						 node, 0, vect_epilogue);
 	      epilogue_cost += record_stmt_cost (cost_vec, 1,
-						 vec_to_scalar, stmt_info, 0,
+						 vec_to_scalar, node, 0,
 						 vect_epilogue);
 	    }
 	}
@@ -5438,12 +5092,12 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	  /* Extraction of scalar elements.  */
 	  epilogue_cost += record_stmt_cost (cost_vec,
 					     2 * estimated_nunits,
-					     vec_to_scalar, stmt_info, 0,
+					     vec_to_scalar, node, 0,
 					     vect_epilogue);
 	  /* Scalar max reductions via COND_EXPR / MAX_EXPR.  */
 	  epilogue_cost += record_stmt_cost (cost_vec,
 					     2 * estimated_nunits - 3,
-					     scalar_stmt, stmt_info, 0,
+					     scalar_stmt, node, 0,
 					     vect_epilogue);
 	}
       else if (reduction_type == EXTRACT_LAST_REDUCTION
@@ -5469,10 +5123,10 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 		 Also requires scalar extract.  */
 	      epilogue_cost += record_stmt_cost (cost_vec,
 						 exact_log2 (nelements) * 2,
-						 vector_stmt, stmt_info, 0,
+						 vector_stmt, node, 0,
 						 vect_epilogue);
 	      epilogue_cost += record_stmt_cost (cost_vec, 1,
-						 vec_to_scalar, stmt_info, 0,
+						 vec_to_scalar, node, 0,
 						 vect_epilogue);
 	    }
 	  else
@@ -5480,7 +5134,7 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	       elements, we have N extracts and N-1 reduction ops.  */
 	    epilogue_cost += record_stmt_cost (cost_vec,
 					       nelements + nelements - 1,
-					       vector_stmt, stmt_info, 0,
+					       vector_stmt, node, 0,
 					       vect_epilogue);
 	}
     }
@@ -7030,7 +6684,7 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
 			       vec_loop_lens *lens)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_node);
   internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in);
 
   gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
@@ -7417,14 +7071,13 @@ vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 
   for (int i = 0; i < (int) gimple_num_ops (stmt) - 1; i++)
     {
-      stmt_vec_info def_stmt_info;
       slp_tree slp_op;
       tree op;
       tree vectype;
       enum vect_def_type dt;
 
-      if (!vect_is_simple_use (loop_vinfo, stmt_info, slp_node, i, &op,
-			       &slp_op, &dt, &vectype, &def_stmt_info))
+      if (!vect_is_simple_use (loop_vinfo, slp_node, i, &op,
+			       &slp_op, &dt, &vectype))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -7493,7 +7146,7 @@ vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
     }
 
   /* Transform via vect_transform_reduction.  */
-  STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+  SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
   return true;
 }
 
@@ -7595,18 +7248,17 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 	      }
 	  /* Analysis for double-reduction is done on the outer
 	     loop PHI, nested cycles have no further restrictions.  */
-	  STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type;
+	  SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
 	}
       else
-	STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+	SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
       return true;
     }
 
-  stmt_vec_info orig_stmt_of_analysis = stmt_info;
   stmt_vec_info phi_info = stmt_info;
   if (!is_a <gphi *> (stmt_info->stmt))
     {
-      STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
       return true;
     }
   if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
@@ -7646,17 +7298,17 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
   unsigned reduc_chain_length = 0;
   bool only_slp_reduc_chain = true;
   stmt_info = NULL;
-  slp_tree slp_for_stmt_info = slp_node_instance->root;
+  slp_tree slp_for_stmt_info = NULL;
+  slp_tree vdef_slp = slp_node_instance->root;
   /* For double-reductions we start SLP analysis at the inner loop LC PHI
      which is the def of the outer loop live stmt.  */
   if (STMT_VINFO_DEF_TYPE (reduc_info) == vect_double_reduction_def)
-    slp_for_stmt_info = SLP_TREE_CHILDREN (slp_for_stmt_info)[0];
+    vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
   while (reduc_def != PHI_RESULT (reduc_def_phi))
     {
       stmt_vec_info def = loop_vinfo->lookup_def (reduc_def);
       stmt_vec_info vdef = vect_stmt_to_vectorize (def);
       int reduc_idx = STMT_VINFO_REDUC_IDX (vdef);
-
       if (reduc_idx == -1)
 	{
 	  if (dump_enabled_p ())
@@ -7673,14 +7325,9 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 	 the SLP node with live lane zero the other live lanes also
 	 need to be identified as part of a reduction to be able
 	 to skip code generation for them.  */
-      if (slp_for_stmt_info)
-	{
-	  for (auto s : SLP_TREE_SCALAR_STMTS (slp_for_stmt_info))
-	    if (STMT_VINFO_LIVE_P (s))
-	      STMT_VINFO_REDUC_DEF (vect_orig_stmt (s)) = phi_info;
-	}
-      else if (STMT_VINFO_LIVE_P (vdef))
-	STMT_VINFO_REDUC_DEF (def) = phi_info;
+      for (auto s : SLP_TREE_SCALAR_STMTS (vdef_slp))
+	if (STMT_VINFO_LIVE_P (s))
+	  STMT_VINFO_REDUC_DEF (vect_orig_stmt (s)) = phi_info;
       gimple_match_op op;
       if (!gimple_extract_op (vdef->stmt, &op))
 	{
@@ -7699,32 +7346,33 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 				 "conversion in the reduction chain.\n");
 	      return false;
 	    }
+	  vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
 	}
       else
 	{
 	  /* First non-conversion stmt.  */
 	  if (!stmt_info)
-	    stmt_info = vdef;
+	    {
+	      stmt_info = vdef;
+	      slp_for_stmt_info = vdef_slp;
+	    }
 
 	  if (lane_reducing_op_p (op.code))
 	    {
-	      enum vect_def_type dt;
-	      tree vectype_op;
-
 	      /* The last operand of lane-reducing operation is for
 		 reduction.  */
 	      gcc_assert (reduc_idx > 0 && reduc_idx == (int) op.num_ops - 1);
 
-	      if (!vect_is_simple_use (op.ops[0], loop_vinfo, &dt, &vectype_op))
-		return false;
-
+	      slp_tree op_node = SLP_TREE_CHILDREN (vdef_slp)[0];
+	      tree vectype_op = SLP_TREE_VECTYPE (op_node);
 	      tree type_op = TREE_TYPE (op.ops[0]);
-
 	      if (!vectype_op)
 		{
 		  vectype_op = get_vectype_for_scalar_type (loop_vinfo,
 							    type_op);
-		  if (!vectype_op)
+		  if (!vectype_op
+		      || !vect_maybe_update_slp_op_vectype (op_node,
+							    vectype_op))
 		    return false;
 		}
 
@@ -7746,14 +7394,14 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 		       < GET_MODE_SIZE (SCALAR_TYPE_MODE (type_op))))
 		vectype_in = vectype_op;
 	    }
-	  else
+	  else if (!vectype_in)
 	    vectype_in = STMT_VINFO_VECTYPE (phi_info);
+	  if (!REDUC_GROUP_FIRST_ELEMENT (vdef))
+	    vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
 	}
 
       reduc_def = op.ops[reduc_idx];
       reduc_chain_length++;
-      if (!stmt_info)
-	slp_for_stmt_info = SLP_TREE_CHILDREN (slp_for_stmt_info)[0];
     }
   /* PHIs should not participate in patterns.  */
   gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
@@ -7805,7 +7453,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
         inside the loop body. The last operand is the reduction variable,
         which is defined by the loop-header-phi.  */
 
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_for_stmt_info);
   STMT_VINFO_REDUC_VECTYPE (reduc_info) = vectype_out;
   STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in;
 
@@ -7855,7 +7503,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 
       stmt_vec_info def_stmt_info;
       enum vect_def_type dt;
-      if (!vect_is_simple_use (loop_vinfo, stmt_info, slp_for_stmt_info,
+      if (!vect_is_simple_use (loop_vinfo, slp_for_stmt_info,
 			       i + opno_adjust, &op.ops[i], &slp_op[i], &dt,
 			       &vectype_op[i], &def_stmt_info))
 	{
@@ -8408,7 +8056,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 	  return false;
 	}
 
-  vect_model_reduction_cost (loop_vinfo, stmt_info, reduc_fn,
+  vect_model_reduction_cost (loop_vinfo, slp_for_stmt_info, reduc_fn,
 			     reduction_type, ncopies, cost_vec);
   /* Cost the reduction op inside the loop if transformed via
      vect_transform_reduction for non-lane-reducing operation.  Otherwise
@@ -8420,7 +8068,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
       && reduction_type == FOLD_LEFT_REDUCTION)
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "using an in-order (fold-left) reduction.\n");
-  STMT_VINFO_TYPE (orig_stmt_of_analysis) = cycle_phi_info_type;
+  SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
 
   /* All but single defuse-cycle optimized and fold-left reductions go
      through their own vectorizable_* routines.  */
@@ -8526,7 +8174,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
 			  stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 			  slp_tree slp_node)
 {
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_node);
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   unsigned vec_num;
 
@@ -8553,7 +8201,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
   tree vectype_in = STMT_VINFO_REDUC_VECTYPE_IN (stmt_info);
 
   if (!vectype_in)
-    vectype_in = STMT_VINFO_VECTYPE (stmt_info);
+    vectype_in = SLP_TREE_VECTYPE (slp_node);
 
   vec_num = vect_get_num_copies (loop_vinfo, slp_node, vectype_in);
 
@@ -8618,7 +8266,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
   if (!cond_fn_p)
     {
       gcc_assert (reduc_index >= 0 && reduc_index <= 2);
-      vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1,
+      vect_get_vec_defs (loop_vinfo, slp_node,
 			 single_defuse_cycle && reduc_index == 0
 			 ? NULL_TREE : op.ops[0], &vec_oprnds[0],
 			 single_defuse_cycle && reduc_index == 1
@@ -8633,19 +8281,19 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
 	 vectype.  */
       gcc_assert (single_defuse_cycle
 		  && (reduc_index == 1 || reduc_index == 2));
-      vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1, op.ops[0],
-			 truth_type_for (vectype_in), &vec_oprnds[0],
+      vect_get_vec_defs (loop_vinfo, slp_node, op.ops[0],
+			 &vec_oprnds[0],
 			 reduc_index == 1 ? NULL_TREE : op.ops[1],
-			 NULL_TREE, &vec_oprnds[1],
+			 &vec_oprnds[1],
 			 reduc_index == 2 ? NULL_TREE : op.ops[2],
-			 NULL_TREE, &vec_oprnds[2]);
+			 &vec_oprnds[2]);
     }
 
   /* For single def-use cycles get one copy of the vectorized reduction
      definition.  */
   if (single_defuse_cycle)
     {
-      vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1,
+      vect_get_vec_defs (loop_vinfo, slp_node,
 			 reduc_index == 0 ? op.ops[0] : NULL_TREE,
 			 &vec_oprnds[0],
 			 reduc_index == 1 ? op.ops[1] : NULL_TREE,
@@ -8873,7 +8521,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
 			  stmt_vec_info stmt_info,
 			  slp_tree slp_node, slp_instance slp_node_instance)
 {
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_node);
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   int i;
   bool nested_cycle = false;
@@ -9104,7 +8752,19 @@ vectorizable_lc_phi (loop_vec_info loop_vinfo,
 			 "incompatible vector types for invariants\n");
       return false;
     }
-  STMT_VINFO_TYPE (stmt_info) = lc_phi_info_type;
+
+  /* ???  This can happen with data vs. mask uses of boolean.  */
+  if (!useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
+				  SLP_TREE_VECTYPE
+				    (SLP_TREE_CHILDREN (slp_node)[0])))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "missed mask promotion\n");
+      return false;
+    }
+
+  SLP_TREE_TYPE (slp_node) = lc_phi_info_type;
   return true;
 }
 
@@ -9114,13 +8774,13 @@ vect_transform_lc_phi (loop_vec_info loop_vinfo,
 		       slp_tree slp_node)
 {
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   tree scalar_dest = gimple_phi_result (stmt_info->stmt);
   basic_block bb = gimple_bb (stmt_info->stmt);
   edge e = single_pred_edge (bb);
   tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
   auto_vec<tree> vec_oprnds;
-  vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1,
+  vect_get_vec_defs (loop_vinfo, slp_node,
 		     gimple_phi_arg_def (stmt_info->stmt, 0), &vec_oprnds);
   for (unsigned i = 0; i < vec_oprnds.length (); i++)
     {
@@ -9137,7 +8797,7 @@ vect_transform_lc_phi (loop_vec_info loop_vinfo,
 
 bool
 vectorizable_phi (vec_info *,
-		  stmt_vec_info stmt_info, gimple **vec_stmt,
+		  stmt_vec_info stmt_info,
 		  slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   if (!is_a <gphi *> (stmt_info->stmt) || !slp_node)
@@ -9148,7 +8808,7 @@ vectorizable_phi (vec_info *,
 
   tree vectype = SLP_TREE_VECTYPE (slp_node);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       slp_tree child;
       unsigned i;
@@ -9189,7 +8849,7 @@ vectorizable_phi (vec_info *,
       if (gimple_phi_num_args (as_a <gphi *> (stmt_info->stmt)) > 1)
 	record_stmt_cost (cost_vec, SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
 			  vector_stmt, stmt_info, vectype, 0, vect_body);
-      STMT_VINFO_TYPE (stmt_info) = phi_info_type;
+      SLP_TREE_TYPE (slp_node) = phi_info_type;
       return true;
     }
 
@@ -9279,8 +8939,7 @@ vectorizable_phi (vec_info *,
 
 bool
 vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
-		     gimple **vec_stmt, slp_tree slp_node,
-		     stmt_vector_for_cost *cost_vec)
+		     slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   if (!loop_vinfo || !is_a<gphi *> (stmt_info->stmt))
     return false;
@@ -9291,14 +8950,10 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
     return false;
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  unsigned ncopies;
-  if (slp_node)
-    ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
+  unsigned ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
   poly_int64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
-  unsigned dist = slp_node ? SLP_TREE_LANES (slp_node) : 1;
+  unsigned dist = SLP_TREE_LANES (slp_node);
   /* We need to be able to make progress with a single vector.  */
   if (maybe_gt (dist * 2, nunits))
     {
@@ -9309,6 +8964,33 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
+  /* We need to be able to build a { ..., a, b } init vector with
+     dist number of distinct trailing values.  Always possible
+     when dist == 1 or when nunits is constant or when the initializations
+     are uniform.  */
+  tree uniform_initval = NULL_TREE;
+  edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
+  for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
+    {
+      gphi *phi = as_a <gphi *> (s->stmt);
+      if (! uniform_initval)
+	uniform_initval = PHI_ARG_DEF_FROM_EDGE (phi, pe);
+      else if (! operand_equal_p (uniform_initval,
+				  PHI_ARG_DEF_FROM_EDGE (phi, pe)))
+	{
+	  uniform_initval = NULL_TREE;
+	  break;
+	}
+    }
+  if (!uniform_initval && !nunits.is_constant ())
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "cannot build initialization vector for "
+			 "first order recurrence\n");
+      return false;
+    }
+
   /* First-order recurrence autovectorization needs to handle permutation
      with indices = [nunits-1, nunits, nunits+1, ...].  */
   vec_perm_builder sel (nunits, 1, 3);
@@ -9316,48 +8998,30 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
     sel.quick_push (nunits - dist + i);
   vec_perm_indices indices (sel, 2, nunits);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
 				 indices))
 	return false;
 
-      if (slp_node)
-	{
-	  /* We eventually need to set a vector type on invariant
-	     arguments.  */
-	  unsigned j;
-	  slp_tree child;
-	  FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
-	    if (!vect_maybe_update_slp_op_vectype
-		  (child, SLP_TREE_VECTYPE (slp_node)))
-	      {
-		if (dump_enabled_p ())
-		  dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				   "incompatible vector types for "
-				   "invariants\n");
-		return false;
-	      }
-	}
+      /* We eventually need to set a vector type on invariant
+	 arguments.  */
+      unsigned j;
+      slp_tree child;
+      FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
+	if (!vect_maybe_update_slp_op_vectype (child, vectype))
+	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "incompatible vector types for "
+			       "invariants\n");
+	    return false;
+	  }
 
       /* Verify we have set up compatible types.  */
       edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
-      tree latch_vectype = NULL_TREE;
-      if (slp_node)
-	{
-	  slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
-	  latch_vectype = SLP_TREE_VECTYPE (latch_def);
-	}
-      else
-	{
-	  tree latch_def = PHI_ARG_DEF_FROM_EDGE (phi, le);
-	  if (TREE_CODE (latch_def) == SSA_NAME)
-	    {
-	      stmt_vec_info latch_def_info = loop_vinfo->lookup_def (latch_def);
-	      latch_def_info = vect_stmt_to_vectorize (latch_def_info);
-	      latch_vectype = STMT_VINFO_VECTYPE (latch_def_info);
-	    }
-	}
+      slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
+      tree latch_vectype = SLP_TREE_VECTYPE (latch_def);
       if (!types_compatible_p (latch_vectype, vectype))
 	return false;
 
@@ -9365,9 +9029,6 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 	 for each copy.  With SLP the prologue value is explicitly
 	 represented and costed separately.  */
       unsigned prologue_cost = 0;
-      if (!slp_node)
-	prologue_cost = record_stmt_cost (cost_vec, 1, scalar_to_vec,
-					  stmt_info, 0, vect_prologue);
       unsigned inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
 					       stmt_info, 0, vect_body);
       if (dump_enabled_p ())
@@ -9376,25 +9037,42 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 			 "prologue_cost = %d .\n", inside_cost,
 			 prologue_cost);
 
-      STMT_VINFO_TYPE (stmt_info) = recurr_info_type;
+      SLP_TREE_TYPE (slp_node) = recurr_info_type;
       return true;
     }
 
-  edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
-  basic_block bb = gimple_bb (phi);
-  tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
-  if (!useless_type_conversion_p (TREE_TYPE (vectype), TREE_TYPE (preheader)))
+  tree vec_init;
+  if (! uniform_initval)
     {
-      gimple_seq stmts = NULL;
-      preheader = gimple_convert (&stmts, TREE_TYPE (vectype), preheader);
-      gsi_insert_seq_on_edge_immediate (pe, stmts);
+      vec<constructor_elt, va_gc> *v = NULL;
+      vec_alloc (v, nunits.to_constant ());
+      for (unsigned i = 0; i < nunits.to_constant () - dist; ++i)
+	CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
+				build_zero_cst (TREE_TYPE (vectype)));
+      for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
+	{
+	  gphi *phi = as_a <gphi *> (s->stmt);
+	  tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
+	  if (!useless_type_conversion_p (TREE_TYPE (vectype),
+					  TREE_TYPE (preheader)))
+	    {
+	      gimple_seq stmts = NULL;
+	      preheader = gimple_convert (&stmts,
+					  TREE_TYPE (vectype), preheader);
+	      gsi_insert_seq_on_edge_immediate (pe, stmts);
+	    }
+	  CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, preheader);
+	}
+      vec_init = build_constructor (vectype, v);
     }
-  tree vec_init = build_vector_from_val (vectype, preheader);
+  else
+    vec_init = uniform_initval;
   vec_init = vect_init_vector (loop_vinfo, stmt_info, vec_init, vectype, NULL);
 
   /* Create the vectorized first-order PHI node.  */
   tree vec_dest = vect_get_new_vect_var (vectype,
 					 vect_simple_var, "vec_recur_");
+  basic_block bb = gimple_bb (phi);
   gphi *new_phi = create_phi_node (vec_dest, bb);
   add_phi_arg (new_phi, vec_init, pe, UNKNOWN_LOCATION);
 
@@ -9419,14 +9097,9 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 				 NULL, perm);
       vect_finish_stmt_generation (loop_vinfo, stmt_info, vperm, &gsi2);
 
-      if (slp_node)
-	slp_node->push_vec_def (vperm);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (vperm);
+      slp_node->push_vec_def (vperm);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
   return true;
 }
 
@@ -9738,7 +9411,7 @@ vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
 static bool
 vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
 				  stmt_vec_info stmt_info,
-				  gimple **vec_stmt, slp_tree slp_node,
+				  slp_tree slp_node,
 				  stmt_vector_for_cost *cost_vec)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -9894,7 +9567,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
       gcc_unreachable ();
     }
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       unsigned inside_cost = 0, prologue_cost = 0;
       /* loop cost for vec_loop. Neg induction doesn't have any
@@ -9917,7 +9590,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
 			 "prologue_cost = %d. \n", inside_cost,
 			 prologue_cost);
 
-      STMT_VINFO_TYPE (stmt_info) = induc_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_nonlinear_induction");
       return true;
     }
@@ -10049,8 +9722,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
 bool
 vectorizable_induction (loop_vec_info loop_vinfo,
 			stmt_vec_info stmt_info,
-			gimple **vec_stmt, slp_tree slp_node,
-			stmt_vector_for_cost *cost_vec)
+			slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   bool nested_in_vect_loop = false;
@@ -10084,7 +9756,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
   /* Handle nonlinear induction in a separate place.  */
   if (induction_type != vect_step_op_add)
     return vectorizable_nonlinear_induction (loop_vinfo, stmt_info,
-					     vec_stmt, slp_node, cost_vec);
+					     slp_node, cost_vec);
 
   tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
@@ -10165,6 +9837,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
     }
   tree stept = TREE_TYPE (step_expr);
   tree step_vectype = get_same_sized_vectype (stept, vectype);
+  stept = TREE_TYPE (step_vectype);
 
   /* Check for target support of the vectorized arithmetic used here.  */
   if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default)
@@ -10187,7 +9860,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
 	}
     }
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       unsigned inside_cost = 0, prologue_cost = 0;
       /* We eventually need to set a vector type on invariant
@@ -10218,7 +9891,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
 			 "prologue_cost = %d .\n", inside_cost,
 			 prologue_cost);
 
-      STMT_VINFO_TYPE (stmt_info) = induc_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_induction");
       return true;
     }
@@ -10619,9 +10292,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
    helper function for vectorizable_live_operation.  */
 
 static tree
-vectorizable_live_operation_1 (loop_vec_info loop_vinfo,
-			       stmt_vec_info stmt_info, basic_block exit_bb,
-			       tree vectype, int ncopies, slp_tree slp_node,
+vectorizable_live_operation_1 (loop_vec_info loop_vinfo, basic_block exit_bb,
+			       tree vectype, slp_tree slp_node,
 			       tree bitsize, tree bitstart, tree vec_lhs,
 			       tree lhs_type, gimple_stmt_iterator *exit_gsi)
 {
@@ -10652,8 +10324,7 @@ vectorizable_live_operation_1 (loop_vec_info loop_vinfo,
 
 	 where VEC_LHS is the vectorized live-out result and MASK is
 	 the loop mask for the final iteration.  */
-      gcc_assert (ncopies == 1
-		  && (!slp_node || SLP_TREE_LANES (slp_node) == 1));
+      gcc_assert (SLP_TREE_LANES (slp_node) == 1);
       gimple_seq tem = NULL;
       gimple_stmt_iterator gsi = gsi_last (tem);
       tree len = vect_get_loop_len (loop_vinfo, &gsi,
@@ -10688,8 +10359,8 @@ vectorizable_live_operation_1 (loop_vec_info loop_vinfo,
 
 	 where VEC_LHS is the vectorized live-out result and MASK is
 	 the loop mask for the final iteration.  */
-      gcc_assert (!slp_node || SLP_TREE_LANES (slp_node) == 1);
-      tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info));
+      gcc_assert (SLP_TREE_LANES (slp_node) == 1);
+      tree scalar_type = TREE_TYPE (vectype);
       gimple_seq tem = NULL;
       gimple_stmt_iterator gsi = gsi_last (tem);
       tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
@@ -10735,11 +10406,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   imm_use_iterator imm_iter;
   tree lhs, lhs_type, bitsize;
-  tree vectype = (slp_node
-		  ? SLP_TREE_VECTYPE (slp_node)
-		  : STMT_VINFO_VECTYPE (stmt_info));
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
-  int ncopies;
   gimple *use_stmt;
   use_operand_p use_p;
   auto_vec<tree> vec_oprnds;
@@ -10758,7 +10426,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	return true;
       /* For SLP reductions we vectorize the epilogue for all involved stmts
 	 together.  */
-      if (slp_node && !REDUC_GROUP_FIRST_ELEMENT (stmt_info) && slp_index != 0)
+      if (!REDUC_GROUP_FIRST_ELEMENT (stmt_info) && slp_index != 0)
 	return true;
       stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
       gcc_assert (reduc_info->is_reduc_info);
@@ -10776,7 +10444,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	 block, but we have to find an alternate exit first.  */
       if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
 	{
-	  slp_tree phis_node = slp_node ? slp_node_instance->reduc_phis : NULL;
+	  slp_tree phis_node = slp_node_instance->reduc_phis;
 	  for (auto exit : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
 	    if (exit != LOOP_VINFO_IV_EXIT (loop_vinfo))
 	      {
@@ -10807,32 +10475,24 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       return true;
     }
 
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  if (slp_node)
-    {
-      gcc_assert (slp_index >= 0);
+  gcc_assert (slp_index >= 0);
 
-      /* Get the last occurrence of the scalar index from the concatenation of
-	 all the slp vectors. Calculate which slp vector it is and the index
-	 within.  */
-      int num_scalar = SLP_TREE_LANES (slp_node);
-      int num_vec = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-      poly_uint64 pos = (num_vec * nunits) - num_scalar + slp_index;
+  /* Get the last occurrence of the scalar index from the concatenation of
+     all the slp vectors. Calculate which slp vector it is and the index
+     within.  */
+  int num_scalar = SLP_TREE_LANES (slp_node);
+  int num_vec = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+  poly_uint64 pos = (num_vec * nunits) - num_scalar + slp_index;
 
-      /* Calculate which vector contains the result, and which lane of
-	 that vector we need.  */
-      if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "Cannot determine which vector holds the"
-			     " final result.\n");
-	  return false;
-	}
+  /* Calculate which vector contains the result, and which lane of
+     that vector we need.  */
+  if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "Cannot determine which vector holds the"
+			 " final result.\n");
+      return false;
     }
 
   if (!vec_stmt_p)
@@ -10840,7 +10500,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       /* No transformation required.  */
       if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
 	{
-	  if (slp_node && SLP_TREE_LANES (slp_node) != 1)
+	  if (SLP_TREE_LANES (slp_node) != 1)
 	    {
 	      if (dump_enabled_p ())
 		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10849,8 +10509,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 				 "the loop.\n");
 	      LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
 	    }
-	  else if (ncopies > 1
-		   || (slp_node && SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) > 1))
+	  else if (SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) > 1)
 	    {
 	      if (dump_enabled_p ())
 		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10860,8 +10519,6 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	    }
 	  else
 	    {
-	      gcc_assert (ncopies == 1
-			  && (!slp_node || SLP_TREE_LANES (slp_node) == 1));
 	      if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
 						  OPTIMIZE_FOR_SPEED))
 		vect_record_loop_mask (loop_vinfo,
@@ -10903,40 +10560,21 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   bitsize = vector_element_bits_tree (vectype);
 
   /* Get the vectorized lhs of STMT and the lane to use (counted in bits).  */
-  tree vec_lhs, vec_lhs0, bitstart;
-  gimple *vec_stmt, *vec_stmt0;
-  if (slp_node)
-    {
-      gcc_assert (!loop_vinfo
-		  || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
-		       && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
-		      || SLP_TREE_LANES (slp_node) == 1));
-
-      /* Get the correct slp vectorized stmt.  */
-      vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
-      vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
+  gcc_assert (!loop_vinfo
+	      || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
+		   && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+		  || SLP_TREE_LANES (slp_node) == 1));
 
-      /* In case we need to early break vectorize also get the first stmt.  */
-      vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
-      vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0);
+  /* Get the correct slp vectorized stmt.  */
+  tree vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
+  gimple *vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
 
-      /* Get entry to use.  */
-      bitstart = bitsize_int (vec_index);
-      bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
-    }
-  else
-    {
-      /* For multiple copies, get the last copy.  */
-      vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last ();
-      vec_lhs = gimple_get_lhs (vec_stmt);
+  /* In case we need to early break vectorize also get the first stmt.  */
+  tree vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
 
-      /* In case we need to early break vectorize also get the first stmt.  */
-      vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-      vec_lhs0 = gimple_get_lhs (vec_stmt0);
-
-      /* Get the last lane in the vector.  */
-      bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1));
-    }
+  /* Get entry to use.  */
+  tree bitstart = bitsize_int (vec_index);
+  bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
 
   if (loop_vinfo)
     {
@@ -10985,8 +10623,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 
 	      gimple_stmt_iterator exit_gsi;
 	      tree new_tree
-		= vectorizable_live_operation_1 (loop_vinfo, stmt_info,
-						 e->dest, vectype, ncopies,
+		= vectorizable_live_operation_1 (loop_vinfo,
+						 e->dest, vectype,
 						 slp_node, bitsize,
 						 tmp_bitstart, tmp_vec_lhs,
 						 lhs_type, &exit_gsi);
@@ -11412,7 +11050,7 @@ vect_get_loop_len (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
 	  factor = exact_div (nunits1, nunits2).to_constant ();
 	  tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
 	  gimple_seq seq = NULL;
-	  loop_len = gimple_build (&seq, RDIV_EXPR, iv_type, loop_len,
+	  loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
 				   build_int_cst (iv_type, factor));
 	  if (seq)
 	    gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
@@ -11472,7 +11110,7 @@ scale_profile_for_vect_loop (class loop *loop, edge exit_e, unsigned vf, bool fl
   profile_count entry_count = loop_preheader_edge (loop)->count ();
 
   /* If we have unreliable loop profile avoid dropping entry
-     count bellow header count.  This can happen since loops
+     count below header count.  This can happen since loops
      has unrealistically low trip counts.  */
   while (vf > 1
 	 && loop->header->count > entry_count
@@ -11653,11 +11291,9 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
 	 updated offset we set using ADVANCE.  Instead we have to make sure the
 	 reference in the data references point to the corresponding copy of
 	 the original in the epilogue.  Make sure to update both
-	 gather/scatters recognized by dataref analysis and also other
-	 refs that get_load_store_type classified as VMAT_GATHER_SCATTER.  */
+	 gather/scatters recognized by dataref analysis.  */
       auto vstmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
-      if (STMT_VINFO_MEMORY_ACCESS_TYPE (vstmt_vinfo) == VMAT_GATHER_SCATTER
-	  || STMT_VINFO_STRIDED_P (vstmt_vinfo)
+      if (STMT_VINFO_STRIDED_P (vstmt_vinfo)
 	  || STMT_VINFO_GATHER_SCATTER_P (vstmt_vinfo))
 	{
 	  /* ???  As we copy epilogues from the main loop incremental
@@ -11679,9 +11315,6 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
 
   /* Remember the advancement made.  */
   LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
-
-  epilogue_vinfo->shared->datarefs_copy.release ();
-  epilogue_vinfo->shared->save_datarefs ();
 }
 
 /*  When vectorizing early break statements instructions that happen before
@@ -11787,7 +11420,8 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
 
   DUMP_VECT_SCOPE ("vec_transform_loop");
 
-  loop_vinfo->shared->check_datarefs ();
+  if (! LOOP_VINFO_EPILOGUE_P (loop_vinfo))
+    loop_vinfo->shared->check_datarefs ();
 
   /* Use the more conservative vectorization threshold.  If the number
      of iterations is constant assume the cost check has been performed
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 0f6d6b7..ffb320f 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -130,7 +130,6 @@ vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
   STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info;
   STMT_VINFO_DEF_TYPE (pattern_stmt_info)
     = STMT_VINFO_DEF_TYPE (orig_stmt_info);
-  STMT_VINFO_TYPE (pattern_stmt_info) = STMT_VINFO_TYPE (orig_stmt_info);
   if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
     {
       gcc_assert (!vectype
@@ -6042,12 +6041,14 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
 
 	  tree vec_els
 	    = vect_get_mask_load_else (elsval, TREE_TYPE (gs_vectype));
-	  pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6, base,
+	  pattern_stmt = gimple_build_call_internal (gs_info.ifn, 7, base,
+						     gs_info.alias_ptr,
 						     offset, scale, zero, mask,
 						     vec_els);
 	}
       else
-	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 4, base,
+	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base,
+						   gs_info.alias_ptr,
 						   offset, scale, zero);
       tree lhs = gimple_get_lhs (stmt_info->stmt);
       tree load_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
@@ -6057,12 +6058,13 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
     {
       tree rhs = vect_get_store_rhs (stmt_info);
       if (mask != NULL)
-	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
-						   base, offset, scale, rhs,
-						   mask);
+	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6,
+						   base, gs_info.alias_ptr,
+						   offset, scale, rhs, mask);
       else
-	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 4,
-						   base, offset, scale, rhs);
+	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
+						   base, gs_info.alias_ptr,
+						   offset, scale, rhs);
     }
   gimple_call_set_nothrow (pattern_stmt, true);
 
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 7a828ca..794a073 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -118,18 +118,19 @@ _slp_tree::_slp_tree ()
   SLP_TREE_CHILDREN (this) = vNULL;
   SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
   SLP_TREE_LANE_PERMUTATION (this) = vNULL;
-  SLP_TREE_SIMD_CLONE_INFO (this) = vNULL;
   SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
   SLP_TREE_CODE (this) = ERROR_MARK;
   this->ldst_lanes = false;
   this->avoid_stlf_fail = false;
   SLP_TREE_VECTYPE (this) = NULL_TREE;
   SLP_TREE_REPRESENTATIVE (this) = NULL;
-  SLP_TREE_MEMORY_ACCESS_TYPE (this) = VMAT_INVARIANT;
+  SLP_TREE_MEMORY_ACCESS_TYPE (this) = VMAT_UNINITIALIZED;
   SLP_TREE_REF_COUNT (this) = 1;
   this->failed = NULL;
   this->max_nunits = 1;
   this->lanes = 0;
+  SLP_TREE_TYPE (this) = undef_vec_info_type;
+  this->data = NULL;
 }
 
 /* Tear down a SLP node.  */
@@ -148,9 +149,10 @@ _slp_tree::~_slp_tree ()
   SLP_TREE_VEC_DEFS (this).release ();
   SLP_TREE_LOAD_PERMUTATION (this).release ();
   SLP_TREE_LANE_PERMUTATION (this).release ();
-  SLP_TREE_SIMD_CLONE_INFO (this).release ();
   if (this->failed)
     free (failed);
+  if (this->data)
+    delete this->data;
 }
 
 /* Push the single SSA definition in DEF to the vector of vector defs.  */
@@ -507,19 +509,21 @@ vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
 	      && (dtb == vect_external_def || dtb == vect_constant_def)));
 }
 
+#define GATHER_SCATTER_OFFSET (-3)
+
 static const int no_arg_map[] = { 0 };
 static const int arg0_map[] = { 1, 0 };
-static const int arg1_map[] = { 1, 1 };
+static const int arg2_map[] = { 1, 2 };
 static const int arg2_arg3_map[] = { 2, 2, 3 };
-static const int arg1_arg3_map[] = { 2, 1, 3 };
-static const int arg1_arg4_arg5_map[] = { 3, 1, 4, 5 };
-static const int arg1_arg3_arg4_map[] = { 3, 1, 3, 4 };
+static const int arg2_arg4_map[] = { 2, 2, 4 };
+static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
+static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
 static const int arg3_arg2_map[] = { 2, 3, 2 };
 static const int op1_op0_map[] = { 2, 1, 0 };
-static const int off_map[] = { 1, -3 };
-static const int off_op0_map[] = { 2, -3, 0 };
-static const int off_arg2_arg3_map[] = { 3, -3, 2, 3 };
-static const int off_arg3_arg2_map[] = { 3, -3, 3, 2 };
+static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
+static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
+static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
+static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
 static const int mask_call_maps[6][7] = {
   { 1, 1, },
   { 2, 1, 2, },
@@ -568,18 +572,18 @@ vect_get_operand_map (const gimple *stmt, bool gather_scatter_p = false,
 	    return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
 
 	  case IFN_GATHER_LOAD:
-	    return arg1_map;
+	    return arg2_map;
 
 	  case IFN_MASK_GATHER_LOAD:
 	  case IFN_MASK_LEN_GATHER_LOAD:
-	    return arg1_arg4_arg5_map;
+	    return arg2_arg5_arg6_map;
 
 	  case IFN_SCATTER_STORE:
-	    return arg1_arg3_map;
+	    return arg2_arg4_map;
 
 	  case IFN_MASK_SCATTER_STORE:
 	  case IFN_MASK_LEN_SCATTER_STORE:
-	    return arg1_arg3_arg4_map;
+	    return arg2_arg4_arg5_map;
 
 	  case IFN_MASK_STORE:
 	    return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
@@ -691,7 +695,7 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap,
     {
       oprnd_info = (*oprnds_info)[i];
       int opno = map ? map[i] : int (i);
-      if (opno == -3)
+      if (opno == GATHER_SCATTER_OFFSET)
 	{
 	  gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
 	  if (!is_a <loop_vec_info> (vinfo)
@@ -1114,6 +1118,16 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
       matches[0] = false;
       return false;
     }
+  if (is_a <bb_vec_info> (vinfo)
+      && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "Build SLP failed: not using single lane "
+			 "vector type %T\n", vectype);
+      matches[0] = false;
+      return false;
+    }
   /* Record nunits required but continue analysis, producing matches[]
      as if nunits was not an issue.  This allows splitting of groups
      to happen.  */
@@ -1126,7 +1140,7 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
       soft_fail_nunits_vectype = nunits_vectype;
     }
 
-  gcc_assert (vectype);
+  gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
   *node_vectype = vectype;
 
   /* For every stmt in NODE find its def stmt/s.  */
@@ -1173,10 +1187,7 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
 
       gcall *call_stmt = dyn_cast <gcall *> (stmt);
       tree lhs = gimple_get_lhs (stmt);
-      if (lhs == NULL_TREE
-	  && (!call_stmt
-	      || !gimple_call_internal_p (stmt)
-	      || !internal_store_fn_p (gimple_call_internal_fn (stmt))))
+      if (lhs == NULL_TREE && !call_stmt)
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -4067,7 +4078,12 @@ vect_build_slp_instance (vec_info *vinfo,
 		  for (unsigned i = 0; i < group_size; ++i)
 		    scalar_stmts.quick_push (next_info);
 		  slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
-		  SLP_TREE_VECTYPE (conv) = STMT_VINFO_VECTYPE (next_info);
+		  SLP_TREE_VECTYPE (conv)
+		    = get_vectype_for_scalar_type (vinfo,
+						   TREE_TYPE
+						     (gimple_assign_lhs
+						       (scalar_def)),
+						   group_size);
 		  SLP_TREE_CHILDREN (conv).quick_push (node);
 		  SLP_INSTANCE_TREE (new_instance) = conv;
 		  /* We also have to fake this conversion stmt as SLP reduction
@@ -4869,9 +4885,11 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 
   /* Find SLP sequences starting from groups of grouped stores.  */
   FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
-    vect_analyze_slp_instance (vinfo, bst_map, first_element,
-			       slp_inst_kind_store, max_tree_size, &limit,
-			       force_single_lane);
+    if (! vect_analyze_slp_instance (vinfo, bst_map, first_element,
+				     slp_inst_kind_store, max_tree_size, &limit,
+				     force_single_lane)
+	&& loop_vinfo)
+      return opt_result::failure_at (vect_location, "SLP build failed.\n");
 
   /* For loops also start SLP discovery from non-grouped stores.  */
   if (loop_vinfo)
@@ -4889,10 +4907,29 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 	    vec<tree> remain = vNULL;
 	    stmts.create (1);
 	    stmts.quick_push (stmt_info);
-	    vect_build_slp_instance (vinfo, slp_inst_kind_store,
-				     stmts, roots, remain, max_tree_size,
-				     &limit, bst_map, NULL, force_single_lane);
+	    if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
+					   stmts, roots, remain, max_tree_size,
+					   &limit, bst_map, NULL,
+					   force_single_lane))
+	      return opt_result::failure_at (vect_location,
+					     "SLP build failed.\n");
 	  }
+
+      stmt_vec_info stmt_info;
+      FOR_EACH_VEC_ELT (LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo), i, stmt_info)
+	{
+	  vec<stmt_vec_info> stmts;
+	  vec<stmt_vec_info> roots = vNULL;
+	  vec<tree> remain = vNULL;
+	  stmts.create (1);
+	  stmts.quick_push (stmt_info);
+	  if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
+					 stmts, roots, remain, max_tree_size,
+					 &limit, bst_map, NULL,
+					 force_single_lane))
+	    return opt_result::failure_at (vect_location,
+					   "SLP build failed.\n");
+	}
     }
 
   if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo))
@@ -4931,6 +4968,9 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 						 max_tree_size, &limit,
 						 force_single_lane))
 	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "SLP discovery of reduction chain failed\n");
 	    /* Dissolve reduction chain group.  */
 	    stmt_vec_info vinfo = first_element;
 	    stmt_vec_info last = NULL;
@@ -4980,12 +5020,14 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 		      vec<tree> remain = vNULL;
 		      stmts.create (1);
 		      stmts.quick_push (next_info);
-		      vect_build_slp_instance (vinfo,
-					       slp_inst_kind_reduc_group,
-					       stmts, roots, remain,
-					       max_tree_size, &limit,
-					       bst_map, NULL,
-					       force_single_lane);
+		      if (! vect_build_slp_instance (vinfo,
+						     slp_inst_kind_reduc_group,
+						     stmts, roots, remain,
+						     max_tree_size, &limit,
+						     bst_map, NULL,
+						     force_single_lane))
+			return opt_result::failure_at (vect_location,
+						       "SLP build failed.\n");
 		    }
 		}
 	    }
@@ -5010,11 +5052,14 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 		  vec<tree> remain = vNULL;
 		  stmts.create (1);
 		  stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
-		  vect_build_slp_instance (vinfo,
-					   slp_inst_kind_reduc_group,
-					   stmts, roots, remain,
-					   max_tree_size, &limit,
-					   bst_map, NULL, force_single_lane);
+		  if (! vect_build_slp_instance (vinfo,
+						 slp_inst_kind_reduc_group,
+						 stmts, roots, remain,
+						 max_tree_size, &limit,
+						 bst_map, NULL,
+						 force_single_lane))
+		    return opt_result::failure_at (vect_location,
+						   "SLP build failed.\n");
 		}
 	    }
 	  saved_stmts.release ();
@@ -5043,11 +5088,14 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 		vec<tree> remain = vNULL;
 		stmts.create (1);
 		stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
-		vect_build_slp_instance (vinfo,
-					 slp_inst_kind_reduc_group,
-					 stmts, roots, remain,
-					 max_tree_size, &limit,
-					 bst_map, NULL, force_single_lane);
+		if (! vect_build_slp_instance (vinfo,
+					       slp_inst_kind_reduc_group,
+					       stmts, roots, remain,
+					       max_tree_size, &limit,
+					       bst_map, NULL,
+					       force_single_lane))
+		  return opt_result::failure_at (vect_location,
+						 "SLP build failed.\n");
 	      }
 	  }
 
@@ -5063,9 +5111,16 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 	  tree args0 = gimple_cond_lhs (stmt);
 	  tree args1 = gimple_cond_rhs (stmt);
 
-	  /* These should be enforced by cond lowering.  */
-	  gcc_assert (gimple_cond_code (stmt) == NE_EXPR);
-	  gcc_assert (zerop (args1));
+	  /* These should be enforced by cond lowering, but if it failed
+	     bail.  */
+	  if (gimple_cond_code (stmt) != NE_EXPR
+	      || TREE_TYPE (args0) != boolean_type_node
+	      || !integer_zerop (args1))
+	    {
+	      roots.release ();
+	      return opt_result::failure_at (vect_location,
+					     "SLP build failed.\n");
+	    }
 
 	  /* An argument without a loop def will be codegened from vectorizing the
 	     root gcond itself.  As such we don't need to try to build an SLP tree
@@ -5082,7 +5137,11 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 					 stmts, roots, remain,
 					 max_tree_size, &limit,
 					 bst_map, NULL, force_single_lane))
-	    roots.release ();
+	    {
+	      roots.release ();
+	      return opt_result::failure_at (vect_location,
+					     "SLP build failed.\n");
+	    }
 	}
 
 	/* Find and create slp instances for inductions that have been forced
@@ -5100,10 +5159,13 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 	      {
 		stmts.create (1);
 		stmts.quick_push (vect_stmt_to_vectorize (lc_info));
-		vect_build_slp_instance (vinfo, slp_inst_kind_reduc_group,
-					 stmts, roots, remain,
-					 max_tree_size, &limit,
-					 bst_map, NULL, force_single_lane);
+		if (! vect_build_slp_instance (vinfo, slp_inst_kind_reduc_group,
+					       stmts, roots, remain,
+					       max_tree_size, &limit,
+					       bst_map, NULL,
+					       force_single_lane))
+		  return opt_result::failure_at (vect_location,
+						 "SLP build failed.\n");
 	      }
 	    /* When the latch def is from a different cycle this can only
 	       be a induction.  Build a simple instance for this.
@@ -5116,10 +5178,13 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 	      {
 		stmts.create (1);
 		stmts.quick_push (stmt_info);
-		vect_build_slp_instance (vinfo, slp_inst_kind_reduc_group,
-					 stmts, roots, remain,
-					 max_tree_size, &limit,
-					 bst_map, NULL, force_single_lane);
+		if (! vect_build_slp_instance (vinfo, slp_inst_kind_reduc_group,
+					       stmts, roots, remain,
+					       max_tree_size, &limit,
+					       bst_map, NULL,
+					       force_single_lane))
+		  return opt_result::failure_at (vect_location,
+						 "SLP build failed.\n");
 	      }
 	  }
     }
@@ -5218,7 +5283,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 		    if (STMT_VINFO_STRIDED_P (stmt_vinfo)
 			|| compare_step_with_zero (vinfo, stmt_vinfo) <= 0
 			|| vect_load_lanes_supported
-			     (STMT_VINFO_VECTYPE (stmt_vinfo),
+			     (SLP_TREE_VECTYPE (load_node),
 			      DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
 			/* ???  During SLP re-discovery with a single lane
 			   a masked grouped load will appear permuted and
@@ -5239,7 +5304,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 			 || SLP_TREE_LANES (load_node) == group_size
 			 || (vect_slp_prefer_store_lanes_p
 			     (vinfo, stmt_vinfo,
-			      STMT_VINFO_VECTYPE (stmt_vinfo), masked,
+			      SLP_TREE_VECTYPE (load_node), masked,
 			      group_size, SLP_TREE_LANES (load_node))));
 		  }
 
@@ -7565,20 +7630,26 @@ vect_make_slp_decision (loop_vec_info loop_vinfo)
   hash_set<slp_tree> visited;
   FOR_EACH_VEC_ELT (slp_instances, i, instance)
     {
-      /* FORNOW: SLP if you can.  */
+      slp_tree root = SLP_INSTANCE_TREE (instance);
+
       /* All unroll factors have the form:
 
 	   GET_MODE_SIZE (vinfo->vector_mode) * X
 
 	 for some rational X, so they must have a common multiple.  */
-      vect_update_slp_vf_for_node (SLP_INSTANCE_TREE (instance),
-				   unrolling_factor, visited);
+      vect_update_slp_vf_for_node (root, unrolling_factor, visited);
 
       /* Mark all the stmts that belong to INSTANCE as PURE_SLP stmts.  Later we
 	 call vect_detect_hybrid_slp () to find stmts that need hybrid SLP and
 	 loop-based vectorization.  Such stmts will be marked as HYBRID.  */
-      vect_mark_slp_stmts (loop_vinfo, SLP_INSTANCE_TREE (instance));
-      decided_to_slp++;
+      vect_mark_slp_stmts (loop_vinfo, root);
+
+      /* If all instances ended up with vector(1) T roots make sure to
+	 not vectorize.  RVV for example relies on loop vectorization
+	 when some instances are essentially kept scalar.  See PR121048.  */
+      if (SLP_TREE_VECTYPE (root)
+	  && known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
+	decided_to_slp++;
     }
 
   LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo) = unrolling_factor;
@@ -7595,186 +7666,6 @@ vect_make_slp_decision (loop_vec_info loop_vinfo)
   return (decided_to_slp > 0);
 }
 
-/* Private data for vect_detect_hybrid_slp.  */
-struct vdhs_data
-{
-  loop_vec_info loop_vinfo;
-  vec<stmt_vec_info> *worklist;
-};
-
-/* Walker for walk_gimple_op.  */
-
-static tree
-vect_detect_hybrid_slp (tree *tp, int *, void *data)
-{
-  walk_stmt_info *wi = (walk_stmt_info *)data;
-  vdhs_data *dat = (vdhs_data *)wi->info;
-
-  if (wi->is_lhs)
-    return NULL_TREE;
-
-  stmt_vec_info def_stmt_info = dat->loop_vinfo->lookup_def (*tp);
-  if (!def_stmt_info)
-    return NULL_TREE;
-  def_stmt_info = vect_stmt_to_vectorize (def_stmt_info);
-  if (PURE_SLP_STMT (def_stmt_info))
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location, "marking hybrid: %G",
-			 def_stmt_info->stmt);
-      STMT_SLP_TYPE (def_stmt_info) = hybrid;
-      dat->worklist->safe_push (def_stmt_info);
-    }
-
-  return NULL_TREE;
-}
-
-/* Look if STMT_INFO is consumed by SLP indirectly and mark it pure_slp
-   if so, otherwise pushing it to WORKLIST.  */
-
-static void
-maybe_push_to_hybrid_worklist (vec_info *vinfo,
-			       vec<stmt_vec_info> &worklist,
-			       stmt_vec_info stmt_info)
-{
-  if (dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location,
-		     "Processing hybrid candidate : %G", stmt_info->stmt);
-  stmt_vec_info orig_info = vect_orig_stmt (stmt_info);
-  imm_use_iterator iter2;
-  ssa_op_iter iter1;
-  use_operand_p use_p;
-  def_operand_p def_p;
-  bool any_def = false;
-  FOR_EACH_PHI_OR_STMT_DEF (def_p, orig_info->stmt, iter1, SSA_OP_DEF)
-    {
-      any_def = true;
-      FOR_EACH_IMM_USE_FAST (use_p, iter2, DEF_FROM_PTR (def_p))
-	{
-	  if (is_gimple_debug (USE_STMT (use_p)))
-	    continue;
-	  stmt_vec_info use_info = vinfo->lookup_stmt (USE_STMT (use_p));
-	  /* An out-of loop use means this is a loop_vect sink.  */
-	  if (!use_info)
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_NOTE, vect_location,
-				 "Found loop_vect sink: %G", stmt_info->stmt);
-	      worklist.safe_push (stmt_info);
-	      return;
-	    }
-	  else if (!STMT_SLP_TYPE (vect_stmt_to_vectorize (use_info)))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_NOTE, vect_location,
-				 "Found loop_vect use: %G", use_info->stmt);
-	      worklist.safe_push (stmt_info);
-	      return;
-	    }
-	}
-    }
-  /* No def means this is a loop_vect sink.  Gimple conditionals also don't have a
-     def but shouldn't be considered sinks.  */
-  if (!any_def && STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "Found loop_vect sink: %G", stmt_info->stmt);
-      worklist.safe_push (stmt_info);
-      return;
-    }
-  if (dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location,
-		     "Marked SLP consumed stmt pure: %G", stmt_info->stmt);
-  STMT_SLP_TYPE (stmt_info) = pure_slp;
-}
-
-/* Find stmts that must be both vectorized and SLPed.  */
-
-void
-vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
-{
-  DUMP_VECT_SCOPE ("vect_detect_hybrid_slp");
-
-  /* All stmts participating in SLP are marked pure_slp, all other
-     stmts are loop_vect.
-     First collect all loop_vect stmts into a worklist.
-     SLP patterns cause not all original scalar stmts to appear in
-     SLP_TREE_SCALAR_STMTS and thus not all of them are marked pure_slp.
-     Rectify this here and do a backward walk over the IL only considering
-     stmts as loop_vect when they are used by a loop_vect stmt and otherwise
-     mark them as pure_slp.  */
-  auto_vec<stmt_vec_info> worklist;
-  for (int i = LOOP_VINFO_LOOP (loop_vinfo)->num_nodes - 1; i >= 0; --i)
-    {
-      basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
-      for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
-	   gsi_next (&gsi))
-	{
-	  gphi *phi = gsi.phi ();
-	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (phi);
-	  if (!STMT_SLP_TYPE (stmt_info) && STMT_VINFO_RELEVANT (stmt_info))
-	    maybe_push_to_hybrid_worklist (loop_vinfo,
-					   worklist, stmt_info);
-	}
-      for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi);
-	   gsi_prev (&gsi))
-	{
-	  gimple *stmt = gsi_stmt (gsi);
-	  if (is_gimple_debug (stmt))
-	    continue;
-	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
-	  if (STMT_VINFO_IN_PATTERN_P (stmt_info))
-	    {
-	      for (gimple_stmt_iterator gsi2
-		     = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
-		   !gsi_end_p (gsi2); gsi_next (&gsi2))
-		{
-		  stmt_vec_info patt_info
-		    = loop_vinfo->lookup_stmt (gsi_stmt (gsi2));
-		  if (!STMT_SLP_TYPE (patt_info)
-		      && STMT_VINFO_RELEVANT (patt_info))
-		    maybe_push_to_hybrid_worklist (loop_vinfo,
-						   worklist, patt_info);
-		}
-	      stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
-	    }
-	  if (!STMT_SLP_TYPE (stmt_info) && STMT_VINFO_RELEVANT (stmt_info))
-	    maybe_push_to_hybrid_worklist (loop_vinfo,
-					   worklist, stmt_info);
-	}
-    }
-
-  /* Now we have a worklist of non-SLP stmts, follow use->def chains and
-     mark any SLP vectorized stmt as hybrid.
-     ???  We're visiting def stmts N times (once for each non-SLP and
-     once for each hybrid-SLP use).  */
-  walk_stmt_info wi;
-  vdhs_data dat;
-  dat.worklist = &worklist;
-  dat.loop_vinfo = loop_vinfo;
-  memset (&wi, 0, sizeof (wi));
-  wi.info = (void *)&dat;
-  while (!worklist.is_empty ())
-    {
-      stmt_vec_info stmt_info = worklist.pop ();
-      /* Since SSA operands are not set up for pattern stmts we need
-	 to use walk_gimple_op.  */
-      wi.is_lhs = 0;
-      walk_gimple_op (stmt_info->stmt, vect_detect_hybrid_slp, &wi);
-      /* For gather/scatter make sure to walk the offset operand, that
-	 can be a scaling and conversion away.  */
-      gather_scatter_info gs_info;
-      if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
-	  && vect_check_gather_scatter (stmt_info, loop_vinfo, &gs_info))
-	{
-	  int dummy;
-	  vect_detect_hybrid_slp (&gs_info.offset, &dummy, &wi);
-	}
-    }
-}
-
-
 /* Initialize a bb_vec_info struct for the statements in BBS basic blocks.  */
 
 _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
@@ -7852,15 +7743,16 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
 				    slp_instance node_instance,
 				    stmt_vector_for_cost *cost_vec)
 {
-  stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
-
   /* Calculate the number of vector statements to be created for the scalar
      stmts in this node.  It is the number of scalar elements in one scalar
      iteration (DR_GROUP_SIZE) multiplied by VF divided by the number of
      elements in a vector.  For single-defuse-cycle, lane-reducing op, and
      PHI statement that starts reduction comprised of only lane-reducing ops,
      the number is more than effective vector statements actually required.  */
-  SLP_TREE_NUMBER_OF_VEC_STMTS (node) = vect_get_num_copies (vinfo, node);
+  if (SLP_TREE_VECTYPE (node))
+    SLP_TREE_NUMBER_OF_VEC_STMTS (node) = vect_get_num_copies (vinfo, node);
+  else
+    SLP_TREE_NUMBER_OF_VEC_STMTS (node) = 0;
 
   /* Handle purely internal nodes.  */
   if (SLP_TREE_CODE (node) == VEC_PERM_EXPR)
@@ -7882,9 +7774,7 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
       return true;
     }
 
-  bool dummy;
-  return vect_analyze_stmt (vinfo, stmt_info, &dummy,
-			    node, node_instance, cost_vec);
+  return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
 }
 
 static int
@@ -8189,8 +8079,7 @@ vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
 	      /* Masked loads can have an undefined (default SSA definition)
 		 else operand.  We do not need to cost it.  */
 	      vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
-	      if ((STMT_VINFO_TYPE (SLP_TREE_REPRESENTATIVE (node))
-		   == load_vec_info_type)
+	      if (SLP_TREE_TYPE (node) == load_vec_info_type
 		  && ((ops.length ()
 		       && TREE_CODE (ops[0]) == SSA_NAME
 		       && SSA_NAME_IS_DEFAULT_DEF (ops[0])
@@ -8201,8 +8090,7 @@ vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
 	      /* For shifts with a scalar argument we don't need
 		 to cost or code-generate anything.
 		 ???  Represent this more explicitely.  */
-	      gcc_assert ((STMT_VINFO_TYPE (SLP_TREE_REPRESENTATIVE (node))
-			   == shift_vec_info_type)
+	      gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
 			  && j == 1);
 	      continue;
 	    }
@@ -8580,7 +8468,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
 	  || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
 	      && !vectorizable_early_exit (vinfo,
 					   SLP_INSTANCE_ROOT_STMTS (instance)[0],
-					   NULL, NULL,
+					   NULL,
 					   SLP_INSTANCE_TREE (instance),
 					   &cost_vec)))
         {
@@ -9502,14 +9390,13 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
 
   slp_instance instance;
   int i;
-  poly_uint64 min_vf = 2;
 
   /* The first group of checks is independent of the vector size.  */
   fatal = true;
 
   /* Analyze the data references.  */
 
-  if (!vect_analyze_data_refs (bb_vinfo, &min_vf, NULL))
+  if (!vect_analyze_data_refs (bb_vinfo, NULL))
     {
       if (dump_enabled_p ())
         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -11222,8 +11109,10 @@ vect_schedule_slp_node (vec_info *vinfo,
 
   stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
 
-  gcc_assert (SLP_TREE_NUMBER_OF_VEC_STMTS (node) != 0);
-  SLP_TREE_VEC_DEFS (node).create (SLP_TREE_NUMBER_OF_VEC_STMTS (node));
+  gcc_assert (!SLP_TREE_VECTYPE (node)
+	      || SLP_TREE_NUMBER_OF_VEC_STMTS (node) != 0);
+  if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) != 0)
+    SLP_TREE_VEC_DEFS (node).create (SLP_TREE_NUMBER_OF_VEC_STMTS (node));
 
   if (SLP_TREE_CODE (node) != VEC_PERM_EXPR
       && STMT_VINFO_DATA_REF (stmt_info))
@@ -11239,9 +11128,9 @@ vect_schedule_slp_node (vec_info *vinfo,
       si = gsi_for_stmt (last_stmt_info->stmt);
     }
   else if (SLP_TREE_CODE (node) != VEC_PERM_EXPR
-	   && (STMT_VINFO_TYPE (stmt_info) == cycle_phi_info_type
-	       || STMT_VINFO_TYPE (stmt_info) == induc_vec_info_type
-	       || STMT_VINFO_TYPE (stmt_info) == phi_info_type))
+	   && (SLP_TREE_TYPE (node) == cycle_phi_info_type
+	       || SLP_TREE_TYPE (node) == induc_vec_info_type
+	       || SLP_TREE_TYPE (node) == phi_info_type))
     {
       /* For PHI node vectorization we do not use the insertion iterator.  */
       si = gsi_none ();
@@ -11261,8 +11150,7 @@ vect_schedule_slp_node (vec_info *vinfo,
 	       last scalar def here.  */
 	    if (SLP_TREE_VEC_DEFS (child).is_empty ())
 	      {
-		gcc_assert (STMT_VINFO_TYPE (SLP_TREE_REPRESENTATIVE (child))
-			    == cycle_phi_info_type);
+		gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
 		gphi *phi = as_a <gphi *>
 			      (vect_find_last_scalar_stmt_in_slp (child)->stmt);
 		if (!last_stmt)
@@ -11303,7 +11191,11 @@ vect_schedule_slp_node (vec_info *vinfo,
 		  && !SSA_NAME_IS_DEFAULT_DEF (def))
 		{
 		  gimple *stmt = SSA_NAME_DEF_STMT (def);
-		  if (!last_stmt)
+		  if (gimple_uid (stmt) == -1u)
+		    /* If the stmt is not inside the region do not
+		       use it as possible insertion point.  */
+		    ;
+		  else if (!last_stmt)
 		    last_stmt = stmt;
 		  else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
 		    last_stmt = stmt;
@@ -11409,7 +11301,7 @@ vect_schedule_slp_node (vec_info *vinfo,
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "------>vectorizing SLP permutation node\n");
-      /* ???  the transform kind is stored to STMT_VINFO_TYPE which might
+      /* ???  the transform kind was stored to STMT_VINFO_TYPE which might
 	 be shared with different SLP nodes (but usually it's the same
 	 operation apart from the case the stmt is only there for denoting
 	 the actual scalar lane defs ...).  So do not call vect_transform_stmt
@@ -11465,8 +11357,6 @@ vect_remove_slp_scalar_calls (vec_info *vinfo,
     {
       if (!stmt_info)
 	continue;
-      if (!PURE_SLP_STMT (stmt_info))
-	continue;
       stmt_info = vect_orig_stmt (stmt_info);
       gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
       if (!stmt || gimple_bb (stmt) == NULL)
@@ -11608,10 +11498,9 @@ vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance i
       auto root_stmt_info = instance->root_stmts[0];
       auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
       gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
-      gimple *vec_stmt = NULL;
       gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
       bool res = vectorizable_early_exit (vinfo, root_stmt_info, &rgsi,
-					  &vec_stmt, node, NULL);
+					  node, NULL);
       gcc_assert (res);
       return;
     }
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 95406b4..f7a052b 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -62,14 +62,6 @@ along with GCC; see the file COPYING3.  If not see
 /* For lang_hooks.types.type_for_mode.  */
 #include "langhooks.h"
 
-/* Return the vectorized type for the given statement.  */
-
-tree
-stmt_vectype (class _stmt_vec_info *stmt_info)
-{
-  return STMT_VINFO_VECTYPE (stmt_info);
-}
-
 /* Return TRUE iff the given statement is in an inner loop relative to
    the loop being vectorized.  */
 bool
@@ -394,6 +386,9 @@ vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
 	  dump_printf_loc (MSG_NOTE, vect_location,
                            "vec_stmt_relevant_p: stmt has vdefs.\n");
 	*relevant = vect_used_in_scope;
+	if (! STMT_VINFO_DATA_REF (stmt_info)
+	    && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF))
+	  LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info);
       }
 
   /* uses outside the loop.  */
@@ -901,13 +896,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
 
 /* Function vect_model_simple_cost.
 
-   Models cost for simple operations, i.e. those that only emit ncopies of a
-   single op.  Right now, this does not account for multiple insns that could
-   be generated for the single vector op.  We will handle that shortly.  */
+   Models cost for simple operations, i.e. those that only emit N operations
+   of the same KIND.  */
 
 static void
-vect_model_simple_cost (vec_info *, int ncopies, enum vect_def_type *dt,
-			int ndts, slp_tree node,
+vect_model_simple_cost (vec_info *, int n, slp_tree node,
 			stmt_vector_for_cost *cost_vec,
 			vect_cost_for_stmt kind = vector_stmt)
 {
@@ -915,22 +908,10 @@ vect_model_simple_cost (vec_info *, int ncopies, enum vect_def_type *dt,
 
   gcc_assert (cost_vec != NULL);
 
-  /* ???  Somehow we need to fix this at the callers.  */
-  if (node)
-    ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (node);
-
-  if (!node)
-    /* Cost the "broadcast" of a scalar operand in to a vector operand.
-       Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
-       cost model.  */
-    for (int i = 0; i < ndts; i++)
-      if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
-	prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
-					   node, 0, vect_prologue);
+  n *= SLP_TREE_NUMBER_OF_VEC_STMTS (node);
 
   /* Pass the inside-of-loop statements to the target-specific cost model.  */
-  inside_cost += record_stmt_cost (cost_vec, ncopies, kind,
-				   node, 0, vect_body);
+  inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
@@ -1269,130 +1250,23 @@ vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
 }
 
 
-/* Function vect_get_vec_defs_for_operand.
-
-   OP is an operand in STMT_VINFO.  This function returns a vector of
-   NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
-
-   In the case that OP is an SSA_NAME which is defined in the loop, then
-   STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
-
-   In case OP is an invariant or constant, a new stmt that creates a vector def
-   needs to be introduced.  VECTYPE may be used to specify a required type for
-   vector invariant.  */
-
-void
-vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info stmt_vinfo,
-			       unsigned ncopies,
-			       tree op, vec<tree> *vec_oprnds, tree vectype)
-{
-  gimple *def_stmt;
-  enum vect_def_type dt;
-  bool is_simple_use;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-
-  if (dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location,
-		     "vect_get_vec_defs_for_operand: %T\n", op);
-
-  stmt_vec_info def_stmt_info;
-  is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt,
-				      &def_stmt_info, &def_stmt);
-  gcc_assert (is_simple_use);
-  if (def_stmt && dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location, "  def_stmt =  %G", def_stmt);
-
-  vec_oprnds->create (ncopies);
-  if (dt == vect_constant_def || dt == vect_external_def)
-    {
-      tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
-      tree vector_type;
-
-      if (vectype)
-	vector_type = vectype;
-      else if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op))
-	       && VECTOR_BOOLEAN_TYPE_P (stmt_vectype))
-	vector_type = truth_type_for (stmt_vectype);
-      else
-	vector_type = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op));
-
-      gcc_assert (vector_type);
-      /* A masked load can have a default SSA definition as else operand.
-	 We should "vectorize" this instead of creating a duplicate from the
-	 scalar default.  */
-      tree vop;
-      if (TREE_CODE (op) == SSA_NAME
-	  && SSA_NAME_IS_DEFAULT_DEF (op)
-	  && VAR_P (SSA_NAME_VAR (op)))
-	vop = get_or_create_ssa_default_def (cfun,
-					     create_tmp_var (vector_type));
-      else
-	vop = vect_init_vector (vinfo, stmt_vinfo, op, vector_type, NULL);
-      while (ncopies--)
-	vec_oprnds->quick_push (vop);
-    }
-  else
-    {
-      def_stmt_info = vect_stmt_to_vectorize (def_stmt_info);
-      gcc_assert (STMT_VINFO_VEC_STMTS (def_stmt_info).length () == ncopies);
-      for (unsigned i = 0; i < ncopies; ++i)
-	vec_oprnds->quick_push (gimple_get_lhs
-				  (STMT_VINFO_VEC_STMTS (def_stmt_info)[i]));
-    }
-}
-
-
 /* Get vectorized definitions for OP0 and OP1.  */
 
 void
-vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node,
-		   unsigned ncopies,
-		   tree op0, tree vectype0, vec<tree> *vec_oprnds0,
-		   tree op1, tree vectype1, vec<tree> *vec_oprnds1,
-		   tree op2, tree vectype2, vec<tree> *vec_oprnds2,
-		   tree op3, tree vectype3, vec<tree> *vec_oprnds3)
-{
-  if (slp_node)
-    {
-      if (op0)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
-      if (op1)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
-      if (op2)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
-      if (op3)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
-    }
-  else
-    {
-      if (op0)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op0, vec_oprnds0, vectype0);
-      if (op1)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op1, vec_oprnds1, vectype1);
-      if (op2)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op2, vec_oprnds2, vectype2);
-      if (op3)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op3, vec_oprnds3, vectype3);
-    }
-}
-
-void
-vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node,
-		   unsigned ncopies,
+vect_get_vec_defs (vec_info *, slp_tree slp_node,
 		   tree op0, vec<tree> *vec_oprnds0,
 		   tree op1, vec<tree> *vec_oprnds1,
 		   tree op2, vec<tree> *vec_oprnds2,
 		   tree op3, vec<tree> *vec_oprnds3)
 {
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		     op0, NULL_TREE, vec_oprnds0,
-		     op1, NULL_TREE, vec_oprnds1,
-		     op2, NULL_TREE, vec_oprnds2,
-		     op3, NULL_TREE, vec_oprnds3);
+  if (op0)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
+  if (op1)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
+  if (op2)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
+  if (op3)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
 }
 
 /* Helper function called by vect_finish_replace_stmt and
@@ -1551,14 +1425,31 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
 				      int group_size,
 				      vect_memory_access_type
 				      memory_access_type,
-				      gather_scatter_info *gs_info,
-				      tree scalar_mask,
+				      const gather_scatter_info *gs_info,
+				      slp_tree mask_node,
 				      vec<int> *elsvals = nullptr)
 {
   /* Invariant loads need no special support.  */
   if (memory_access_type == VMAT_INVARIANT)
     return;
 
+  /* Figure whether the mask is uniform.  scalar_mask is used to
+     populate the scalar_cond_masked_set.  */
+  tree scalar_mask = NULL_TREE;
+  if (mask_node)
+    for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
+      {
+	tree def = vect_get_slp_scalar_def (mask_node, i);
+	if (!def
+	    || (scalar_mask && def != scalar_mask))
+	  {
+	    scalar_mask = NULL;
+	    break;
+	  }
+	else
+	  scalar_mask = def;
+      }
+
   unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node, vectype);
   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
   vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
@@ -1566,8 +1457,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
   bool is_load = (vls_type == VLS_LOAD);
   if (memory_access_type == VMAT_LOAD_STORE_LANES)
     {
-      if (slp_node)
-	nvectors /= group_size;
+      nvectors /= group_size;
       internal_fn ifn
 	= (is_load ? vect_load_lanes_supported (vectype, group_size, true,
 						elsvals)
@@ -1622,8 +1512,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
       return;
     }
 
-  if (memory_access_type != VMAT_CONTIGUOUS
-      && memory_access_type != VMAT_CONTIGUOUS_PERMUTE)
+  if (memory_access_type != VMAT_CONTIGUOUS)
     {
       /* Element X of the data must come from iteration i * VF + X of the
 	 scalar loop.  We need more work to support other mappings.  */
@@ -1646,7 +1535,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
     }
 
   /* We might load more scalars than we need for permuting SLP loads.
-     We checked in get_group_load_store_type that the extra elements
+     We checked in get_load_store_type that the extra elements
      don't leak into a new vector.  */
   auto group_memory_nvectors = [](poly_uint64 size, poly_uint64 nunits)
   {
@@ -1734,7 +1623,7 @@ prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
    else values will be stored in the vector ELSVALS points to.  */
 
 static bool
-vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
+vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
 				     loop_vec_info loop_vinfo, bool masked_p,
 				     gather_scatter_info *gs_info,
 				     vec<int> *elsvals)
@@ -1752,7 +1641,6 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
     }
 
   /* Get the number of bits in an element.  */
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
   unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
 
@@ -1803,9 +1691,11 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
       /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
 	 but we don't need to store that here.  */
       gs_info->base = NULL_TREE;
+      gs_info->alias_ptr = build_int_cst
+	(reference_alias_ptr_type (DR_REF (dr)),
+	 get_object_alignment (DR_REF (dr)));
       gs_info->element_type = TREE_TYPE (vectype);
       gs_info->offset = fold_convert (offset_type, step);
-      gs_info->offset_dt = vect_constant_def;
       gs_info->scale = scale;
       gs_info->memory_type = memory_type;
       return true;
@@ -1829,22 +1719,35 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
    else values will be stored in the vector ELSVALS points to.  */
 
 static bool
-vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info,
+vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
 				    loop_vec_info loop_vinfo, bool masked_p,
 				    gather_scatter_info *gs_info,
-				    vec<int> *elsvals)
+				    vec<int> *elsvals,
+				    unsigned int group_size,
+				    bool single_element_p)
 {
   if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info, elsvals)
       || gs_info->ifn == IFN_LAST)
-    return vect_truncate_gather_scatter_offset (stmt_info, loop_vinfo,
-						masked_p, gs_info, elsvals);
+    {
+      if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
+						masked_p, gs_info, elsvals))
+	return false;
+    }
+  else
+    {
+      tree old_offset_type = TREE_TYPE (gs_info->offset);
+      tree new_offset_type = TREE_TYPE (gs_info->offset_vectype);
 
-  tree old_offset_type = TREE_TYPE (gs_info->offset);
-  tree new_offset_type = TREE_TYPE (gs_info->offset_vectype);
+      gcc_assert (TYPE_PRECISION (new_offset_type)
+		  >= TYPE_PRECISION (old_offset_type));
+      gs_info->offset = fold_convert (new_offset_type, gs_info->offset);
+    }
 
-  gcc_assert (TYPE_PRECISION (new_offset_type)
-	      >= TYPE_PRECISION (old_offset_type));
-  gs_info->offset = fold_convert (new_offset_type, gs_info->offset);
+  if (!single_element_p
+      && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
+						   gs_info->scale,
+						   group_size))
+    return false;
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
@@ -2032,38 +1935,45 @@ vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype)
   return NULL_TREE;
 }
 
-/* A subroutine of get_load_store_type, with a subset of the same
-   arguments.  Handle the case where STMT_INFO is part of a grouped load
-   or store.
+/* Analyze load or store SLP_NODE of type VLS_TYPE.  Return true
+   if there is a memory access type that the vectorized form can use,
+   storing it in *MEMORY_ACCESS_TYPE if so.  If we decide to use gathers
+   or scatters, fill in GS_INFO accordingly.  In addition
+   *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
+   the target does not support the alignment scheme.  *MISALIGNMENT
+   is set according to the alignment of the access (including
+   DR_MISALIGNMENT_UNKNOWN when it is unknown).
 
-   For stores, the statements in the group are all consecutive
-   and there is no gap at the end.  For loads, the statements in the
-   group might not be consecutive; there can be gaps between statements
-   as well as at the end.
+   MASKED_P is true if the statement is conditional on a vectorized mask.
+   VECTYPE is the vector type that the vectorized statements will use.
 
-   If we can use gather/scatter and ELSVALS is nonzero the supported
-   else values will be stored in the vector ELSVALS points to.
-*/
+   If ELSVALS is nonzero the supported else values will be stored in the
+   vector ELSVALS points to.  */
 
 static bool
-get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
-			   tree vectype, slp_tree slp_node,
-			   bool masked_p, vec_load_store_type vls_type,
-			   vect_memory_access_type *memory_access_type,
-			   poly_int64 *poffset,
-			   dr_alignment_support *alignment_support_scheme,
-			   int *misalignment,
-			   gather_scatter_info *gs_info,
-			   internal_fn *lanes_ifn,
-			   vec<int> *elsvals)
+get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
+		     tree vectype, slp_tree slp_node,
+		     bool masked_p, vec_load_store_type vls_type,
+		     vect_memory_access_type *memory_access_type,
+		     poly_int64 *poffset,
+		     dr_alignment_support *alignment_support_scheme,
+		     int *misalignment,
+		     gather_scatter_info *gs_info,
+		     internal_fn *lanes_ifn,
+		     vec<int> *elsvals = nullptr)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+  poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
   class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
   stmt_vec_info first_stmt_info;
   unsigned int group_size;
   unsigned HOST_WIDE_INT gap;
   bool single_element_p;
   poly_int64 neg_ldst_offset = 0;
+
+  *misalignment = DR_MISALIGNMENT_UNKNOWN;
+  *poffset = 0;
+
   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
     {
       first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
@@ -2080,7 +1990,6 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
       single_element_p = true;
     }
   dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
-  poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
   /* True if the vectorized statements would access beyond the last
      statement in the group.  */
@@ -2106,6 +2015,48 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
        separated by the stride, until we have a complete vector.
        Fall back to scalar accesses if that isn't possible.  */
     *memory_access_type = VMAT_STRIDED_SLP;
+  else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+    {
+      *memory_access_type = VMAT_GATHER_SCATTER;
+      if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info,
+				      elsvals))
+	gcc_unreachable ();
+      slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
+      tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
+      gs_info->offset_vectype = offset_vectype;
+      /* When using internal functions, we rely on pattern recognition
+	 to convert the type of the offset to the type that the target
+	 requires, with the result being a call to an internal function.
+	 If that failed for some reason (e.g. because another pattern
+	 took priority), just handle cases in which the offset already
+	 has the right type.  */
+      if (GATHER_SCATTER_IFN_P (*gs_info)
+	  && !is_gimple_call (stmt_info->stmt)
+	  && !tree_nop_conversion_p (TREE_TYPE (gs_info->offset),
+				     TREE_TYPE (offset_vectype)))
+	{
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			     "%s offset requires a conversion\n",
+			     vls_type == VLS_LOAD ? "gather" : "scatter");
+	  return false;
+	}
+      else if (GATHER_SCATTER_EMULATED_P (*gs_info))
+	{
+	  if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
+	      || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
+	      || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
+	      || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
+				       TYPE_VECTOR_SUBPARTS (vectype)))
+	    {
+	      if (dump_enabled_p ())
+		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+				 "unsupported vector types for emulated "
+				 "gather.\n");
+	      return false;
+	    }
+	}
+    }
   else
     {
       int cmp = compare_step_with_zero (vinfo, stmt_info);
@@ -2349,19 +2300,21 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
      allows us to use contiguous accesses.  */
   if ((*memory_access_type == VMAT_ELEMENTWISE
        || *memory_access_type == VMAT_STRIDED_SLP)
-      && single_element_p
+      && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
       && SLP_TREE_LANES (slp_node) == 1
       && loop_vinfo
-      && vect_use_strided_gather_scatters_p (stmt_info, loop_vinfo,
-					     masked_p, gs_info, elsvals))
+      && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
+					     masked_p, gs_info, elsvals,
+					     group_size, single_element_p))
     *memory_access_type = VMAT_GATHER_SCATTER;
 
   if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
       || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
     *poffset = neg_ldst_offset;
 
-  if (*memory_access_type == VMAT_GATHER_SCATTER
-      || *memory_access_type == VMAT_ELEMENTWISE
+  if (*memory_access_type == VMAT_ELEMENTWISE
+      || (*memory_access_type == VMAT_GATHER_SCATTER
+	  && GATHER_SCATTER_LEGACY_P (*gs_info))
       || *memory_access_type == VMAT_STRIDED_SLP
       || *memory_access_type == VMAT_INVARIANT)
     {
@@ -2370,30 +2323,15 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
     }
   else
     {
-      *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
+      if (*memory_access_type == VMAT_GATHER_SCATTER
+	  && !first_dr_info)
+	*misalignment = DR_MISALIGNMENT_UNKNOWN;
+      else
+	*misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
       *alignment_support_scheme
-	= vect_supportable_dr_alignment (vinfo, first_dr_info, vectype,
-					 *misalignment);
-    }
-
-  if (vls_type != VLS_LOAD && first_stmt_info == stmt_info)
-    {
-      /* STMT is the leader of the group. Check the operands of all the
-	 stmts of the group.  */
-      stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
-      while (next_stmt_info)
-	{
-	  tree op = vect_get_store_rhs (next_stmt_info);
-	  enum vect_def_type dt;
-	  if (!vect_is_simple_use (op, vinfo, &dt))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "use not simple.\n");
-	      return false;
-	    }
-	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
-	}
+	= vect_supportable_dr_alignment
+	   (vinfo, first_dr_info, vectype, *misalignment,
+	    *memory_access_type == VMAT_GATHER_SCATTER ? gs_info : nullptr);
     }
 
   if (overrun_p)
@@ -2406,104 +2344,6 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
       LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
     }
 
-  return true;
-}
-
-/* Analyze load or store statement STMT_INFO of type VLS_TYPE.  Return true
-   if there is a memory access type that the vectorized form can use,
-   storing it in *MEMORY_ACCESS_TYPE if so.  If we decide to use gathers
-   or scatters, fill in GS_INFO accordingly.  In addition
-   *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
-   the target does not support the alignment scheme.  *MISALIGNMENT
-   is set according to the alignment of the access (including
-   DR_MISALIGNMENT_UNKNOWN when it is unknown).
-
-   SLP says whether we're performing SLP rather than loop vectorization.
-   MASKED_P is true if the statement is conditional on a vectorized mask.
-   VECTYPE is the vector type that the vectorized statements will use.
-   NCOPIES is the number of vector statements that will be needed.
-
-   If ELSVALS is nonzero the supported else values will be stored in the
-   vector ELSVALS points to.  */
-
-static bool
-get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
-		     tree vectype, slp_tree slp_node,
-		     bool masked_p, vec_load_store_type vls_type,
-		     unsigned int,
-		     vect_memory_access_type *memory_access_type,
-		     poly_int64 *poffset,
-		     dr_alignment_support *alignment_support_scheme,
-		     int *misalignment,
-		     gather_scatter_info *gs_info,
-		     internal_fn *lanes_ifn,
-		     vec<int> *elsvals = nullptr)
-{
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
-  *misalignment = DR_MISALIGNMENT_UNKNOWN;
-  *poffset = 0;
-  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-    {
-      *memory_access_type = VMAT_GATHER_SCATTER;
-      if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info,
-				      elsvals))
-	gcc_unreachable ();
-      /* When using internal functions, we rely on pattern recognition
-	 to convert the type of the offset to the type that the target
-	 requires, with the result being a call to an internal function.
-	 If that failed for some reason (e.g. because another pattern
-	 took priority), just handle cases in which the offset already
-	 has the right type.  */
-      else if (gs_info->ifn != IFN_LAST
-	       && !is_gimple_call (stmt_info->stmt)
-	       && !tree_nop_conversion_p (TREE_TYPE (gs_info->offset),
-					  TREE_TYPE (gs_info->offset_vectype)))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "%s offset requires a conversion\n",
-			     vls_type == VLS_LOAD ? "gather" : "scatter");
-	  return false;
-	}
-      else if (!vect_is_simple_use (gs_info->offset, vinfo,
-				    &gs_info->offset_dt,
-				    &gs_info->offset_vectype))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "%s index use not simple.\n",
-			     vls_type == VLS_LOAD ? "gather" : "scatter");
-	  return false;
-	}
-      else if (gs_info->ifn == IFN_LAST && !gs_info->decl)
-	{
-	  if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
-	      || !TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype).is_constant ()
-	      || VECTOR_BOOLEAN_TYPE_P (gs_info->offset_vectype)
-	      || !constant_multiple_p (TYPE_VECTOR_SUBPARTS
-					 (gs_info->offset_vectype),
-				       TYPE_VECTOR_SUBPARTS (vectype)))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "unsupported vector types for emulated "
-				 "gather.\n");
-	      return false;
-	    }
-	}
-      /* Gather-scatter accesses perform only component accesses, alignment
-	 is irrelevant for them.  */
-      *alignment_support_scheme = dr_unaligned_supported;
-    }
-  else if (!get_group_load_store_type (vinfo, stmt_info, vectype, slp_node,
-				       masked_p,
-				       vls_type, memory_access_type, poffset,
-				       alignment_support_scheme,
-				       misalignment, gs_info, lanes_ifn,
-				       elsvals))
-    return false;
-
   if ((*memory_access_type == VMAT_ELEMENTWISE
        || *memory_access_type == VMAT_STRIDED_SLP)
       && !nunits.is_constant ())
@@ -2515,7 +2355,6 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
-
   /* Checks if all scalar iterations are known to be inbounds.  */
   bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
 
@@ -2535,17 +2374,18 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
 			   "alignment. With non-contiguous memory vectorization"
 			   " could read out of bounds at %G ",
 			   STMT_VINFO_STMT (stmt_info));
-	if (inbounds)
-	  LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
-	else
-	  return false;
+      if (inbounds)
+	LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
+      else
+	return false;
     }
 
   /* If this DR needs alignment for correctness, we must ensure the target
      alignment is a constant power-of-two multiple of the amount read per
      vector iteration or force masking.  */
   if (dr_safe_speculative_read_required (stmt_info)
-      && *alignment_support_scheme == dr_aligned)
+      && (*alignment_support_scheme == dr_aligned
+	  && *memory_access_type != VMAT_GATHER_SCATTER))
     {
       /* We can only peel for loops, of course.  */
       gcc_checking_assert (loop_vinfo);
@@ -2648,9 +2488,6 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
   /* FIXME: At the moment the cost model seems to underestimate the
      cost of using elementwise accesses.  This check preserves the
      traditional behavior until that can be fixed.  */
-  stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
-  if (!first_stmt_info)
-    first_stmt_info = stmt_info;
   if (*memory_access_type == VMAT_ELEMENTWISE
       && !STMT_VINFO_STRIDED_P (first_stmt_info)
       && !(stmt_info == DR_GROUP_FIRST_ELEMENT (stmt_info)
@@ -2667,21 +2504,21 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
 
 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
    conditional operation STMT_INFO.  When returning true, store the mask
-   in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
-   vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
-   to the mask in *MASK_NODE if MASK_NODE is not NULL.  */
+   in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
+   the vectorized mask in *MASK_VECTYPE_OUT.  */
 
 static bool
-vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
+vect_check_scalar_mask (vec_info *vinfo,
 			slp_tree slp_node, unsigned mask_index,
-			tree *mask, slp_tree *mask_node,
+			slp_tree *mask_node,
 			vect_def_type *mask_dt_out, tree *mask_vectype_out)
 {
   enum vect_def_type mask_dt;
   tree mask_vectype;
   slp_tree mask_node_1;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, mask_index,
-			   mask, &mask_node_1, &mask_dt, &mask_vectype))
+  tree mask_;
+  if (!vect_is_simple_use (vinfo, slp_node, mask_index,
+			   &mask_, &mask_node_1, &mask_dt, &mask_vectype))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2690,7 +2527,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
     }
 
   if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
-      && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (*mask)))
+      && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2698,19 +2535,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
-  /* If the caller is not prepared for adjusting an external/constant
-     SLP mask vector type fail.  */
-  if (slp_node
-      && !mask_node
-      && SLP_TREE_DEF_TYPE (mask_node_1) != vect_internal_def)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "SLP mask argument is not vectorized.\n");
-      return false;
-    }
-
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (!mask_vectype)
     mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
 						  mask_node_1);
@@ -2737,11 +2562,11 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
 
   *mask_dt_out = mask_dt;
   *mask_vectype_out = mask_vectype;
-  if (mask_node)
-    *mask_node = mask_node_1;
+  *mask_node = mask_node_1;
   return true;
 }
 
+
 /* Return true if stored value is suitable for vectorizing store
    statement STMT_INFO.  When returning true, store the scalar stored
    in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
@@ -2750,7 +2575,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
 
 static bool
 vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
-		      slp_tree slp_node, tree *rhs, slp_tree *rhs_node,
+		      slp_tree slp_node, slp_tree *rhs_node,
 		      vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
 		      vec_load_store_type *vls_type_out)
 {
@@ -2761,14 +2586,14 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
 	  && internal_store_fn_p (gimple_call_internal_fn (call)))
 	op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
     }
-  if (slp_node)
-    op_no = vect_slp_child_index_for_operand
-	      (stmt_info->stmt, op_no, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
+  op_no = vect_slp_child_index_for_operand
+	    (stmt_info->stmt, op_no, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
 
   enum vect_def_type rhs_dt;
   tree rhs_vectype;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, op_no,
-			   rhs, rhs_node, &rhs_dt, &rhs_vectype))
+  tree rhs;
+  if (!vect_is_simple_use (vinfo, slp_node, op_no,
+			   &rhs, rhs_node, &rhs_dt, &rhs_vectype))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2779,7 +2604,7 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
   /* In the case this is a store from a constant make sure
      native_encode_expr can handle it.  */
   if (rhs_dt == vect_constant_def
-      && CONSTANT_CLASS_P (*rhs) && native_encode_expr (*rhs, NULL, 64) == 0)
+      && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2787,7 +2612,7 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
     {
       if (dump_enabled_p ())
@@ -2893,11 +2718,11 @@ vect_get_mask_load_else (int elsval, tree type)
 
 static gimple *
 vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
+				 tree vectype,
 				 gimple_stmt_iterator *gsi,
-				 gather_scatter_info *gs_info,
+				 const gather_scatter_info *gs_info,
 				 tree ptr, tree offset, tree mask)
 {
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl));
   tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
   tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
@@ -2993,7 +2818,7 @@ vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
 static gimple *
 vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
 				   gimple_stmt_iterator *gsi,
-				   gather_scatter_info *gs_info,
+				   const gather_scatter_info *gs_info,
 				   tree ptr, tree offset, tree oprnd, tree mask)
 {
   tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
@@ -3074,9 +2899,8 @@ vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
    containing loop.  */
 
 static void
-vect_get_gather_scatter_ops (loop_vec_info loop_vinfo,
-			     class loop *loop, stmt_vec_info stmt_info,
-			     slp_tree slp_node, gather_scatter_info *gs_info,
+vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
+			     const gather_scatter_info *gs_info,
 			     tree *dataref_ptr, vec<tree> *vec_offset)
 {
   gimple_seq stmts = NULL;
@@ -3088,16 +2912,7 @@ vect_get_gather_scatter_ops (loop_vec_info loop_vinfo,
       new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
       gcc_assert (!new_bb);
     }
-  if (slp_node)
-    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
-  else
-    {
-      unsigned ncopies
-	= vect_get_num_copies (loop_vinfo, gs_info->offset_vectype);
-      vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, ncopies,
-				     gs_info->offset, vec_offset,
-				     gs_info->offset_vectype);
-    }
+  vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
 }
 
 /* Prepare to implement a grouped or strided load or store using
@@ -3110,15 +2925,14 @@ vect_get_gather_scatter_ops (loop_vec_info loop_vinfo,
    I * DR_STEP / SCALE.  */
 
 static void
-vect_get_strided_load_store_ops (stmt_vec_info stmt_info,
+vect_get_strided_load_store_ops (stmt_vec_info stmt_info, tree vectype,
 				 loop_vec_info loop_vinfo,
 				 gimple_stmt_iterator *gsi,
-				 gather_scatter_info *gs_info,
+				 const gather_scatter_info *gs_info,
 				 tree *dataref_bump, tree *vec_offset,
 				 vec_loop_lens *loop_lens)
 {
   struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
 
   if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
     {
@@ -3243,29 +3057,17 @@ vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi,
 static bool
 vectorizable_bswap (vec_info *vinfo,
 		    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		    gimple **vec_stmt, slp_tree slp_node,
+		    slp_tree slp_node,
 		    slp_tree *slp_op,
 		    tree vectype_in, stmt_vector_for_cost *cost_vec)
 {
   tree op, vectype;
   gcall *stmt = as_a <gcall *> (stmt_info->stmt);
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  unsigned ncopies;
 
   op = gimple_call_arg (stmt, 0);
-  vectype = STMT_VINFO_VECTYPE (stmt_info);
+  vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
-  /* Multiple types in SLP are handled by creating the appropriate number of
-     vectorized stmts for each SLP node.  Hence, NCOPIES is always 1 in
-     case of SLP.  */
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
     {
       if (dump_enabled_p ())
@@ -3295,10 +3097,9 @@ vectorizable_bswap (vec_info *vinfo,
   if (!can_vec_perm_const_p (vmode, vmode, indices))
     return false;
 
-  if (! vec_stmt)
+  if (cost_vec)
     {
-      if (slp_node
-	  && !vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
+      if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -3306,13 +3107,12 @@ vectorizable_bswap (vec_info *vinfo,
 	  return false;
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = call_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_bswap");
       record_stmt_cost (cost_vec,
 			1, vector_stmt, stmt_info, 0, vect_prologue);
       record_stmt_cost (cost_vec,
-			slp_node
-			? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) : ncopies,
+			SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
 			vec_perm, stmt_info, 0, vect_body);
       return true;
     }
@@ -3321,8 +3121,7 @@ vectorizable_bswap (vec_info *vinfo,
 
   /* Transform.  */
   vec<tree> vec_oprnds = vNULL;
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		     op, &vec_oprnds);
+  vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds);
   /* Arguments are ready. create the new vector stmt.  */
   unsigned i;
   tree vop;
@@ -3341,15 +3140,9 @@ vectorizable_bswap (vec_info *vinfo,
       new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
 						   vectype, tem2));
       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds.release ();
   return true;
 }
@@ -3389,14 +3182,14 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in,
 static bool
 vectorizable_call (vec_info *vinfo,
 		   stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		   gimple **vec_stmt, slp_tree slp_node,
+		   slp_tree slp_node,
 		   stmt_vector_for_cost *cost_vec)
 {
   gcall *stmt;
   tree vec_dest;
   tree scalar_dest;
   tree op;
-  tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
+  tree vec_oprnd0 = NULL_TREE;
   tree vectype_out, vectype_in;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
@@ -3408,19 +3201,16 @@ vectorizable_call (vec_info *vinfo,
 	vect_unknown_def_type };
   tree vectypes[ARRAY_SIZE (dt)] = {};
   slp_tree slp_op[ARRAY_SIZE (dt)] = {};
-  int ndts = ARRAY_SIZE (dt);
-  int ncopies, j;
   auto_vec<tree, 8> vargs;
   enum { NARROW, NONE, WIDEN } modifier;
   size_t i, nargs;
-  tree lhs;
   tree clz_ctz_arg1 = NULL_TREE;
 
   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is STMT_INFO a vectorizable call?   */
@@ -3440,7 +3230,7 @@ vectorizable_call (vec_info *vinfo,
 
   gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
 
-  vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  vectype_out = SLP_TREE_VECTYPE (slp_node);
 
   /* Process function arguments.  */
   rhs_type = NULL_TREE;
@@ -3471,19 +3261,25 @@ vectorizable_call (vec_info *vinfo,
 
   int mask_opno = -1;
   if (internal_fn_p (cfn))
-    mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
+    {
+      /* We can only handle direct internal masked calls here,
+	 vectorizable_simd_clone_call is for the rest.  */
+      if (cfn == CFN_MASK_CALL)
+	return false;
+      mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
+    }
 
   for (i = 0; i < nargs; i++)
     {
       if ((int) i == mask_opno)
 	{
-	  if (!vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_opno,
-				       &op, &slp_op[i], &dt[i], &vectypes[i]))
+	  if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
+				       &slp_op[i], &dt[i], &vectypes[i]))
 	    return false;
 	  continue;
 	}
 
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       i, &op, &slp_op[i], &dt[i], &vectypes[i]))
 	{
 	  if (dump_enabled_p ())
@@ -3519,7 +3315,7 @@ vectorizable_call (vec_info *vinfo,
      from the scalar type.  */
   if (!vectype_in)
     vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype_in);
   if (!vectype_in)
     {
@@ -3539,13 +3335,14 @@ vectorizable_call (vec_info *vinfo,
       return false;
     }
 
-  if (vect_emulated_vector_p (vectype_in) || vect_emulated_vector_p (vectype_out))
-  {
+  if (vect_emulated_vector_p (vectype_in)
+      || vect_emulated_vector_p (vectype_out))
+    {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "use emulated vector type for call\n");
       return false;
-  }
+    }
 
   /* FORNOW */
   nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
@@ -3600,7 +3397,7 @@ vectorizable_call (vec_info *vinfo,
   if (ifn == IFN_LAST && !fndecl)
     {
       if (cfn == CFN_GOMP_SIMD_LANE
-	  && (!slp_node || SLP_TREE_LANES (slp_node) == 1)
+	  && SLP_TREE_LANES (slp_node) == 1
 	  && loop_vinfo
 	  && LOOP_VINFO_LOOP (loop_vinfo)->simduid
 	  && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
@@ -3616,7 +3413,7 @@ vectorizable_call (vec_info *vinfo,
 		   || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
 		   || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
 		   || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
-	return vectorizable_bswap (vinfo, stmt_info, gsi, vec_stmt, slp_node,
+	return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
 				   slp_op, vectype_in, cost_vec);
       else
 	{
@@ -3627,42 +3424,27 @@ vectorizable_call (vec_info *vinfo,
 	}
     }
 
-  if (slp_node)
-    ncopies = 1;
-  else if (modifier == NARROW && ifn == IFN_LAST)
-    ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
-
-  /* Sanity check: make sure that at least one copy of the vectorized stmt
-     needs to be generated.  */
-  gcc_assert (ncopies >= 1);
-
   int reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
   internal_fn cond_fn = get_conditional_internal_fn (ifn);
   internal_fn cond_len_fn = get_len_internal_fn (ifn);
   int len_opno = internal_fn_len_index (cond_len_fn);
   vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
   vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node)
-	for (i = 0; i < nargs; ++i)
-	  if (!vect_maybe_update_slp_op_vectype (slp_op[i],
-						 vectypes[i]
-						 ? vectypes[i] : vectype_in))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "incompatible vector types for invariants\n");
-	      return false;
-	    }
-      STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
+      for (i = 0; i < nargs; ++i)
+	if (!vect_maybe_update_slp_op_vectype (slp_op[i],
+					       vectypes[i]
+					       ? vectypes[i] : vectype_in))
+	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "incompatible vector types for invariants\n");
+	    return false;
+	  }
+      SLP_TREE_TYPE (slp_node) = call_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_call");
-      vect_model_simple_cost (vinfo, ncopies, dt, ndts, slp_node, cost_vec);
-      if (ifn != IFN_LAST && modifier == NARROW && !slp_node)
-	record_stmt_cost (cost_vec, ncopies / 2,
-			  vec_promote_demote, stmt_info, 0, vect_body);
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
 
       if (loop_vinfo
 	  && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
@@ -3684,10 +3466,7 @@ vectorizable_call (vec_info *vinfo,
 	    }
 	  else
 	    {
-	      unsigned int nvectors
-		= (slp_node
-		   ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node)
-		   : ncopies);
+	      unsigned int nvectors = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 	      tree scalar_mask = NULL_TREE;
 	      if (mask_opno >= 0)
 		scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
@@ -3740,220 +3519,112 @@ vectorizable_call (vec_info *vinfo,
       tree prev_res = NULL_TREE;
       vargs.safe_grow (vect_nargs, true);
       auto_vec<vec<tree> > vec_defs (nargs);
-      for (j = 0; j < ncopies; ++j)
+
+      /* Build argument list for the vectorized call.  */
+      if (cfn == CFN_GOMP_SIMD_LANE)
+	{
+	  for (i = 0; i < SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); ++i)
+	    {
+	      /* ???  For multi-lane SLP we'd need to build
+		 { 0, 0, .., 1, 1, ... }.  */
+	      tree cst = build_index_vector (vectype_out,
+					     i * nunits_out, 1);
+	      tree new_var
+		= vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
+	      gimple *init_stmt = gimple_build_assign (new_var, cst);
+	      vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
+	      new_temp = make_ssa_name (vec_dest);
+	      gimple *new_stmt = gimple_build_assign (new_temp, new_var);
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	      slp_node->push_vec_def (new_stmt);
+	    }
+	}
+      else
 	{
-	  /* Build argument list for the vectorized call.  */
-	  if (slp_node)
+	  vec<tree> vec_oprnds0;
+	  vect_get_slp_defs (vinfo, slp_node, &vec_defs);
+	  vec_oprnds0 = vec_defs[0];
+
+	  /* Arguments are ready.  Create the new vector stmt.  */
+	  FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
 	    {
-	      if (cfn == CFN_GOMP_SIMD_LANE)
+	      int varg = 0;
+	      if (masked_loop_p && reduc_idx >= 0)
+		{
+		  unsigned int vec_num = vec_oprnds0.length ();
+		  vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi, masks,
+						      vec_num, vectype_out, i);
+		}
+	      size_t k;
+	      for (k = 0; k < nargs; k++)
 		{
-		  for (i = 0; i < SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); ++i)
+		  vec<tree> vec_oprndsk = vec_defs[k];
+		  vargs[varg++] = vec_oprndsk[i];
+		}
+	      if (masked_loop_p && reduc_idx >= 0)
+		vargs[varg++] = vargs[reduc_idx + 1];
+	      if (clz_ctz_arg1)
+		vargs[varg++] = clz_ctz_arg1;
+
+	      gimple *new_stmt;
+	      if (modifier == NARROW)
+		{
+		  /* We don't define any narrowing conditional functions
+		     at present.  */
+		  gcc_assert (mask_opno < 0);
+		  tree half_res = make_ssa_name (vectype_in);
+		  gcall *call = gimple_build_call_internal_vec (ifn, vargs);
+		  gimple_call_set_lhs (call, half_res);
+		  gimple_call_set_nothrow (call, true);
+		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+		  if ((i & 1) == 0)
 		    {
-		      /* ???  For multi-lane SLP we'd need to build
-			 { 0, 0, .., 1, 1, ... }.  */
-		      tree cst = build_index_vector (vectype_out,
-						     i * nunits_out, 1);
-		      tree new_var
-			  = vect_get_new_ssa_name (vectype_out, vect_simple_var,
-						   "cst_");
-		      gimple *init_stmt = gimple_build_assign (new_var, cst);
-		      vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
-		      new_temp = make_ssa_name (vec_dest);
-		      gimple *new_stmt
-			= gimple_build_assign (new_temp, new_var);
-		      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
-						   gsi);
-		      slp_node->push_vec_def (new_stmt);
+		      prev_res = half_res;
+		      continue;
 		    }
-		  continue;
+		  new_temp = make_ssa_name (vec_dest);
+		  new_stmt = vect_gimple_build (new_temp, convert_code,
+						prev_res, half_res);
+		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 		}
-
-	      vec<tree> vec_oprnds0;
-	      vect_get_slp_defs (vinfo, slp_node, &vec_defs);
-	      vec_oprnds0 = vec_defs[0];
-
-	      /* Arguments are ready.  Create the new vector stmt.  */
-	      FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
+	      else
 		{
-		  int varg = 0;
-		  if (masked_loop_p && reduc_idx >= 0)
+		  if (len_opno >= 0 && len_loop_p)
 		    {
 		      unsigned int vec_num = vec_oprnds0.length ();
-		      /* Always true for SLP.  */
-		      gcc_assert (ncopies == 1);
-		      vargs[varg++] = vect_get_loop_mask (loop_vinfo,
-							  gsi, masks, vec_num,
-							  vectype_out, i);
+		      tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
+						    vec_num, vectype_out, i, 1);
+		      signed char biasval
+			= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+		      tree bias = build_int_cst (intQI_type_node, biasval);
+		      vargs[len_opno] = len;
+		      vargs[len_opno + 1] = bias;
 		    }
-		  size_t k;
-		  for (k = 0; k < nargs; k++)
+		  else if (mask_opno >= 0 && masked_loop_p)
 		    {
-		      vec<tree> vec_oprndsk = vec_defs[k];
-		      vargs[varg++] = vec_oprndsk[i];
+		      unsigned int vec_num = vec_oprnds0.length ();
+		      tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
+						      vec_num, vectype_out, i);
+		      vargs[mask_opno]
+			= prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
+					    vargs[mask_opno], gsi);
 		    }
-		  if (masked_loop_p && reduc_idx >= 0)
-		    vargs[varg++] = vargs[reduc_idx + 1];
-		  if (clz_ctz_arg1)
-		    vargs[varg++] = clz_ctz_arg1;
 
-		  gimple *new_stmt;
-		  if (modifier == NARROW)
-		    {
-		      /* We don't define any narrowing conditional functions
-			 at present.  */
-		      gcc_assert (mask_opno < 0);
-		      tree half_res = make_ssa_name (vectype_in);
-		      gcall *call
-			= gimple_build_call_internal_vec (ifn, vargs);
-		      gimple_call_set_lhs (call, half_res);
-		      gimple_call_set_nothrow (call, true);
-		      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		      if ((i & 1) == 0)
-			{
-			  prev_res = half_res;
-			  continue;
-			}
-		      new_temp = make_ssa_name (vec_dest);
-		      new_stmt = vect_gimple_build (new_temp, convert_code,
-						    prev_res, half_res);
-		      vect_finish_stmt_generation (vinfo, stmt_info,
-						   new_stmt, gsi);
-		    }
+		  gcall *call;
+		  if (ifn != IFN_LAST)
+		    call = gimple_build_call_internal_vec (ifn, vargs);
 		  else
-		    {
-		      if (len_opno >= 0 && len_loop_p)
-			{
-			  unsigned int vec_num = vec_oprnds0.length ();
-			  /* Always true for SLP.  */
-			  gcc_assert (ncopies == 1);
-			  tree len
-			    = vect_get_loop_len (loop_vinfo, gsi, lens, vec_num,
-						 vectype_out, i, 1);
-			  signed char biasval
-			    = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
-			  tree bias = build_int_cst (intQI_type_node, biasval);
-			  vargs[len_opno] = len;
-			  vargs[len_opno + 1] = bias;
-			}
-		      else if (mask_opno >= 0 && masked_loop_p)
-			{
-			  unsigned int vec_num = vec_oprnds0.length ();
-			  /* Always true for SLP.  */
-			  gcc_assert (ncopies == 1);
-			  tree mask = vect_get_loop_mask (loop_vinfo,
-							  gsi, masks, vec_num,
-							  vectype_out, i);
-			  vargs[mask_opno] = prepare_vec_mask
-			    (loop_vinfo, TREE_TYPE (mask), mask,
-			     vargs[mask_opno], gsi);
-			}
-
-		      gcall *call;
-		      if (ifn != IFN_LAST)
-			call = gimple_build_call_internal_vec (ifn, vargs);
-		      else
-			call = gimple_build_call_vec (fndecl, vargs);
-		      new_temp = make_ssa_name (vec_dest, call);
-		      gimple_call_set_lhs (call, new_temp);
-		      gimple_call_set_nothrow (call, true);
-		      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		      new_stmt = call;
-		    }
-		  slp_node->push_vec_def (new_stmt);
-		}
-	      continue;
-	    }
-
-	  int varg = 0;
-	  if (masked_loop_p && reduc_idx >= 0)
-	    vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies,
-						vectype_out, j);
-	  for (i = 0; i < nargs; i++)
-	    {
-	      op = gimple_call_arg (stmt, i);
-	      if (j == 0)
-		{
-		  vec_defs.quick_push (vNULL);
-		  vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-						 op, &vec_defs[i],
-						 vectypes[i]);
-		}
-	      vargs[varg++] = vec_defs[i][j];
-	    }
-	  if (masked_loop_p && reduc_idx >= 0)
-	    vargs[varg++] = vargs[reduc_idx + 1];
-	  if (clz_ctz_arg1)
-	    vargs[varg++] = clz_ctz_arg1;
-
-	  if (len_opno >= 0 && len_loop_p)
-	    {
-	      tree len = vect_get_loop_len (loop_vinfo, gsi, lens, ncopies,
-					    vectype_out, j, 1);
-	      signed char biasval
-		= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
-	      tree bias = build_int_cst (intQI_type_node, biasval);
-	      vargs[len_opno] = len;
-	      vargs[len_opno + 1] = bias;
-	    }
-	  else if (mask_opno >= 0 && masked_loop_p)
-	    {
-	      tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies,
-					      vectype_out, j);
-	      vargs[mask_opno]
-		= prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
-				    vargs[mask_opno], gsi);
-	    }
-
-	  gimple *new_stmt;
-	  if (cfn == CFN_GOMP_SIMD_LANE)
-	    {
-	      tree cst = build_index_vector (vectype_out, j * nunits_out, 1);
-	      tree new_var
-		= vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
-	      gimple *init_stmt = gimple_build_assign (new_var, cst);
-	      vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
-	      new_temp = make_ssa_name (vec_dest);
-	      new_stmt = gimple_build_assign (new_temp, new_var);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-	    }
-	  else if (modifier == NARROW)
-	    {
-	      /* We don't define any narrowing conditional functions at
-		 present.  */
-	      gcc_assert (mask_opno < 0);
-	      tree half_res = make_ssa_name (vectype_in);
-	      gcall *call = gimple_build_call_internal_vec (ifn, vargs);
-	      gimple_call_set_lhs (call, half_res);
-	      gimple_call_set_nothrow (call, true);
-	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-	      if ((j & 1) == 0)
-		{
-		  prev_res = half_res;
-		  continue;
+		    call = gimple_build_call_vec (fndecl, vargs);
+		  new_temp = make_ssa_name (vec_dest, call);
+		  gimple_call_set_lhs (call, new_temp);
+		  gimple_call_set_nothrow (call, true);
+		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+		  new_stmt = call;
 		}
-	      new_temp = make_ssa_name (vec_dest);
-	      new_stmt = vect_gimple_build (new_temp, convert_code, prev_res,
-					    half_res);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-	    }
-	  else
-	    {
-	      gcall *call;
-	      if (ifn != IFN_LAST)
-		call = gimple_build_call_internal_vec (ifn, vargs);
-	      else
-		call = gimple_build_call_vec (fndecl, vargs);
-	      new_temp = make_ssa_name (vec_dest, call);
-	      gimple_call_set_lhs (call, new_temp);
-	      gimple_call_set_nothrow (call, true);
-	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-	      new_stmt = call;
+	      slp_node->push_vec_def (new_stmt);
 	    }
-
-	  if (j == (modifier == NARROW ? 1 : 0))
-	    *vec_stmt = new_stmt;
-	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
 	}
+
       for (i = 0; i < nargs; i++)
 	{
 	  vec<tree> vec_oprndsi = vec_defs[i];
@@ -3965,73 +3636,36 @@ vectorizable_call (vec_info *vinfo,
       auto_vec<vec<tree> > vec_defs (nargs);
       /* We don't define any narrowing conditional functions at present.  */
       gcc_assert (mask_opno < 0);
-      for (j = 0; j < ncopies; ++j)
-	{
-	  /* Build argument list for the vectorized call.  */
-	  if (j == 0)
-	    vargs.create (nargs * 2);
-	  else
-	    vargs.truncate (0);
-
-	  if (slp_node)
-	    {
-	      vec<tree> vec_oprnds0;
 
-	      vect_get_slp_defs (vinfo, slp_node, &vec_defs);
-	      vec_oprnds0 = vec_defs[0];
+      /* Build argument list for the vectorized call.  */
+      vargs.create (nargs * 2);
 
-	      /* Arguments are ready.  Create the new vector stmt.  */
-	      for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
-		{
-		  size_t k;
-		  vargs.truncate (0);
-		  for (k = 0; k < nargs; k++)
-		    {
-		      vec<tree> vec_oprndsk = vec_defs[k];
-		      vargs.quick_push (vec_oprndsk[i]);
-		      vargs.quick_push (vec_oprndsk[i + 1]);
-		    }
-		  gcall *call;
-		  if (ifn != IFN_LAST)
-		    call = gimple_build_call_internal_vec (ifn, vargs);
-		  else
-		    call = gimple_build_call_vec (fndecl, vargs);
-		  new_temp = make_ssa_name (vec_dest, call);
-		  gimple_call_set_lhs (call, new_temp);
-		  gimple_call_set_nothrow (call, true);
-		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		  slp_node->push_vec_def (call);
-		}
-	      continue;
-	    }
+      vect_get_slp_defs (vinfo, slp_node, &vec_defs);
+      vec<tree> vec_oprnds0 = vec_defs[0];
 
-	  for (i = 0; i < nargs; i++)
+      /* Arguments are ready.  Create the new vector stmt.  */
+      for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
+	{
+	  size_t k;
+	  vargs.truncate (0);
+	  for (k = 0; k < nargs; k++)
 	    {
-	      op = gimple_call_arg (stmt, i);
-	      if (j == 0)
-		{
-		  vec_defs.quick_push (vNULL);
-		  vect_get_vec_defs_for_operand (vinfo, stmt_info, 2 * ncopies,
-						 op, &vec_defs[i], vectypes[i]);
-		}
-	      vec_oprnd0 = vec_defs[i][2*j];
-	      vec_oprnd1 = vec_defs[i][2*j+1];
-
-	      vargs.quick_push (vec_oprnd0);
-	      vargs.quick_push (vec_oprnd1);
+	      vec<tree> vec_oprndsk = vec_defs[k];
+	      vargs.quick_push (vec_oprndsk[i]);
+	      vargs.quick_push (vec_oprndsk[i + 1]);
 	    }
-
-	  gcall *new_stmt = gimple_build_call_vec (fndecl, vargs);
-	  new_temp = make_ssa_name (vec_dest, new_stmt);
-	  gimple_call_set_lhs (new_stmt, new_temp);
-	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-
-	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+	  gcall *call;
+	  if (ifn != IFN_LAST)
+	    call = gimple_build_call_internal_vec (ifn, vargs);
+	  else
+	    call = gimple_build_call_vec (fndecl, vargs);
+	  new_temp = make_ssa_name (vec_dest, call);
+	  gimple_call_set_lhs (call, new_temp);
+	  gimple_call_set_nothrow (call, true);
+	  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+	  slp_node->push_vec_def (call);
 	}
 
-      if (!slp_node)
-	*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
       for (i = 0; i < nargs; i++)
 	{
 	  vec<tree> vec_oprndsi = vec_defs[i];
@@ -4044,21 +3678,6 @@ vectorizable_call (vec_info *vinfo,
 
   vargs.release ();
 
-  /* The call in STMT might prevent it from being removed in dce.
-     We however cannot remove it here, due to the way the ssa name
-     it defines is mapped to the new definition.  So just replace
-     rhs of the statement with something harmless.  */
-
-  if (slp_node)
-    return true;
-
-  stmt_info = vect_orig_stmt (stmt_info);
-  lhs = gimple_get_lhs (stmt_info->stmt);
-
-  gassign *new_stmt
-    = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
-  vinfo->replace_stmt (gsi, stmt_info, new_stmt);
-
   return true;
 }
 
@@ -4152,12 +3771,12 @@ vect_simd_lane_linear (tree op, class loop *loop,
 static bool
 vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			      gimple_stmt_iterator *gsi,
-			      gimple **vec_stmt, slp_tree slp_node,
-			      stmt_vector_for_cost *)
+			      slp_tree slp_node,
+			      stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
   tree scalar_dest;
-  tree op, type;
+  tree op;
   tree vec_oprnd0 = NULL_TREE;
   tree vectype;
   poly_uint64 nunits;
@@ -4169,7 +3788,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   auto_vec<simd_call_arg_info> arginfo;
   vec<tree> vargs = vNULL;
   size_t i, nargs;
-  tree lhs, rtype, ratype;
+  tree rtype, ratype;
   vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
   int masked_call_offset = 0;
 
@@ -4199,7 +3818,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   if (gimple_call_lhs (stmt)
@@ -4208,7 +3827,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 
   gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
 
-  vectype = STMT_VINFO_VECTYPE (stmt_info);
+  vectype = SLP_TREE_VECTYPE (slp_node);
 
   if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
     return false;
@@ -4220,10 +3839,9 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   if (nargs == 0)
     return false;
 
-  vec<tree>& simd_clone_info = (slp_node ? SLP_TREE_SIMD_CLONE_INFO (slp_node)
-				: STMT_VINFO_SIMD_CLONE_INFO (stmt_info));
-  if (!vec_stmt)
-    simd_clone_info.truncate (0);
+  vect_simd_clone_data _data;
+  vect_simd_clone_data &data = slp_node->get_data (_data);
+  vec<tree>& simd_clone_info = data.simd_clone_info;
   arginfo.reserve (nargs, true);
   auto_vec<slp_tree> slp_op;
   slp_op.safe_grow_cleared (nargs);
@@ -4238,10 +3856,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
       thisarginfo.op = NULL_TREE;
       thisarginfo.simd_lane_linear = false;
 
-      int op_no = i + masked_call_offset;
-      if (slp_node)
-	op_no = vect_slp_child_index_for_operand (stmt, op_no, false);
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      int op_no = vect_slp_child_index_for_operand (stmt,
+						    i + masked_call_offset,
+						    false);
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       op_no, &op, &slp_op[i],
 			       &thisarginfo.dt, &thisarginfo.vectype)
 	  || thisarginfo.dt == vect_uninitialized_def)
@@ -4259,10 +3877,9 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	     at analysis time, handling conflicts via
 	     vect_maybe_update_slp_op_vectype.  At transform time
 	     we have a vector type recorded for SLP.  */
-	  gcc_assert (!vec_stmt
-		      || !slp_node
+	  gcc_assert (cost_vec
 		      || thisarginfo.vectype != NULL_TREE);
-	  if (!vec_stmt)
+	  if (cost_vec)
 	    thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
 							       TREE_TYPE (op),
 							       slp_node);
@@ -4272,7 +3889,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 
       /* For linear arguments, the analyze phase should have saved
 	 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO.  */
-      if (vec_stmt
+      if (!cost_vec
 	  && i * 3 + 4 <= simd_clone_info.length ()
 	  && simd_clone_info[i * 3 + 2])
 	{
@@ -4296,7 +3913,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			       thisarginfo.op, bias);
 	    }
 	}
-      else if (!vec_stmt
+      else if (cost_vec
 	       && thisarginfo.dt != vect_constant_def
 	       && thisarginfo.dt != vect_external_def
 	       && loop_vinfo
@@ -4316,7 +3933,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	 linear too.  */
       if (POINTER_TYPE_P (TREE_TYPE (op))
 	  && !thisarginfo.linear_step
-	  && !vec_stmt
+	  && cost_vec
 	  && thisarginfo.dt != vect_constant_def
 	  && thisarginfo.dt != vect_external_def
 	  && loop_vinfo
@@ -4327,10 +3944,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
     }
 
   poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
-  unsigned group_size = slp_node ? SLP_TREE_LANES (slp_node) : 1;
+  unsigned group_size = SLP_TREE_LANES (slp_node);
   unsigned int badness = 0;
   struct cgraph_node *bestn = NULL;
-  if (vec_stmt)
+  if (!cost_vec)
     bestn = cgraph_node::get (simd_clone_info[0]);
   else
     for (struct cgraph_node *n = node->simd_clones; n != NULL;
@@ -4357,7 +3974,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	if (n->simdclone->inbranch)
 	  this_badness += 8192;
 
-	/* If STMT_VINFO_VECTYPE has not been set yet pass the general vector
+	/* If SLP_TREE_VECTYPE has not been set yet pass the general vector
 	   mode,  which for targets that use it will determine what ISA we can
 	   vectorize this code with.  */
 	machine_mode vector_mode = vinfo->vector_mode;
@@ -4542,10 +4159,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 
   fndecl = bestn->decl;
   nunits = bestn->simdclone->simdlen;
-  if (slp_node)
-    ncopies = vector_unroll_factor (vf * group_size, nunits);
-  else
-    ncopies = vector_unroll_factor (vf, nunits);
+  ncopies = vector_unroll_factor (vf * group_size, nunits);
 
   /* If the function isn't const, only allow it in simd loops where user
      has asserted that at least nunits consecutive iterations can be
@@ -4558,17 +4172,16 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
      needs to be generated.  */
   gcc_assert (ncopies >= 1);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node)
-	for (unsigned i = 0; i < nargs; ++i)
-	  if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "incompatible vector types for invariants\n");
-	      return false;
-	    }
+      for (unsigned i = 0; i < nargs; ++i)
+	if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
+	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "incompatible vector types for invariants\n");
+	    return false;
+	  }
       /* When the original call is pure or const but the SIMD ABI dictates
 	 an aggregate return we will have to use a virtual definition and
 	 in a loop eventually even need to add a virtual PHI.  That's
@@ -4580,7 +4193,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
       /* ???  For SLP code-gen we end up inserting after the last
 	 vector argument def rather than at the original call position
 	 so automagic virtual operand updating doesn't work.  */
-      if (gimple_vuse (stmt) && slp_node)
+      if (gimple_vuse (stmt))
 	vinfo->any_known_not_updated_vssa = true;
       simd_clone_info.safe_push (bestn->decl);
       for (i = 0; i < bestn->simdclone->nargs; i++)
@@ -4624,9 +4237,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	  LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = call_simd_clone_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
+      slp_node->data = new vect_simd_clone_data (std::move (_data));
       DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
-/*      vect_model_simple_cost (vinfo, ncopies, dt, slp_node, cost_vec); */
+/*      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec); */
       return true;
     }
 
@@ -4654,13 +4268,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   auto_vec<vec<tree> > vec_oprnds;
   auto_vec<unsigned> vec_oprnds_i;
   vec_oprnds_i.safe_grow_cleared (nargs, true);
-  if (slp_node)
-    {
-      vec_oprnds.reserve_exact (nargs);
-      vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
-    }
-  else
-    vec_oprnds.safe_grow_cleared (nargs, true);
+  vec_oprnds.reserve_exact (nargs);
+  vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
   for (j = 0; j < ncopies; ++j)
     {
       poly_uint64 callee_nelements;
@@ -4695,10 +4304,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		      gcc_assert ((k & (k - 1)) == 0);
 		      if (m == 0)
 			{
-			  if (!slp_node)
-			    vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							   ncopies * o / k, op,
-							   &vec_oprnds[i]);
 			  vec_oprnds_i[i] = 0;
 			  vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
 			}
@@ -4735,11 +4340,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			{
 			  if (m == 0 && l == 0)
 			    {
-			      if (!slp_node)
-				vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							       k * o * ncopies,
-							       op,
-							       &vec_oprnds[i]);
 			      vec_oprnds_i[i] = 0;
 			      vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
 			    }
@@ -4803,14 +4403,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			  /* The SIMD clone function has the same number of
 			     elements as the current function.  */
 			  if (m == 0)
-			    {
-			      if (!slp_node)
-				vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							       o * ncopies,
-							       op,
-							       &vec_oprnds[i]);
-			      vec_oprnds_i[i] = 0;
-			    }
+			    vec_oprnds_i[i] = 0;
 			  vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
 			  if (loop_vinfo
 			      && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
@@ -4861,14 +4454,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		  for (m = j * o; m < (j + 1) * o; m++)
 		    {
 		      if (m == 0)
-			{
-			  if (!slp_node)
-			    vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							   o * ncopies,
-							   op,
-							   &vec_oprnds[i]);
-			  vec_oprnds_i[i] = 0;
-			}
+			vec_oprnds_i[i] = 0;
 		      if (maybe_lt (atype_subparts,
 				    TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
 			{
@@ -5103,13 +4689,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		  new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
 		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 
-		  if (j == 0 && l == 0)
-		    *vec_stmt = new_stmt;
-		  if (slp_node)
-		    SLP_TREE_VEC_DEFS (slp_node)
-		      .quick_push (gimple_assign_lhs (new_stmt));
-		  else
-		    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+		  SLP_TREE_VEC_DEFS (slp_node)
+		    .quick_push (gimple_assign_lhs (new_stmt));
 		}
 
 	      if (ratype)
@@ -5152,13 +4733,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		= gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
 	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 
-	      if ((unsigned) j == k - 1)
-		*vec_stmt = new_stmt;
-	      if (slp_node)
-		SLP_TREE_VEC_DEFS (slp_node)
-		  .quick_push (gimple_assign_lhs (new_stmt));
-	      else
-		STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+	      SLP_TREE_VEC_DEFS (slp_node)
+		.quick_push (gimple_assign_lhs (new_stmt));
 	      continue;
 	    }
 	  else if (ratype)
@@ -5179,12 +4755,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	    }
 	}
 
-      if (j == 0)
-	*vec_stmt = new_stmt;
-      if (slp_node)
+      if (gimple_get_lhs (new_stmt))
 	SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
     }
 
   for (i = 0; i < nargs; ++i)
@@ -5197,26 +4769,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   /* Mark the clone as no longer being a candidate for GC.  */
   bestn->gc_candidate = false;
 
-  /* The call in STMT might prevent it from being removed in dce.
-     We however cannot remove it here, due to the way the ssa name
-     it defines is mapped to the new definition.  So just replace
-     rhs of the statement with something harmless.  */
-
-  if (slp_node)
-    return true;
-
-  gimple *new_stmt;
-  if (scalar_dest)
-    {
-      type = TREE_TYPE (scalar_dest);
-      lhs = gimple_call_lhs (vect_orig_stmt (stmt_info)->stmt);
-      new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
-    }
-  else
-    new_stmt = gimple_build_nop ();
-  vinfo->replace_stmt (gsi, vect_orig_stmt (stmt_info), new_stmt);
-  unlink_stmt_vdef (stmt);
-
   return true;
 }
 
@@ -5287,12 +4839,8 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
       else
 	{
 	  /* This is the last step of the conversion sequence. Store the
-	     vectors in SLP_NODE or in vector info of the scalar statement
-	     (or in STMT_VINFO_RELATED_STMT chain).  */
-	  if (slp_node)
-	    slp_node->push_vec_def (new_stmt);
-	  else
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+	     vectors in SLP_NODE.  */
+	  slp_node->push_vec_def (new_stmt);
 	}
     }
 
@@ -5437,7 +4985,7 @@ vect_create_half_widening_stmts (vec_info *vinfo,
 static bool
 vectorizable_conversion (vec_info *vinfo,
 			 stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt, slp_tree slp_node,
+			 slp_tree slp_node,
 			 stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest, cvt_op = NULL_TREE;
@@ -5448,7 +4996,6 @@ vectorizable_conversion (vec_info *vinfo,
   code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
   tree new_temp;
   enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 2;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
   tree vectype_out, vectype_in;
@@ -5467,6 +5014,7 @@ vectorizable_conversion (vec_info *vinfo,
   vec<tree> vec_oprnds1 = vNULL;
   tree vop0;
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
+  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   int multi_step_cvt = 0;
   vec<tree> interm_types = vNULL;
   tree intermediate_type, cvt_type = NULL_TREE;
@@ -5479,7 +5027,7 @@ vectorizable_conversion (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   gimple* stmt = stmt_info->stmt;
@@ -5523,7 +5071,7 @@ vectorizable_conversion (vec_info *vinfo,
 
   /* Check the operands of the operation.  */
   slp_tree slp_op0, slp_op1 = NULL;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &op0, &slp_op0, &dt[0], &vectype_in))
     {
       if (dump_enabled_p ())
@@ -5562,7 +5110,7 @@ vectorizable_conversion (vec_info *vinfo,
       op1 = is_gimple_assign (stmt) ? gimple_assign_rhs2 (stmt) :
 				     gimple_call_arg (stmt, 0);
       tree vectype1_in;
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1,
+      if (!vect_is_simple_use (vinfo, slp_node, 1,
 			       &op1, &slp_op1, &dt[1], &vectype1_in))
 	{
           if (dump_enabled_p ())
@@ -5580,7 +5128,7 @@ vectorizable_conversion (vec_info *vinfo,
      from the scalar type.  */
   if (!vectype_in)
     vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype_in);
   if (!vectype_in)
     {
@@ -5809,7 +5357,21 @@ vectorizable_conversion (vec_info *vinfo,
       gcc_unreachable ();
     }
 
-  if (!vec_stmt)		/* transformation not required.  */
+  if (modifier == WIDEN
+      && loop_vinfo
+      && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
+      && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
+	  || widening_evenodd_fn_p (code1)))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "can't use a fully-masked loop because"
+			 " widening operation on even/odd elements"
+			 " mixes up lanes.\n");
+      LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+    }
+
+  if (cost_vec)		/* transformation not required.  */
     {
       if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
 	  || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
@@ -5822,13 +5384,13 @@ vectorizable_conversion (vec_info *vinfo,
       DUMP_VECT_SCOPE ("vectorizable_conversion");
       if (modifier == NONE)
         {
-	  STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
+	  SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
 	  vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
-				  dt, ndts, slp_node, cost_vec);
+				  slp_node, cost_vec);
 	}
       else if (modifier == NARROW_SRC || modifier == NARROW_DST)
 	{
-	  STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
+	  SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
 	  /* The final packing step produces one vector result per copy.  */
 	  unsigned int nvectors = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 	  vect_model_promotion_demotion_cost (stmt_info, dt, nvectors,
@@ -5837,7 +5399,7 @@ vectorizable_conversion (vec_info *vinfo,
 	}
       else
 	{
-	  STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
+	  SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
 	  /* The initial unpacking step produces two vector results
 	     per copy.  MULTI_STEP_CVT is 0 for a single conversion,
 	     so >> MULTI_STEP_CVT divides by 2^(number of steps - 1).  */
@@ -5895,8 +5457,7 @@ vectorizable_conversion (vec_info *vinfo,
   switch (modifier)
     {
     case NONE:
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
-			 op0, vectype_in, &vec_oprnds0);
+      vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0);
       /* vec_dest is intermediate type operand when multi_step_cvt.  */
       if (multi_step_cvt)
 	{
@@ -5931,10 +5492,9 @@ vectorizable_conversion (vec_info *vinfo,
 	 of elements that we can fit in a vectype (nunits), we have to
 	 generate more than one vector stmt - i.e - we need to "unroll"
 	 the vector stmt by a factor VF/nunits.  */
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
-			 op0, vectype_in, &vec_oprnds0,
+      vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0,
 			 code == WIDEN_LSHIFT_EXPR ? NULL_TREE : op1,
-			 vectype_in, &vec_oprnds1);
+			 &vec_oprnds1);
       if (code == WIDEN_LSHIFT_EXPR)
 	{
 	  int oprnds_size = vec_oprnds0.length ();
@@ -5985,8 +5545,7 @@ vectorizable_conversion (vec_info *vinfo,
 	 of elements that we can fit in a vectype (nunits), we have to
 	 generate more than one vector stmt - i.e - we need to "unroll"
 	 the vector stmt by a factor VF/nunits.  */
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
-			 op0, vectype_in, &vec_oprnds0);
+      vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0);
       /* Arguments are ready.  Create the new vector stmts.  */
       if (cvt_type && modifier == NARROW_DST)
 	FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
@@ -6065,17 +5624,14 @@ vect_nop_conversion_p (stmt_vec_info stmt_info)
 static bool
 vectorizable_assignment (vec_info *vinfo,
 			 stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt, slp_tree slp_node,
+			 slp_tree slp_node,
 			 stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
   tree scalar_dest;
   tree op;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   tree new_temp;
   enum vect_def_type dt[1] = {vect_unknown_def_type};
-  int ndts = 1;
-  int ncopies;
   int i;
   vec<tree> vec_oprnds = vNULL;
   tree vop;
@@ -6087,7 +5643,7 @@ vectorizable_assignment (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is vectorizable assignment?  */
@@ -6108,21 +5664,11 @@ vectorizable_assignment (vec_info *vinfo,
 	|| CONVERT_EXPR_CODE_P (code)))
     return false;
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
-  /* Multiple types in SLP are handled by creating the appropriate number of
-     vectorized stmts for each SLP node.  Hence, NCOPIES is always 1 in
-     case of SLP.  */
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   slp_tree slp_op;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &op, &slp_op,
+  if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
 			   &dt[0], &vectype_in))
     {
       if (dump_enabled_p ())
@@ -6179,20 +5725,19 @@ vectorizable_assignment (vec_info *vinfo,
       return false;
     }
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node
-	  && !vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
+      if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			     "incompatible vector types for invariants\n");
 	  return false;
 	}
-      STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_assignment");
       if (!vect_nop_conversion_p (stmt_info))
-	vect_model_simple_cost (vinfo, ncopies, dt, ndts, slp_node, cost_vec);
+	vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
       return true;
     }
 
@@ -6204,7 +5749,7 @@ vectorizable_assignment (vec_info *vinfo,
   vec_dest = vect_create_destination_var (scalar_dest, vectype);
 
   /* Handle use.  */
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op, &vec_oprnds);
+  vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds);
 
   /* Arguments are ready. create the new vector stmt.  */
   FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
@@ -6216,13 +5761,8 @@ vectorizable_assignment (vec_info *vinfo,
       new_temp = make_ssa_name (vec_dest, new_stmt);
       gimple_assign_set_lhs (new_stmt, new_temp);
       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
 
   vec_oprnds.release ();
   return true;
@@ -6264,7 +5804,7 @@ vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
 static bool
 vectorizable_shift (vec_info *vinfo,
 		    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		    gimple **vec_stmt, slp_tree slp_node,
+		    slp_tree slp_node,
 		    stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
@@ -6272,7 +5812,6 @@ vectorizable_shift (vec_info *vinfo,
   tree op0, op1 = NULL;
   tree vec_oprnd1 = NULL_TREE;
   tree vectype;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   enum tree_code code;
   machine_mode vec_mode;
   tree new_temp;
@@ -6280,12 +5819,10 @@ vectorizable_shift (vec_info *vinfo,
   int icode;
   machine_mode optab_op2_mode;
   enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 2;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
   tree vectype_out;
   tree op1_vectype;
-  int ncopies;
   int i;
   vec<tree> vec_oprnds0 = vNULL;
   vec<tree> vec_oprnds1 = vNULL;
@@ -6300,7 +5837,7 @@ vectorizable_shift (vec_info *vinfo,
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
       && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is STMT a vectorizable binary/unary operation?   */
@@ -6318,7 +5855,7 @@ vectorizable_shift (vec_info *vinfo,
     return false;
 
   scalar_dest = gimple_assign_lhs (stmt);
-  vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  vectype_out = SLP_TREE_VECTYPE (slp_node);
   if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
     {
       if (dump_enabled_p ())
@@ -6328,7 +5865,7 @@ vectorizable_shift (vec_info *vinfo,
     }
 
   slp_tree slp_op0;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &op0, &slp_op0, &dt[0], &vectype))
     {
       if (dump_enabled_p ())
@@ -6340,7 +5877,7 @@ vectorizable_shift (vec_info *vinfo,
      from the scalar type.  */
   if (!vectype)
     vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype);
   if (!vectype)
     {
@@ -6357,7 +5894,7 @@ vectorizable_shift (vec_info *vinfo,
 
   stmt_vec_info op1_def_stmt_info;
   slp_tree slp_op1;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1, &op1, &slp_op1,
+  if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
 			   &dt[1], &op1_vectype, &op1_def_stmt_info))
     {
       if (dump_enabled_p ())
@@ -6366,23 +5903,13 @@ vectorizable_shift (vec_info *vinfo,
       return false;
     }
 
-  /* Multiple types in SLP are handled by creating the appropriate number of
-     vectorized stmts for each SLP node.  Hence, NCOPIES is always 1 in
-     case of SLP.  */
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   /* Determine whether the shift amount is a vector, or scalar.  If the
      shift/rotate amount is a vector, use the vector/vector shift optabs.  */
 
   if ((dt[1] == vect_internal_def
        || dt[1] == vect_induction_def
        || dt[1] == vect_nested_cycle)
-      && (!slp_node || SLP_TREE_LANES (slp_node) == 1))
+      && SLP_TREE_LANES (slp_node) == 1)
     scalar_shift_arg = false;
   else if (dt[1] == vect_constant_def
 	   || dt[1] == vect_external_def
@@ -6391,29 +5918,26 @@ vectorizable_shift (vec_info *vinfo,
       /* In SLP, need to check whether the shift count is the same,
 	 in loops if it is a constant or invariant, it is always
 	 a scalar shift.  */
-      if (slp_node)
-	{
-	  vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
-	  stmt_vec_info slpstmt_info;
+      vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
+      stmt_vec_info slpstmt_info;
 
-	  FOR_EACH_VEC_ELT (stmts, k, slpstmt_info)
-	    if (slpstmt_info)
-	      {
-		gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt);
-		if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
-		  scalar_shift_arg = false;
-	      }
+      FOR_EACH_VEC_ELT (stmts, k, slpstmt_info)
+	if (slpstmt_info)
+	  {
+	    gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt);
+	    if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
+	      scalar_shift_arg = false;
+	  }
 
-	  /* For internal SLP defs we have to make sure we see scalar stmts
-	     for all vector elements.
-	     ???  For different vectors we could resort to a different
-	     scalar shift operand but code-generation below simply always
-	     takes the first.  */
-	  if (dt[1] == vect_internal_def
-	      && maybe_ne (nunits_out * SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
-			   stmts.length ()))
-	    scalar_shift_arg = false;
-	}
+      /* For internal SLP defs we have to make sure we see scalar stmts
+	 for all vector elements.
+	 ???  For different vectors we could resort to a different
+	 scalar shift operand but code-generation below simply always
+	 takes the first.  */
+      if (dt[1] == vect_internal_def
+	  && maybe_ne (nunits_out * SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
+		       stmts.length ()))
+	scalar_shift_arg = false;
 
       /* If the shift amount is computed by a pattern stmt we cannot
          use the scalar amount directly thus give up and use a vector
@@ -6447,8 +5971,7 @@ vectorizable_shift (vec_info *vinfo,
 			TYPE_VECTOR_SUBPARTS (vectype))
 	   || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
       if (incompatible_op1_vectype_p
-	  && (!slp_node
-	      || SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
+	  && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
 	      || slp_op1->refcnt != 1))
 	{
 	  if (dump_enabled_p ())
@@ -6529,16 +6052,15 @@ vectorizable_shift (vec_info *vinfo,
   if (vect_emulated_vector_p (vectype))
     return false;
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node
-	  && (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
-	      || ((!scalar_shift_arg || dt[1] == vect_internal_def)
-		  && (!incompatible_op1_vectype_p
-		      || dt[1] == vect_constant_def)
-		  && !vect_maybe_update_slp_op_vectype
-			(slp_op1,
-			 incompatible_op1_vectype_p ? vectype : op1_vectype))))
+      if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
+	  || ((!scalar_shift_arg || dt[1] == vect_internal_def)
+	      && (!incompatible_op1_vectype_p
+		  || dt[1] == vect_constant_def)
+	      && !vect_maybe_update_slp_op_vectype
+			 (slp_op1,
+			  incompatible_op1_vectype_p ? vectype : op1_vectype)))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -6546,24 +6068,20 @@ vectorizable_shift (vec_info *vinfo,
 	  return false;
 	}
       /* Now adjust the constant shift amount in place.  */
-      if (slp_node
-	  && incompatible_op1_vectype_p
+      if (incompatible_op1_vectype_p
 	  && dt[1] == vect_constant_def)
-	{
-	  for (unsigned i = 0;
-	       i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i)
-	    {
-	      SLP_TREE_SCALAR_OPS (slp_op1)[i]
-		= fold_convert (TREE_TYPE (vectype),
-				SLP_TREE_SCALAR_OPS (slp_op1)[i]);
-	      gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
-			   == INTEGER_CST));
-	    }
-	}
-      STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
+	for (unsigned i = 0;
+	     i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i)
+	  {
+	    SLP_TREE_SCALAR_OPS (slp_op1)[i]
+	      = fold_convert (TREE_TYPE (vectype),
+			      SLP_TREE_SCALAR_OPS (slp_op1)[i]);
+	    gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
+			 == INTEGER_CST));
+	  }
+      SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_shift");
-      vect_model_simple_cost (vinfo, ncopies, dt,
-			      scalar_shift_arg ? 1 : ndts, slp_node, cost_vec);
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
       return true;
     }
 
@@ -6573,15 +6091,6 @@ vectorizable_shift (vec_info *vinfo,
     dump_printf_loc (MSG_NOTE, vect_location,
                      "transform binary/unary operation.\n");
 
-  if (incompatible_op1_vectype_p && !slp_node)
-    {
-      gcc_assert (!scalar_shift_arg && was_scalar_shift_arg);
-      op1 = fold_convert (TREE_TYPE (vectype), op1);
-      if (dt[1] != vect_constant_def)
-	op1 = vect_init_vector (vinfo, stmt_info, op1,
-				TREE_TYPE (vectype), NULL);
-    }
-
   /* Handle def.  */
   vec_dest = vect_create_destination_var (scalar_dest, vectype);
 
@@ -6598,7 +6107,7 @@ vectorizable_shift (vec_info *vinfo,
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "operand 1 using scalar mode.\n");
 	  vec_oprnd1 = op1;
-	  vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : ncopies);
+	  vec_oprnds1.create (SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node));
 	  vec_oprnds1.quick_push (vec_oprnd1);
 	      /* Store vec_oprnd1 for every vector stmt to be created.
 		 We check during the analysis that all the shift arguments
@@ -6606,11 +6115,11 @@ vectorizable_shift (vec_info *vinfo,
 		 TODO: Allow different constants for different vector
 		 stmts generated for an SLP instance.  */
 	  for (k = 0;
-	       k < (slp_node ? slp_node->vec_stmts_size - 1 : ncopies - 1); k++)
+	       k < SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) - 1; k++)
 	    vec_oprnds1.quick_push (vec_oprnd1);
 	}
     }
-  else if (!scalar_shift_arg && slp_node && incompatible_op1_vectype_p)
+  else if (!scalar_shift_arg && incompatible_op1_vectype_p)
     {
       if (was_scalar_shift_arg)
 	{
@@ -6638,7 +6147,7 @@ vectorizable_shift (vec_info *vinfo,
   /* vec_oprnd1 is available if operand 1 should be of a scalar-type
      (a special case for certain kind of vector shifts); otherwise,
      operand 1 should be of a vector type (the usual case).  */
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+  vect_get_vec_defs (vinfo, slp_node,
 		     op0, &vec_oprnds0,
 		     vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1);
 
@@ -6666,15 +6175,9 @@ vectorizable_shift (vec_info *vinfo,
       new_temp = make_ssa_name (vec_dest, new_stmt);
       gimple_assign_set_lhs (new_stmt, new_temp);
       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds0.release ();
   vec_oprnds1.release ();
 
@@ -6692,7 +6195,7 @@ vectorizable_shift (vec_info *vinfo,
 static bool
 vectorizable_operation (vec_info *vinfo,
 			stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			gimple **vec_stmt, slp_tree slp_node,
+			slp_tree slp_node,
 			stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
@@ -6708,11 +6211,9 @@ vectorizable_operation (vec_info *vinfo,
   bool target_support_p;
   enum vect_def_type dt[3]
     = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 3;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
   tree vectype_out;
-  int vec_num;
   int i;
   vec<tree> vec_oprnds0 = vNULL;
   vec<tree> vec_oprnds1 = vNULL;
@@ -6724,7 +6225,7 @@ vectorizable_operation (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is STMT a vectorizable binary/unary operation?   */
@@ -6791,7 +6292,7 @@ vectorizable_operation (vec_info *vinfo,
     }
 
   slp_tree slp_op0;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &op0, &slp_op0, &dt[0], &vectype))
     {
       if (dump_enabled_p ())
@@ -6825,7 +6326,7 @@ vectorizable_operation (vec_info *vinfo,
 	vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
 					       slp_node);
     }
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype);
   if (!vectype)
     {
@@ -6847,7 +6348,7 @@ vectorizable_operation (vec_info *vinfo,
   slp_tree slp_op1 = NULL, slp_op2 = NULL;
   if (op_type == binary_op || op_type == ternary_op)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       1, &op1, &slp_op1, &dt[1], &vectype2))
 	{
 	  if (dump_enabled_p ())
@@ -6865,7 +6366,7 @@ vectorizable_operation (vec_info *vinfo,
     }
   if (op_type == ternary_op)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       2, &op2, &slp_op2, &dt[2], &vectype3))
 	{
 	  if (dump_enabled_p ())
@@ -6884,7 +6385,7 @@ vectorizable_operation (vec_info *vinfo,
 
   /* Multiple types in SLP are handled by creating the appropriate number of
      vectorized stmts for each SLP node.  */
-  vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+  auto vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 
   /* Reject attempts to combine mask types with nonmask types, e.g. if
      we have an AND between a (nonmask) boolean loaded from memory and
@@ -6931,7 +6432,7 @@ vectorizable_operation (vec_info *vinfo,
 	       || !target_support_p)
 	      && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
 	  /* Check only during analysis.  */
-	  || (!vec_stmt && !vect_can_vectorize_without_simd_p (code)))
+	  || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf (MSG_NOTE, "using word mode not possible.\n");
@@ -6960,7 +6461,7 @@ vectorizable_operation (vec_info *vinfo,
   bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
 			    || reduc_idx >= 0);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       if (loop_vinfo
 	  && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
@@ -6997,9 +6498,9 @@ vectorizable_operation (vec_info *vinfo,
 	  return false;
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = op_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_operation");
-      vect_model_simple_cost (vinfo, 1, dt, ndts, slp_node, cost_vec);
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
       if (using_emulated_vectors_p)
 	{
 	  /* The above vect_model_simple_cost call handles constants
@@ -7059,7 +6560,7 @@ vectorizable_operation (vec_info *vinfo,
   else
     vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
 
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
+  vect_get_vec_defs (vinfo, slp_node,
 		     op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2);
   /* Arguments are ready.  Create the new vector stmt.  */
   FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
@@ -7236,7 +6737,7 @@ vectorizable_operation (vec_info *vinfo,
 	      && code == BIT_AND_EXPR
 	      && VECTOR_BOOLEAN_TYPE_P (vectype))
 	    {
-	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, 1 }))
+	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
 		{
 		  mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
 					     vec_num, vectype, i);
@@ -7245,7 +6746,7 @@ vectorizable_operation (vec_info *vinfo,
 					   vop0, gsi);
 		}
 
-	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, 1 }))
+	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
 		{
 		  mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
 					     vec_num, vectype, i);
@@ -7538,7 +7039,8 @@ scan_store_can_perm_p (tree vectype, tree init,
 
 static bool
 check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
-		  enum vect_def_type rhs_dt, slp_tree slp_node, tree mask,
+		  enum vect_def_type rhs_dt, slp_tree slp_node,
+		  slp_tree mask_node,
 		  vect_memory_access_type memory_access_type)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -7546,13 +7048,14 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
   tree ref_type;
 
   gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
-  if ((slp_node && SLP_TREE_LANES (slp_node) > 1)
-      || mask
+  if (SLP_TREE_LANES (slp_node) > 1
+      || mask_node
       || memory_access_type != VMAT_CONTIGUOUS
       || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
       || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
       || loop_vinfo == NULL
       || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
+      || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
       || STMT_VINFO_GROUPED_ACCESS (stmt_info)
       || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
       || !integer_zerop (DR_INIT (dr_info->dr))
@@ -7942,17 +7445,16 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
 
 static bool
 vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
-			 slp_tree slp_node, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt, int ncopies)
+			 slp_tree slp_node, gimple_stmt_iterator *gsi)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
   tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
-		     "transform scan store. ncopies = %d\n", ncopies);
+		     "transform scan store.\n");
 
   gimple *stmt = STMT_VINFO_STMT (stmt_info);
   tree rhs = gimple_assign_rhs1 (stmt);
@@ -8063,8 +7565,8 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
       /* We want to lookup the vector operands of the reduction, not those
 	 of the store - for SLP we have to use the proper SLP node for the
 	 lookup, which should be the single child of the scan store.  */
-      vect_get_vec_defs (vinfo, stmt_info, SLP_TREE_CHILDREN (slp_node)[0],
-			 ncopies, rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
+      vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
+			 rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
       /* ???  For SLP we do not key the def on 'rhs1' or 'rhs2' but get
 	 them in SLP child order.  So we have to swap here with logic
 	 similar to above.  */
@@ -8078,7 +7580,7 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
     }
   else
-    vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+    vect_get_vec_defs (vinfo, slp_node,
 		       rhs2, &vec_oprnds3);
   for (unsigned j = 0; j < vec_oprnds3.length (); j++)
     {
@@ -8099,11 +7601,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
 	  gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    {
-	      STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
-	      *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-	    }
 	}
 
       tree v = vec_oprnd2;
@@ -8117,11 +7614,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 					   ? zero_vec : vec_oprnd1, v,
 					   perms[i]);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    {
-	      STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
-	      *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-	    }
 
 	  if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
 	    {
@@ -8138,8 +7630,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 				       new_temp, vec_oprnd1);
 	      vect_finish_stmt_generation (vinfo, stmt_info,
 							   g, gsi);
-	      if (! slp_node)
-		STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 	      new_temp = new_temp2;
 	    }
 
@@ -8157,8 +7647,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  tree new_temp2 = make_ssa_name (vectype);
 	  g = gimple_build_assign (new_temp2, code, v, new_temp);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 
 	  v = new_temp2;
 	}
@@ -8166,8 +7654,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
       tree new_temp = make_ssa_name (vectype);
       gimple *g = gimple_build_assign (new_temp, code, orig, v);
       vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-      if (! slp_node)
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 
       tree last_perm_arg = new_temp;
       /* For exclusive scan, new_temp computed above is the exclusive scan
@@ -8178,16 +7664,12 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  last_perm_arg = make_ssa_name (vectype);
 	  g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 	}
 
       orig = make_ssa_name (vectype);
       g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
 			       last_perm_arg, perms[units_log2]);
       vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-      if (! slp_node)
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 
       if (!inscan_var_store)
 	{
@@ -8197,8 +7679,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
 	  g = gimple_build_assign (data_ref, new_temp);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 	}
     }
 
@@ -8214,8 +7694,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
 	gimple *g = gimple_build_assign (data_ref, orig);
 	vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	if (! slp_node)
-	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
       }
   return true;
 }
@@ -8232,7 +7710,7 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 static bool
 vectorizable_store (vec_info *vinfo,
 		    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		    gimple **vec_stmt, slp_tree slp_node,
+		    slp_tree slp_node,
 		    stmt_vector_for_cost *cost_vec)
 {
   tree data_ref;
@@ -8263,12 +7741,12 @@ vectorizable_store (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is vectorizable store? */
 
-  tree mask = NULL_TREE, mask_vectype = NULL_TREE;
+  tree mask_vectype = NULL_TREE;
   slp_tree mask_node = NULL;
   if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
     {
@@ -8300,16 +7778,12 @@ vectorizable_store (vec_info *vinfo,
 	mask_index = vect_slp_child_index_for_operand
 		    (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (mask_index >= 0
-	  && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index,
-				      &mask, &mask_node, &mask_dt,
+	  && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
+				      &mask_node, &mask_dt,
 				      &mask_vectype))
 	return false;
     }
 
-  /* Cannot have hybrid store SLP -- that would mean storing to the
-     same location twice.  */
-  gcc_assert (PURE_SLP_STMT (stmt_info));
-
   tree vectype = SLP_TREE_VECTYPE (stmt_info), rhs_vectype = NULL_TREE;
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
@@ -8332,10 +7806,9 @@ vectorizable_store (vec_info *vinfo,
       return false;
     }
 
-  tree op;
   slp_tree op_node;
   if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
-			     &op, &op_node, &rhs_dt, &rhs_vectype, &vls_type))
+			     &op_node, &rhs_dt, &rhs_vectype, &vls_type))
     return false;
 
   elem_type = TREE_TYPE (vectype);
@@ -8349,8 +7822,8 @@ vectorizable_store (vec_info *vinfo,
   int misalignment;
   poly_int64 poffset;
   internal_fn lanes_ifn;
-  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask, vls_type,
-			    1, &memory_access_type, &poffset,
+  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
+			    vls_type, &memory_access_type, &poffset,
 			    &alignment_support_scheme, &misalignment, &gs_info,
 			    &lanes_ifn))
     return false;
@@ -8364,7 +7837,7 @@ vectorizable_store (vec_info *vinfo,
       return false;
     }
 
-  if (mask)
+  if (mask_node)
     {
       if (memory_access_type == VMAT_CONTIGUOUS)
 	{
@@ -8375,7 +7848,8 @@ vectorizable_store (vec_info *vinfo,
 	}
       else if (memory_access_type != VMAT_LOAD_STORE_LANES
 	       && (memory_access_type != VMAT_GATHER_SCATTER
-		   || (gs_info.decl && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
+		   || (GATHER_SCATTER_LEGACY_P (gs_info)
+		       && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -8383,8 +7857,7 @@ vectorizable_store (vec_info *vinfo,
 	  return false;
 	}
       else if (memory_access_type == VMAT_GATHER_SCATTER
-	       && gs_info.ifn == IFN_LAST
-	       && !gs_info.decl)
+	       && GATHER_SCATTER_EMULATED_P (gs_info))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -8416,17 +7889,16 @@ vectorizable_store (vec_info *vinfo,
       group_size = vec_num = 1;
     }
 
-  if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && !vec_stmt)
+  if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
     {
-      if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node, mask,
-			     memory_access_type))
+      if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
+			     mask_node, memory_access_type))
 	return false;
     }
 
-  bool costing_p = !vec_stmt;
+  bool costing_p = cost_vec;
   if (costing_p) /* transformation not required.  */
     {
-      STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
       SLP_TREE_MEMORY_ACCESS_TYPE (slp_node) = memory_access_type;
 
       if (loop_vinfo
@@ -8434,10 +7906,10 @@ vectorizable_store (vec_info *vinfo,
 	check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
 					      vls_type, group_size,
 					      memory_access_type, &gs_info,
-					      mask);
+					      mask_node);
 
       if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
-	  || (mask
+	  || (mask_node
 	      && !vect_maybe_update_slp_op_vectype (mask_node,
 						    mask_vectype)))
 	{
@@ -8449,16 +7921,15 @@ vectorizable_store (vec_info *vinfo,
 
       if (dump_enabled_p ()
 	  && memory_access_type != VMAT_ELEMENTWISE
-	  && memory_access_type != VMAT_GATHER_SCATTER
 	  && memory_access_type != VMAT_STRIDED_SLP
 	  && memory_access_type != VMAT_INVARIANT
 	  && alignment_support_scheme != dr_aligned)
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "Vectorizing an unaligned access.\n");
 
-      STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = store_vec_info_type;
     }
-  gcc_assert (memory_access_type == SLP_TREE_MEMORY_ACCESS_TYPE (stmt_info));
+  gcc_assert (memory_access_type == SLP_TREE_MEMORY_ACCESS_TYPE (slp_node));
 
   /* Transform.  */
 
@@ -8486,8 +7957,7 @@ vectorizable_store (vec_info *vinfo,
 
 	  return true;
 	}
-      return vectorizable_scan_store (vinfo, stmt_info, slp_node,
-				      gsi, vec_stmt, 1);
+      return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
     }
 
   /* FORNOW */
@@ -8503,7 +7973,6 @@ vectorizable_store (vec_info *vinfo,
   gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
 	      || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
   first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
-  op = vect_get_store_rhs (first_stmt_info);
 
   ref_type = get_group_alias_ptr_type (first_stmt_info);
 
@@ -8683,15 +8152,13 @@ vectorizable_store (vec_info *vinfo,
 	}
 
       alias_off = build_int_cst (ref_type, 0);
-      stmt_vec_info next_stmt_info = first_stmt_info;
       auto_vec<tree> vec_oprnds;
       /* For costing some adjacent vector stores, we'd like to cost with
 	 the total number of them once instead of cost each one by one. */
       unsigned int n_adjacent_stores = 0;
       running_off = offvar;
       if (!costing_p)
-	vect_get_vec_defs (vinfo, next_stmt_info, slp_node, ncopies, op,
-			   &vec_oprnds);
+	vect_get_slp_defs (op_node, &vec_oprnds);
       unsigned int group_el = 0;
       unsigned HOST_WIDE_INT elsz
 	= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
@@ -8766,7 +8233,7 @@ vectorizable_store (vec_info *vinfo,
 	      else
 		inside_cost
 		  += record_stmt_cost (cost_vec, n_adjacent_stores,
-				       scalar_store, stmt_info, 0, vect_body);
+				       scalar_store, slp_node, 0, vect_body);
 	      /* Only need vector extracting when there are more
 		 than one stores.  */
 	      if (nstores > 1)
@@ -8806,7 +8273,7 @@ vectorizable_store (vec_info *vinfo,
      realignment.  vect_supportable_dr_alignment always returns either
      dr_aligned or dr_unaligned_supported for masked operations.  */
   gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
-	       && !mask
+	       && !mask_node
 	       && !loop_masks)
 	      || alignment_support_scheme == dr_aligned
 	      || alignment_support_scheme == dr_unaligned_supported);
@@ -8826,7 +8293,8 @@ vectorizable_store (vec_info *vinfo,
     {
       aggr_type = elem_type;
       if (!costing_p)
-	vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
+	vect_get_strided_load_store_ops (stmt_info, vectype, loop_vinfo,
+					 gsi, &gs_info,
 					 &bump, &vec_offset, loop_lens);
     }
   else
@@ -8840,7 +8308,7 @@ vectorizable_store (vec_info *vinfo,
 					    memory_access_type, loop_lens);
     }
 
-  if (mask && !costing_p)
+  if (mask_node && !costing_p)
     LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
 
   /* In case the vectorization factor (VF) is bigger than the number
@@ -8848,39 +8316,6 @@ vectorizable_store (vec_info *vinfo,
      more than one vector stmt - i.e - we need to "unroll" the
      vector stmt by a factor VF/nunits.  */
 
-  /* In case of interleaving (non-unit grouped access):
-
-        S1:  &base + 2 = x2
-        S2:  &base = x0
-        S3:  &base + 1 = x1
-        S4:  &base + 3 = x3
-
-     We create vectorized stores starting from base address (the access of the
-     first stmt in the chain (S2 in the above example), when the last store stmt
-     of the chain (S4) is reached:
-
-        VS1: &base = vx2
-	VS2: &base + vec_size*1 = vx0
-	VS3: &base + vec_size*2 = vx1
-	VS4: &base + vec_size*3 = vx3
-
-     Then permutation statements are generated:
-
-	VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
-	VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
-	...
-
-     And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
-     (the order of the data-refs in the output of vect_permute_store_chain
-     corresponds to the order of scalar stmts in the interleaving chain - see
-     the documentation of vect_permute_store_chain()).
-
-     In case of both multiple types and interleaving, above vector stores and
-     permutation stmts are created for every copy.  The result vector stmts are
-     put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
-     STMT_VINFO_RELATED_STMT for the next copies.
-  */
-
   auto_vec<tree> dr_chain (group_size);
   auto_vec<tree> vec_masks;
   tree vec_mask = NULL;
@@ -8913,7 +8348,7 @@ vectorizable_store (vec_info *vinfo,
 	    {
 	      if (!costing_p)
 		{
-		  if (mask)
+		  if (mask_node)
 		    {
 		      vect_get_slp_defs (mask_node, &vec_masks);
 		      vec_mask = vec_masks[0];
@@ -8927,7 +8362,7 @@ vectorizable_store (vec_info *vinfo,
 	  else if (!costing_p)
 	    {
 	      gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
-	      if (mask)
+	      if (mask_node)
 		vec_mask = vec_masks[j];
 	      dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
 					     stmt_info, bump);
@@ -9062,12 +8497,11 @@ vectorizable_store (vec_info *vinfo,
 		     DR_CHAIN is of size 1.  */
 		  gcc_assert (group_size == 1);
 		  vect_get_slp_defs (op_node, gvec_oprnds[0]);
-		  if (mask)
+		  if (mask_node)
 		    vect_get_slp_defs (mask_node, &vec_masks);
 
 		  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-		    vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
-						 slp_node, &gs_info,
+		    vect_get_gather_scatter_ops (loop, slp_node, &gs_info,
 						 &dataref_ptr, &vec_offsets);
 		  else
 		    dataref_ptr
@@ -9089,7 +8523,7 @@ vectorizable_store (vec_info *vinfo,
 	  if (!costing_p)
 	    {
 	      vec_oprnd = (*gvec_oprnds[0])[j];
-	      if (mask)
+	      if (mask_node)
 		vec_mask = vec_masks[j];
 	      /* We should have catched mismatched types earlier.  */
 	      gcc_assert (useless_type_conversion_p (vectype,
@@ -9110,7 +8544,7 @@ vectorizable_store (vec_info *vinfo,
 					       final_mask, vec_mask, gsi);
 	    }
 
-	  if (gs_info.ifn != IFN_LAST)
+	  if (GATHER_SCATTER_IFN_P (gs_info))
 	    {
 	      if (costing_p)
 		{
@@ -9150,30 +8584,37 @@ vectorizable_store (vec_info *vinfo,
 		{
 		  if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
 		    call = gimple_build_call_internal (
-			    IFN_MASK_LEN_SCATTER_STORE, 7, dataref_ptr,
+			    IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
+			    gs_info.alias_ptr,
 			    vec_offset, scale, vec_oprnd, final_mask, final_len,
 			    bias);
 		  else
 		    /* Non-vector offset indicates that prefer to take
 		       MASK_LEN_STRIDED_STORE instead of the
-		       IFN_MASK_SCATTER_STORE with direct stride arg.  */
+		       IFN_MASK_SCATTER_STORE with direct stride arg.
+		       Similar to the gather case we have checked the
+		       alignment for a scatter already and assume
+		       that the strided store has the same requirements.  */
 		    call = gimple_build_call_internal (
 			    IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
 			    vec_offset, vec_oprnd, final_mask, final_len, bias);
 		}
 	      else if (final_mask)
 		call = gimple_build_call_internal
-			     (IFN_MASK_SCATTER_STORE, 5, dataref_ptr,
+			     (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
+			      gs_info.alias_ptr,
 			      vec_offset, scale, vec_oprnd, final_mask);
 	      else
-		call = gimple_build_call_internal (IFN_SCATTER_STORE, 4,
-						   dataref_ptr, vec_offset,
+		call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
+						   dataref_ptr,
+						   gs_info.alias_ptr,
+						   vec_offset,
 						   scale, vec_oprnd);
 	      gimple_call_set_nothrow (call, true);
 	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
 	      new_stmt = call;
 	    }
-	   else if (gs_info.decl)
+	  else if (GATHER_SCATTER_LEGACY_P (gs_info))
 	    {
 	      /* The builtin decls path for scatter is legacy, x86 only.  */
 	      gcc_assert (nunits.is_constant ()
@@ -9370,7 +8811,6 @@ vectorizable_store (vec_info *vinfo,
 
   gcc_assert (memory_access_type == VMAT_CONTIGUOUS
 	      || memory_access_type == VMAT_CONTIGUOUS_DOWN
-	      || memory_access_type == VMAT_CONTIGUOUS_PERMUTE
 	      || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
 
   unsigned inside_cost = 0, prologue_cost = 0;
@@ -9383,42 +8823,11 @@ vectorizable_store (vec_info *vinfo,
   if (!costing_p)
     {
       /* Get vectorized arguments for SLP_NODE.  */
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1, op,
-			 &vec_oprnds, mask, &vec_masks);
+      vect_get_slp_defs (op_node, &vec_oprnds);
       vec_oprnd = vec_oprnds[0];
-      if (mask)
-	vec_mask = vec_masks[0];
-    }
-  else
-    {
-      /* For interleaved stores we collect vectorized defs for all the
-	 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
-	 input to vect_permute_store_chain().
-
-	 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
-	 is of size 1.  */
-      stmt_vec_info next_stmt_info = first_stmt_info;
-      for (i = 0; i < group_size; i++)
-	{
-	  /* Since gaps are not supported for interleaved stores,
-	     DR_GROUP_SIZE is the exact number of stmts in the chain.
-	     Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
-	     that there is no interleaving, DR_GROUP_SIZE is 1,
-	     and only one iteration of the loop will be executed.  */
-	  op = vect_get_store_rhs (next_stmt_info);
-	  if (!costing_p)
-	    {
-	      vect_get_vec_defs_for_operand (vinfo, next_stmt_info,
-					     1, op, gvec_oprnds[i]);
-	      vec_oprnd = (*gvec_oprnds[i])[0];
-	      dr_chain.quick_push (vec_oprnd);
-	    }
-	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
-	}
-      if (mask && !costing_p)
+      if (mask_node)
 	{
-	  vect_get_vec_defs_for_operand (vinfo, stmt_info, 1,
-					 mask, &vec_masks, mask_vectype);
+	  vect_get_slp_defs (mask_node, &vec_masks);
 	  vec_mask = vec_masks[0];
 	}
     }
@@ -9448,25 +8857,7 @@ vectorizable_store (vec_info *vinfo,
 					    simd_lane_access_p, bump);
 
   new_stmt = NULL;
-  if (grouped_store)
-    {
-      /* Permute.  */
-      gcc_assert (memory_access_type == VMAT_CONTIGUOUS_PERMUTE);
-      if (costing_p)
-	{
-	  int group_size = DR_GROUP_SIZE (first_stmt_info);
-	  int nstmts = ceil_log2 (group_size) * group_size;
-	  inside_cost += record_stmt_cost (cost_vec, nstmts, vec_perm,
-					   slp_node, 0, vect_body);
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location, "vect_model_store_cost: "
-			     "strided group_size = %d .\n", group_size);
-	}
-      else
-	vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info,
-				  gsi, &result_chain);
-    }
-
+  gcc_assert (!grouped_store);
   for (i = 0; i < vec_num; i++)
     {
       if (!costing_p)
@@ -9480,10 +8871,7 @@ vectorizable_store (vec_info *vinfo,
 	  else
 	    {
 	      tree perm_mask = perm_mask_for_reverse (vectype);
-	      tree perm_dest
-		= vect_create_destination_var (vect_get_store_rhs (stmt_info),
-					       vectype);
-	      tree new_temp = make_ssa_name (perm_dest);
+	      tree new_temp = make_ssa_name (vectype);
 
 	      /* Generate the permute statement.  */
 	      gimple *perm_stmt
@@ -9819,7 +9207,7 @@ hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
 static bool
 vectorizable_load (vec_info *vinfo,
 		   stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		   gimple **vec_stmt, slp_tree slp_node,
+		   slp_tree slp_node,
 		   stmt_vector_for_cost *cost_vec)
 {
   tree scalar_dest;
@@ -9863,18 +9251,18 @@ vectorizable_load (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   if (!STMT_VINFO_DATA_REF (stmt_info))
     return false;
 
-  tree mask = NULL_TREE, mask_vectype = NULL_TREE;
+  tree mask_vectype = NULL_TREE;
   tree els = NULL_TREE; tree els_vectype = NULL_TREE;
 
   int mask_index = -1;
   int els_index = -1;
-  slp_tree slp_op = NULL;
+  slp_tree mask_node = NULL;
   slp_tree els_op = NULL;
   if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
     {
@@ -9912,8 +9300,8 @@ vectorizable_load (vec_info *vinfo,
 	mask_index = vect_slp_child_index_for_operand
 		    (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (mask_index >= 0
-	  && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index,
-				      &mask, &slp_op, &mask_dt, &mask_vectype))
+	  && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
+				      &mask_node, &mask_dt, &mask_vectype))
 	return false;
 
       els_index = internal_fn_else_index (ifn);
@@ -9921,12 +9309,12 @@ vectorizable_load (vec_info *vinfo,
 	els_index = vect_slp_child_index_for_operand
 	  (call, els_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (els_index >= 0
-	  && !vect_is_simple_use (vinfo, stmt_info, slp_node, els_index,
+	  && !vect_is_simple_use (vinfo, slp_node, els_index,
 				  &els, &els_op, &els_dt, &els_vectype))
 	return false;
     }
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
   if (loop_vinfo)
@@ -9996,8 +9384,8 @@ vectorizable_load (vec_info *vinfo,
   auto_vec<int> elsvals;
   int maskload_elsval = 0;
   bool need_zeroing = false;
-  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask, VLS_LOAD,
-			    1, &memory_access_type, &poffset,
+  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
+			    VLS_LOAD, &memory_access_type, &poffset,
 			    &alignment_support_scheme, &misalignment, &gs_info,
 			    &lanes_ifn, &elsvals))
     return false;
@@ -10011,7 +9399,7 @@ vectorizable_load (vec_info *vinfo,
     = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
 
   /* ???  The following checks should really be part of
-     get_group_load_store_type.  */
+     get_load_store_type.  */
   if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
       && !((memory_access_type == VMAT_ELEMENTWISE
 	    || memory_access_type == VMAT_GATHER_SCATTER)
@@ -10064,7 +9452,7 @@ vectorizable_load (vec_info *vinfo,
       return false;
     }
 
-  if (mask)
+  if (mask_node)
     {
       if (memory_access_type == VMAT_CONTIGUOUS)
 	{
@@ -10084,8 +9472,7 @@ vectorizable_load (vec_info *vinfo,
 	  return false;
 	}
       else if (memory_access_type == VMAT_GATHER_SCATTER
-	       && gs_info.ifn == IFN_LAST
-	       && !gs_info.decl)
+	       && GATHER_SCATTER_EMULATED_P (gs_info))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10102,12 +9489,12 @@ vectorizable_load (vec_info *vinfo,
 	}
     }
 
-  bool costing_p = !vec_stmt;
+  bool costing_p = cost_vec;
 
   if (costing_p) /* transformation not required.  */
     {
-      if (mask
-	  && !vect_maybe_update_slp_op_vectype (slp_op,
+      if (mask_node
+	  && !vect_maybe_update_slp_op_vectype (mask_node,
 						mask_vectype))
 	{
 	  if (dump_enabled_p ())
@@ -10123,7 +9510,7 @@ vectorizable_load (vec_info *vinfo,
 	check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
 					      VLS_LOAD, group_size,
 					      memory_access_type, &gs_info,
-					      mask, &elsvals);
+					      mask_node, &elsvals);
 
       if (dump_enabled_p ()
 	  && memory_access_type != VMAT_ELEMENTWISE
@@ -10137,7 +9524,7 @@ vectorizable_load (vec_info *vinfo,
       if (memory_access_type == VMAT_LOAD_STORE_LANES)
 	vinfo->any_known_not_updated_vssa = true;
 
-      STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = load_vec_info_type;
     }
   else
     {
@@ -10147,7 +9534,7 @@ vectorizable_load (vec_info *vinfo,
 	check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
 					      VLS_LOAD, group_size,
 					      memory_access_type, &gs_info,
-					      mask, &elsvals);
+					      mask_node, &elsvals);
     }
 
   /* If the type needs padding we must zero inactive elements.
@@ -10182,7 +9569,7 @@ vectorizable_load (vec_info *vinfo,
 
   if (memory_access_type == VMAT_INVARIANT)
     {
-      gcc_assert (!grouped_load && !mask && !bb_vinfo);
+      gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
       /* If we have versioned for aliasing or the loop doesn't
 	 have any data dependencies that would preclude this,
 	 then we are sure this is a loop invariant load and
@@ -10635,7 +10022,6 @@ vectorizable_load (vec_info *vinfo,
       vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
     }
 
-  gcc_assert (alignment_support_scheme);
   vec_loop_masks *loop_masks
     = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
        ? &LOOP_VINFO_MASKS (loop_vinfo)
@@ -10655,10 +10041,12 @@ vectorizable_load (vec_info *vinfo,
 
   /* Targets with store-lane instructions must not require explicit
      realignment.  vect_supportable_dr_alignment always returns either
-     dr_aligned or dr_unaligned_supported for masked operations.  */
+     dr_aligned or dr_unaligned_supported for (non-length) masked
+     operations.  */
   gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
-	       && !mask
+	       && !mask_node
 	       && !loop_masks)
+	      || memory_access_type == VMAT_GATHER_SCATTER
 	      || alignment_support_scheme == dr_aligned
 	      || alignment_support_scheme == dr_unaligned_supported);
 
@@ -10694,39 +10082,6 @@ vectorizable_load (vec_info *vinfo,
         S2:     z = x + 1       -               -
   */
 
-  /* In case of interleaving (non-unit grouped access):
-
-     S1:  x2 = &base + 2
-     S2:  x0 = &base
-     S3:  x1 = &base + 1
-     S4:  x3 = &base + 3
-
-     Vectorized loads are created in the order of memory accesses
-     starting from the access of the first stmt of the chain:
-
-     VS1: vx0 = &base
-     VS2: vx1 = &base + vec_size*1
-     VS3: vx3 = &base + vec_size*2
-     VS4: vx4 = &base + vec_size*3
-
-     Then permutation statements are generated:
-
-     VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
-     VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
-       ...
-
-     And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
-     (the order of the data-refs in the output of vect_permute_load_chain
-     corresponds to the order of scalar stmts in the interleaving chain - see
-     the documentation of vect_permute_load_chain()).
-     The generation of permutation stmts and recording them in
-     STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
-
-     In case of both multiple types and interleaving, the vector loads and
-     permutation stmts above are created for every copy.  The result vector
-     stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
-     corresponding STMT_VINFO_RELATED_STMT for the next copies.  */
-
   /* If the data reference is aligned (dr_aligned) or potentially unaligned
      on a target that supports unaligned accesses (dr_unaligned_supported)
      we generate the following code:
@@ -10808,32 +10163,10 @@ vectorizable_load (vec_info *vinfo,
 
   tree bump;
   tree vec_offset = NULL_TREE;
-  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-    {
-      aggr_type = NULL_TREE;
-      bump = NULL_TREE;
-    }
-  else if (memory_access_type == VMAT_GATHER_SCATTER)
-    {
-      aggr_type = elem_type;
-      if (!costing_p)
-	vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
-					 &bump, &vec_offset, loop_lens);
-    }
-  else
-    {
-      if (memory_access_type == VMAT_LOAD_STORE_LANES)
-	aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
-      else
-	aggr_type = vectype;
-      if (!costing_p)
-	bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
-					    memory_access_type, loop_lens);
-    }
 
   auto_vec<tree> vec_offsets;
   auto_vec<tree> vec_masks;
-  if (mask && !costing_p)
+  if (mask_node && !costing_p)
     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
 		       &vec_masks);
 
@@ -10844,6 +10177,11 @@ vectorizable_load (vec_info *vinfo,
       gcc_assert (alignment_support_scheme == dr_aligned
 		  || alignment_support_scheme == dr_unaligned_supported);
 
+      aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
+      if (!costing_p)
+	bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
+					    memory_access_type, loop_lens);
+
       unsigned int inside_cost = 0, prologue_cost = 0;
       /* For costing some adjacent vector loads, we'd like to cost with
 	 the total number of them once instead of cost each one by one. */
@@ -10896,7 +10234,7 @@ vectorizable_load (vec_info *vinfo,
 	      dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
 					     stmt_info, bump);
 	    }
-	  if (mask)
+	  if (mask_node)
 	    vec_mask = vec_masks[j];
 
 	  tree vec_array = create_vector_array (vectype, group_size);
@@ -11003,25 +10341,33 @@ vectorizable_load (vec_info *vinfo,
 
   if (memory_access_type == VMAT_GATHER_SCATTER)
     {
-      gcc_assert (alignment_support_scheme == dr_aligned
-		  || alignment_support_scheme == dr_unaligned_supported);
       gcc_assert (!grouped_load && !slp_perm);
 
-      unsigned int inside_cost = 0, prologue_cost = 0;
-
       /* 1. Create the vector or array pointer update chain.  */
-      if (!costing_p)
+      if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
 	{
-	  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-	    vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
-					 slp_node, &gs_info, &dataref_ptr,
+	  aggr_type = NULL_TREE;
+	  bump = NULL_TREE;
+	  if (!costing_p)
+	    vect_get_gather_scatter_ops (loop, slp_node, &gs_info, &dataref_ptr,
 					 &vec_offsets);
-	  else
-	    dataref_ptr
-	      = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
-					  at_loop, offset, &dummy, gsi,
-					  &ptr_incr, false, bump);
 	}
+      else
+	{
+	  aggr_type = elem_type;
+	  if (!costing_p)
+	    {
+	      vect_get_strided_load_store_ops (stmt_info, vectype, loop_vinfo,
+					       gsi, &gs_info,
+					       &bump, &vec_offset, loop_lens);
+	      dataref_ptr
+		  = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
+					      at_loop, offset, &dummy, gsi,
+					      &ptr_incr, false, bump);
+	    }
+	}
+
+      unsigned int inside_cost = 0, prologue_cost = 0;
 
       gimple *new_stmt = NULL;
       for (i = 0; i < vec_num; i++)
@@ -11031,7 +10377,7 @@ vectorizable_load (vec_info *vinfo,
 	  tree bias = NULL_TREE;
 	  if (!costing_p)
 	    {
-	      if (mask)
+	      if (mask_node)
 		vec_mask = vec_masks[i];
 	      if (loop_masks)
 		final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
@@ -11047,7 +10393,7 @@ vectorizable_load (vec_info *vinfo,
 
 	  /* 2. Create the vector-load in the loop.  */
 	  unsigned HOST_WIDE_INT align;
-	  if (gs_info.ifn != IFN_LAST)
+	  if (GATHER_SCATTER_IFN_P (gs_info))
 	    {
 	      if (costing_p)
 		{
@@ -11093,7 +10439,8 @@ vectorizable_load (vec_info *vinfo,
 		{
 		  if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
 		    call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
-						       8, dataref_ptr,
+						       9, dataref_ptr,
+						       gs_info.alias_ptr,
 						       vec_offset, scale, zero,
 						       final_mask, vec_els,
 						       final_len, bias);
@@ -11108,18 +10455,20 @@ vectorizable_load (vec_info *vinfo,
 		}
 	      else if (final_mask)
 		call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
-						   6, dataref_ptr,
+						   7, dataref_ptr,
+						   gs_info.alias_ptr,
 						   vec_offset, scale,
 						   zero, final_mask, vec_els);
 	      else
-		call = gimple_build_call_internal (IFN_GATHER_LOAD, 4,
-						   dataref_ptr, vec_offset,
-						   scale, zero);
+		call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
+						   dataref_ptr,
+						   gs_info.alias_ptr,
+						   vec_offset, scale, zero);
 	      gimple_call_set_nothrow (call, true);
 	      new_stmt = call;
 	      data_ref = NULL_TREE;
 	    }
-	  else if (gs_info.decl)
+	  else if (GATHER_SCATTER_LEGACY_P (gs_info))
 	    {
 	      /* The builtin decls path for gather is legacy, x86 only.  */
 	      gcc_assert (!final_len && nunits.is_constant ());
@@ -11136,7 +10485,7 @@ vectorizable_load (vec_info *vinfo,
 	      if (known_eq (nunits, offset_nunits))
 		{
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
 				dataref_ptr, vec_offsets[i], final_mask);
 		  data_ref = NULL_TREE;
 		}
@@ -11146,7 +10495,7 @@ vectorizable_load (vec_info *vinfo,
 		     lanes but the builtins will produce full vectype
 		     data with just the lower lanes filled.  */
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
 				dataref_ptr, vec_offsets[2 * i], final_mask);
 		  tree low = make_ssa_name (vectype);
 		  gimple_set_lhs (new_stmt, low);
@@ -11185,7 +10534,8 @@ vectorizable_load (vec_info *vinfo,
 		    }
 
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info, dataref_ptr,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
+				dataref_ptr,
 				vec_offsets[2 * i + 1], final_mask);
 		  tree high = make_ssa_name (vectype);
 		  gimple_set_lhs (new_stmt, high);
@@ -11228,7 +10578,7 @@ vectorizable_load (vec_info *vinfo,
 						   new_stmt, gsi);
 		    }
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
 				dataref_ptr, vec_offset, final_mask);
 		  data_ref = NULL_TREE;
 		}
@@ -11339,6 +10689,11 @@ vectorizable_load (vec_info *vinfo,
       return true;
     }
 
+  aggr_type = vectype;
+  if (!costing_p)
+    bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
+					memory_access_type, loop_lens);
+
   poly_uint64 group_elt = 0;
   unsigned int inside_cost = 0, prologue_cost = 0;
   /* For costing some adjacent vector loads, we'd like to cost with
@@ -11418,7 +10773,7 @@ vectorizable_load (vec_info *vinfo,
 
       if (!costing_p)
 	{
-	  if (mask)
+	  if (mask_node)
 	    vec_mask = vec_masks[i];
 	  if (loop_masks)
 	    final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
@@ -11809,18 +11164,12 @@ vectorizable_load (vec_info *vinfo,
 	 alignment support schemes.  */
       if (costing_p)
 	{
-	  /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
-	     only need to take care of the first stmt, whose
-	     stmt_info is first_stmt_info, vec_num iterating on it
-	     will cover the cost for the remaining, it's consistent
-	     with transforming.  For the prologue cost for realign,
+	  /* For the prologue cost for realign,
 	     we only need to count it once for the whole group.  */
 	  bool first_stmt_info_p = first_stmt_info == stmt_info;
 	  bool add_realign_cost = first_stmt_info_p && i == 0;
 	  if (memory_access_type == VMAT_CONTIGUOUS
-	      || memory_access_type == VMAT_CONTIGUOUS_REVERSE
-	      || (memory_access_type == VMAT_CONTIGUOUS_PERMUTE
-		  && (!grouped_load || first_stmt_info_p)))
+	      || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
 	    {
 	      /* Leave realign cases alone to keep them simple.  */
 	      if (alignment_support_scheme == dr_explicit_realign_optimized
@@ -11908,7 +11257,7 @@ vectorizable_load (vec_info *vinfo,
 	}
 
       /* Collect vector loads and later create their permutation in
-	 vect_transform_grouped_load ().  */
+	 vect_transform_slp_perm_load.  */
       if (!costing_p && (grouped_load || slp_perm))
 	dr_chain.quick_push (new_temp);
 
@@ -11977,8 +11326,7 @@ vectorizable_load (vec_info *vinfo,
   if (costing_p)
     {
       gcc_assert (memory_access_type == VMAT_CONTIGUOUS
-		  || memory_access_type == VMAT_CONTIGUOUS_REVERSE
-		  || memory_access_type == VMAT_CONTIGUOUS_PERMUTE);
+		  || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
       if (n_adjacent_loads > 0)
 	vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
 			    alignment_support_scheme, misalignment, false,
@@ -12008,7 +11356,7 @@ vectorizable_load (vec_info *vinfo,
    condition operands are supportable using vec_is_simple_use.  */
 
 static bool
-vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
+vect_is_simple_cond (tree cond, vec_info *vinfo,
 		     slp_tree slp_node, tree *comp_vectype,
 		     enum vect_def_type *dts, tree vectype)
 {
@@ -12020,7 +11368,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
   if (TREE_CODE (cond) == SSA_NAME
       && VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond)))
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &cond,
+      if (!vect_is_simple_use (vinfo, slp_node, 0, &cond,
 			       &slp_op, &dts[0], comp_vectype)
 	  || !*comp_vectype
 	  || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
@@ -12036,7 +11384,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
 
   if (TREE_CODE (lhs) == SSA_NAME)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0,
+      if (!vect_is_simple_use (vinfo, slp_node, 0,
 			       &lhs, &slp_op, &dts[0], &vectype1))
 	return false;
     }
@@ -12048,7 +11396,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
 
   if (TREE_CODE (rhs) == SSA_NAME)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1,
+      if (!vect_is_simple_use (vinfo, slp_node, 1,
 			       &rhs, &slp_op, &dts[1], &vectype2))
 	return false;
     }
@@ -12101,7 +11449,6 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
 static bool
 vectorizable_condition (vec_info *vinfo,
 			stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			gimple **vec_stmt,
 			slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   tree scalar_dest = NULL_TREE;
@@ -12117,9 +11464,6 @@ vectorizable_condition (vec_info *vinfo,
   enum vect_def_type dts[4]
     = {vect_unknown_def_type, vect_unknown_def_type,
        vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 4;
-  int ncopies;
-  int vec_num;
   enum tree_code code, cond_code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
   int i;
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
@@ -12149,7 +11493,7 @@ vectorizable_condition (vec_info *vinfo,
     = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)) != NULL;
   if (for_reduction)
     {
-      if (slp_node && SLP_TREE_LANES (slp_node) > 1)
+      if (SLP_TREE_LANES (slp_node) > 1)
 	return false;
       reduc_info = info_for_reduction (vinfo, stmt_info);
       reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
@@ -12163,38 +11507,25 @@ vectorizable_condition (vec_info *vinfo,
 	return false;
     }
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
 
-  if (slp_node)
-    {
-      ncopies = 1;
-      vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-    }
-  else
-    {
-      ncopies = vect_get_num_copies (loop_vinfo, vectype);
-      vec_num = 1;
-    }
-
-  gcc_assert (ncopies >= 1);
-  if (for_reduction && ncopies > 1)
-    return false; /* FORNOW */
+  int vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 
   cond_expr = gimple_assign_rhs1 (stmt);
   gcc_assert (! COMPARISON_CLASS_P (cond_expr));
 
-  if (!vect_is_simple_cond (cond_expr, vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_cond (cond_expr, vinfo, slp_node,
 			    &comp_vectype, &dts[0], vectype)
       || !comp_vectype)
     return false;
 
   unsigned op_adjust = COMPARISON_CLASS_P (cond_expr) ? 1 : 0;
   slp_tree then_slp_node, else_slp_node;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1 + op_adjust,
+  if (!vect_is_simple_use (vinfo, slp_node, 1 + op_adjust,
 			   &then_clause, &then_slp_node, &dts[2], &vectype1))
     return false;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 2 + op_adjust,
+  if (!vect_is_simple_use (vinfo, slp_node, 2 + op_adjust,
 			   &else_clause, &else_slp_node, &dts[3], &vectype2))
     return false;
 
@@ -12242,9 +11573,8 @@ vectorizable_condition (vec_info *vinfo,
 	}
       /* ???  The vectorized operand query below doesn't allow swapping
 	 this way for SLP.  */
-      if (slp_node)
-	return false;
-      std::swap (then_clause, else_clause);
+      return false;
+      /* std::swap (then_clause, else_clause); */
     }
 
   if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype))
@@ -12298,7 +11628,7 @@ vectorizable_condition (vec_info *vinfo,
       return false;
     }
 
-  if (!vec_stmt)
+  if (cost_vec)
     {
       if (bitop1 != NOP_EXPR)
 	{
@@ -12329,14 +11659,13 @@ vectorizable_condition (vec_info *vinfo,
 		       || !expand_vec_cond_expr_p (vectype, vec_cmp_type))))
 	return false;
 
-      if (slp_node
-	  && (!vect_maybe_update_slp_op_vectype
-		 (SLP_TREE_CHILDREN (slp_node)[0], comp_vectype)
-	      || (op_adjust == 1
-		  && !vect_maybe_update_slp_op_vectype
-			(SLP_TREE_CHILDREN (slp_node)[1], comp_vectype))
-	      || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
-	      || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype)))
+      if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
+					     comp_vectype)
+	  || (op_adjust == 1
+	      && !vect_maybe_update_slp_op_vectype
+			      (SLP_TREE_CHILDREN (slp_node)[1], comp_vectype))
+	  || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
+	  || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -12353,11 +11682,11 @@ vectorizable_condition (vec_info *vinfo,
 						  vectype, OPTIMIZE_FOR_SPEED))
 		vect_record_loop_len (loop_vinfo,
 				      &LOOP_VINFO_LENS (loop_vinfo),
-				      ncopies * vec_num, vectype, 1);
+				      vec_num, vectype, 1);
 	      else
 		vect_record_loop_mask (loop_vinfo,
 				       &LOOP_VINFO_MASKS (loop_vinfo),
-				       ncopies * vec_num, vectype, NULL);
+				       vec_num, vectype, NULL);
 	    }
 	  /* Extra inactive lanes should be safe for vect_nested_cycle.  */
 	  else if (STMT_VINFO_DEF_TYPE (reduc_info) != vect_nested_cycle)
@@ -12370,9 +11699,8 @@ vectorizable_condition (vec_info *vinfo,
 	    }
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
-      vect_model_simple_cost (vinfo, ncopies, dts, ndts, slp_node,
-			      cost_vec, kind);
+      SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
       return true;
     }
 
@@ -12401,7 +11729,7 @@ vectorizable_condition (vec_info *vinfo,
 	masks = &LOOP_VINFO_MASKS (loop_vinfo);
       else
 	{
-	  scalar_cond_masked_key cond (cond_expr, ncopies);
+	  scalar_cond_masked_key cond (cond_expr, 1);
 	  if (loop_vinfo->scalar_cond_masked_set.contains (cond))
 	    masks = &LOOP_VINFO_MASKS (loop_vinfo);
 	  else
@@ -12436,18 +11764,18 @@ vectorizable_condition (vec_info *vinfo,
 
   /* Handle cond expr.  */
   if (masked)
-    vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		       cond_expr, comp_vectype, &vec_oprnds0,
-		       then_clause, vectype, &vec_oprnds2,
+    vect_get_vec_defs (vinfo, slp_node,
+		       cond_expr, &vec_oprnds0,
+		       then_clause, &vec_oprnds2,
 		       reduction_type != EXTRACT_LAST_REDUCTION
-		       ? else_clause : NULL, vectype, &vec_oprnds3);
+		       ? else_clause : NULL, &vec_oprnds3);
   else
-    vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		       cond_expr0, comp_vectype, &vec_oprnds0,
-		       cond_expr1, comp_vectype, &vec_oprnds1,
-		       then_clause, vectype, &vec_oprnds2,
+    vect_get_vec_defs (vinfo, slp_node,
+		       cond_expr0, &vec_oprnds0,
+		       cond_expr1, &vec_oprnds1,
+		       then_clause, &vec_oprnds2,
 		       reduction_type != EXTRACT_LAST_REDUCTION
-		       ? else_clause : NULL, vectype, &vec_oprnds3);
+		       ? else_clause : NULL, &vec_oprnds3);
 
   if (reduction_type == EXTRACT_LAST_REDUCTION)
     vec_else_clause = else_clause;
@@ -12564,7 +11892,7 @@ vectorizable_condition (vec_info *vinfo,
 	      if (lens)
 		{
 		  len = vect_get_loop_len (loop_vinfo, gsi, lens,
-					   vec_num * ncopies, vectype, i, 1);
+					   vec_num, vectype, i, 1);
 		  signed char biasval
 		    = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
 		  bias = build_int_cst (intQI_type_node, biasval);
@@ -12578,7 +11906,7 @@ vectorizable_condition (vec_info *vinfo,
 	  if (masks)
 	    {
 	      tree loop_mask
-		= vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num * ncopies,
+		= vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
 				      vectype, i);
 	      tree tmp2 = make_ssa_name (vec_cmp_type);
 	      gassign *g
@@ -12630,15 +11958,9 @@ vectorizable_condition (vec_info *vinfo,
 					  vec_then_clause, vec_else_clause);
 	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 	}
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds0.release ();
   vec_oprnds1.release ();
   vec_oprnds2.release ();
@@ -12658,18 +11980,15 @@ vectorizable_condition (vec_info *vinfo,
 static bool
 vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 			   stmt_vec_info stmt_info, tree_code code,
-			   gimple_stmt_iterator *gsi, gimple **vec_stmt,
+			   gimple_stmt_iterator *gsi,
 			   slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   tree lhs, rhs1, rhs2;
   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
   tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
   tree new_temp;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 2;
   poly_uint64 nunits;
-  int ncopies;
   enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
   int i;
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
@@ -12687,22 +12006,15 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
   mask_type = vectype;
   nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   if (TREE_CODE_CLASS (code) != tcc_comparison)
     return false;
 
   slp_tree slp_rhs1, slp_rhs2;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
     return false;
 
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
     return false;
 
@@ -12768,7 +12080,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	}
     }
 
-  if (!vec_stmt)
+  if (cost_vec)
     {
       if (bitop1 == NOP_EXPR)
 	{
@@ -12793,9 +12105,8 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	}
 
       /* Put types on constant and invariant SLP children.  */
-      if (slp_node
-	  && (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
-	      || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype)))
+      if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
+	  || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -12803,8 +12114,8 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	  return false;
 	}
 
-      vect_model_simple_cost (vinfo, ncopies * (1 + (bitop2 != NOP_EXPR)),
-			      dts, ndts, slp_node, cost_vec);
+      vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
+			      slp_node, cost_vec);
       return true;
     }
 
@@ -12815,9 +12126,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
   if (lhs)
     mask = vect_create_destination_var (lhs, mask_type);
 
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		     rhs1, vectype, &vec_oprnds0,
-		     rhs2, vectype, &vec_oprnds1);
+  vect_get_vec_defs (vinfo, slp_node, rhs1, &vec_oprnds0, rhs2, &vec_oprnds1);
   if (swap_p)
     std::swap (vec_oprnds0, vec_oprnds1);
 
@@ -12856,15 +12165,9 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 	    }
 	}
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds0.release ();
   vec_oprnds1.release ();
 
@@ -12882,7 +12185,6 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 static bool
 vectorizable_comparison (vec_info *vinfo,
 			 stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt,
 			 slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
@@ -12898,13 +12200,13 @@ vectorizable_comparison (vec_info *vinfo,
     return false;
 
   enum tree_code code = gimple_assign_rhs_code (stmt);
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
-				  vec_stmt, slp_node, cost_vec))
+				  slp_node, cost_vec))
     return false;
 
-  if (!vec_stmt)
-    STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type;
+  if (cost_vec)
+    SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
 
   return true;
 }
@@ -12914,7 +12216,7 @@ vectorizable_comparison (vec_info *vinfo,
 
 bool
 vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
-			 gimple_stmt_iterator *gsi, gimple **vec_stmt,
+			 gimple_stmt_iterator *gsi,
 			 slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -12932,48 +12234,17 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 
   auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
 
-  tree vectype = NULL_TREE;
-  slp_tree slp_op0;
-  tree op0;
-  enum vect_def_type dt0;
-
-  /* Early break gcond kind SLP trees can be root only and have no children,
-     for instance in the case where the argument is an external.  If that's
-     the case there is no operand to analyse use of.  */
-  if ((!slp_node || !SLP_TREE_CHILDREN (slp_node).is_empty ())
-      && !vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &op0, &slp_op0, &dt0,
-			      &vectype))
-    {
-      if (dump_enabled_p ())
-	  dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			   "use not simple.\n");
-	return false;
-    }
-
   /* For SLP we don't want to use the type of the operands of the SLP node, when
      vectorizing using SLP slp_node will be the children of the gcond and we
      want to use the type of the direct children which since the gcond is root
      will be the current node, rather than a child node as vect_is_simple_use
      assumes.  */
-  if (slp_node)
-    vectype = SLP_TREE_VECTYPE (slp_node);
-
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (!vectype)
     return false;
 
   machine_mode mode = TYPE_MODE (vectype);
-  int ncopies, vec_num;
-
-  if (slp_node)
-    {
-      ncopies = 1;
-      vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-    }
-  else
-    {
-      ncopies = vect_get_num_copies (loop_vinfo, vectype);
-      vec_num = 1;
-    }
+  int vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 
   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
   vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
@@ -13003,7 +12274,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 					exit_true_edge->dest);
 
   /* Analyze only.  */
-  if (!vec_stmt)
+  if (cost_vec)
     {
       if (direct_optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
 	{
@@ -13015,31 +12286,17 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 	  return false;
 	}
 
-      if (ncopies > 1
-	  && direct_optab_handler (ior_optab, mode) == CODE_FOR_nothing)
-	{
-	  if (dump_enabled_p ())
-	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			       "can't vectorize early exit because the "
-			       "target does not support boolean vector IOR "
-			       "for type %T.\n",
-			       vectype);
-	  return false;
-	}
-
       if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
-				      vec_stmt, slp_node, cost_vec))
+				      slp_node, cost_vec))
 	return false;
 
       if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
 	{
 	  if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
 					      OPTIMIZE_FOR_SPEED))
-	    vect_record_loop_len (loop_vinfo, lens, ncopies * vec_num,
-				  vectype, 1);
+	    vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
 	  else
-	    vect_record_loop_mask (loop_vinfo, masks, ncopies * vec_num,
-				   vectype, NULL);
+	    vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
 	}
 
       return true;
@@ -13059,28 +12316,13 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
      vectorized.  It's not very clean to do this here, But the masking code below is
      complex and this keeps it all in one place to ease fixes and backports.  Once we
      drop the non-SLP loop vect or split vectorizable_* this can be simplified.  */
-  if (!slp_node)
-    {
-      if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
-				      vec_stmt, slp_node, cost_vec))
-	gcc_unreachable ();
-    }
 
   gimple *stmt = STMT_VINFO_STMT (stmt_info);
   basic_block cond_bb = gimple_bb (stmt);
   gimple_stmt_iterator  cond_gsi = gsi_last_bb (cond_bb);
 
   auto_vec<tree> stmts;
-
-  if (slp_node)
-    stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
-  else
-    {
-      auto vec_stmts = STMT_VINFO_VEC_STMTS (stmt_info);
-      stmts.reserve_exact (vec_stmts.length ());
-      for (auto stmt : vec_stmts)
-	stmts.quick_push (gimple_assign_lhs (stmt));
-    }
+  stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
 
   /* If we're comparing against a previous forall we need to negate the resullts
      before we do the final comparison or reduction.  */
@@ -13119,7 +12361,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 	for (unsigned i = 0; i < stmts.length (); i++)
 	  {
 	    tree stmt_mask
-	      = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies * vec_num,
+	      = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
 				    vectype, i);
 	    stmt_mask
 	      = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
@@ -13130,7 +12372,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 	for (unsigned i = 0; i < stmts.length (); i++)
 	  {
 	    tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
-						    lens, ncopies * vec_num,
+						    lens, vec_num,
 						    vectype, stmts[i], i, 1);
 
 	    workset.quick_push (len_mask);
@@ -13155,13 +12397,13 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
       if (masked_loop_p)
 	{
 	  tree mask
-	    = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies, vectype, 0);
+	    = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
 	  new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
 				       new_temp, &cond_gsi);
 	}
       else if (len_loop_p)
 	new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
-					   ncopies, vectype, new_temp, 0, 1);
+					   1, vectype, new_temp, 0, 1);
     }
 
   gcc_assert (new_temp);
@@ -13169,13 +12411,8 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
   gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
   update_stmt (orig_stmt);
 
-  if (slp_node)
-    SLP_TREE_VEC_DEFS (slp_node).truncate (0);
-   else
-    STMT_VINFO_VEC_STMTS (stmt_info).truncate (0);
-
-  if (!slp_node)
-    *vec_stmt = orig_stmt;
+  /* ??? */
+  SLP_TREE_VEC_DEFS (slp_node).truncate (0);
 
   return true;
 }
@@ -13186,37 +12423,27 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
    VEC_STMT_P is as for vectorizable_live_operation.  */
 
 static bool
-can_vectorize_live_stmts (vec_info *vinfo, stmt_vec_info stmt_info,
+can_vectorize_live_stmts (vec_info *vinfo,
 			  slp_tree slp_node, slp_instance slp_node_instance,
 			  bool vec_stmt_p,
 			  stmt_vector_for_cost *cost_vec)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  if (slp_node)
-    {
-      stmt_vec_info slp_stmt_info;
-      unsigned int i;
-      FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
-	{
-	  if (slp_stmt_info
-	      && (STMT_VINFO_LIVE_P (slp_stmt_info)
-		  || (loop_vinfo
-		      && LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
-		      && STMT_VINFO_DEF_TYPE (slp_stmt_info)
-		      == vect_induction_def))
-	      && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
-					       slp_node_instance, i,
-					       vec_stmt_p, cost_vec))
-	    return false;
-	}
+  stmt_vec_info slp_stmt_info;
+  unsigned int i;
+  FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
+    {
+      if (slp_stmt_info
+	  && (STMT_VINFO_LIVE_P (slp_stmt_info)
+	      || (loop_vinfo
+		  && LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
+		  && STMT_VINFO_DEF_TYPE (slp_stmt_info)
+		  == vect_induction_def))
+	  && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
+					   slp_node_instance, i,
+					   vec_stmt_p, cost_vec))
+	return false;
     }
-  else if ((STMT_VINFO_LIVE_P (stmt_info)
-	    || (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
-		&& STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def))
-	   && !vectorizable_live_operation (vinfo, stmt_info,
-					    slp_node, slp_node_instance, -1,
-					    vec_stmt_p, cost_vec))
-    return false;
 
   return true;
 }
@@ -13225,115 +12452,42 @@ can_vectorize_live_stmts (vec_info *vinfo, stmt_vec_info stmt_info,
 
 opt_result
 vect_analyze_stmt (vec_info *vinfo,
-		   stmt_vec_info stmt_info, bool *need_to_vectorize,
 		   slp_tree node, slp_instance node_instance,
 		   stmt_vector_for_cost *cost_vec)
 {
+  stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
   enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
   bool ok;
-  gimple_seq pattern_def_seq;
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
 		     stmt_info->stmt);
 
   if (gimple_has_volatile_ops (stmt_info->stmt))
-    return opt_result::failure_at (stmt_info->stmt,
-				   "not vectorized:"
-				   " stmt has volatile operands: %G\n",
-				   stmt_info->stmt);
-
-  if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-      && node == NULL
-      && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
     {
-      gimple_stmt_iterator si;
-
-      for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
-	{
-	  stmt_vec_info pattern_def_stmt_info
-	    = vinfo->lookup_stmt (gsi_stmt (si));
-	  if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
-	      || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
-	    {
-	      /* Analyze def stmt of STMT if it's a pattern stmt.  */
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_NOTE, vect_location,
-				 "==> examining pattern def statement: %G",
-				 pattern_def_stmt_info->stmt);
-
-	      opt_result res
-		= vect_analyze_stmt (vinfo, pattern_def_stmt_info,
-				     need_to_vectorize, node, node_instance,
-				     cost_vec);
-	      if (!res)
-		return res;
-	    }
-	}
+      /* ???  This shouldn't really happen, volatile stmts should
+	 not end up in the SLP graph.  */
+      return opt_result::failure_at (stmt_info->stmt,
+				     "not vectorized:"
+				     " stmt has volatile operands: %G\n",
+				     stmt_info->stmt);
     }
 
-  /* Skip stmts that do not need to be vectorized. In loops this is expected
-     to include:
-     - the COND_EXPR which is the loop exit condition
-     - any LABEL_EXPRs in the loop
-     - computations that are used only for array indexing or loop control.
-     In basic blocks we only analyze statements that are a part of some SLP
-     instance, therefore, all the statements are relevant.
-
-     Pattern statement needs to be analyzed instead of the original statement
-     if the original statement is not relevant.  Otherwise, we analyze both
-     statements.  In basic blocks we are called from some SLP instance
-     traversal, don't analyze pattern stmts instead, the pattern stmts
-     already will be part of SLP instance.  */
-
-  stmt_vec_info pattern_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
+  /* Skip stmts that do not need to be vectorized.  */
   if (!STMT_VINFO_RELEVANT_P (stmt_info)
       && !STMT_VINFO_LIVE_P (stmt_info))
     {
-      if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-	  && pattern_stmt_info
-	  && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
-	      || STMT_VINFO_LIVE_P (pattern_stmt_info)))
-        {
-          /* Analyze PATTERN_STMT instead of the original stmt.  */
-	  stmt_info = pattern_stmt_info;
-          if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "==> examining pattern statement: %G",
-			     stmt_info->stmt);
-        }
-      else
-        {
-          if (dump_enabled_p ())
-            dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
-
-	  if (node)
-	    return opt_result::failure_at (stmt_info->stmt,
-					   "not vectorized:"
-					   " irrelevant stmt as SLP node %p "
-					   "representative.\n",
-					   (void *)node);
-          return opt_result::success ();
-        }
-    }
-  else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-	   && node == NULL
-	   && pattern_stmt_info
-	   && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
-	       || STMT_VINFO_LIVE_P (pattern_stmt_info)))
-    {
-      /* Analyze PATTERN_STMT too.  */
       if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "==> examining pattern statement: %G",
-			 pattern_stmt_info->stmt);
+	dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
 
-      opt_result res
-	= vect_analyze_stmt (vinfo, pattern_stmt_info, need_to_vectorize, node,
-			     node_instance, cost_vec);
-      if (!res)
-	return res;
+      /* ???  This shouldn't really happen, irrelevant stmts should
+	 not end up in the SLP graph.  */
+      return opt_result::failure_at (stmt_info->stmt,
+				     "not vectorized:"
+				     " irrelevant stmt as SLP node %p "
+				     "representative.\n",
+				     (void *)node);
     }
 
   switch (STMT_VINFO_DEF_TYPE (stmt_info))
@@ -13369,8 +12523,7 @@ vect_analyze_stmt (vec_info *vinfo,
     }
 
   tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
-  if (node)
-    STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (node);
+  STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (node);
 
   if (STMT_VINFO_RELEVANT_P (stmt_info))
     {
@@ -13378,23 +12531,8 @@ vect_analyze_stmt (vec_info *vinfo,
       gcc_assert (STMT_VINFO_VECTYPE (stmt_info)
 		  || gimple_code (stmt_info->stmt) == GIMPLE_COND
 		  || (call && gimple_call_lhs (call) == NULL_TREE));
-      *need_to_vectorize = true;
-    }
-
-  if (PURE_SLP_STMT (stmt_info) && !node)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "handled only by SLP analysis\n");
-      return opt_result::success ();
     }
 
-  /* When we arrive here with a non-SLP statement and we are supposed
-     to use SLP for everything fail vectorization.  */
-  if (!node)
-    return opt_result::failure_at (stmt_info->stmt,
-				   "needs non-SLP handling\n");
-
   ok = true;
   if (!bb_vinfo
       && (STMT_VINFO_RELEVANT_P (stmt_info)
@@ -13402,64 +12540,63 @@ vect_analyze_stmt (vec_info *vinfo,
     /* Prefer vectorizable_call over vectorizable_simd_clone_call so
        -mveclibabi= takes preference over library functions with
        the simd attribute.  */
-    ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec)
-	  || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, NULL, node,
+    ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
+	  || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
 					   cost_vec)
 	  || vectorizable_conversion (vinfo, stmt_info,
-				      NULL, NULL, node, cost_vec)
+				      NULL, node, cost_vec)
 	  || vectorizable_operation (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
+				     NULL, node, cost_vec)
 	  || vectorizable_assignment (vinfo, stmt_info,
-				      NULL, NULL, node, cost_vec)
-	  || vectorizable_load (vinfo, stmt_info, NULL, NULL, node, cost_vec)
-	  || vectorizable_store (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+				      NULL, node, cost_vec)
+	  || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
+	  || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
 	  || vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
 					 stmt_info, node, cost_vec)
 	  || vectorizable_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
 				     node, node_instance, cost_vec)
 	  || vectorizable_induction (as_a <loop_vec_info> (vinfo), stmt_info,
-				     NULL, node, cost_vec)
-	  || vectorizable_shift (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+				     node, cost_vec)
+	  || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
 	  || vectorizable_condition (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
-	  || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
+				     NULL, node, cost_vec)
+	  || vectorizable_comparison (vinfo, stmt_info, NULL, node,
 				      cost_vec)
 	  || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
 				  stmt_info, node)
 	  || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
-				   stmt_info, NULL, node, cost_vec)
-	  || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+				   stmt_info, node, cost_vec)
+	  || vectorizable_early_exit (vinfo, stmt_info, NULL, node,
 				      cost_vec));
   else
     {
       if (bb_vinfo)
-	ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+	ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
 	      || vectorizable_simd_clone_call (vinfo, stmt_info,
-					       NULL, NULL, node, cost_vec)
-	      || vectorizable_conversion (vinfo, stmt_info, NULL, NULL, node,
+					       NULL, node, cost_vec)
+	      || vectorizable_conversion (vinfo, stmt_info, NULL, node,
 					  cost_vec)
 	      || vectorizable_shift (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
+				     NULL, node, cost_vec)
 	      || vectorizable_operation (vinfo, stmt_info,
-					 NULL, NULL, node, cost_vec)
-	      || vectorizable_assignment (vinfo, stmt_info, NULL, NULL, node,
+					 NULL, node, cost_vec)
+	      || vectorizable_assignment (vinfo, stmt_info, NULL, node,
 					  cost_vec)
 	      || vectorizable_load (vinfo, stmt_info,
-				    NULL, NULL, node, cost_vec)
+				    NULL, node, cost_vec)
 	      || vectorizable_store (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
+				     NULL, node, cost_vec)
 	      || vectorizable_condition (vinfo, stmt_info,
-					 NULL, NULL, node, cost_vec)
-	      || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
+					 NULL, node, cost_vec)
+	      || vectorizable_comparison (vinfo, stmt_info, NULL, node,
 					  cost_vec)
-	      || vectorizable_phi (vinfo, stmt_info, NULL, node, cost_vec)
-	      || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+	      || vectorizable_phi (vinfo, stmt_info, node, cost_vec)
+	      || vectorizable_early_exit (vinfo, stmt_info, NULL, node,
 					  cost_vec));
 
     }
 
-  if (node)
-    STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
+  STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
 
   if (!ok)
     return opt_result::failure_at (stmt_info->stmt,
@@ -13470,11 +12607,12 @@ vect_analyze_stmt (vec_info *vinfo,
   /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
       need extra handling, except for vectorizable reductions.  */
   if (!bb_vinfo
-      && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
-      && STMT_VINFO_TYPE (stmt_info) != lc_phi_info_type
-      && (!node || !node->ldst_lanes || SLP_TREE_CODE (node) == VEC_PERM_EXPR)
+      && SLP_TREE_TYPE (node) != reduc_vec_info_type
+      && (SLP_TREE_TYPE (node) != lc_phi_info_type
+	  || STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def)
+      && (!node->ldst_lanes || SLP_TREE_CODE (node) == VEC_PERM_EXPR)
       && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
-				    stmt_info, node, node_instance,
+				    node, node_instance,
 				    false, cost_vec))
     return opt_result::failure_at (stmt_info->stmt,
 				   "not vectorized:"
@@ -13495,57 +12633,49 @@ vect_transform_stmt (vec_info *vinfo,
 		     slp_tree slp_node, slp_instance slp_node_instance)
 {
   bool is_store = false;
-  gimple *vec_stmt = NULL;
   bool done;
 
-  gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info));
+  gcc_assert (slp_node);
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "------>vectorizing statement: %G", stmt_info->stmt);
 
   tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
-  if (slp_node)
-    STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (slp_node);
+  STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (slp_node);
 
-  switch (STMT_VINFO_TYPE (stmt_info))
+  switch (SLP_TREE_TYPE (slp_node))
     {
     case type_demotion_vec_info_type:
     case type_promotion_vec_info_type:
     case type_conversion_vec_info_type:
-      done = vectorizable_conversion (vinfo, stmt_info,
-				      gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case induc_vec_info_type:
       done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
-				     stmt_info, &vec_stmt, slp_node,
-				     NULL);
+				     stmt_info, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case shift_vec_info_type:
-      done = vectorizable_shift (vinfo, stmt_info,
-				 gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case op_vec_info_type:
-      done = vectorizable_operation (vinfo, stmt_info, gsi, &vec_stmt, slp_node,
-				     NULL);
+      done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case assignment_vec_info_type:
-      done = vectorizable_assignment (vinfo, stmt_info,
-				      gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case load_vec_info_type:
-      done = vectorizable_load (vinfo, stmt_info, gsi, &vec_stmt, slp_node,
-				NULL);
+      done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
@@ -13561,32 +12691,28 @@ vect_transform_stmt (vec_info *vinfo,
 	;
       else
 	{
-	  done = vectorizable_store (vinfo, stmt_info,
-				     gsi, &vec_stmt, slp_node, NULL);
+	  done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
 	  gcc_assert (done);
 	  is_store = true;
 	}
       break;
 
     case condition_vec_info_type:
-      done = vectorizable_condition (vinfo, stmt_info,
-				     gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case comparison_vec_info_type:
-      done = vectorizable_comparison (vinfo, stmt_info, gsi, &vec_stmt,
-				      slp_node, NULL);
+      done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case call_vec_info_type:
-      done = vectorizable_call (vinfo, stmt_info,
-				gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
       break;
 
     case call_simd_clone_vec_info_type:
-      done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi, &vec_stmt,
+      done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
 					   slp_node, NULL);
       break;
 
@@ -13610,18 +12736,17 @@ vect_transform_stmt (vec_info *vinfo,
 
     case recurr_info_type:
       done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
-				  stmt_info, &vec_stmt, slp_node, NULL);
+				  stmt_info, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case phi_info_type:
-      done = vectorizable_phi (vinfo, stmt_info, &vec_stmt, slp_node, NULL);
+      done = vectorizable_phi (vinfo, stmt_info, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case loop_exit_ctrl_vec_info_type:
-      done = vectorizable_early_exit (vinfo, stmt_info, gsi, &vec_stmt,
-				      slp_node, NULL);
+      done = vectorizable_early_exit (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
@@ -13636,23 +12761,18 @@ vect_transform_stmt (vec_info *vinfo,
       done = true;
     }
 
-  if (!slp_node && vec_stmt)
-    gcc_assert (STMT_VINFO_VEC_STMTS (stmt_info).exists ());
-
-  if (STMT_VINFO_TYPE (stmt_info) != store_vec_info_type
-      && (!slp_node
-	  || !slp_node->ldst_lanes
+  if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
+      && (!slp_node->ldst_lanes
 	  || SLP_TREE_CODE (slp_node) == VEC_PERM_EXPR))
     {
       /* Handle stmts whose DEF is used outside the loop-nest that is
 	 being vectorized.  */
-      done = can_vectorize_live_stmts (vinfo, stmt_info, slp_node,
+      done = can_vectorize_live_stmts (vinfo, slp_node,
 				       slp_node_instance, true, NULL);
       gcc_assert (done);
     }
 
-  if (slp_node)
-    STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
+  STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
 
   return is_store;
 }
@@ -14087,123 +13207,49 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
 
 /* Function vect_is_simple_use.
 
-   Same as vect_is_simple_use but also determines the vector operand
-   type of OPERAND and stores it to *VECTYPE.  If the definition of
-   OPERAND is vect_uninitialized_def, vect_constant_def or
-   vect_external_def *VECTYPE will be set to NULL_TREE and the caller
-   is responsible to compute the best suited vector type for the
-   scalar operand.  */
-
-bool
-vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
-		    tree *vectype, stmt_vec_info *def_stmt_info_out,
-		    gimple **def_stmt_out)
-{
-  stmt_vec_info def_stmt_info;
-  gimple *def_stmt;
-  if (!vect_is_simple_use (operand, vinfo, dt, &def_stmt_info, &def_stmt))
-    return false;
-
-  if (def_stmt_out)
-    *def_stmt_out = def_stmt;
-  if (def_stmt_info_out)
-    *def_stmt_info_out = def_stmt_info;
-
-  /* Now get a vector type if the def is internal, otherwise supply
-     NULL_TREE and leave it up to the caller to figure out a proper
-     type for the use stmt.  */
-  if (*dt == vect_internal_def
-      || *dt == vect_induction_def
-      || *dt == vect_reduction_def
-      || *dt == vect_double_reduction_def
-      || *dt == vect_nested_cycle
-      || *dt == vect_first_order_recurrence)
-    {
-      *vectype = STMT_VINFO_VECTYPE (def_stmt_info);
-      gcc_assert (*vectype != NULL_TREE);
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "vect_is_simple_use: vectype %T\n", *vectype);
-    }
-  else if (*dt == vect_uninitialized_def
-	   || *dt == vect_constant_def
-	   || *dt == vect_external_def)
-    *vectype = NULL_TREE;
-  else
-    gcc_unreachable ();
-
-  return true;
-}
-
-/* Function vect_is_simple_use.
-
    Same as vect_is_simple_use but determines the operand by operand
    position OPERAND from either STMT or SLP_NODE, filling in *OP
    and *SLP_DEF (when SLP_NODE is not NULL).  */
 
 bool
-vect_is_simple_use (vec_info *vinfo, stmt_vec_info stmt, slp_tree slp_node,
+vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
 		    unsigned operand, tree *op, slp_tree *slp_def,
 		    enum vect_def_type *dt,
 		    tree *vectype, stmt_vec_info *def_stmt_info_out)
 {
-  if (slp_node)
-    {
-      slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
-      *slp_def = child;
-      *vectype = SLP_TREE_VECTYPE (child);
-      if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
-	{
-	  /* ???  VEC_PERM nodes might be intermediate and their lane value
-	     have no representative (nor do we build a VEC_PERM stmt for
-	     the actual operation).  Note for two-operator nodes we set
-	     a representative but leave scalar stmts empty as we'd only
-	     have one for a subset of lanes.  Ideally no caller would
-	     require *op for internal defs.  */
-	  if (SLP_TREE_REPRESENTATIVE (child))
-	    {
-	      *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
-	      return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
-	    }
-	  else
-	    {
-	      gcc_assert (SLP_TREE_CODE (child) == VEC_PERM_EXPR);
-	      *op = error_mark_node;
-	      *dt = vect_internal_def;
-	      if (def_stmt_info_out)
-		*def_stmt_info_out = NULL;
-	      return true;
-	    }
+  slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
+  *slp_def = child;
+  *vectype = SLP_TREE_VECTYPE (child);
+  if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
+    {
+      /* ???  VEC_PERM nodes might be intermediate and their lane value
+	 have no representative (nor do we build a VEC_PERM stmt for
+	 the actual operation).  Note for two-operator nodes we set
+	 a representative but leave scalar stmts empty as we'd only
+	 have one for a subset of lanes.  Ideally no caller would
+	 require *op for internal defs.  */
+      if (SLP_TREE_REPRESENTATIVE (child))
+	{
+	  *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
+	  return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
 	}
       else
 	{
+	  gcc_assert (SLP_TREE_CODE (child) == VEC_PERM_EXPR);
+	  *op = error_mark_node;
+	  *dt = vect_internal_def;
 	  if (def_stmt_info_out)
 	    *def_stmt_info_out = NULL;
-	  *op = SLP_TREE_SCALAR_OPS (child)[0];
-	  *dt = SLP_TREE_DEF_TYPE (child);
 	  return true;
 	}
     }
   else
     {
-      *slp_def = NULL;
-      if (gassign *ass = dyn_cast <gassign *> (stmt->stmt))
-	{
-	  if (gimple_assign_rhs_code (ass) == COND_EXPR
-	      && COMPARISON_CLASS_P (gimple_assign_rhs1 (ass)))
-	    gcc_unreachable ();
-	  else if (gimple_assign_rhs_code (ass) == VIEW_CONVERT_EXPR)
-	    *op = TREE_OPERAND (gimple_assign_rhs1 (ass), 0);
-	  else
-	    *op = gimple_op (ass, operand + 1);
-	}
-      else if (gcond *cond = dyn_cast <gcond *> (stmt->stmt))
-	*op = gimple_op (cond, operand);
-      else if (gcall *call = dyn_cast <gcall *> (stmt->stmt))
-	*op = gimple_call_arg (call, operand);
-      else
-	gcc_unreachable ();
-      return vect_is_simple_use (*op, vinfo, dt, vectype, def_stmt_info_out);
+      if (def_stmt_info_out)
+	*def_stmt_info_out = NULL;
+      *op = SLP_TREE_SCALAR_OPS (child)[0];
+      *dt = SLP_TREE_DEF_TYPE (child);
+      return true;
     }
 }
 
@@ -14410,6 +13456,8 @@ supportable_widening_operation (vec_info *vinfo,
 
       internal_fn lo, hi, even, odd;
       lookup_hilo_internal_fn (ifn, &lo, &hi);
+      if (BYTES_BIG_ENDIAN)
+	std::swap (lo, hi);
       *code1 = as_combined_fn (lo);
       *code2 = as_combined_fn (hi);
       optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index 2f77e46..50985a6 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -715,7 +715,6 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   stmt_vec_info res = XCNEW (class _stmt_vec_info);
   res->stmt = stmt;
 
-  STMT_VINFO_TYPE (res) = undef_vec_info_type;
   STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
   STMT_VINFO_VECTORIZABLE (res) = true;
   STMT_VINFO_REDUC_TYPE (res) = TREE_CODE_REDUCTION;
@@ -724,7 +723,6 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   STMT_VINFO_REDUC_IDX (res) = -1;
   STMT_VINFO_SLP_VECT_ONLY (res) = false;
   STMT_VINFO_SLP_VECT_ONLY_PATTERN (res) = false;
-  STMT_VINFO_VEC_STMTS (res) = vNULL;
   res->reduc_initial_values = vNULL;
   res->reduc_scalar_results = vNULL;
 
@@ -735,7 +733,7 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   else
     STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
 
-  STMT_SLP_TYPE (res) = loop_vect;
+  STMT_SLP_TYPE (res) = not_vect;
 
   /* This is really "uninitialized" until vect_compute_data_ref_alignment.  */
   res->dr_aux.misalignment = DR_MISALIGNMENT_UNINITIALIZED;
@@ -790,8 +788,6 @@ vec_info::free_stmt_vec_info (stmt_vec_info stmt_info)
 
   stmt_info->reduc_initial_values.release ();
   stmt_info->reduc_scalar_results.release ();
-  STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release ();
-  STMT_VINFO_VEC_STMTS (stmt_info).release ();
   free (stmt_info);
 }
 
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 66a2964..52e1075 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -173,6 +173,8 @@ struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
 /* Describes how we're going to vectorize an individual load or store,
    or a group of loads or stores.  */
 enum vect_memory_access_type {
+  VMAT_UNINITIALIZED,
+
   /* An access to an invariant address.  This is used only for loads.  */
   VMAT_INVARIANT,
 
@@ -184,11 +186,6 @@ enum vect_memory_access_type {
      of invariants.  */
   VMAT_CONTIGUOUS_DOWN,
 
-  /* A simple contiguous access in which the elements need to be permuted
-     after loading or before storing.  Only used for loop vectorization;
-     SLP uses separate permutes.  */
-  VMAT_CONTIGUOUS_PERMUTE,
-
   /* A simple contiguous access in which the elements need to be reversed
      after loading or before storing.  */
   VMAT_CONTIGUOUS_REVERSE,
@@ -210,6 +207,32 @@ enum vect_memory_access_type {
   VMAT_GATHER_SCATTER
 };
 
+/*-----------------------------------------------------------------*/
+/* Info on vectorized defs.                                        */
+/*-----------------------------------------------------------------*/
+enum stmt_vec_info_type {
+  undef_vec_info_type = 0,
+  load_vec_info_type,
+  store_vec_info_type,
+  shift_vec_info_type,
+  op_vec_info_type,
+  call_vec_info_type,
+  call_simd_clone_vec_info_type,
+  assignment_vec_info_type,
+  condition_vec_info_type,
+  comparison_vec_info_type,
+  reduc_vec_info_type,
+  induc_vec_info_type,
+  type_promotion_vec_info_type,
+  type_demotion_vec_info_type,
+  type_conversion_vec_info_type,
+  cycle_phi_info_type,
+  lc_phi_info_type,
+  phi_info_type,
+  recurr_info_type,
+  loop_exit_ctrl_vec_info_type
+};
+
 /************************************************************************
   SLP
  ************************************************************************/
@@ -218,6 +241,23 @@ typedef auto_vec<std::pair<unsigned, unsigned>, 16> auto_lane_permutation_t;
 typedef vec<unsigned> load_permutation_t;
 typedef auto_vec<unsigned, 16> auto_load_permutation_t;
 
+struct vect_data {
+  virtual ~vect_data () = default;
+};
+
+/* Analysis data from vectorizable_simd_clone_call for
+   call_simd_clone_vec_info_type.  */
+struct vect_simd_clone_data : vect_data {
+  virtual ~vect_simd_clone_data () = default;
+  vect_simd_clone_data () = default;
+  vect_simd_clone_data (vect_simd_clone_data &&other) = default;
+
+  /* Selected SIMD clone's function info.  First vector element
+     is SIMD clone's function decl, followed by a pair of trees (base + step)
+     for linear arguments (pair of NULLs for other arguments).  */
+  auto_vec<tree> simd_clone_info;
+};
+
 /* A computation tree of an SLP instance.  Each node corresponds to a group of
    stmts to be packed in a SIMD stmt.  */
 struct _slp_tree {
@@ -246,11 +286,6 @@ struct _slp_tree {
      denotes the number of output lanes.  */
   lane_permutation_t lane_permutation;
 
-  /* Selected SIMD clone's function info.  First vector element
-     is SIMD clone's function decl, followed by a pair of trees (base + step)
-     for linear arguments (pair of NULLs for other arguments).  */
-  vec<tree> simd_clone_info;
-
   tree vectype;
   /* Vectorized defs.  */
   vec<tree> vec_defs;
@@ -284,6 +319,14 @@ struct _slp_tree {
      for loop vectorization.  */
   vect_memory_access_type memory_access_type;
 
+  /* The kind of operation as determined by analysis and optional
+     kind specific data.  */
+  enum stmt_vec_info_type type;
+  vect_data *data;
+
+  template <class T>
+  T& get_data (T& else_) { return data ? *static_cast <T *> (data) : else_; }
+
   /* If not NULL this is a cached failed SLP discovery attempt with
      the lanes that failed during SLP discovery as 'false'.  This is
      a copy of the matches array.  */
@@ -362,13 +405,13 @@ public:
 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S)          (S)->vec_stmts_size
 #define SLP_TREE_LOAD_PERMUTATION(S)             (S)->load_permutation
 #define SLP_TREE_LANE_PERMUTATION(S)             (S)->lane_permutation
-#define SLP_TREE_SIMD_CLONE_INFO(S)              (S)->simd_clone_info
 #define SLP_TREE_DEF_TYPE(S)			 (S)->def_type
 #define SLP_TREE_VECTYPE(S)			 (S)->vectype
 #define SLP_TREE_REPRESENTATIVE(S)		 (S)->representative
 #define SLP_TREE_LANES(S)			 (S)->lanes
 #define SLP_TREE_CODE(S)			 (S)->code
 #define SLP_TREE_MEMORY_ACCESS_TYPE(S)		 (S)->memory_access_type
+#define SLP_TREE_TYPE(S)			 (S)->type
 
 enum vect_partial_vector_style {
     vect_partial_vectors_none,
@@ -904,6 +947,10 @@ public:
      stmt in the chain.  */
   auto_vec<stmt_vec_info> reduction_chains;
 
+  /* Defs that could not be analyzed such as OMP SIMD calls without
+     a LHS.  */
+  auto_vec<stmt_vec_info> alternate_defs;
+
   /* Cost vector for a single scalar iteration.  */
   auto_vec<stmt_info_for_cost> scalar_cost_vec;
 
@@ -1143,6 +1190,7 @@ public:
 #define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
 #define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L)  (L)->inv_pattern_def_seq
 #define LOOP_VINFO_DRS_ADVANCED_BY(L)      (L)->drs_advanced_by
+#define LOOP_VINFO_ALTERNATE_DEFS(L)       (L)->alternate_defs
 
 #define LOOP_VINFO_FULLY_MASKED_P(L)		\
   (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L)	\
@@ -1168,6 +1216,10 @@ public:
    || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L)		\
    || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
 
+#define LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING(L)	\
+  ((L)->may_misalign_stmts.length () > 0		\
+   && !LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (L))
+
 #define LOOP_VINFO_NITERS_KNOWN_P(L)          \
   (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
 
@@ -1216,32 +1268,6 @@ public:
 #define BB_VINFO_DATAREFS(B)         (B)->shared->datarefs
 #define BB_VINFO_DDRS(B)             (B)->shared->ddrs
 
-/*-----------------------------------------------------------------*/
-/* Info on vectorized defs.                                        */
-/*-----------------------------------------------------------------*/
-enum stmt_vec_info_type {
-  undef_vec_info_type = 0,
-  load_vec_info_type,
-  store_vec_info_type,
-  shift_vec_info_type,
-  op_vec_info_type,
-  call_vec_info_type,
-  call_simd_clone_vec_info_type,
-  assignment_vec_info_type,
-  condition_vec_info_type,
-  comparison_vec_info_type,
-  reduc_vec_info_type,
-  induc_vec_info_type,
-  type_promotion_vec_info_type,
-  type_demotion_vec_info_type,
-  type_conversion_vec_info_type,
-  cycle_phi_info_type,
-  lc_phi_info_type,
-  phi_info_type,
-  recurr_info_type,
-  loop_exit_ctrl_vec_info_type
-};
-
 /* Indicates whether/how a variable is used in the scope of loop/basic
    block.  */
 enum vect_relevant {
@@ -1267,26 +1293,12 @@ enum vect_relevant {
   vect_used_in_scope
 };
 
-/* The type of vectorization that can be applied to the stmt: regular loop-based
-   vectorization; pure SLP - the stmt is a part of SLP instances and does not
-   have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
-   a part of SLP instance and also must be loop-based vectorized, since it has
-   uses outside SLP sequences.
-
-   In the loop context the meanings of pure and hybrid SLP are slightly
-   different. By saying that pure SLP is applied to the loop, we mean that we
-   exploit only intra-iteration parallelism in the loop; i.e., the loop can be
-   vectorized without doing any conceptual unrolling, cause we don't pack
-   together stmts from different iterations, only within a single iteration.
-   Loop hybrid SLP means that we exploit both intra-iteration and
-   inter-iteration parallelism (e.g., number of elements in the vector is 4
-   and the slp-group-size is 2, in which case we don't have enough parallelism
-   within an iteration, so we obtain the rest of the parallelism from subsequent
-   iterations by unrolling the loop by 2).  */
+/* The type of vectorization.  pure_slp means the stmt is covered by the
+   SLP graph, not_vect means it is not.  This is mostly used by BB
+   vectorization.  */
 enum slp_vect_type {
-  loop_vect = 0,
+  not_vect = 0,
   pure_slp,
-  hybrid
 };
 
 /* Says whether a statement is a load, a store of a vectorized statement
@@ -1334,8 +1346,6 @@ typedef struct data_reference *dr_p;
 class _stmt_vec_info {
 public:
 
-  enum stmt_vec_info_type type;
-
   /* Indicates whether this stmts is part of a computation whose result is
      used outside the loop.  */
   bool live;
@@ -1359,9 +1369,6 @@ public:
   /* The vector type to be used for the LHS of this statement.  */
   tree vectype;
 
-  /* The vectorized stmts.  */
-  vec<gimple *> vec_stmts;
-
   /* The following is relevant only for stmts that contain a non-scalar
      data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
      at most one such data-ref.  */
@@ -1398,11 +1405,6 @@ public:
      pattern statement.  */
   gimple_seq pattern_def_seq;
 
-  /* Selected SIMD clone's function info.  First vector element
-     is SIMD clone's function decl, followed by a pair of trees (base + step)
-     for linear arguments (pair of NULLs for other arguments).  */
-  vec<tree> simd_clone_info;
-
   /* Classify the def of this stmt.  */
   enum vect_def_type def_type;
 
@@ -1441,10 +1443,6 @@ public:
   /* For both loads and stores.  */
   unsigned simd_lane_access_p : 3;
 
-  /* Classifies how the load or store is going to be implemented
-     for loop vectorization.  */
-  vect_memory_access_type memory_access_type;
-
   /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used.  */
   tree induc_cond_initial_val;
 
@@ -1557,6 +1555,10 @@ struct gather_scatter_info {
   /* The loop-invariant base value.  */
   tree base;
 
+  /* The TBBA alias pointer the value of which determines the alignment
+     of the scalar accesses.  */
+  tree alias_ptr;
+
   /* The original scalar offset, which is a non-loop-invariant SSA_NAME.  */
   tree offset;
 
@@ -1564,9 +1566,6 @@ struct gather_scatter_info {
      being added to the base.  */
   int scale;
 
-  /* The definition type for the vectorized offset.  */
-  enum vect_def_type offset_dt;
-
   /* The type of the vectorized offset.  */
   tree offset_vectype;
 
@@ -1578,17 +1577,14 @@ struct gather_scatter_info {
 };
 
 /* Access Functions.  */
-#define STMT_VINFO_TYPE(S)                 (S)->type
 #define STMT_VINFO_STMT(S)                 (S)->stmt
 #define STMT_VINFO_RELEVANT(S)             (S)->relevant
 #define STMT_VINFO_LIVE_P(S)               (S)->live
 #define STMT_VINFO_VECTYPE(S)              (S)->vectype
-#define STMT_VINFO_VEC_STMTS(S)            (S)->vec_stmts
 #define STMT_VINFO_VECTORIZABLE(S)         (S)->vectorizable
 #define STMT_VINFO_DATA_REF(S)             ((S)->dr_aux.dr + 0)
 #define STMT_VINFO_GATHER_SCATTER_P(S)	   (S)->gather_scatter_p
 #define STMT_VINFO_STRIDED_P(S)	   	   (S)->strided_p
-#define STMT_VINFO_MEMORY_ACCESS_TYPE(S)   (S)->memory_access_type
 #define STMT_VINFO_SIMD_LANE_ACCESS_P(S)   (S)->simd_lane_access_p
 #define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
 #define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
@@ -1614,7 +1610,6 @@ struct gather_scatter_info {
 #define STMT_VINFO_IN_PATTERN_P(S)         (S)->in_pattern_p
 #define STMT_VINFO_RELATED_STMT(S)         (S)->related_stmt
 #define STMT_VINFO_PATTERN_DEF_SEQ(S)      (S)->pattern_def_seq
-#define STMT_VINFO_SIMD_CLONE_INFO(S)	   (S)->simd_clone_info
 #define STMT_VINFO_DEF_TYPE(S)             (S)->def_type
 #define STMT_VINFO_GROUPED_ACCESS(S) \
   ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
@@ -1651,10 +1646,17 @@ struct gather_scatter_info {
 
 #define STMT_VINFO_RELEVANT_P(S)          ((S)->relevant != vect_unused_in_scope)
 
-#define HYBRID_SLP_STMT(S)                ((S)->slp_type == hybrid)
 #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
 #define STMT_SLP_TYPE(S)                   (S)->slp_type
 
+#define GATHER_SCATTER_LEGACY_P(info) ((info).decl != NULL_TREE \
+				       && (info).ifn == IFN_LAST)
+#define GATHER_SCATTER_IFN_P(info) ((info).decl == NULL_TREE \
+				    && (info).ifn != IFN_LAST)
+#define GATHER_SCATTER_EMULATED_P(info) ((info).decl == NULL_TREE \
+					 && (info).ifn == IFN_LAST)
+
+
 /* Contains the scalar or vector costs for a vec_info.  */
 class vector_costs
 {
@@ -1714,7 +1716,8 @@ public:
   unsigned int outside_cost () const;
   unsigned int total_cost () const;
   unsigned int suggested_unroll_factor () const;
-  machine_mode suggested_epilogue_mode () const;
+  machine_mode suggested_epilogue_mode (int &masked) const;
+  bool costing_for_scalar () const { return m_costing_for_scalar; }
 
 protected:
   unsigned int record_stmt_cost (stmt_vec_info, vect_cost_model_location,
@@ -1738,8 +1741,13 @@ protected:
   unsigned int m_suggested_unroll_factor;
 
   /* The suggested mode to be used for a vectorized epilogue or VOIDmode,
-     determined at finish_cost.  */
+     determined at finish_cost.  m_masked_epilogue specifies whether the
+     epilogue should use masked vectorization, regardless of the
+     --param vect-partial-vector-usage default.  If -1 then the
+     --param setting takes precedence.  If the user explicitly specified
+     --param vect-partial-vector-usage then that takes precedence.  */
   machine_mode m_suggested_epilogue_mode;
+  int m_masked_epilogue;
 
   /* True if finish_cost has been called.  */
   bool m_finished;
@@ -1755,6 +1763,7 @@ vector_costs::vector_costs (vec_info *vinfo, bool costing_for_scalar)
     m_costs (),
     m_suggested_unroll_factor(1),
     m_suggested_epilogue_mode(VOIDmode),
+    m_masked_epilogue (-1),
     m_finished (false)
 {
 }
@@ -1815,9 +1824,10 @@ vector_costs::suggested_unroll_factor () const
 /* Return the suggested epilogue mode.  */
 
 inline machine_mode
-vector_costs::suggested_epilogue_mode () const
+vector_costs::suggested_epilogue_mode (int &masked_p) const
 {
   gcc_checking_assert (m_finished);
+  masked_p = m_masked_epilogue;
   return m_suggested_epilogue_mode;
 }
 
@@ -1990,6 +2000,13 @@ add_stmt_cost (vector_costs *costs, int count,
 	       tree vectype, int misalign,
 	       enum vect_cost_model_location where)
 {
+  /* Even though a vector type might be set on stmt do not pass that on when
+     costing the scalar IL.  A SLP node shouldn't have been recorded.  */
+  if (costs->costing_for_scalar ())
+    {
+      vectype = NULL_TREE;
+      gcc_checking_assert (node == NULL);
+    }
   unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
 					misalign, where);
   if (dump_file && (dump_flags & TDF_DETAILS))
@@ -2415,10 +2432,7 @@ extern bool vect_chooses_same_modes_p (machine_mode, machine_mode);
 extern bool vect_get_loop_mask_type (loop_vec_info);
 extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
 				stmt_vec_info * = NULL, gimple ** = NULL);
-extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
-				tree *, stmt_vec_info * = NULL,
-				gimple ** = NULL);
-extern bool vect_is_simple_use (vec_info *, stmt_vec_info, slp_tree,
+extern bool vect_is_simple_use (vec_info *, slp_tree,
 				unsigned, tree *, slp_tree *,
 				enum vect_def_type *,
 				tree *, stmt_vec_info * = NULL);
@@ -2479,18 +2493,11 @@ extern void vect_finish_stmt_generation (vec_info *, stmt_vec_info, gimple *,
 					 gimple_stmt_iterator *);
 extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *);
 extern tree vect_get_store_rhs (stmt_vec_info);
-void vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info, unsigned,
-				    tree op, vec<tree> *, tree = NULL);
-void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
+void vect_get_vec_defs (vec_info *, slp_tree,
 			tree, vec<tree> *,
 			tree = NULL, vec<tree> * = NULL,
 			tree = NULL, vec<tree> * = NULL,
 			tree = NULL, vec<tree> * = NULL);
-void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
-			tree, tree, vec<tree> *,
-			tree = NULL, tree = NULL, vec<tree> * = NULL,
-			tree = NULL, tree = NULL, vec<tree> * = NULL,
-			tree = NULL, tree = NULL, vec<tree> * = NULL);
 extern tree vect_init_vector (vec_info *, stmt_vec_info, tree, tree,
                               gimple_stmt_iterator *);
 extern tree vect_get_slp_vect_def (slp_tree, unsigned);
@@ -2499,8 +2506,7 @@ extern bool vect_transform_stmt (vec_info *, stmt_vec_info,
 				 slp_tree, slp_instance);
 extern void vect_remove_stores (vec_info *, stmt_vec_info);
 extern bool vect_nop_conversion_p (stmt_vec_info);
-extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *,
-				     slp_tree,
+extern opt_result vect_analyze_stmt (vec_info *, slp_tree,
 				     slp_instance, stmt_vector_for_cost *);
 extern void vect_get_load_cost (vec_info *, stmt_vec_info, slp_tree, int,
 				dr_alignment_support, int, bool,
@@ -2528,7 +2534,8 @@ extern bool ref_within_array_bound (gimple *, tree);
 /* In tree-vect-data-refs.cc.  */
 extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
 extern enum dr_alignment_support vect_supportable_dr_alignment
-				   (vec_info *, dr_vec_info *, tree, int);
+				   (vec_info *, dr_vec_info *, tree, int,
+				     gather_scatter_info * = nullptr);
 extern tree vect_get_smallest_scalar_type (stmt_vec_info, tree);
 extern opt_result vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *);
 extern bool vect_slp_analyze_instance_dependence (vec_info *, slp_instance);
@@ -2546,7 +2553,7 @@ extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
 extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
 						 vec<data_reference_p> *,
 						 vec<int> *, int);
-extern opt_result vect_analyze_data_refs (vec_info *, poly_uint64 *, bool *);
+extern opt_result vect_analyze_data_refs (vec_info *, bool *);
 extern void vect_record_base_alignments (vec_info *);
 extern tree vect_create_data_ref_ptr (vec_info *,
 				      stmt_vec_info, tree, class loop *, tree,
@@ -2562,17 +2569,10 @@ extern internal_fn vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, boo
 extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
 extern internal_fn vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT,
 					      bool, vec<int> * = nullptr);
-extern void vect_permute_store_chain (vec_info *, vec<tree> &,
-				      unsigned int, stmt_vec_info,
-				      gimple_stmt_iterator *, vec<tree> *);
 extern tree vect_setup_realignment (vec_info *,
 				    stmt_vec_info, gimple_stmt_iterator *,
 				    tree *, enum dr_alignment_support, tree,
 	                            class loop **);
-extern void vect_transform_grouped_load (vec_info *, stmt_vec_info, vec<tree>,
-					 int, gimple_stmt_iterator *);
-extern void vect_record_grouped_load_vectors (vec_info *,
-					      stmt_vec_info, vec<tree>);
 extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
 extern tree vect_get_new_ssa_name (tree, enum vect_var_kind,
 				   const char * = NULL);
@@ -2641,8 +2641,7 @@ extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info,
 				    slp_tree, slp_instance,
 				    stmt_vector_for_cost *);
 extern bool vectorizable_induction (loop_vec_info, stmt_vec_info,
-				    gimple **, slp_tree,
-				    stmt_vector_for_cost *);
+				    slp_tree, stmt_vector_for_cost *);
 extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info,
 				      gimple_stmt_iterator *,
 				      slp_tree);
@@ -2650,12 +2649,12 @@ extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info,
 				      slp_tree, slp_instance);
 extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
 extern bool vect_transform_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
-extern bool vectorizable_phi (vec_info *, stmt_vec_info, gimple **, slp_tree,
+extern bool vectorizable_phi (vec_info *, stmt_vec_info, slp_tree,
 			      stmt_vector_for_cost *);
 extern bool vectorizable_recurr (loop_vec_info, stmt_vec_info,
-				  gimple **, slp_tree, stmt_vector_for_cost *);
+				  slp_tree, stmt_vector_for_cost *);
 extern bool vectorizable_early_exit (vec_info *, stmt_vec_info,
-				     gimple_stmt_iterator *, gimple **,
+				     gimple_stmt_iterator *,
 				     slp_tree, stmt_vector_for_cost *);
 extern bool vect_emulated_vector_p (tree);
 extern bool vect_can_vectorize_without_simd_p (tree_code);
@@ -2682,7 +2681,7 @@ extern bool vect_slp_analyze_operations (vec_info *);
 extern void vect_schedule_slp (vec_info *, const vec<slp_instance> &);
 extern opt_result vect_analyze_slp (vec_info *, unsigned, bool);
 extern bool vect_make_slp_decision (loop_vec_info);
-extern void vect_detect_hybrid_slp (loop_vec_info);
+extern bool vect_detect_hybrid_slp (loop_vec_info);
 extern void vect_optimize_slp (vec_info *);
 extern void vect_gather_slp_loads (vec_info *);
 extern tree vect_get_slp_scalar_def (slp_tree, unsigned);
@@ -2833,18 +2832,6 @@ vect_is_reduction (stmt_vec_info stmt_info)
   return STMT_VINFO_REDUC_IDX (stmt_info) >= 0;
 }
 
-/* Returns the memory acccess type being used to vectorize the statement.  If
-   SLP this is read from NODE, otherwise it's read from the STMT_VINFO.  */
-
-inline vect_memory_access_type
-vect_mem_access_type (stmt_vec_info stmt_info, slp_tree node)
-{
-  if (node)
-    return SLP_TREE_MEMORY_ACCESS_TYPE (node);
-  else
-    return STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info);
-}
-
 /* If STMT_INFO describes a reduction, return the vect_reduction_type
    of the reduction it describes, otherwise return -1.  */
 inline int
diff --git a/gcc/tree.cc b/gcc/tree.cc
index 6a055c8..0f02924 100644
--- a/gcc/tree.cc
+++ b/gcc/tree.cc
@@ -32,6 +32,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "backend.h"
 #include "target.h"
+#include "memmodel.h"
+#include "tm_p.h"
 #include "tree.h"
 #include "gimple.h"
 #include "tree-pass.h"
@@ -15418,6 +15420,65 @@ get_target_clone_attr_len (tree arglist)
   return str_len_sum;
 }
 
+/* Returns an auto_vec of string_slices containing the version strings from
+   ARGLIST.  DEFAULT_COUNT is incremented for each default version found.  */
+
+auto_vec<string_slice>
+get_clone_attr_versions (const tree arglist, int *default_count)
+{
+  gcc_assert (TREE_CODE (arglist) == TREE_LIST);
+  auto_vec<string_slice> versions;
+
+  static const char separator_str[] = {TARGET_CLONES_ATTR_SEPARATOR, 0};
+  string_slice separators = string_slice (separator_str);
+
+  for (tree arg = arglist; arg; arg = TREE_CHAIN (arg))
+    {
+      string_slice str = string_slice (TREE_STRING_POINTER (TREE_VALUE (arg)));
+      while (str.is_valid ())
+	{
+	  string_slice attr = string_slice::tokenize (&str, separators);
+	  attr = attr.strip ();
+
+	  if (attr == "default" && default_count)
+	    (*default_count)++;
+	  versions.safe_push (attr);
+	}
+    }
+  return versions;
+}
+
+/* Returns an auto_vec of string_slices containing the version strings from
+   the target_clone attribute from DECL.  DEFAULT_COUNT is incremented for each
+   default version found.  */
+auto_vec<string_slice>
+get_clone_versions (const tree decl, int *default_count)
+{
+  tree attr = lookup_attribute ("target_clones", DECL_ATTRIBUTES (decl));
+  if (!attr)
+    return auto_vec<string_slice> ();
+  tree arglist = TREE_VALUE (attr);
+  return get_clone_attr_versions (arglist, default_count);
+}
+
+/* If DECL has a target_version attribute, returns a string_slice containing the
+   attribute value.  Otherwise, returns string_slice::invalid.
+   Only works for target_version due to target attributes allowing multiple
+   string arguments to specify one target.  */
+string_slice
+get_target_version (const tree decl)
+{
+  gcc_assert (!TARGET_HAS_FMV_TARGET_ATTRIBUTE);
+
+  tree attr = lookup_attribute ("target_version", DECL_ATTRIBUTES (decl));
+
+  if (!attr)
+    return string_slice::invalid ();
+
+  return string_slice (TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))))
+	   .strip ();
+}
+
 void
 tree_cc_finalize (void)
 {
diff --git a/gcc/tree.h b/gcc/tree.h
index e87fa0f..c0e434b 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -22,6 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "tree-core.h"
 #include "options.h"
+#include "vec.h"
 
 /* Convert a target-independent built-in function code to a combined_fn.  */
 
@@ -2232,6 +2233,12 @@ class auto_suppress_location_wrappers
 #define OMP_CLAUSE_OPERAND(NODE, I)				\
 	OMP_CLAUSE_ELT_CHECK (NODE, I)
 
+/* True if the clause decl NODE contains an OpenMP iterator.  */
+#define OMP_ITERATOR_DECL_P(NODE) \
+	(TREE_CODE (NODE) == TREE_LIST				\
+	 && TREE_PURPOSE (NODE)					\
+	 && TREE_CODE (TREE_PURPOSE (NODE)) == TREE_VEC)
+
 /* In a BLOCK (scope) node:
    Variables declared in the scope NODE.  */
 #define BLOCK_VARS(NODE) (BLOCK_CHECK (NODE)->block.vars)
@@ -7082,4 +7089,14 @@ extern tree get_attr_nonstring_decl (tree, tree * = NULL);
 
 extern int get_target_clone_attr_len (tree);
 
+/* Returns the version string for a decl with target_version attribute.
+   Returns an invalid string_slice if no attribute is present.  */
+extern string_slice get_target_version (const tree);
+/* Returns a vector of the version strings from a target_clones attribute on
+   a decl.  Can also record the number of default versions found.  */
+extern auto_vec<string_slice> get_clone_versions (const tree, int * = NULL);
+/* Returns a vector of the version strings from a target_clones attribute
+   directly.  */
+extern auto_vec<string_slice> get_clone_attr_versions (const tree, int *);
+
 #endif  /* GCC_TREE_H  */
diff --git a/gcc/value-range.h b/gcc/value-range.h
index 5c358f3..3bc02db 100644
--- a/gcc/value-range.h
+++ b/gcc/value-range.h
@@ -324,7 +324,7 @@ public:
   virtual void update_bitmask (const class irange_bitmask &) override;
   virtual irange_bitmask get_bitmask () const override;
 
-  virtual void verify_range () const;
+  virtual void verify_range () const override;
 protected:
   void maybe_resize (int needed);
   virtual void set (tree, tree, value_range_kind = VR_RANGE) override;
@@ -422,7 +422,7 @@ public:
   bool contains_p (const wide_int &) const;
   wide_int lower_bound () const;
   wide_int upper_bound () const;
-  virtual void verify_range () const;
+  virtual void verify_range () const final override;
   irange_bitmask get_bitmask () const final override;
   void update_bitmask (const irange_bitmask &) final override;
 protected:
@@ -594,7 +594,7 @@ public:
   bool nan_signbit_p (bool &signbit) const;
   bool known_isnormal () const;
   bool known_isdenormal_or_zero () const;
-  virtual void verify_range () const;
+  virtual void verify_range () const override;
 protected:
   virtual bool contains_p (tree cst) const override;
   virtual void set (tree, tree, value_range_kind = VR_RANGE) override;
diff --git a/gcc/varasm.cc b/gcc/varasm.cc
index 10c1d2e..000ad9e 100644
--- a/gcc/varasm.cc
+++ b/gcc/varasm.cc
@@ -871,7 +871,7 @@ mergeable_string_section (tree decl ATTRIBUTE_UNUSED,
   if (HAVE_GAS_SHF_MERGE && flag_merge_constants
       && TREE_CODE (decl) == STRING_CST
       && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
-      && align <= 256
+      && align <= MAX_ALIGN_MERGABLE
       && (len = int_size_in_bytes (TREE_TYPE (decl))) > 0
       && TREE_STRING_LENGTH (decl) == len)
     {
@@ -885,7 +885,7 @@ mergeable_string_section (tree decl ATTRIBUTE_UNUSED,
 
       mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (TREE_TYPE (decl)));
       modesize = GET_MODE_BITSIZE (mode);
-      if (modesize >= 8 && modesize <= 256
+      if (modesize >= 8 && modesize <= MAX_ALIGN_MERGABLE
 	  && (modesize & (modesize - 1)) == 0)
 	{
 	  if (align < modesize)
@@ -919,16 +919,14 @@ mergeable_string_section (tree decl ATTRIBUTE_UNUSED,
 /* Return the section to use for constant merging.  */
 
 section *
-mergeable_constant_section (machine_mode mode ATTRIBUTE_UNUSED,
-			    unsigned HOST_WIDE_INT align ATTRIBUTE_UNUSED,
-			    unsigned int flags ATTRIBUTE_UNUSED)
+mergeable_constant_section (unsigned HOST_WIDE_INT size_bits,
+			    unsigned HOST_WIDE_INT align,
+			    unsigned int flags)
 {
   if (HAVE_GAS_SHF_MERGE && flag_merge_constants
-      && mode != VOIDmode
-      && mode != BLKmode
-      && known_le (GET_MODE_BITSIZE (mode), align)
+      && size_bits <= align
       && align >= 8
-      && align <= 256
+      && align <= MAX_ALIGN_MERGABLE
       && (align & (align - 1)) == 0)
     {
       const char *prefix = function_mergeable_rodata_prefix ();
@@ -940,6 +938,38 @@ mergeable_constant_section (machine_mode mode ATTRIBUTE_UNUSED,
     }
   return readonly_data_section;
 }
+
+
+/* Return the section to use for constant merging. Like the above
+   but the size stored as a tree.  */
+static section *
+mergeable_constant_section (tree size_bits,
+			    unsigned HOST_WIDE_INT align,
+			    unsigned int flags)
+{
+  if (!size_bits || !tree_fits_uhwi_p (size_bits))
+    return readonly_data_section;
+  return mergeable_constant_section (tree_to_uhwi (size_bits), align, flags);
+}
+
+
+/* Return the section to use for constant merging. Like the above
+   but given a mode rather than the size.  */
+
+section *
+mergeable_constant_section (machine_mode mode,
+			    unsigned HOST_WIDE_INT align,
+			    unsigned int flags)
+{
+  /* If the mode is unknown (BLK or VOID), then return a non mergable section.  */
+  if (mode == BLKmode || mode == VOIDmode)
+    return readonly_data_section;
+  unsigned HOST_WIDE_INT size;
+  if (!GET_MODE_BITSIZE (mode).is_constant (&size))
+    return readonly_data_section;
+  return mergeable_constant_section (size, align, flags);
+}
+
 
 /* Given NAME, a putative register name, discard any customary prefixes.  */
 
@@ -1734,7 +1764,8 @@ assemble_asm (tree asm_str)
 	  constraints[i]
 	    = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (tail)));
 	  if (!parse_output_constraint (&constraints[i], i, ninputs, noutputs,
-					&allows_mem, &allows_reg, &is_inout))
+					&allows_mem, &allows_reg, &is_inout,
+					nullptr))
 	    goto done;
 	  if (is_inout)
 	    {
@@ -1776,7 +1807,7 @@ assemble_asm (tree asm_str)
 	    = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (tail)));
 	  if (!parse_input_constraint (&constraints[i + noutputs], i,
 				       ninputs, noutputs, 0, constraints,
-				       &allows_mem, &allows_reg))
+				       &allows_mem, &allows_reg, nullptr))
 	    goto done;
 	  if (strchr (constraints[i], '%'))
 	    {
@@ -2907,7 +2938,7 @@ assemble_name_resolve (const char *name)
       ultimate_transparent_alias_target (&id);
       if (id != id_orig)
 	name = IDENTIFIER_POINTER (id);
-      gcc_assert (! TREE_CHAIN (id));
+      gcc_assert (!IDENTIFIER_TRANSPARENT_ALIAS (id));
     }
 
   return name;
@@ -7452,7 +7483,7 @@ default_elf_select_section (tree decl, int reloc,
     case SECCAT_RODATA_MERGE_STR_INIT:
       return mergeable_string_section (DECL_INITIAL (decl), align, 0);
     case SECCAT_RODATA_MERGE_CONST:
-      return mergeable_constant_section (DECL_MODE (decl), align, 0);
+      return mergeable_constant_section (DECL_SIZE (decl), align, 0);
     case SECCAT_SRODATA:
       sname = ".sdata2";
       break;
diff --git a/gcc/vec.cc b/gcc/vec.cc
index 55f5f3d..38314d7 100644
--- a/gcc/vec.cc
+++ b/gcc/vec.cc
@@ -176,6 +176,74 @@ dump_vec_loc_statistics (void)
   vec_mem_desc.dump (VEC_ORIGIN);
 }
 
+/* Gets the next token from STR delimited by DELIMS (deliminator not included
+   in returned string).
+
+   Updates STR to be the remaining string after the given token.
+
+   STR and DELIMS must both be valid string_slices.
+
+   If there aren't any of the chars in DELIM in STR (ie no more tokens in STR)
+   then returns the string, and updates STR to be invalid.  */
+string_slice
+string_slice::tokenize (string_slice *str, string_slice delims)
+{
+  const char *ptr = str->begin ();
+
+  gcc_assert (str->is_valid () && delims.is_valid ());
+
+  for (; ptr < str->end (); ptr++)
+    for (char c : delims)
+      if (*ptr == c)
+	{
+	  /* Update the input string to be the remaining string.  */
+	  const char *str_begin = str->begin ();
+	  *str = string_slice (ptr  + 1, str->end ());
+	  return string_slice (str_begin, ptr);
+	}
+
+  /* If no deliminators between the start and end, return the whole string.  */
+  string_slice res = *str;
+  *str = string_slice::invalid ();
+  return res;
+}
+
+/* Compares the string_slices STR1 and STR2 giving a lexograpical ordering.
+   Returns -1 if STR1 comes before STR2, 1 if STR1 comes after, and 0 if the
+   string_slices have the same contents.  */
+
+int
+string_slice::strcmp (string_slice str1, string_slice str2)
+{
+  for (unsigned int i = 0; i < str1.size () && i < str2.size (); i++)
+    {
+      if (str1[i] < str2[i])
+	return -1;
+      if (str1[i] > str2[i])
+	return 1;
+    }
+
+  if (str1.size () < str2.size ())
+    return -1;
+  if (str1.size () > str2.size ())
+    return 1;
+  return 0;
+}
+
+string_slice
+string_slice::strip ()
+{
+  const char *start = this->begin ();
+  const char *end = this->end ();
+
+  while (start < end && ISSPACE (*start))
+    start++;
+  while (end > start && ISSPACE (*(end-1)))
+    end--;
+
+  return string_slice (start, end);
+}
+
 #if CHECKING_P
 /* Report qsort comparator CMP consistency check failure with P1, P2, P3 as
    witness elements.  */
@@ -584,6 +652,159 @@ test_auto_alias ()
   ASSERT_EQ (val, 0);
 }
 
+static void
+test_string_slice_initializers ()
+{
+  string_slice str1 = string_slice ();
+  ASSERT_TRUE (str1.is_valid ());
+  ASSERT_EQ (str1.size (), 0);
+
+  string_slice str2 = string_slice ("Test string");
+  ASSERT_TRUE (str2.is_valid ());
+  ASSERT_EQ (str2.size (), 11);
+
+  string_slice str3 = "Test string the second";
+  ASSERT_TRUE (str3.is_valid ());
+  ASSERT_EQ (str3.size (), 22);
+
+  string_slice str4 = string_slice ("Test string", 4);
+  ASSERT_TRUE (str4.is_valid ());
+  ASSERT_EQ (str4.size (), 4);
+}
+
+static void
+test_string_slice_tokenize ()
+{
+  string_slice test_string_slice = "";
+  string_slice test_delims = ",";
+
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_FALSE (test_string_slice.is_valid ());
+
+  test_string_slice = ",";
+  test_delims = ",";
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     string_slice (""));
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     string_slice (""));
+  ASSERT_FALSE (test_string_slice.is_valid ());
+
+  test_string_slice = ",test.,.test, ,  test  ";
+  test_delims = ",.";
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "test");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "test");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), " ");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "  test  ");
+  ASSERT_FALSE (test_string_slice.is_valid ());
+
+  const char *test_string
+    = "This is the test string, it \0 is for testing, 123 ,,";
+  test_string_slice = string_slice (test_string, 52);
+  test_delims = string_slice (",\0", 2);
+
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "This is the test string");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     " it ");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     " is for testing");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     " 123 ");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "");
+  ASSERT_FALSE (test_string_slice.is_valid ());
+}
+
+static void
+test_string_slice_strcmp ()
+{
+  ASSERT_EQ (string_slice::strcmp (string_slice (),
+				   string_slice ()), 0);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("test"),
+				   string_slice ()), 1);
+  ASSERT_EQ (string_slice::strcmp (string_slice (),
+				   string_slice ("test")), -1);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("test"),
+				   string_slice ("test")), 0);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("a"),
+				   string_slice ("b")), -1);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("b"),
+				   string_slice ("a")), 1);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("ab", 1),
+				   string_slice ("a")), 0);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("ab", 2),
+				   string_slice ("a")), 1);
+}
+
+static void
+test_string_slice_equality ()
+{
+  ASSERT_TRUE (string_slice () == string_slice ());
+  ASSERT_FALSE (string_slice ("test") == string_slice ());
+  ASSERT_FALSE ("test" == string_slice ());
+  ASSERT_FALSE (string_slice () == string_slice ("test"));
+  ASSERT_FALSE (string_slice () == "test");
+  ASSERT_TRUE (string_slice ("test") == string_slice ("test"));
+  ASSERT_TRUE ("test" == string_slice ("test"));
+  ASSERT_TRUE (string_slice ("test") == "test");
+  ASSERT_FALSE (string_slice ("a") == string_slice ("b"));
+  ASSERT_FALSE ("a" == string_slice ("b"));
+  ASSERT_FALSE (string_slice ("a") == "b");
+  ASSERT_FALSE (string_slice ("b") == string_slice ("a"));
+  ASSERT_TRUE (string_slice ("ab", 1) == string_slice ("a"));
+  ASSERT_TRUE (string_slice ("ab", 1) == "a");
+  ASSERT_FALSE (string_slice ("ab", 2) == string_slice ("a"));
+  ASSERT_FALSE (string_slice ("ab", 2) == "a");
+}
+
+static void
+test_string_slice_inequality ()
+{
+  ASSERT_FALSE (string_slice () != string_slice ());
+  ASSERT_TRUE (string_slice ("test") != string_slice ());
+  ASSERT_TRUE ("test" != string_slice ());
+  ASSERT_TRUE (string_slice () != string_slice ("test"));
+  ASSERT_TRUE (string_slice () != "test");
+  ASSERT_FALSE (string_slice ("test") != string_slice ("test"));
+  ASSERT_FALSE ("test" != string_slice ("test"));
+  ASSERT_FALSE (string_slice ("test") != "test");
+  ASSERT_TRUE (string_slice ("a") != string_slice ("b"));
+  ASSERT_TRUE ("a" != string_slice ("b"));
+  ASSERT_TRUE (string_slice ("a") != "b");
+  ASSERT_TRUE (string_slice ("b") != string_slice ("a"));
+  ASSERT_FALSE (string_slice ("ab", 1) != string_slice ("a"));
+  ASSERT_FALSE (string_slice ("ab", 1) != "a");
+  ASSERT_TRUE (string_slice ("ab", 2) != string_slice ("a"));
+  ASSERT_TRUE (string_slice ("ab", 2) != "a");
+}
+
+static void
+test_string_slice_invalid ()
+{
+  ASSERT_FALSE (string_slice::invalid ().is_valid ());
+  ASSERT_FALSE (string_slice (NULL, 1).is_valid ());
+  ASSERT_TRUE (string_slice (NULL, (size_t) 0).is_valid ());
+  ASSERT_TRUE (string_slice ("Test", (size_t) 0).is_valid ());
+  ASSERT_TRUE (string_slice ().is_valid ());
+}
+
+static void
+test_string_slice_strip ()
+{
+  ASSERT_EQ (string_slice ("   test   ").strip (), string_slice ("test"));
+  ASSERT_EQ (string_slice ("\t   test string\t   \n ").strip (),
+	     string_slice ("test string"));
+  ASSERT_EQ (string_slice ("test").strip (), string_slice ("test"));
+  ASSERT_EQ (string_slice ().strip (), string_slice ());
+  ASSERT_EQ (string_slice ("\t  \n \t   ").strip (), string_slice ());
+}
+
 /* Run all of the selftests within this file.  */
 
 void
@@ -604,6 +825,13 @@ vec_cc_tests ()
   test_reverse ();
   test_auto_delete_vec ();
   test_auto_alias ();
+  test_string_slice_initializers ();
+  test_string_slice_tokenize ();
+  test_string_slice_strcmp ();
+  test_string_slice_equality ();
+  test_string_slice_inequality ();
+  test_string_slice_invalid ();
+  test_string_slice_strip ();
 }
 
 } // namespace selftest
diff --git a/gcc/vec.h b/gcc/vec.h
index 7e112d1..0ea7a49 100644
--- a/gcc/vec.h
+++ b/gcc/vec.h
@@ -2495,4 +2495,54 @@ make_array_slice (T *base, unsigned int size)
 # pragma GCC poison m_vec m_vecpfx m_vecdata
 #endif
 
+/* string_slice inherits from array_slice, specifically to refer to a substring
+   of a character array.
+   It includes some string like helpers.  */
+class string_slice : public array_slice<const char>
+{
+public:
+  string_slice () : array_slice<const char> () {}
+  string_slice (const char *str) : array_slice (str, strlen (str)) {}
+  explicit string_slice (const char *str, size_t len)
+    : array_slice (str, len) {}
+  explicit string_slice (const char *start, const char *end)
+    : array_slice (start, end - start) {}
+
+  friend bool operator== (const string_slice &lhs, const string_slice &rhs)
+  {
+    if (!lhs.is_valid () || !rhs.is_valid ())
+      return false;
+    if (lhs.size () != rhs.size ())
+      return false;
+    /* Case where either is a NULL pointer and therefore, as both are valid,
+       both are empty slices with length 0.  */
+    if (lhs.size () == 0)
+      return true;
+    return memcmp (lhs.begin (), rhs.begin (), lhs.size ()) == 0;
+  }
+
+  friend bool operator!= (const string_slice &lhs, const string_slice &rhs)
+  {
+    return !(lhs == rhs);
+  }
+
+  /* Returns an invalid string_slice.  */
+  static string_slice invalid ()
+  {
+    return string_slice (nullptr, ~0U);
+  }
+
+  /* tokenize is used to split a string by some deliminator into
+     string_slice's.  Similarly to the posix strtok_r.but without modifying the
+     input string, and returning all tokens which may be empty in the case
+     of an empty input string of consecutive deliminators.  */
+  static string_slice tokenize (string_slice *str, string_slice delims);
+
+  /* Removes white space from the front and back of the string_slice.  */
+  string_slice strip ();
+
+  /* Compares two string_slices in lexographical ordering.  */
+  static int strcmp (string_slice str1, string_slice str2);
+};
+
 #endif // GCC_VEC_H
diff --git a/gcc/warning-control.cc b/gcc/warning-control.cc
index 48446a7..e04ef69 100644
--- a/gcc/warning-control.cc
+++ b/gcc/warning-control.cc
@@ -28,7 +28,7 @@
 #include "gimple.h"
 #include "cgraph.h"
 #include "hash-map.h"
-#include "diagnostic-spec.h"
+#include "gcc-diagnostic-spec.h"
 
 /* Return the no-warning bit for EXPR.  */